diff --git a/src/cuda/GPU_Microbenchmark/Makefile b/src/cuda/GPU_Microbenchmark/Makefile index b3ad4de64..91eb0ed7b 100644 --- a/src/cuda/GPU_Microbenchmark/Makefile +++ b/src/cuda/GPU_Microbenchmark/Makefile @@ -2,6 +2,9 @@ BASE_DIR := $(shell pwd) BIN_DIR := $(BASE_DIR)/bin ALL_SUB_DIRS = $(wildcard ubench/*/*/) GMMA_DIRS = $(filter %gmma/,$(ALL_SUB_DIRS)) +# ubench/wgmma/wgmma builds wgmma_verify (standalone, no CUTLASS); its CuTe +# benches are guarded inside its own Makefile, so keep it in the default build. +GMMA_DIRS := $(filter-out ubench/wgmma/wgmma/,$(GMMA_DIRS)) SUB_DIRS = $(filter-out $(GMMA_DIRS),$(ALL_SUB_DIRS)) SUB_DIRS_ALL = $(SUB_DIRS:%=all-%) SUB_DIRS_CLEAN = $(SUB_DIRS:%=clean-%) diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/Makefile b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/Makefile new file mode 100644 index 000000000..e20ac26ab --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/Makefile @@ -0,0 +1,44 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin + +CC := nvcc +EXE := gpgpusim_griddepcontrol_semantics +SRC := griddepcontrol_semantics.cu + +# Programmatic dependent launch / grid dependency control is Hopper-era CUDA +# functionality. Keep the default narrow, but allow callers to override ARCH. +ARCH ?= sm_90 + +CUDA_CPPFLAGS = +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +CUDA_SRC_FILES := $(filter %.cu, $(SRC)) +CUDA_LIB_FILES := $(CUDA_SRC_FILES:.cu=.a) + +.DEFAULT_GOAL := all + +all: $(BIN_DIR)/$(EXE) + +$(BIN_DIR): + mkdir -p $@ + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(BIN_DIR)/$(EXE): $(CUDA_LIB_FILES) | $(BIN_DIR) + $(CC) $(NVCC_FLAGS) $^ -o $@ -lcudart --cudart shared + +run: $(BIN_DIR)/$(EXE) + $(BIN_DIR)/$(EXE) + +ptx: $(BIN_DIR)/$(EXE) + cuobjdump -ptx $(BIN_DIR)/$(EXE) | tee $(EXE).ptx + +sass: $(BIN_DIR)/$(EXE) + cuobjdump -sass $(BIN_DIR)/$(EXE) | tee $(EXE).sass + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(EXE) $(CUDA_LIB_FILES) + rm -f $(BIN_DIR)/$(EXE) + +.PHONY: all clean run ptx sass diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/griddepcontrol_semantics.cu b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/griddepcontrol_semantics.cu new file mode 100644 index 000000000..7a04624e3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/griddepcontrol_semantics.cu @@ -0,0 +1,324 @@ +// griddepcontrol_semantics.cu +// Semantic tests for CUDA Programmatic Dependent Launch / grid dependency +// control. +// +// Covers: +// 1. Explicit trigger smoke test +// 2. Implicit trigger smoke test +// 3. Large memory visibility: pre-trigger and post-trigger writes +// 4. Uneven CTA trigger timing: dependent launch must wait for all primary +// CTAs to trigger +// 5. No-attribute control: without programmatic stream serialization, normal +// stream ordering applies +// +// Notes for simulator bring-up: +// - overlap_seen is intentionally a soft observation, not a hard requirement. +// - Correctness is based on data visibility after +// cudaGridDependencySynchronize(). +// - The memory-visibility checks intentionally do not rely on __threadfence() +// around primary writes. + +#include + +#include +#include +#include + +#define CHECK_CUDA(call) \ + do { \ + cudaError_t err__ = (call); \ + if (err__ != cudaSuccess) { \ + fprintf(stderr, "%s:%d: CUDA error: %s\n", \ + __FILE__, __LINE__, cudaGetErrorString(err__)); \ + std::exit(1); \ + } \ + } while (0) + +static constexpr int BLOCKS = 2; +static constexpr int THREADS = 32; +static constexpr int WORDS_PER_BLOCK = 256; +static constexpr int PRE_SPIN = 200; +static constexpr int POST_TRIGGER_SPIN = 200; +static constexpr int LONG_SPIN = 4000; + +// Device-global witness flags. +// secondary_started: set by secondary as soon as it begins executing +// overlap_seen: set by primary if it sees secondary_started before +// finishing +__device__ volatile int secondary_started; +__device__ volatile int overlap_seen; + +__device__ __forceinline__ void burn_cycles(int n) { + volatile int x = threadIdx.x + blockIdx.x; + for (int i = 0; i < n; ++i) { + x = x * 1664525 + 1013904223; + } +} + +__device__ __forceinline__ int pre_pattern(int block, int i) { + return 0x10000000 ^ (block * 0x1009) ^ i; +} + +__device__ __forceinline__ int post_pattern(int block, int i) { + return 0x20000000 ^ (block * 0x2003) ^ i; +} + +__device__ __forceinline__ void write_pre_region(int *buf) { + int base = blockIdx.x * WORDS_PER_BLOCK; + for (int i = threadIdx.x; i < WORDS_PER_BLOCK / 2; i += blockDim.x) { + buf[base + i] = pre_pattern(blockIdx.x, i); + } +} + +__device__ __forceinline__ void write_post_region(int *buf) { + int base = blockIdx.x * WORDS_PER_BLOCK; + for (int i = WORDS_PER_BLOCK / 2 + threadIdx.x; i < WORDS_PER_BLOCK; + i += blockDim.x) { + buf[base + i] = post_pattern(blockIdx.x, i); + } +} + +__device__ __forceinline__ void observe_secondary_and_finish(int *done) { + for (int i = 0; i < POST_TRIGGER_SPIN; ++i) { + if (threadIdx.x == 0 && secondary_started) { + overlap_seen = 1; + } + burn_cycles(1); + } + + if (threadIdx.x == 0) { + done[blockIdx.x] = 1; + } +} + +// Explicit primary: write first region, trigger dependent launch eligibility, +// then continue with post-trigger writes. +__global__ void primary_explicit_memory(int *done, int *buf) { + burn_cycles(PRE_SPIN); + write_pre_region(buf); + + if (threadIdx.x == 0) { + cudaTriggerProgrammaticLaunchCompletion(); + } + + write_post_region(buf); + observe_secondary_and_finish(done); +} + +// Implicit primary: no explicit trigger; release is implicit at primary +// completion. +__global__ void primary_implicit_memory(int *done, int *buf) { + burn_cycles(PRE_SPIN); + write_pre_region(buf); + write_post_region(buf); + observe_secondary_and_finish(done); +} + +// Uneven explicit trigger timing: +// block 0 writes pre, triggers early, then spins and writes post +// block 1 writes pre, spins first, triggers late, then writes post +// The secondary may start only after all primary CTAs have reached +// trigger/completion eligibility. +__global__ void primary_explicit_uneven(int *done, int *buf) { + write_pre_region(buf); + + if (blockIdx.x == 0) { + if (threadIdx.x == 0) { + cudaTriggerProgrammaticLaunchCompletion(); + } + burn_cycles(LONG_SPIN); + } else { + burn_cycles(LONG_SPIN); + if (threadIdx.x == 0) { + cudaTriggerProgrammaticLaunchCompletion(); + } + } + + write_post_region(buf); + observe_secondary_and_finish(done); +} + +__global__ void secondary_check_full(const int *done, + const int *buf, + int *errors, + int *presync_counter) { + if (threadIdx.x == 0 && blockIdx.x == 0) { + secondary_started = 1; + } + + if (threadIdx.x == 0) { + atomicAdd(presync_counter, 1); + } + + // The key operation under test. + cudaGridDependencySynchronize(); + + // Check original-style per-primary-CTA completion witness. + int tid = blockIdx.x * blockDim.x + threadIdx.x; + if (tid < BLOCKS) { + if (done[tid] != 1) { + atomicAdd(errors, 1); + } + } + + // Check larger memory visibility, including pre-trigger and post-trigger + // writes. + int total_words = BLOCKS * WORDS_PER_BLOCK; + for (int idx = tid; idx < total_words; idx += gridDim.x * blockDim.x) { + int b = idx / WORDS_PER_BLOCK; + int i = idx % WORDS_PER_BLOCK; + int expected = + (i < WORDS_PER_BLOCK / 2) ? pre_pattern(b, i) : post_pattern(b, i); + if (buf[idx] != expected) { + atomicAdd(errors, 1); + } + } +} + +static void reset_device_flags() { + int zero = 0; + CHECK_CUDA(cudaMemcpyToSymbol(secondary_started, &zero, sizeof(int))); + CHECK_CUDA(cudaMemcpyToSymbol(overlap_seen, &zero, sizeof(int))); +} + +static int copy_overlap_seen_to_host() { + int h_overlap = 0; + CHECK_CUDA(cudaMemcpyFromSymbol(&h_overlap, overlap_seen, sizeof(int))); + return h_overlap; +} + +static int copy_secondary_started_to_host() { + int h_started = 0; + CHECK_CUDA(cudaMemcpyFromSymbol(&h_started, secondary_started, sizeof(int))); + return h_started; +} + +enum class PrimaryKind { + ExplicitMemory, + ImplicitMemory, + ExplicitUneven, +}; + +static void launch_primary(PrimaryKind kind, + int *d_done, + int *d_buf, + cudaStream_t stream) { + switch (kind) { + case PrimaryKind::ExplicitMemory: + primary_explicit_memory<<>>(d_done, + d_buf); + break; + case PrimaryKind::ImplicitMemory: + primary_implicit_memory<<>>(d_done, + d_buf); + break; + case PrimaryKind::ExplicitUneven: + primary_explicit_uneven<<>>(d_done, + d_buf); + break; + } + CHECK_CUDA(cudaGetLastError()); +} + +static void run_test(const char *name, + PrimaryKind kind, + bool enable_pdl_attribute) { + int *d_done = nullptr; + int *d_buf = nullptr; + int *d_errors = nullptr; + int *d_presync = nullptr; + + const int total_words = BLOCKS * WORDS_PER_BLOCK; + + CHECK_CUDA(cudaMalloc(&d_done, BLOCKS * sizeof(int))); + CHECK_CUDA(cudaMalloc(&d_buf, total_words * sizeof(int))); + CHECK_CUDA(cudaMalloc(&d_errors, sizeof(int))); + CHECK_CUDA(cudaMalloc(&d_presync, sizeof(int))); + + CHECK_CUDA(cudaMemset(d_done, 0, BLOCKS * sizeof(int))); + CHECK_CUDA(cudaMemset(d_buf, 0, total_words * sizeof(int))); + CHECK_CUDA(cudaMemset(d_errors, 0, sizeof(int))); + CHECK_CUDA(cudaMemset(d_presync, 0, sizeof(int))); + + reset_device_flags(); + + cudaStream_t stream; + CHECK_CUDA(cudaStreamCreate(&stream)); + + cudaLaunchAttribute attr{}; + attr.id = cudaLaunchAttributeProgrammaticStreamSerialization; + attr.val.programmaticStreamSerializationAllowed = + enable_pdl_attribute ? 1 : 0; + + cudaLaunchConfig_t cfg{}; + cfg.gridDim = dim3(1); + cfg.blockDim = dim3(128); + cfg.dynamicSmemBytes = 0; + cfg.stream = stream; + cfg.attrs = enable_pdl_attribute ? &attr : nullptr; + cfg.numAttrs = enable_pdl_attribute ? 1 : 0; + + launch_primary(kind, d_done, d_buf, stream); + + CHECK_CUDA(cudaLaunchKernelEx(&cfg, secondary_check_full, d_done, d_buf, + d_errors, d_presync)); + CHECK_CUDA(cudaStreamSynchronize(stream)); + + int h_errors = -1; + int h_presync = -1; + int h_overlap = copy_overlap_seen_to_host(); + int h_secondary_started = copy_secondary_started_to_host(); + std::vector h_done(BLOCKS, 0); + + CHECK_CUDA(cudaMemcpy(&h_errors, d_errors, sizeof(int), + cudaMemcpyDeviceToHost)); + CHECK_CUDA(cudaMemcpy(&h_presync, d_presync, sizeof(int), + cudaMemcpyDeviceToHost)); + CHECK_CUDA(cudaMemcpy(h_done.data(), d_done, BLOCKS * sizeof(int), + cudaMemcpyDeviceToHost)); + + bool all_done = true; + for (int i = 0; i < BLOCKS; ++i) { + if (h_done[i] != 1) { + all_done = false; + break; + } + } + + printf("[%s] pdl_attr=%d, secondary_started=%d, presync_counter=%d, " + "overlap_seen=%d, errors=%d, all_done=%s\n", + name, enable_pdl_attribute ? 1 : 0, h_secondary_started, h_presync, + h_overlap, h_errors, all_done ? "true" : "false"); + + CHECK_CUDA(cudaStreamDestroy(stream)); + CHECK_CUDA(cudaFree(d_done)); + CHECK_CUDA(cudaFree(d_buf)); + CHECK_CUDA(cudaFree(d_errors)); + CHECK_CUDA(cudaFree(d_presync)); + + // Hard correctness requirements: + // 1. dependent reads after cudaGridDependencySynchronize() must be correct + // 2. all primary blocks must eventually finish + if (h_errors != 0 || !all_done) { + fprintf(stderr, "[%s] FAILED: semantic correctness check failed\n", + name); + std::exit(2); + } + + // Soft observations: + // - explicit PDL cases may show overlap_seen=1, but this is + // scheduling/resource dependent. + // - no_attr_control is expected to have overlap_seen=0 under normal + // stream serialization, but correctness should not depend on asserting + // that in this portable test. +} + +int main() { + run_test("explicit_basic_memory", PrimaryKind::ExplicitMemory, true); + run_test("explicit_uneven_trigger", PrimaryKind::ExplicitUneven, true); + run_test("implicit_memory", PrimaryKind::ImplicitMemory, true); + run_test("no_attribute_control", PrimaryKind::ExplicitMemory, false); + + printf("PASS\n"); + return 0; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/Makefile b/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/Makefile new file mode 100644 index 000000000..38058ccf7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/Makefile @@ -0,0 +1,37 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# ld.global.L2::128B microbenchmark: issues `ld.global.L2::128B.u32` via +# explicit inline PTX (plus a plain `ld.global.u32` control path) and verifies +# from the host that the load still returns correct data — proving the +# .L2::128B prefetch hint is functionally inert while the underlying global +# load performs a real memory transaction. Single source -> single executable. +ALL_BENCHES := ld_global_l2_128b + +# .L2::128B is a PTX .level::prefetch_size hint (PTX ISA 7.4+, SM_75+). +# Target Ampere and newer to match the rest of the modern ubench suite. +# Override on the command line, e.g. `make ARCH=sm_90a`, for a single GPU. +ARCH ?= sm_80 sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS (code=sm_XX) and PTX (code=compute_XX) for each arch. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/ld_global_l2_128b.cu b/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/ld_global_l2_128b.cu new file mode 100644 index 000000000..57b008819 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/ld_global_l2_128b.cu @@ -0,0 +1,182 @@ +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// ld_global_l2_128b.cu — exercise `ld.global.L2::128B.u32` and verify from the +// CPU that the *load* still works functionally while the L2::128B qualifier is +// treated as a no-op performance hint. +// +// CONTEXT (CUTLASS / GPGPU-Sim) +// ----------------------------- +// CUTLASS emits loads of the form: +// +// ld.global.L2::128B.u32 %r, [%addr]; +// +// `.L2::128B` is a PTX `.level::prefetch_size` hint (PTX ISA 7.4+): it asks the +// hardware to prefetch 128B into L2 alongside the load. It does NOT change the +// data returned by the load — only (maybe) timing on silicon. +// +// In GPGPU-Sim the qualifier is already recognized: +// ptx.l : `.L2::128B` -> L2_OPTION +// ptx_ir : L2_OPTION falls into the "recognized, no special action" path +// so the hint is effectively NOP'd for functional mode, while `ld.global.u32` +// still performs a real global memory read. +// +// WHAT THIS TEST CHECKS +// --------------------- +// 1. The simulator PARSES `ld.global.L2::128B.u32` without error (a missing +// lexer/option rule would abort PTX parsing). +// 2. The underlying global load still happens: each thread reads a known value +// from a host-initialized global buffer via the hinted load, transforms it, +// and writes to an output buffer. The host checks every slot — proving a +// real memory transaction occurred, not a silent no-op of the whole insn. +// 3. A plain `ld.global.u32` control path on the same input produces identical +// results, confirming L2::128B does not affect functional correctness. +// +// make KERNEL=ld_global_l2_128b +// ./run.sh ld_global_l2_128b +// ───────────────────────────────────────────────────────────────────────────── + +// The instruction under test: load with the L2::128B prefetch hint. +__device__ __forceinline__ unsigned ld_global_l2_128b(const unsigned *ptr) { + unsigned val; + asm volatile("ld.global.L2::128B.u32 %0, [%1];" + : "=r"(val) + : "l"(ptr)); + return val; +} + +// Control: same load without the prefetch hint. +__device__ __forceinline__ unsigned ld_global_plain(const unsigned *ptr) { + unsigned val; + asm volatile("ld.global.u32 %0, [%1];" + : "=r"(val) + : "l"(ptr)); + return val; +} + +// Each thread: +// 1. loads in[tid] via ld.global.L2::128B.u32 -> out_hinted[tid] +// 2. loads in[tid] via ld.global.u32 -> out_plain[tid] +// 3. stores (loaded + 1) so the host sees a write that depends on the load +__global__ void ld_global_l2_128b_kernel(const unsigned *in, + unsigned *out_hinted, + unsigned *out_plain) { + unsigned tid = blockIdx.x * blockDim.x + threadIdx.x; + + unsigned hinted = ld_global_l2_128b(&in[tid]); + unsigned plain = ld_global_plain(&in[tid]); + + // Depend on the loaded values so neither load can be DCE'd, and so the + // host-visible stores are genuine memory transactions seeded by the loads. + out_hinted[tid] = hinted + 1u; + out_plain[tid] = plain + 1u; +} + +int main() { + const int block_size = 256; + const int num_blocks = 4; + const int total_threads = num_blocks * block_size; + + unsigned *h_in = (unsigned *)malloc(total_threads * sizeof(unsigned)); + unsigned *h_hinted = (unsigned *)malloc(total_threads * sizeof(unsigned)); + unsigned *h_plain = (unsigned *)malloc(total_threads * sizeof(unsigned)); + unsigned *d_in = NULL; + unsigned *d_hinted = NULL; + unsigned *d_plain = NULL; + + for (int i = 0; i < total_threads; ++i) { + h_in[i] = (unsigned)(i * 3 + 7); // matches produce(tid) + } + + cudaMalloc(&d_in, total_threads * sizeof(unsigned)); + cudaMalloc(&d_hinted, total_threads * sizeof(unsigned)); + cudaMalloc(&d_plain, total_threads * sizeof(unsigned)); + + // Sentinel-fill outputs so unwritten slots are obvious (0xFF -> all bits 1). + cudaMemset(d_hinted, 0xFF, total_threads * sizeof(unsigned)); + cudaMemset(d_plain, 0xFF, total_threads * sizeof(unsigned)); + + cudaMemcpy(d_in, h_in, total_threads * sizeof(unsigned), + cudaMemcpyHostToDevice); + + printf("Launching ld_global_l2_128b_kernel: %d blocks x %d threads\n" + " path A: ld.global.L2::128B.u32 (hint under test)\n" + " path B: ld.global.u32 (plain control)\n\n", + num_blocks, block_size); + + ld_global_l2_128b_kernel<<>>(d_in, d_hinted, d_plain); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("Kernel launch error: %s\n", cudaGetErrorString(err)); + return 1; + } + cudaDeviceSynchronize(); + + cudaMemcpy(h_hinted, d_hinted, total_threads * sizeof(unsigned), + cudaMemcpyDeviceToHost); + cudaMemcpy(h_plain, d_plain, total_threads * sizeof(unsigned), + cudaMemcpyDeviceToHost); + + int failures = 0; + + // Check 1: hinted load produced the correct functional value for every tid. + printf("Check 1 — ld.global.L2::128B.u32 functional correctness:\n"); + { + int ok = 1; + for (int i = 0; i < total_threads; ++i) { + unsigned expected = h_in[i] + 1u; + unsigned got = h_hinted[i]; + if (got != expected) { + if (failures < 12) { + printf(" [FAIL] tid %d: got %u, expected %u " + "(input was %u)\n", + i, got, expected, h_in[i]); + } + ok = 0; + failures++; + } + } + if (ok) { + printf(" [PASS] all %d threads: hinted load returned input+1\n", + total_threads); + } + } + printf("\n"); + + // Check 2: plain load matches the hinted path exactly. + printf("Check 2 — L2::128B hint is functionally equivalent to plain ld:\n"); + { + int ok = 1; + for (int i = 0; i < total_threads; ++i) { + if (h_hinted[i] != h_plain[i]) { + if (failures < 12) { + printf(" [FAIL] tid %d: hinted=%u plain=%u\n", + i, h_hinted[i], h_plain[i]); + } + ok = 0; + failures++; + } + } + if (ok) { + printf(" [PASS] all %d threads: hinted == plain " + "(prefetch hint did not change results)\n", + total_threads); + } + } + + printf("\n==> %s\n", + failures == 0 ? "ALL CHECKS PASSED" : "SOME CHECKS FAILED"); + printf(" (a passing run confirms L2::128B was parsed/ignored as a\n" + " prefetch hint, while ld.global still performed a real load\n" + " that the host can observe through the output buffer)\n"); + + cudaFree(d_in); + cudaFree(d_hinted); + cudaFree(d_plain); + free(h_in); + free(h_hinted); + free(h_plain); + + return failures == 0 ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile new file mode 100644 index 000000000..9bd7b10d1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile @@ -0,0 +1,38 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# mapa.shared::cluster.u32 functional microbenchmarks. Each remaps a +# shared-window address onto a cluster CTA rank and verifies rank sensitivity +# plus remote-store / local-load semantics: +# mapa_shared_cluster - 2-CTA cluster, Gates A/B/C +# mapa_shared_cluster_extended - 8-CTA cluster (mirror pairing) + full +# getctarank round-trip (Gate A+) +# Both build into standalone executables dropped in ../../../bin/. +ALL_BENCHES := mapa_shared_cluster mapa_shared_cluster_extended + +# mapa.shared::cluster is Hopper+ PTX (PTX ISA 7.8+, SM_90 and above). +# Override on the command line, e.g. `make ARCH=sm_90a`, for a single GPU. +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS (code=sm_XX) and PTX (code=compute_XX) for each arch. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster.cu b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster.cu new file mode 100644 index 000000000..13c08eb98 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster.cu @@ -0,0 +1,262 @@ +#include +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// mapa_shared_cluster.cu — careful functional gate for `mapa.shared::cluster.u32` +// +// WHAT mapa DOES (PTX ISA 7.8+, sm_90+) +// -------------------------------------- +// mapa.shared::cluster.u32 d, a, b; +// +// Remap shared-window address `a` onto cluster CTA rank `b`. Does NOT move data; +// a later ld/st.shared::cluster uses `d` to touch CTA b's copy of that offset. +// +// WHY THE PREVIOUS UBENCH WAS TOO WEAK +// ------------------------------------ +// It wrote each CTA's magic into its *own* slot, then remote-loaded via mapa. +// A stub mapa that returns `a` unchanged (ignoring rank) can still look plausible +// when combined with incidental cluster-address encoding elsewhere. This version +// closes those gaps: +// +// Gate A — RANK SENSITIVITY (address-only) +// From the same local address, mapa(., 0) and mapa(., 1) MUST differ. +// The current GPGPU-Sim stub ignores operand b and copies a → a, so +// mapped0 == mapped1 and this gate fails hard. +// +// Gate B — REMOTE STORE, LOCAL LOAD (end-to-end) +// Each CTA writes its magic into the *peer* slot via +// st.shared::cluster [mapa(local, peer)] +// then reads its *own* slot with an ordinary shared load (no mapa). +// Expected local value == peer's magic. Identity mapa writes own slot +// instead, so the local load sees own magic and fails. +// +// Gate C — SELF MAP IS IDENTITY-ISH +// mapa(local, own_rank) must still address this CTA's slot (local load +// through that mapped addr returns what we wrote remotely into ourselves, +// i.e. the peer's magic after Gate B). +// +// Host checks all three after D2H. Poison-fill so a silent no-op is obvious. +// +// make KERNEL=mapa_shared_cluster +// ./run.sh mapa_shared_cluster +// ./run.sh mapa_shared_cluster silicon +// ───────────────────────────────────────────────────────────────────────────── + +#define CLUSTER_SIZE 2 +#define BLOCK_SIZE 32 +#define MAGIC_BASE 0xA000u +#define POISON 0xFFFFFFFFu + +__device__ __forceinline__ unsigned cluster_ctarank() { + unsigned rank; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(rank)); + return rank; +} + +// Instruction under test: remap shared-window address `addr` onto CTA `rank`. +__device__ __forceinline__ unsigned mapa_shared_cluster_u32(unsigned addr, + unsigned rank) { + unsigned mapped; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" + : "=r"(mapped) + : "r"(addr), "r"(rank)); + return mapped; +} + +__device__ __forceinline__ void st_shared_cluster_u32(unsigned addr, + unsigned val) { + asm volatile("st.shared::cluster.u32 [%0], %1;" + : + : "r"(addr), "r"(val) + : "memory"); +} + +__device__ __forceinline__ unsigned ld_shared_cluster_u32(unsigned addr) { + unsigned val; + asm volatile("ld.shared::cluster.u32 %0, [%1];" + : "=r"(val) + : "r"(addr) + : "memory"); + return val; +} + +__global__ void __cluster_dims__(CLUSTER_SIZE, 1, 1) +mapa_shared_cluster_kernel(unsigned *d_local_after, // Gate B result + unsigned *d_self_via_mapa, // Gate C result + unsigned *d_mapped_rank0, // Gate A + unsigned *d_mapped_rank1, // Gate A + unsigned *d_local_addr, + unsigned *d_mapped_peer) { + __shared__ unsigned slot; + + const unsigned tid = threadIdx.x; + const unsigned rank = cluster_ctarank(); + const unsigned peer = rank ^ 1u; + + // Shared-window address of our local slot (CUTLASS / PTX mapa input form). + const unsigned local_addr = + static_cast(__cvta_generic_to_shared(&slot)); + + // Start from a known empty slot so Gate B cannot see a self-written magic. + if (tid == 0) { + slot = 0u; + } + __syncthreads(); + + // Cluster barrier so every CTA's zeroing is visible before remote stores. + // (GPGPU-Sim currently lexes these to NOP_OP; silicon is a real barrier.) + asm volatile("barrier.cluster.arrive;" ::: "memory"); + asm volatile("barrier.cluster.wait;" ::: "memory"); + + if (tid == 0) { + // ── Gate B: write *into the peer CTA* via mapa ───────────────────── + // (Gate A address checks are recorded after the barrier below.) + const unsigned mapped_peer = mapa_shared_cluster_u32(local_addr, peer); + st_shared_cluster_u32(mapped_peer, MAGIC_BASE + rank); + } + + // Make remote stores visible before anyone reads their local slot. + asm volatile("barrier.cluster.arrive;" ::: "memory"); + asm volatile("barrier.cluster.wait;" ::: "memory"); + __syncthreads(); + + if (tid == 0) { + // Ordinary .shared::cta load of *our* slot — peer should have written + // MAGIC_BASE + peer into it through Gate B's remapped store. + const unsigned local_after = slot; + + // ── Gate C: self-map still names our own slot ────────────────────── + const unsigned mapped_self = + mapa_shared_cluster_u32(local_addr, rank); + const unsigned self_via_mapa = ld_shared_cluster_u32(mapped_self); + + // Recompute peer map for the host-visible address dump (same as Gate B). + const unsigned mapped_peer = mapa_shared_cluster_u32(local_addr, peer); + const unsigned mapped0 = mapa_shared_cluster_u32(local_addr, 0u); + const unsigned mapped1 = mapa_shared_cluster_u32(local_addr, 1u); + + d_local_after[rank] = local_after; + d_self_via_mapa[rank] = self_via_mapa; + d_mapped_rank0[rank] = mapped0; + d_mapped_rank1[rank] = mapped1; + d_local_addr[rank] = local_addr; + d_mapped_peer[rank] = mapped_peer; + } +} + +int main() { + setvbuf(stdout, NULL, _IONBF, 0); + + printf("=== mapa.shared::cluster.u32 careful functional ubench ===\n"); + printf("cluster=%d CTAs, block=%d threads, magic_base=0x%X\n\n", + CLUSTER_SIZE, BLOCK_SIZE, MAGIC_BASE); + + const int n = CLUSTER_SIZE; + const size_t bytes = n * sizeof(unsigned); + + unsigned *h_local_after = (unsigned *)malloc(bytes); + unsigned *h_self_via_mapa = (unsigned *)malloc(bytes); + unsigned *h_mapped0 = (unsigned *)malloc(bytes); + unsigned *h_mapped1 = (unsigned *)malloc(bytes); + unsigned *h_local_addr = (unsigned *)malloc(bytes); + unsigned *h_mapped_peer = (unsigned *)malloc(bytes); + + unsigned *d_local_after = NULL, *d_self_via_mapa = NULL; + unsigned *d_mapped0 = NULL, *d_mapped1 = NULL; + unsigned *d_local_addr = NULL, *d_mapped_peer = NULL; + + cudaMalloc(&d_local_after, bytes); + cudaMalloc(&d_self_via_mapa, bytes); + cudaMalloc(&d_mapped0, bytes); + cudaMalloc(&d_mapped1, bytes); + cudaMalloc(&d_local_addr, bytes); + cudaMalloc(&d_mapped_peer, bytes); + + cudaMemset(d_local_after, 0xFF, bytes); + cudaMemset(d_self_via_mapa, 0xFF, bytes); + cudaMemset(d_mapped0, 0xFF, bytes); + cudaMemset(d_mapped1, 0xFF, bytes); + cudaMemset(d_local_addr, 0xFF, bytes); + cudaMemset(d_mapped_peer, 0xFF, bytes); + + mapa_shared_cluster_kernel<<>>( + d_local_after, d_self_via_mapa, d_mapped0, d_mapped1, + d_local_addr, d_mapped_peer); + cudaError_t err = cudaDeviceSynchronize(); + + int fails = 0; + if (err != cudaSuccess) { + printf("launch/sync error: %s\n", cudaGetErrorString(err)); + fails++; + } else { + cudaMemcpy(h_local_after, d_local_after, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_self_via_mapa, d_self_via_mapa, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_mapped0, d_mapped0, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_mapped1, d_mapped1, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_local_addr, d_local_addr, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_mapped_peer, d_mapped_peer, bytes, cudaMemcpyDeviceToHost); + + printf("per-CTA results (after D2H):\n"); + for (int r = 0; r < n; ++r) { + // After remote stores, my slot holds the peer CTA's magic. + const unsigned expect_local = MAGIC_BASE + (unsigned)(r ^ 1); + const unsigned expect_self = expect_local; // self-map → same slot + + const int wrote = + (h_local_after[r] != POISON) && (h_self_via_mapa[r] != POISON) && + (h_mapped0[r] != POISON) && (h_mapped1[r] != POISON); + + // Gate A: mapa(.,0) and mapa(.,1) must disagree. + const int rank_sensitive = (h_mapped0[r] != h_mapped1[r]); + + // Gate A': peer map must differ from self map when peer != self. + const unsigned mapped_self_expect = + (r == 0) ? h_mapped0[r] : h_mapped1[r]; + const int peer_addr_differs = + (h_mapped_peer[r] != mapped_self_expect); + + // Gate B / C value checks. + const int local_ok = (h_local_after[r] == expect_local); + const int self_ok = (h_self_via_mapa[r] == expect_self); + + if (!wrote) fails++; + if (!rank_sensitive) fails++; + if (!peer_addr_differs) fails++; + if (!local_ok) fails++; + if (!self_ok) fails++; + + printf(" CTA rank %d:\n", r); + printf(" local_addr = 0x%08X\n", h_local_addr[r]); + printf(" mapa(local, 0) = 0x%08X\n", h_mapped0[r]); + printf(" mapa(local, 1) = 0x%08X [%s]\n", + h_mapped1[r], + rank_sensitive ? "rank-sensitive ok" : "RANK IGNORED"); + printf(" mapped_peer_addr = 0x%08X [%s]\n", + h_mapped_peer[r], + peer_addr_differs ? "differs from self ok" : "SAME AS SELF"); + printf(" local_after = 0x%08X (expected 0x%08X) [%s]\n", + h_local_after[r], expect_local, + local_ok ? "ok" : "MISMATCH"); + printf(" self_via_mapa = 0x%08X (expected 0x%08X) [%s]\n", + h_self_via_mapa[r], expect_self, + self_ok ? "ok" : "MISMATCH"); + } + } + + cudaFree(d_local_after); + cudaFree(d_self_via_mapa); + cudaFree(d_mapped0); + cudaFree(d_mapped1); + cudaFree(d_local_addr); + cudaFree(d_mapped_peer); + free(h_local_after); + free(h_self_via_mapa); + free(h_mapped0); + free(h_mapped1); + free(h_local_addr); + free(h_mapped_peer); + + printf("\nRESULT: %s\n", fails == 0 ? "PASSED" : "FAILED"); + return fails == 0 ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster_extended.cu b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster_extended.cu new file mode 100644 index 000000000..5e5fc004f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster_extended.cu @@ -0,0 +1,311 @@ +#include +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// mapa_shared_cluster_extended.cu — wide functional gate for +// `mapa.shared::cluster.u32`, extending mapa_shared_cluster.cu from a 2-CTA +// cluster to a full 8-CTA cluster. +// +// WHAT mapa DOES (PTX ISA 7.8+, sm_90+) +// -------------------------------------- +// mapa.shared::cluster.u32 d, a, b; +// +// Remap shared-window address `a` onto cluster CTA rank `b`. Does NOT move data; +// a later ld/st.shared::cluster uses `d` to touch CTA b's copy of that offset. +// +// WHY AN 8-CTA VERSION IS A STRONGER TEST +// --------------------------------------- +// The 2-CTA test can only ever map to "the one other CTA", so it cannot tell a +// correct rank->CTA mapping apart from "always target the single peer". With 8 +// CTAs we pin the mapping down far more tightly: +// +// Pairing — MIRROR ACROSS THE CLUSTER +// rank r talks to rank (CLUSTER_SIZE-1 - r): 0<->7, 1<->6, 2<->5, 3<->4. +// peer() is an involution with peer(r) != r for all r, so the 8 CTAs form +// 4 disjoint pairs and every slot has exactly one remote writer. +// +// Gate A — RANK SENSITIVITY (address-only) +// For each CTA, mapa(local, peer) MUST differ from mapa(local, self). +// +// Gate A+ — FULL RANK ROUND-TRIP (the real upgrade) +// For every rank k in [0, CLUSTER_SIZE), getctarank(mapa(local, k)) == k. +// This proves mapa lands on the *correct* CTA for all 8 ranks, not merely +// that two addresses differ. An off-by-one or "always the peer" mapping is +// caught here. +// +// Gate B — REMOTE STORE, LOCAL LOAD (end-to-end) +// Each CTA writes its magic into its *peer* slot via +// st.shared::cluster [mapa(local, peer)] +// then reads its *own* slot with an ordinary shared load (no mapa). +// Expected local value == peer's magic. A no-op mapa writes its own slot +// instead, so the local load sees its own magic and fails. +// +// Gate C — SELF MAP IS IDENTITY-ISH +// mapa(local, own_rank) must still address this CTA's slot (local load +// through that mapped addr returns what the peer wrote into us, Gate B). +// +// Host checks all gates after D2H. Poison-fill so a silent no-op is obvious. +// +// NOTE (silicon): 8 CTAs is the portable maximum cluster size, so no +// non-portable opt-in is required. All CTAs of a cluster must be co-resident +// in one GPC on Hopper (sm_90); GPGPU-Sim models the 8-CTA cluster directly +// (needs >= 8 SMs, which the H100 config provides). +// +// make KERNEL=mapa_shared_cluster_extended +// ./run.sh mapa_shared_cluster_extended +// ./run.sh mapa_shared_cluster_extended silicon +// ───────────────────────────────────────────────────────────────────────────── + +#define CLUSTER_SIZE 8 +#define BLOCK_SIZE 32 +#define MAGIC_BASE 0xC000u +#define POISON 0xFFFFFFFFu + +// Mirror pairing across the cluster: 0<->7, 1<->6, 2<->5, 3<->4. +__device__ __host__ __forceinline__ unsigned mirror_peer(unsigned rank) { + return (CLUSTER_SIZE - 1u) - rank; +} + +__device__ __forceinline__ unsigned cluster_ctarank() { + unsigned rank; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(rank)); + return rank; +} + +// Instruction under test: remap shared-window address `addr` onto CTA `rank`. +__device__ __forceinline__ unsigned mapa_shared_cluster_u32(unsigned addr, + unsigned rank) { + unsigned mapped; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" + : "=r"(mapped) + : "r"(addr), "r"(rank)); + return mapped; +} + +// Inverse of mapa's rank encoding: which cluster rank owns this mapped address. +__device__ __forceinline__ unsigned getctarank_shared_cluster_u32(unsigned addr) { + unsigned rank; + asm volatile("getctarank.shared::cluster.u32 %0, %1;" + : "=r"(rank) + : "r"(addr)); + return rank; +} + +__device__ __forceinline__ void st_shared_cluster_u32(unsigned addr, + unsigned val) { + asm volatile("st.shared::cluster.u32 [%0], %1;" + : + : "r"(addr), "r"(val) + : "memory"); +} + +__device__ __forceinline__ unsigned ld_shared_cluster_u32(unsigned addr) { + unsigned val; + asm volatile("ld.shared::cluster.u32 %0, [%1];" + : "=r"(val) + : "r"(addr) + : "memory"); + return val; +} + +__global__ void __cluster_dims__(CLUSTER_SIZE, 1, 1) +mapa_shared_cluster_extended_kernel(unsigned *d_local_after, // Gate B + unsigned *d_self_via_mapa, // Gate C + unsigned *d_mapped_peer, // Gate A + unsigned *d_mapped_self, // Gate A + unsigned *d_local_addr, + unsigned *d_roundtrip_fail, // Gate A+ + unsigned *d_peer) { + __shared__ unsigned slot; + + const unsigned tid = threadIdx.x; + const unsigned rank = cluster_ctarank(); + const unsigned peer = mirror_peer(rank); + + // Shared-window address of our local slot (CUTLASS / PTX mapa input form). + const unsigned local_addr = + static_cast(__cvta_generic_to_shared(&slot)); + + // Start from a known empty slot so Gate B cannot see a self-written magic. + if (tid == 0) { + slot = 0u; + } + __syncthreads(); + + // Cluster barrier so every CTA's zeroing is globally visible before any + // remote store — otherwise a late zero could clobber a peer's magic. + asm volatile("barrier.cluster.arrive;" ::: "memory"); + asm volatile("barrier.cluster.wait;" ::: "memory"); + + if (tid == 0) { + // ── Gate B: write *into the peer CTA* via mapa ───────────────────── + const unsigned mapped_peer = mapa_shared_cluster_u32(local_addr, peer); + st_shared_cluster_u32(mapped_peer, MAGIC_BASE + rank); + } + + // Make remote stores visible before anyone reads their local slot. + asm volatile("barrier.cluster.arrive;" ::: "memory"); + asm volatile("barrier.cluster.wait;" ::: "memory"); + __syncthreads(); + + if (tid == 0) { + // Ordinary .shared::cta load of *our* slot — our mirror peer should have + // written MAGIC_BASE + peer into it through Gate B's remapped store. + const unsigned local_after = slot; + + // ── Gate C: self-map still names our own slot ────────────────────── + const unsigned mapped_self = + mapa_shared_cluster_u32(local_addr, rank); + const unsigned self_via_mapa = ld_shared_cluster_u32(mapped_self); + + // Recompute peer map for the host-visible address dump (same as Gate B). + const unsigned mapped_peer = mapa_shared_cluster_u32(local_addr, peer); + + // ── Gate A+: every rank must round-trip through mapa/getctarank ──── + unsigned rt_fail = 0u; + #pragma unroll + for (unsigned k = 0; k < CLUSTER_SIZE; ++k) { + const unsigned m = mapa_shared_cluster_u32(local_addr, k); + if (getctarank_shared_cluster_u32(m) != k) { + rt_fail++; + } + } + + d_local_after[rank] = local_after; + d_self_via_mapa[rank] = self_via_mapa; + d_mapped_peer[rank] = mapped_peer; + d_mapped_self[rank] = mapped_self; + d_local_addr[rank] = local_addr; + d_roundtrip_fail[rank] = rt_fail; + d_peer[rank] = peer; + } +} + +int main() { + setvbuf(stdout, NULL, _IONBF, 0); + + printf("=== mapa.shared::cluster.u32 extended (8-CTA) functional ubench ===\n"); + printf("cluster=%d CTAs, block=%d threads, magic_base=0x%X\n", + CLUSTER_SIZE, BLOCK_SIZE, MAGIC_BASE); + printf("pairing: rank r <-> (%d - r) [0<->%d, 1<->%d, ...]\n\n", + CLUSTER_SIZE - 1, CLUSTER_SIZE - 1, CLUSTER_SIZE - 2); + + const int n = CLUSTER_SIZE; + const size_t bytes = n * sizeof(unsigned); + + unsigned *h_local_after = (unsigned *)malloc(bytes); + unsigned *h_self_via_mapa = (unsigned *)malloc(bytes); + unsigned *h_mapped_peer = (unsigned *)malloc(bytes); + unsigned *h_mapped_self = (unsigned *)malloc(bytes); + unsigned *h_local_addr = (unsigned *)malloc(bytes); + unsigned *h_roundtrip = (unsigned *)malloc(bytes); + unsigned *h_peer = (unsigned *)malloc(bytes); + + unsigned *d_local_after = NULL, *d_self_via_mapa = NULL; + unsigned *d_mapped_peer = NULL, *d_mapped_self = NULL; + unsigned *d_local_addr = NULL, *d_roundtrip = NULL, *d_peer = NULL; + + cudaMalloc(&d_local_after, bytes); + cudaMalloc(&d_self_via_mapa, bytes); + cudaMalloc(&d_mapped_peer, bytes); + cudaMalloc(&d_mapped_self, bytes); + cudaMalloc(&d_local_addr, bytes); + cudaMalloc(&d_roundtrip, bytes); + cudaMalloc(&d_peer, bytes); + + cudaMemset(d_local_after, 0xFF, bytes); + cudaMemset(d_self_via_mapa, 0xFF, bytes); + cudaMemset(d_mapped_peer, 0xFF, bytes); + cudaMemset(d_mapped_self, 0xFF, bytes); + cudaMemset(d_local_addr, 0xFF, bytes); + cudaMemset(d_roundtrip, 0xFF, bytes); + cudaMemset(d_peer, 0xFF, bytes); + + // 8 CTAs is the portable maximum cluster size, so no non-portable opt-in + // is needed here. + mapa_shared_cluster_extended_kernel<<>>( + d_local_after, d_self_via_mapa, d_mapped_peer, d_mapped_self, + d_local_addr, d_roundtrip, d_peer); + cudaError_t err = cudaDeviceSynchronize(); + + int fails = 0; + if (err != cudaSuccess) { + printf("launch/sync error: %s\n", cudaGetErrorString(err)); + fails++; + } else { + cudaMemcpy(h_local_after, d_local_after, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_self_via_mapa, d_self_via_mapa, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_mapped_peer, d_mapped_peer, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_mapped_self, d_mapped_self, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_local_addr, d_local_addr, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_roundtrip, d_roundtrip, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_peer, d_peer, bytes, cudaMemcpyDeviceToHost); + + printf("per-CTA results (after D2H):\n"); + for (int r = 0; r < n; ++r) { + const unsigned expect_peer = mirror_peer((unsigned)r); + // After the remote store, my slot holds my mirror peer's magic. + const unsigned expect_local = MAGIC_BASE + expect_peer; + const unsigned expect_self = expect_local; // self-map -> same slot + + const int wrote = + (h_local_after[r] != POISON) && (h_self_via_mapa[r] != POISON) && + (h_mapped_peer[r] != POISON) && (h_mapped_self[r] != POISON) && + (h_local_addr[r] != POISON) && (h_roundtrip[r] != POISON); + + // Gate A: peer map must differ from self map (peer != self always). + const int peer_addr_differs = + (h_mapped_peer[r] != h_mapped_self[r]); + + // Gate A+: every rank round-tripped through mapa/getctarank. + const int roundtrip_ok = (h_roundtrip[r] == 0u); + + // Gate B / C value checks. + const int peer_ok = (h_peer[r] == expect_peer); + const int local_ok = (h_local_after[r] == expect_local); + const int self_ok = (h_self_via_mapa[r] == expect_self); + + if (!wrote) fails++; + if (!peer_addr_differs) fails++; + if (!roundtrip_ok) fails++; + if (!peer_ok) fails++; + if (!local_ok) fails++; + if (!self_ok) fails++; + + printf(" CTA rank %2d (peer %2d):\n", r, expect_peer); + printf(" local_addr = 0x%08X\n", h_local_addr[r]); + printf(" mapped_self = 0x%08X\n", h_mapped_self[r]); + printf(" mapped_peer = 0x%08X [%s]\n", + h_mapped_peer[r], + peer_addr_differs ? "differs from self ok" : "SAME AS SELF"); + printf(" roundtrip_fail = %u/%d [%s]\n", + h_roundtrip[r], CLUSTER_SIZE, + roundtrip_ok ? "all ranks map correctly" : "RANK MAP WRONG"); + printf(" local_after = 0x%08X (expected 0x%08X) [%s]\n", + h_local_after[r], expect_local, + local_ok ? "ok" : "MISMATCH"); + printf(" self_via_mapa = 0x%08X (expected 0x%08X) [%s]\n", + h_self_via_mapa[r], expect_self, + self_ok ? "ok" : "MISMATCH"); + } + } + + cudaFree(d_local_after); + cudaFree(d_self_via_mapa); + cudaFree(d_mapped_peer); + cudaFree(d_mapped_self); + cudaFree(d_local_addr); + cudaFree(d_roundtrip); + cudaFree(d_peer); + free(h_local_after); + free(h_self_via_mapa); + free(h_mapped_peer); + free(h_mapped_self); + free(h_local_addr); + free(h_roundtrip); + free(h_peer); + + printf("\nRESULT: %s\n", fails == 0 ? "PASSED" : "FAILED"); + return fails == 0 ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/Makefile b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/Makefile new file mode 100644 index 000000000..7fe7b8680 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/Makefile @@ -0,0 +1,33 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := cluster_ctarank cluster_ctarank_3d + +# Thread-block clusters and %cluster_ctarank require SM_90 or newer. +ARCH ?= sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),\ + -gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) \ + -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mkdir -p $(BIN_DIR) + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/README.md b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/README.md new file mode 100644 index 000000000..2098d418a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/README.md @@ -0,0 +1,38 @@ +# `%cluster_ctarank` microbenchmark + +This benchmark launches multiple one-dimensional thread-block clusters and +records the PTX `%cluster_ctarank` special register once per CTA. For a cluster +containing `N` CTAs, the expected ranks are `0` through `N - 1`; the sequence +starts again at zero in the next cluster. It also reads `%ctaid.x` directly and +checks that it equals CUDA's `blockIdx.x`, the CTA's grid-wide x coordinate. + +Build for an RTX 5070/5070 Ti (`sm_120`) and run the default case (four clusters, +four CTAs per cluster): + +```sh +make +../../../bin/cluster_ctarank +``` + +Choose another number of clusters and CTAs per cluster at runtime: + +```sh +../../../bin/cluster_ctarank 3 8 +``` + +The grid always contains `clusters * ctas-per-cluster` CTAs. The second argument +is limited to the portable maximum of eight CTAs per cluster. + +## Three-dimensional test + +`cluster_ctarank_3d` uses a `2 x 2 x 2` cluster shape and a `4 x 4 x 4` grid, +giving eight clusters with eight CTAs each. It reads `%ctaid.x`, `%ctaid.y`, and +`%ctaid.z` directly and compares them with the corresponding `blockIdx` +components. It also checks the flattened cluster-local rank: + +```text +rank = local_x + 2 * (local_y + 2 * local_z) +``` + +Run it with: + diff --git a/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank.cu b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank.cu new file mode 100644 index 000000000..5c12a383a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank.cu @@ -0,0 +1,191 @@ +#include + +#include +#include +#include +#include + +namespace { +/* + modify these values using cli args to test different numbers of clusters and + ctas per clusters. + example: 4 clusters, 8 ctas per cluster + 'cluster_ctarank 4 8' +*/ +constexpr unsigned kDefaultClusters = 4; +constexpr unsigned kDefaultCTAsPerCluster = 4; +constexpr unsigned kThreadsPerCTA = 32; + +// struct used to report results +struct CTAResult { + unsigned block_idx; + unsigned ctaid_x; + unsigned cluster_ctarank; +}; + +//error checking macro +#define CHECK_CUDA(call) \ + do { \ + const cudaError_t error = (call); \ + if (error != cudaSuccess) { \ + std::fprintf(stderr, "%s:%d: CUDA error: %s\n", __FILE__, \ + __LINE__, cudaGetErrorString(error)); \ + std::exit(EXIT_FAILURE); \ + } \ + } while (0) + +// gpu fn to read the %cluster_ctarank register +__device__ __forceinline__ unsigned read_cluster_ctarank() { + unsigned rank; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(rank)); + return rank; +} +// gpu fn to read the %ctaid register in x dimension only +__device__ __forceinline__ unsigned read_ctaid_x() { + unsigned ctaid_x; + asm volatile("mov.u32 %0, %%ctaid.x;" : "=r"(ctaid_x)); + return ctaid_x; +} + +__global__ void cluster_ctarank_kernel(CTAResult *results) { + // The special register is the same for every thread in the CTA, so only read if + // thread 0 to get single print and avoid interleaving of prints + // blockIdx.x == ctaid.x + if (threadIdx.x == 0) { + results[blockIdx.x] = + {blockIdx.x, read_ctaid_x(), read_cluster_ctarank()}; + } +} + +//parse cli args and ensure positive +unsigned parse_positive(const char *text, const char *name) { + errno = 0; + char *end = nullptr; + const unsigned long value = std::strtoul(text, &end, 10); + if (errno != 0 || end == text || *end != '\0' || value == 0 || + value > UINT_MAX) { + std::fprintf(stderr, "%s must be a positive integer (got '%s')\n", + name, text); + std::exit(EXIT_FAILURE); + } + return static_cast(value); +} + +} // namespace + +int main(int argc, char **argv) { + if (argc > 3) { + std::fprintf(stderr, "usage: %s [clusters] [ctas-per-cluster]\n", + argv[0]); + return EXIT_FAILURE; + } + + const unsigned num_clusters = + argc >= 2 ? parse_positive(argv[1], "clusters") : kDefaultClusters; + const unsigned ctas_per_cluster = + argc >= 3 ? parse_positive(argv[2], "ctas-per-cluster") + : kDefaultCTAsPerCluster; + + int device = 0; + CHECK_CUDA(cudaGetDevice(&device)); + + // check clusters are supported + cudaDeviceProp properties{}; + CHECK_CUDA(cudaGetDeviceProperties(&properties, device)); + + int cluster_launch_supported = 0; + CHECK_CUDA(cudaDeviceGetAttribute(&cluster_launch_supported, + cudaDevAttrClusterLaunch, device)); + if (!cluster_launch_supported) { + std::fprintf(stderr, "%s does not support thread-block clusters\n", + properties.name); + return EXIT_FAILURE; + } + const unsigned total_ctas = num_clusters * ctas_per_cluster; + + //descirbe requested cluster shape + cudaLaunchAttribute attribute{}; + attribute.id = cudaLaunchAttributeClusterDimension; + attribute.val.clusterDim.x = ctas_per_cluster; + attribute.val.clusterDim.y = 1; + attribute.val.clusterDim.z = 1; + + // build kernel launch config + cudaLaunchConfig_t config{}; + config.gridDim = dim3(total_ctas, 1, 1); + config.blockDim = dim3(kThreadsPerCTA, 1, 1); + config.attrs = &attribute; + config.numAttrs = 1; + + //check supported cluster size + int max_cluster_size = 0; + CHECK_CUDA(cudaOccupancyMaxPotentialClusterSize( + &max_cluster_size, + (void*)cluster_ctarank_kernel, + &config)); + + printf("Maximum supported cluster size: %d\n", max_cluster_size); + + // verify ctas/cluster requested is valid + if (ctas_per_cluster > static_cast(max_cluster_size)) { + std::fprintf(stderr, + "Requested %u CTAs/cluster, but this GPU supports at most %d\n", + ctas_per_cluster, max_cluster_size); + return EXIT_FAILURE; + } + + // calculate total number of CTAS and malloc enough space for device results + const size_t bytes = total_ctas * sizeof(CTAResult); + CTAResult *device_results = nullptr; + CHECK_CUDA(cudaMalloc(&device_results, bytes)); + + //launch kernel + CHECK_CUDA(cudaLaunchKernelEx( + &config, + cluster_ctarank_kernel, + device_results)); + CHECK_CUDA(cudaDeviceSynchronize()); + + CTAResult *host_results = + static_cast(std::malloc(bytes)); + if (host_results == nullptr) { + std::fprintf(stderr, "host allocation failed\n"); + CHECK_CUDA(cudaFree(device_results)); + return EXIT_FAILURE; + } + CHECK_CUDA(cudaMemcpy(host_results, device_results, bytes, + cudaMemcpyDeviceToHost)); + + std::printf("GPU: %s (compute capability %d.%d)\n", properties.name, + properties.major, properties.minor); + std::printf("launch: %u clusters x %u CTAs/cluster = %u CTAs\n\n", + num_clusters, ctas_per_cluster, total_ctas); + std::printf("logical_cluster blockIdx.x %%ctaid.x " + "%%cluster_ctarank expected_rank\n"); + + unsigned failures = 0; + for (unsigned block = 0; block < total_ctas; ++block) { + const unsigned logical_cluster = block / ctas_per_cluster; + const unsigned expected_rank = block % ctas_per_cluster; + const CTAResult result = host_results[block]; + const bool pass = result.block_idx == block && + result.ctaid_x == result.block_idx && + result.cluster_ctarank == expected_rank; + std::printf("%15u %10u %8u %18u %13u %s\n", logical_cluster, + result.block_idx, result.ctaid_x, result.cluster_ctarank, + expected_rank, + pass ? "PASS" : "FAIL"); + failures += !pass; + } + + std::free(host_results); + CHECK_CUDA(cudaFree(device_results)); + + if (failures != 0) { + std::printf("\nFAIL: %u CTA result(s) did not match\n", failures); + return EXIT_FAILURE; + } + std::printf("\nPASS: %%ctaid.x matched blockIdx.x and %%cluster_ctarank " + "restarted at zero in every cluster\n"); + return EXIT_SUCCESS; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu new file mode 100644 index 000000000..296f56025 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu @@ -0,0 +1,191 @@ +#include + +#include +#include + +namespace { + +constexpr unsigned kClusterX = 2; +constexpr unsigned kClusterY = 2; +constexpr unsigned kClusterZ = 2; +constexpr unsigned kGridX = 4; +constexpr unsigned kGridY = 4; +constexpr unsigned kGridZ = 4; +constexpr unsigned kThreadsPerCta = 32; +constexpr unsigned kTotalCtas = kGridX * kGridY * kGridZ; +constexpr unsigned kCtasPerCluster = kClusterX * kClusterY * kClusterZ; +// struct used to report results +struct CtaResult { + unsigned block_x; + unsigned block_y; + unsigned block_z; + unsigned ctaid_x; + unsigned ctaid_y; + unsigned ctaid_z; + unsigned cluster_ctarank; +}; +//error checking macro +#define CHECK_CUDA(call) \ + do { \ + const cudaError_t error = (call); \ + if (error != cudaSuccess) { \ + std::fprintf(stderr, "%s:%d: CUDA error: %s\n", __FILE__, \ + __LINE__, cudaGetErrorString(error)); \ + std::exit(EXIT_FAILURE); \ + } \ + } while (0) + +// gpu fns to read the %ctaid register in each dimension +__device__ __forceinline__ unsigned read_ctaid_x() { + unsigned value; + asm volatile("mov.u32 %0, %%ctaid.x;" : "=r"(value)); + return value; +} + +__device__ __forceinline__ unsigned read_ctaid_y() { + unsigned value; + asm volatile("mov.u32 %0, %%ctaid.y;" : "=r"(value)); + return value; +} + +__device__ __forceinline__ unsigned read_ctaid_z() { + unsigned value; + asm volatile("mov.u32 %0, %%ctaid.z;" : "=r"(value)); + return value; +} + +// gpu fn to read the %cluster_ctarank register +__device__ __forceinline__ unsigned read_cluster_ctarank() { + unsigned value; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(value)); + return value; +} + +__global__ void cluster_ctarank_3d_kernel(CtaResult *results) { + // The special register is the same for every thread in the CTA, so only read if + // thread 0 to get single print and avoid interleaving of prints + if (threadIdx.x != 0) { + return; + } + + const unsigned linearized_block_idx = + blockIdx.x + gridDim.x * (blockIdx.y + gridDim.y * blockIdx.z); + results[linearized_block_idx] = {blockIdx.x, + blockIdx.y, + blockIdx.z, + read_ctaid_x(), + read_ctaid_y(), + read_ctaid_z(), + read_cluster_ctarank()}; +} + +} // namespace + +int main() { + int device = 0; + CHECK_CUDA(cudaGetDevice(&device)); + + cudaDeviceProp properties{}; + CHECK_CUDA(cudaGetDeviceProperties(&properties, device)); + + int cluster_launch_supported = 0; + CHECK_CUDA(cudaDeviceGetAttribute(&cluster_launch_supported, + cudaDevAttrClusterLaunch, device)); + if (!cluster_launch_supported) { + std::fprintf(stderr, "%s does not support thread-block clusters\n", + properties.name); + return EXIT_FAILURE; + } + + CtaResult *device_results = nullptr; + CHECK_CUDA(cudaMalloc(&device_results, kTotalCtas * sizeof(CtaResult))); + + cudaLaunchAttribute attribute{}; + attribute.id = cudaLaunchAttributeClusterDimension; + attribute.val.clusterDim.x = kClusterX; + attribute.val.clusterDim.y = kClusterY; + attribute.val.clusterDim.z = kClusterZ; + + cudaLaunchConfig_t config{}; + config.gridDim = dim3(kGridX, kGridY, kGridZ); + config.blockDim = dim3(kThreadsPerCta, 1, 1); + config.attrs = &attribute; + config.numAttrs = 1; + + //check supported cluster size + int max_cluster_size = 0; + CHECK_CUDA(cudaOccupancyMaxPotentialClusterSize( + &max_cluster_size, + (void*)cluster_ctarank_3d_kernel, + &config)); + + printf("Maximum supported cluster size: %d\n", max_cluster_size); + + // verify ctas/cluster requested is valid + if (kCtasPerCluster > static_cast(max_cluster_size)) { + std::fprintf(stderr, + "Requested %u CTAs/cluster, but this GPU supports at most %d\n", + kTotalCtas, max_cluster_size); + return EXIT_FAILURE; + } + + CHECK_CUDA(cudaLaunchKernelEx(&config, cluster_ctarank_3d_kernel, + device_results)); + CHECK_CUDA(cudaDeviceSynchronize()); + + CtaResult host_results[kTotalCtas]; + CHECK_CUDA(cudaMemcpy(host_results, device_results, sizeof(host_results), + cudaMemcpyDeviceToHost)); + CHECK_CUDA(cudaFree(device_results)); + + std::printf("GPU: %s (compute capability %d.%d)\n", properties.name, + properties.major, properties.minor); + std::printf("grid=(%u,%u,%u), cluster=(%u,%u,%u), total=%u CTAs\n\n", + kGridX, kGridY, kGridZ, kClusterX, kClusterY, kClusterZ, + kTotalCtas); + std::printf("cluster(x,y,z) blockIdx(x,y,z) %%ctaid(x,y,z) " + "%%cluster_ctarank expected\n"); + + unsigned failures = 0; + //walk through every cta in grid + for (unsigned z = 0; z < kGridZ; ++z) { + for (unsigned y = 0; y < kGridY; ++y) { + for (unsigned x = 0; x < kGridX; ++x) { + const unsigned index = x + kGridX * (y + kGridY * z); //convert 3d corrd to 1d index + const CtaResult result = host_results[index]; + //figure out which cluster has the cta we care abt + const unsigned cluster_x = x / kClusterX; + const unsigned cluster_y = y / kClusterY; + const unsigned cluster_z = z / kClusterZ; + //figure out the local coordinate of the cta we care abt within ints cluster + const unsigned local_x = x % kClusterX; + const unsigned local_y = y % kClusterY; + const unsigned local_z = z % kClusterZ; + //from this, we have the expected rank + const unsigned expected_rank = + local_x + kClusterX * (local_y + kClusterY * local_z); + + const bool pass = result.block_x == x && result.block_y == y && + result.block_z == z && result.ctaid_x == x && + result.ctaid_y == y && result.ctaid_z == z && //ctaid is same as blockidx + result.cluster_ctarank == expected_rank; //does our expected rank match cluster ctarank? + std::printf("(%u,%u,%u) (%u,%u,%u) " + "(%u,%u,%u) %2u %2u %s\n", + cluster_x, cluster_y, cluster_z, result.block_x, + result.block_y, result.block_z, result.ctaid_x, + result.ctaid_y, result.ctaid_z, + result.cluster_ctarank, expected_rank, + pass ? "PASS" : "FAIL"); + failures += !pass; + } + } + } + + if (failures != 0) { + std::printf("\nFAIL: %u CTA result(s) did not match\n", failures); + return EXIT_FAILURE; + } + std::printf("\nPASS: 3D %%ctaid matched blockIdx and all cluster-local " + "ranks matched\n"); + return EXIT_SUCCESS; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/Makefile b/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/Makefile new file mode 100644 index 000000000..07368a3f1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/Makefile @@ -0,0 +1,26 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := stmatrix_verify + +ARCH ?= sm_90a +CC := nvcc + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/stmatrix_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/stmatrix_verify.cu new file mode 100644 index 000000000..f1cf48174 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/stmatrix_verify.cu @@ -0,0 +1,278 @@ +// ============================================================================= +// stmatrix_verify.cu +// +// Verifies stmatrix.sync.aligned.m8n8.shared.b16 for all Hopper sm_90a variants: +// x1, x2, x4 × non-transposed and .trans +// +// Matrix model: M_m[r][c] = (uint16_t)(m*64 + r*8 + c) (values 0..255) +// +// Non-transposed layout: smem[m*64 + r*8 + c] = M_m[r][c] (row-major) +// Transposed layout: smem[m*64 + c*8 + r] = M_m[r][c] (column-major) +// +// For each test, one warp sets up registers, calls stmatrix, copies smem to +// global memory, then the host verifies every element. +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Device helper: generic-pointer → 32-bit shared-memory address +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_u32(const void *ptr) { + uint32_t a; + asm volatile("{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }" + : "=r"(a) : "l"(ptr)); + return a; +} + +// --------------------------------------------------------------------------- +// stmatrix wrappers (inline asm, one per register count × .trans) +// --------------------------------------------------------------------------- +__device__ __forceinline__ +void stm_x1(uint32_t addr, uint32_t r0) { + asm volatile("stmatrix.sync.aligned.x1.m8n8.shared.b16 [%0], {%1};" + :: "r"(addr), "r"(r0) : "memory"); +} +__device__ __forceinline__ +void stm_x1t(uint32_t addr, uint32_t r0) { + asm volatile("stmatrix.sync.aligned.x1.trans.m8n8.shared.b16 [%0], {%1};" + :: "r"(addr), "r"(r0) : "memory"); +} +__device__ __forceinline__ +void stm_x2(uint32_t addr, uint32_t r0, uint32_t r1) { + asm volatile("stmatrix.sync.aligned.x2.m8n8.shared.b16 [%0], {%1,%2};" + :: "r"(addr), "r"(r0), "r"(r1) : "memory"); +} +__device__ __forceinline__ +void stm_x2t(uint32_t addr, uint32_t r0, uint32_t r1) { + asm volatile("stmatrix.sync.aligned.x2.trans.m8n8.shared.b16 [%0], {%1,%2};" + :: "r"(addr), "r"(r0), "r"(r1) : "memory"); +} +__device__ __forceinline__ +void stm_x4(uint32_t addr, uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3) { + asm volatile("stmatrix.sync.aligned.x4.m8n8.shared.b16 [%0], {%1,%2,%3,%4};" + :: "r"(addr), "r"(r0), "r"(r1), "r"(r2), "r"(r3) : "memory"); +} +__device__ __forceinline__ +void stm_x4t(uint32_t addr, uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3) { + asm volatile("stmatrix.sync.aligned.x4.trans.m8n8.shared.b16 [%0], {%1,%2,%3,%4};" + :: "r"(addr), "r"(r0), "r"(r1), "r"(r2), "r"(r3) : "memory"); +} + +// --------------------------------------------------------------------------- +// Data helpers +// --------------------------------------------------------------------------- + +// Value model: M_m[r][c] = m*64 + r*8 + c +__host__ __device__ __forceinline__ +uint16_t mat_val(int m, int r, int c) { return (uint16_t)(m*64 + r*8 + c); } + +// Non-transposed register for matrix m, thread T: +// lo = M_m[T/4][(T%4)*2], hi = M_m[T/4][(T%4)*2+1] +__device__ __forceinline__ +uint32_t reg_nontrans(int m, int T) { + uint16_t lo = mat_val(m, T/4, (T%4)*2); + uint16_t hi = mat_val(m, T/4, (T%4)*2 + 1); + return (uint32_t)lo | ((uint32_t)hi << 16); +} + +// Transposed register for matrix m, thread T: +// lo = M_m[(T%4)*2][T/4], hi = M_m[(T%4)*2+1][T/4] +__device__ __forceinline__ +uint32_t reg_trans(int m, int T) { + uint16_t lo = mat_val(m, (T%4)*2, T/4); + uint16_t hi = mat_val(m, (T%4)*2 + 1, T/4); + return (uint32_t)lo | ((uint32_t)hi << 16); +} + +// Address register for thread T (for non-transposed and transposed alike): +// Threads 0-7: address provider for matrix 0, row/col (T%8). +// Threads 8-15: address provider for matrix 1. +// Threads 16-23, 24-31: matrices 2, 3. +// Formula: smem_base + (T/8)*128 + (T%8)*16 +__device__ __forceinline__ +uint32_t addr_reg(uint32_t smem_base, int T) { + return smem_base + (T/8)*128 + (T%8)*16; +} + +// --------------------------------------------------------------------------- +// Kernels +// (each launched with 1 block × 32 threads, dynamic shared memory) +// --------------------------------------------------------------------------- + +__global__ void kernel_x1(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[128]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_nontrans(0, T); + stm_x1(addr, r0); + __syncthreads(); + for (int i = T; i < 64; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +__global__ void kernel_x1t(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[128]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_trans(0, T); + stm_x1t(addr, r0); + __syncthreads(); + for (int i = T; i < 64; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +__global__ void kernel_x2(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[256]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_nontrans(0, T); + uint32_t r1 = reg_nontrans(1, T); + stm_x2(addr, r0, r1); + __syncthreads(); + for (int i = T; i < 128; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +__global__ void kernel_x2t(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[256]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_trans(0, T); + uint32_t r1 = reg_trans(1, T); + stm_x2t(addr, r0, r1); + __syncthreads(); + for (int i = T; i < 128; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +__global__ void kernel_x4(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[512]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_nontrans(0, T); + uint32_t r1 = reg_nontrans(1, T); + uint32_t r2 = reg_nontrans(2, T); + uint32_t r3 = reg_nontrans(3, T); + stm_x4(addr, r0, r1, r2, r3); + __syncthreads(); + for (int i = T; i < 256; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +__global__ void kernel_x4t(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[512]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_trans(0, T); + uint32_t r1 = reg_trans(1, T); + uint32_t r2 = reg_trans(2, T); + uint32_t r3 = reg_trans(3, T); + stm_x4t(addr, r0, r1, r2, r3); + __syncthreads(); + for (int i = T; i < 256; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +// --------------------------------------------------------------------------- +// Host: expected-value helpers +// --------------------------------------------------------------------------- + +// Non-transposed: smem[m*64 + r*8 + c] = M_m[r][c] +static uint16_t expected_nontrans(int num_matrices, int idx) { + int m = idx / 64; + int local = idx % 64; + int r = local / 8; + int c = local % 8; + (void)num_matrices; + return mat_val(m, r, c); +} + +// Transposed: smem[m*64 + c*8 + r] = M_m[r][c] +// → smem[m*64 + i] = M_m[i%8][i/8] +static uint16_t expected_trans(int num_matrices, int idx) { + int m = idx / 64; + int local = idx % 64; + int r = local % 8; // row index + int c = local / 8; // column index + (void)num_matrices; + return mat_val(m, r, c); +} + +// --------------------------------------------------------------------------- +// Host: run one test variant +// --------------------------------------------------------------------------- +typedef void (*kernel_fn)(uint16_t *); + +static bool run_test(const char *name, + kernel_fn kernel, + int num_matrices, + bool is_trans) { + const int nelems = num_matrices * 64; + uint16_t *d_out; + cudaMalloc(&d_out, nelems * sizeof(uint16_t)); + cudaMemset(d_out, 0, nelems * sizeof(uint16_t)); + + kernel<<<1, 32>>>(d_out); + cudaDeviceSynchronize(); + + std::vector h_out(nelems); + cudaMemcpy(h_out.data(), d_out, nelems * sizeof(uint16_t), + cudaMemcpyDeviceToHost); + cudaFree(d_out); + + int errors = 0; + int first_bad = -1; + uint16_t first_got = 0, first_exp = 0; + for (int i = 0; i < nelems; i++) { + uint16_t exp = is_trans ? expected_trans(num_matrices, i) + : expected_nontrans(num_matrices, i); + if (h_out[i] != exp) { + if (errors == 0) { + first_bad = i; + first_got = h_out[i]; + first_exp = exp; + } + errors++; + } + } + + bool pass = (errors == 0); + printf("[%-44s] %s", name, pass ? "PASS" : "FAIL"); + if (!pass) + printf(" first mismatch: smem[%d] = %u, expected %u (%d/%d wrong)", + first_bad, first_got, first_exp, errors, nelems); + printf("\n"); + return pass; +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("stmatrix_verify — %s (SM %d.%d)\n", + prop.name, prop.major, prop.minor); + printf("============================================================\n"); + + bool all_pass = true; + + all_pass &= run_test("x1 non-transposed (1 matrix)", kernel_x1, 1, false); + all_pass &= run_test("x1 transposed (1 matrix)", kernel_x1t, 1, true); + all_pass &= run_test("x2 non-transposed (2 matrices)", kernel_x2, 2, false); + all_pass &= run_test("x2 transposed (2 matrices)", kernel_x2t, 2, true); + all_pass &= run_test("x4 non-transposed (4 matrices)", kernel_x4, 4, false); + all_pass &= run_test("x4 transposed (4 matrices)", kernel_x4t, 4, true); + + printf("============================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/system/bar_sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/system/bar_sync/Makefile new file mode 100644 index 000000000..3f01a50f7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/system/bar_sync/Makefile @@ -0,0 +1,36 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# bar.sync microbenchmark: exercises the CTA-wide barrier (PTX `bar.sync 0;`, +# i.e. __syncthreads()) via explicit inline PTX and verifies the result on the +# host. The single source file bar_sync.cu builds a single executable. +ALL_BENCHES := bar_sync + +# bar.sync is a universal barrier instruction (present on every SM generation), +# so we target a broad range while matching the architectures used by the rest +# of the tma/ suite. Override on the command line, e.g. `make ARCH=sm_90a`, to +# build for a single GPU. +ARCH ?= sm_80 sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS (code=sm_XX) and PTX (code=compute_XX) for each arch. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/system/bar_sync/bar_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/system/bar_sync/bar_sync.cu new file mode 100644 index 000000000..6219893aa --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/system/bar_sync/bar_sync.cu @@ -0,0 +1,242 @@ +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// bar_sync.cu — exercise the CTA-wide barrier (PTX `bar.sync`, i.e. the C-level +// __syncthreads()) and verify from the CPU that it actually synchronized every +// warp in the block. +// +// WHY THIS IS A REAL TEST +// ----------------------- +// In GPGPU-Sim each warp (32 lanes) executes in lockstep, but *different warps +// in the same CTA are scheduled independently* — there is no ordering between +// warp 0 and warp 7 unless you force one. bar.sync is that force: it is a gate +// that no warp may pass until EVERY warp in the block has arrived. +// +// A barrier test only proves something if the result is correct *only when the +// barrier works*. So every thread here consumes data produced by a thread in a +// DIFFERENT warp: +// +// Phase 1 (cross-warp exchange): +// shared[tid] = value(tid) // each warp produces its slice +// __syncthreads() // <-- all warps must finish +// exchange_out[tid] = shared[BS-1-tid] // read the *reversed* slot, +// // always owned by another warp +// +// Phase 2 (tree reduction — many barriers across warps): +// shared[tid] = value(tid) +// for (stride = BS/2; stride > 0; stride >>= 1) { +// if (tid < stride) shared[tid] += shared[tid + stride]; +// __syncthreads() // <-- between every step +// } +// block_sum = shared[0] +// +// Without the barriers, the reversed reads and the reduction steps race on +// shared memory produced by other warps, and the CPU checks below fail. +// +// CONTROL EXPERIMENT +// ------------------ +// Compile normally -> barriers ON -> ALL CHECKS PASSED. +// Compile with -DUSE_BARRIER=0 -> barriers stripped -> checks are expected to +// FAIL (or read garbage), demonstrating that bar.sync is doing real work in the +// simulator rather than being a no-op. +// +// make KERNEL=bar_sync +// ./run.sh bar_sync +// # control: +// nvcc -DUSE_BARRIER=0 ... bar_sync.cu -o bar_sync_nobar +// ───────────────────────────────────────────────────────────────────────────── + +#define WARP_SIZE 32 + +// Barriers are on by default; -DUSE_BARRIER=0 strips them for the control run. +#ifndef USE_BARRIER +#define USE_BARRIER 1 +#endif + +// Emit the CTA-wide barrier as EXPLICIT inline PTX so there is no ambiguity +// about which instruction is under test. `bar.sync 0;` is barrier number 0 with +// the implicit "all threads in the CTA" count — exactly what __syncthreads() +// lowers to. In GPGPU-Sim's PTX front-end (src/cuda-sim/ptx.l) the mnemonic +// `bar` lexes to BAR_OP and the `.sync` suffix to SYNC_OPTION, which the decoder +// (cuda-sim.cc set_bar_type) turns into bar_type = SYNC and the shader executes +// via barrier_set_t::warp_reaches_barrier. The "memory" clobber prevents the +// compiler from reordering shared-memory accesses across the barrier, so the +// cross-warp dependency below stays a genuine test. +#if USE_BARRIER +#define BARRIER() asm volatile("bar.sync 0;" ::: "memory") +#else +#define BARRIER() ((void)0) +#endif + +// The per-thread "work" result. Kept simple and deterministic so the CPU can +// predict the exact expected value for every slot. +__device__ __forceinline__ int produce(int global_tid) { + return global_tid + 1; // value(tid) = tid + 1 (nonzero, so 0 == "unwritten") +} + +// Deliberately skew when each warp arrives, so the warps do NOT march in +// lockstep. Later warps (higher warp id) start their shared-memory writes much +// later. This matters because Phase 1 has warp 0 consume the slot produced by +// the *last* warp: if that producer is delayed and there is no barrier, the +// consumer races ahead and reads stale data. With the barrier, everyone waits +// regardless of skew, so the result is still correct. The spin reads the cycle +// counter each iteration, so it cannot be optimized away and it advances real +// simulated time. +__device__ __forceinline__ void skew_by_warp(int warp_id) { + long long start = clock64(); + long long budget = (long long)warp_id * 4000; + while (clock64() - start < budget) { + // busy-wait; the loop condition has a side effect (%clock64 read) + } +} + +__global__ void bar_sync_kernel(int *exchange_out, long long *block_sums, + int block_size) { + extern __shared__ int smem[]; + + const int local_tid = threadIdx.x; + const int global_tid = blockIdx.x * blockDim.x + threadIdx.x; + const int warp_id = local_tid / WARP_SIZE; + + // Force the warps out of lockstep so the cross-warp dependency below is a + // genuine race when the barrier is absent. + skew_by_warp(warp_id); + + // ── Phase 1: every warp produces its slice of shared memory ────────────── + smem[local_tid] = produce(global_tid); + + // Barrier #1: no warp may read another warp's slot until all warps wrote. + BARRIER(); + + // Cross-warp exchange: read the mirrored slot. For a 256-thread block, + // lane 0 (warp 0) reads slot 255 (warp 7), and vice-versa — every thread + // reads a slot owned by a *different* warp. This is only correct if the + // barrier above completed for the whole CTA. + int mirror = block_size - 1 - local_tid; + exchange_out[global_tid] = smem[mirror]; + + // Barrier #2: make sure all mirror-reads are done before Phase 2 overwrites + // shared memory. + BARRIER(); + + // ── Phase 2: block-wide tree reduction (a barrier between each step) ────── + smem[local_tid] = produce(global_tid); + BARRIER(); + + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (local_tid < stride) { + smem[local_tid] += smem[local_tid + stride]; + } + // Each step consumes partial sums written by threads that may live in + // other warps, so a barrier is required after every step. + BARRIER(); + } + + if (local_tid == 0) { + block_sums[blockIdx.x] = (long long)smem[0]; + } +} + +int main() { + const int block_size = 256; // 8 warps per block + const int num_blocks = 4; // 4 CTAs -> tests per-CTA scoping + const int total_threads = num_blocks * block_size; + const size_t smem_bytes = block_size * sizeof(int); + + // ── Allocations ────────────────────────────────────────────────────────── + int *h_exchange = (int *)malloc(total_threads * sizeof(int)); + long long *h_sums = (long long *)malloc(num_blocks * sizeof(long long)); + int *d_exchange = NULL; + long long *d_sums = NULL; + cudaMalloc(&d_exchange, total_threads * sizeof(int)); + cudaMalloc(&d_sums, num_blocks * sizeof(long long)); + + // Sentinel-fill so any unwritten slot is obvious (0xFF bytes -> -1). + cudaMemset(d_exchange, 0xFF, total_threads * sizeof(int)); + cudaMemset(d_sums, 0xFF, num_blocks * sizeof(long long)); + + // ── Launch ─────────────────────────────────────────────────────────────── + printf("Launching bar_sync_kernel: %d blocks x %d threads (%d warps/block), " + "barriers %s\n\n", + num_blocks, block_size, block_size / WARP_SIZE, + USE_BARRIER ? "ON" : "OFF (control experiment)"); + + bar_sync_kernel<<>>(d_exchange, d_sums, + block_size); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("Kernel launch error: %s\n", cudaGetErrorString(err)); + return 1; + } + cudaDeviceSynchronize(); + + // ── GPU -> CPU transfer ──────────────────────────────────────────────────── + cudaMemcpy(h_exchange, d_exchange, total_threads * sizeof(int), + cudaMemcpyDeviceToHost); + cudaMemcpy(h_sums, d_sums, num_blocks * sizeof(long long), + cudaMemcpyDeviceToHost); + + // ── Host-side verification ───────────────────────────────────────────────── + int failures = 0; + + // Check 1: cross-warp exchange. For block b, local thread L must have read + // the mirrored producer value = produce(global index of (block_size-1-L)). + printf("Phase 1 — cross-warp shared-memory exchange (reversed index):\n"); + for (int b = 0; b < num_blocks; ++b) { + int block_ok = 1; + for (int l = 0; l < block_size; ++l) { + int global_tid = b * block_size + l; + int mirror_local = block_size - 1 - l; + int mirror_global = b * block_size + mirror_local; + int expected = mirror_global + 1; // produce(mirror_global) + int got = h_exchange[global_tid]; + if (got != expected) { + if (failures < 12) { // cap the noise + printf(" [FAIL] block %d lane %d: got %d, expected %d " + "(reversed slot owned by warp %d)\n", + b, l, got, expected, mirror_local / WARP_SIZE); + } + block_ok = 0; + failures++; + } + } + if (block_ok) { + printf(" [PASS] block %d: all %d threads read the correct " + "cross-warp mirrored value\n", b, block_size); + } + } + printf("\n"); + + // Check 2: block-wide reduction. Sum of produce(tid) over the block. + // produce(tid) = tid + 1, tids in block b are [b*BS .. b*BS+BS-1]. + printf("Phase 2 — block-wide tree reduction (barrier between each step):\n"); + for (int b = 0; b < num_blocks; ++b) { + long long expected = 0; + for (int l = 0; l < block_size; ++l) { + expected += (b * block_size + l) + 1; // produce(global_tid) + } + long long got = h_sums[b]; + if (got != expected) { + printf(" [FAIL] block %d: sum got %lld, expected %lld\n", + b, got, expected); + failures++; + } else { + printf(" [PASS] block %d: reduced sum = %lld\n", b, got); + } + } + + printf("\n==> %s\n", failures == 0 ? "ALL CHECKS PASSED" : "SOME CHECKS FAILED"); +#if !USE_BARRIER + printf(" (barriers were OFF; failures/garbage here confirm bar.sync is " + "doing real work)\n"); +#endif + + // ── Cleanup ──────────────────────────────────────────────────────────────── + cudaFree(d_exchange); + cudaFree(d_sums); + free(h_exchange); + free(h_sums); + + return failures == 0 ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/system/bar_warp_sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/system/bar_warp_sync/Makefile new file mode 100644 index 000000000..2f23c4b14 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/system/bar_warp_sync/Makefile @@ -0,0 +1,35 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# bar.warp.sync microbenchmark: issues the warp-level barrier (PTX +# `bar.warp.sync`, i.e. __syncwarp()) via explicit inline PTX and verifies an +# intra-warp exchange on the host. Single source -> single executable. +ALL_BENCHES := bar_warp_sync + +# bar.warp.sync is a universal instruction, so target a broad range while +# matching the architectures used by the rest of the tma/ suite. Override on the +# command line, e.g. `make ARCH=sm_90a`, to build for a single GPU. +ARCH ?= sm_80 sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS (code=sm_XX) and PTX (code=compute_XX) for each arch. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/system/bar_warp_sync/bar_warp_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/system/bar_warp_sync/bar_warp_sync.cu new file mode 100644 index 000000000..bb9057f30 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/system/bar_warp_sync/bar_warp_sync.cu @@ -0,0 +1,135 @@ +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// bar_warp_sync.cu — exercise the WARP-level barrier (PTX `bar.warp.sync`, i.e. +// the C-level __syncwarp()) and verify from the CPU that the program is still +// correct. +// +// WHY THIS IS DIFFERENT FROM bar.sync +// ----------------------------------- +// bar.sync is a CTA-wide barrier: it synchronizes *all warps* in the block, and +// GPGPU-Sim must implement it because warps are scheduled independently. +// +// bar.warp.sync only synchronizes the 32 lanes *within a single warp*. GPGPU-Sim +// already executes every warp in lockstep (all 32 lanes step together) and +// reconverges divergent lanes with the SIMT/PDOM stack, so the synchronization +// bar.warp.sync would provide is ALREADY guaranteed by the model. The +// instruction therefore has no work to do and is NOP'd in the PTX front-end +// (src/cuda-sim/ptx.l: `bar\.warp[...]* -> NOP_OP`). Its real purpose on silicon +// is Independent Thread Scheduling (Volta+), which the SIMT-stack model does not +// implement. +// +// WHAT THIS TEST CHECKS +// --------------------- +// 1. The simulator PARSES and NOPs `bar.warp.sync` without error (a missing +// lexer rule would abort PTX parsing). +// 2. A warp-internal data exchange guarded by __syncwarp() produces the correct +// result. Each lane writes its value to shared memory, __syncwarp(), then +// reads the MIRRORED lane *within the same warp* (lane l reads lane 31-l). +// Because the exchange stays inside one warp, lockstep execution makes it +// correct — which is exactly why the barrier can be a NOP here. +// +// Note: unlike the bar.sync test, there is no cross-warp dependency and no need +// to skew warps — the whole point is that intra-warp ordering is implicit. +// ───────────────────────────────────────────────────────────────────────────── + +#define WARP_SIZE 32 +#define FULL_MASK 0xffffffffu + +// Explicit inline PTX for the warp barrier. `bar.warp.sync ;` is +// what __syncwarp() lowers to. In GPGPU-Sim this lexes to NOP_OP (see ptx.l), +// so it is dropped; the "memory" clobber still prevents the compiler from +// reordering the shared-memory accesses around it. +__device__ __forceinline__ void warp_sync(unsigned mask) { + asm volatile("bar.warp.sync %0;" ::"r"(mask) : "memory"); +} + +__device__ __forceinline__ int produce(int global_tid) { + return global_tid + 1; // nonzero, so 0 means "unwritten" +} + +__global__ void bar_warp_sync_kernel(int *exchange_out) { + extern __shared__ int smem[]; + + const int local_tid = threadIdx.x; + const int global_tid = blockIdx.x * blockDim.x + threadIdx.x; + const int lane = local_tid % WARP_SIZE; + const int warp_base = local_tid - lane; // first local tid of this warp + + // Each lane publishes its value into this warp's slice of shared memory. + smem[local_tid] = produce(global_tid); + + // Warp-level barrier: all 32 lanes have published before anyone reads. + warp_sync(FULL_MASK); + + // Intra-warp mirrored read: lane l reads lane (31 - l) of the SAME warp. + int partner = warp_base + (WARP_SIZE - 1 - lane); + exchange_out[global_tid] = smem[partner]; +} + +int main() { + const int block_size = 256; // 8 warps per block + const int num_blocks = 2; + const int total_threads = num_blocks * block_size; + const size_t smem_bytes = block_size * sizeof(int); + + int *h_exchange = (int *)malloc(total_threads * sizeof(int)); + int *d_exchange = NULL; + cudaMalloc(&d_exchange, total_threads * sizeof(int)); + cudaMemset(d_exchange, 0xFF, total_threads * sizeof(int)); // -1 sentinel + + printf("Launching bar_warp_sync_kernel: %d blocks x %d threads " + "(%d warps/block)\n\n", + num_blocks, block_size, block_size / WARP_SIZE); + + bar_warp_sync_kernel<<>>(d_exchange); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("Kernel launch error: %s\n", cudaGetErrorString(err)); + return 1; + } + cudaDeviceSynchronize(); + + cudaMemcpy(h_exchange, d_exchange, total_threads * sizeof(int), + cudaMemcpyDeviceToHost); + + // Verify: each lane l must have read the value produced by lane (31 - l) + // of the SAME warp. + int failures = 0; + printf("Intra-warp mirrored exchange (lane l reads lane 31-l of its warp):\n"); + for (int b = 0; b < num_blocks; ++b) { + for (int w = 0; w < block_size / WARP_SIZE; ++w) { + int warp_ok = 1; + for (int lane = 0; lane < WARP_SIZE; ++lane) { + int local_tid = w * WARP_SIZE + lane; + int global_tid = b * block_size + local_tid; + int partner_lane = WARP_SIZE - 1 - lane; + int partner_global = b * block_size + w * WARP_SIZE + partner_lane; + int expected = partner_global + 1; // produce(partner) + int got = h_exchange[global_tid]; + if (got != expected) { + if (failures < 12) { + printf(" [FAIL] block %d warp %d lane %d: got %d, " + "expected %d\n", b, w, lane, got, expected); + } + warp_ok = 0; + failures++; + } + } + if (warp_ok) { + printf(" [PASS] block %d warp %d: all 32 lanes read the " + "correct mirrored lane\n", b, w); + } + } + } + + printf("\n==> %s\n", failures == 0 ? "ALL CHECKS PASSED" : "SOME CHECKS FAILED"); + printf(" (a passing run confirms bar.warp.sync was parsed and NOP'd " + "correctly, and intra-warp lockstep ordering holds)\n"); + + cudaFree(d_exchange); + free(h_exchange); + + return failures == 0 ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/system/deviceQuery/Makefile b/src/cuda/GPU_Microbenchmark/ubench/system/deviceQuery/Makefile index 167a71704..ef6ca28da 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/system/deviceQuery/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/system/deviceQuery/Makefile @@ -2,7 +2,7 @@ SRC = deviceQuery.cpp EXE = deviceQuery -INCLUDE := -I../../../../cuda-samples/Common/ +INCLUDE := -I../../../../NVIDIA_CUDA-11.0_Samples/common/inc/ NVCC_FLGAS = include ../../../common/common.mk diff --git a/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/Makefile b/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/Makefile new file mode 100644 index 000000000..ce625f3df --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/Makefile @@ -0,0 +1,13 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +SRC := independent_thread_scheduling.cu +EXE := independent_thread_scheduling + +# Independent thread scheduling was introduced with Volta (SM 70). Keep PTX +# for simulation and SASS for validation on NVIDIA hardware. +GENCODE_SM70 ?= -gencode=arch=compute_70,code=\"sm_70,compute_70\" +GENCODE_SM90 ?= -gencode=arch=compute_90,code=\"sm_90,compute_90\" +NVCC_FLAGS := $(GENCODE_SM70) $(GENCODE_SM90) -std=c++17 + +include ../../../common/common.mk diff --git a/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/independent_thread_scheduling.cu b/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/independent_thread_scheduling.cu new file mode 100644 index 000000000..13884d208 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/independent_thread_scheduling.cu @@ -0,0 +1,188 @@ +#include + +#include +#include +#include + +namespace { + +constexpr int kWarpSize = 32; +constexpr int kUnset = -1; + +#define CUDA_CHECK(call) \ + do { \ + cudaError_t error = (call); \ + if (error != cudaSuccess) { \ + std::fprintf(stderr, "CUDA error at %s:%d: %s\n", __FILE__, __LINE__, \ + cudaGetErrorString(error)); \ + std::exit(EXIT_FAILURE); \ + } \ + } while (0) + +// An atomic read makes the polling operation visible to both real hardware +// and the timing simulator. Volta+ may schedule a runnable sibling lane while +// the current divergent path repeatedly executes this loop. +__device__ __forceinline__ int poll(const int *address) { + return *reinterpret_cast(address); +} + +// Lanes 1..31 can reach the wait before lane 0 reaches the publishing path. +// A non-preemptive PDOM stack can pin those waiters above lane 0 forever. +__global__ void forward_publish_kernel(int *output) { + __shared__ int ready; + __shared__ int payload; + const int lane = threadIdx.x; + + if (lane == 0) { + ready = 0; + payload = 0; + } + __syncthreads(); + + if (lane != 0) { + while (poll(&ready) == 0) { + } + output[lane] = payload + lane; + } else { + payload = 1000; + *reinterpret_cast(&ready) = 1; + output[0] = 1000; + } +} + +// Mirror the predicate and producer lane. This catches implementations that +// accidentally make progress only for one branch direction or lane ordering. +__global__ void reverse_publish_kernel(int *output) { + __shared__ int ready; + __shared__ int payload; + const int lane = threadIdx.x; + + if (lane == 0) { + ready = 0; + payload = 0; + } + __syncthreads(); + + if (lane != kWarpSize - 1) { + while (poll(&ready) == 0) { + } + output[lane] = payload - lane; + } else { + payload = 2000; + *reinterpret_cast(&ready) = 1; + output[lane] = 2000 - lane; + } +} + +// The active owner changes 32 times. Each lane must leave the polling path, +// publish its trace slot, and wake the next lane. This exercises many dynamic +// split masks rather than a single producer/consumer split. +__global__ void token_ring_kernel(int *output) { + __shared__ int token; + const int lane = threadIdx.x; + + if (lane == 0) *reinterpret_cast(&token) = 0; + __syncthreads(); + + while (poll(&token) != lane) { + } + output[lane] = 3000 + lane; + *reinterpret_cast(&token) = lane + 1; +} + +enum TestId { kRing, kForward, kReverse, kTestCount }; + +const char *const kTestNames[kTestCount] = { + "token-ring", "forward-publish", "reverse-publish"}; + +int expected_value(TestId test, int lane) { + switch (test) { + case kForward: + return 1000 + lane; + case kReverse: + return 2000 - lane; + case kRing: + return 3000 + lane; + default: + return kUnset; + } +} + +void launch(TestId test, int *device_output) { + switch (test) { + case kForward: + forward_publish_kernel<<<1, kWarpSize>>>(device_output); + break; + case kReverse: + reverse_publish_kernel<<<1, kWarpSize>>>(device_output); + break; + case kRing: + token_ring_kernel<<<1, kWarpSize>>>(device_output); + break; + default: + std::abort(); + } +} + +bool run_test(TestId test) { + int *device_output = nullptr; + int host_output[kWarpSize]; + CUDA_CHECK(cudaMalloc(&device_output, sizeof(host_output))); + CUDA_CHECK(cudaMemset(device_output, 0xff, sizeof(host_output))); + + launch(test, device_output); + CUDA_CHECK(cudaGetLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + CUDA_CHECK(cudaMemcpy(host_output, device_output, sizeof(host_output), + cudaMemcpyDeviceToHost)); + CUDA_CHECK(cudaFree(device_output)); + + int failures = 0; + for (int lane = 0; lane < kWarpSize; ++lane) { + const int expected = expected_value(test, lane); + if (host_output[lane] != expected) { + std::printf(" lane %2d: got %d, expected %d\n", lane, + host_output[lane], expected); + ++failures; + } + } + std::printf("[%s] %s\n", failures == 0 ? "PASS" : "FAIL", + kTestNames[test]); + return failures == 0; +} + +int find_test(const char *name) { + for (int i = 0; i < kTestCount; ++i) { + if (std::strcmp(name, kTestNames[i]) == 0) return i; + } + return -1; +} + +} // namespace + +int main(int argc, char **argv) { + int first = 0; + int last = kTestCount; + if (argc == 2) { + first = find_test(argv[1]); + if (first < 0) { + std::fprintf(stderr, "Unknown test '%s'. Available tests:\n", argv[1]); + for (const char *name : kTestNames) std::fprintf(stderr, " %s\n", name); + return EXIT_FAILURE; + } + last = first + 1; + } else if (argc != 1) { + std::fprintf(stderr, "Usage: %s [test-name]\n", argv[0]); + return EXIT_FAILURE; + } + + int failures = 0; + for (int test = first; test < last; ++test) { + if (!run_test(static_cast(test))) ++failures; + } + + std::printf("RESULT: %s (%d/%d tests passed)\n", + failures == 0 ? "PASSED" : "FAILED", + last - first - failures, last - first); + return failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile new file mode 100644 index 000000000..a9eb5283f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile @@ -0,0 +1,31 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# elect.sync microbenchmark. +ALL_BENCHES := elect_sync + +# elect.sync is a Hopper+ PTX instruction (SM_90a and above). +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS and PTX. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu new file mode 100644 index 000000000..1b46b1aae --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu @@ -0,0 +1,160 @@ +#include +#include + +// Verify elect.sync by forcing a *specific* leader lane in each warp. The +// instruction has TWO results we want to check: +// +// 1. The predicate p -> true only for the elected leader lane. +// 2. The register rx -> the laneid of the elected leader, returned to +// EVERY participating thread in the membermask +// (not just the leader). +// +// We test both with one slot per thread in two contiguous buffers: +// - d_leader[tid] : leader writes 500, every other (participating) thread 0. +// - d_rx[tid] : each participating thread writes the laneid it read from +// rx; non-participating threads write the sentinel -1. +// +// elect.sync elects the lowest active lane in the membermask. So to make lane w +// the leader of warp w, we pass a membermask with the low w bits cleared +// (0xFFFFFFFF << w); the lowest remaining active lane is then exactly w: +// warp 0 -> lane 0 leader, warp 1 -> lane 1 leader, ... warp w -> lane w. +// +// After the kernel runs, the host checks each warp's 32-slot region: +// - d_leader: a single 500 at lane == warp id, 0 at the other participants. +// - d_rx : every participating lane (l >= warp id) reads rx == warp id. + +#define WARP_SIZE 32 + +__global__ void elect_sync_verify_kernel(int *d_leader, int *d_rx) { + unsigned int tid = blockIdx.x * blockDim.x + threadIdx.x; + unsigned int warp_id = tid / WARP_SIZE; + unsigned int lane = threadIdx.x % WARP_SIZE; // lane index within the warp + + // We want lane == warp_id to be the elected leader of warp w. + // + // elect.sync requires that EVERY thread executing it is named in the + // membermask, and that all named threads are converged/active. The mask + // 0xFFFFFFFF << warp_id names lanes [warp_id .. 31], so we must guard the + // instruction so that only those lanes actually execute it. Inside the + // branch the converged active set is exactly the masked set, which is the + // well-defined contract for elect.sync. It then elects the lowest active + // lane — which is exactly lane == warp_id. + unsigned int member_mask = 0xFFFFFFFFu << warp_id; + + int value = 0; + int elected_lane = -1; // sentinel for threads that don't participate + if (lane >= warp_id) { + unsigned int is_leader; + unsigned int rx; // laneid of the elected leader + asm volatile( + "{\n\t" + " .reg .pred p;\n\t" // predicate: true only for the leader + " elect.sync %0|p, %2;\n\t" // rx = elected laneid, p = is-leader + " selp.u32 %1, 1, 0, p;\n\t" // is_leader = p ? 1 : 0 + "}" + : "=r"(rx), // %0 + "=r"(is_leader) // %1 + : "r"(member_mask) // %2 + ); + elected_lane = (int)rx; + if (is_leader) value = 500; + } + + // Leader writes 500, everyone else writes 0 — contiguous, one slot per thread. + d_leader[tid] = value; + // Each participating thread records the laneid it got back from rx. + d_rx[tid] = elected_lane; +} + +int main() { + const int total_warps = 5; // need warp 4 to exist + const int total_threads = total_warps * WARP_SIZE; + + // ── Allocations ────────────────────────────────────────────────────────── + int *h_leader = (int *)malloc(total_threads * sizeof(int)); + int *h_rx = (int *)malloc(total_threads * sizeof(int)); + int *d_leader = NULL; + int *d_rx = NULL; + cudaMalloc(&d_leader, total_threads * sizeof(int)); + cudaMalloc(&d_rx, total_threads * sizeof(int)); + + // Sentinel-fill so any unwritten slot is obvious (0xFF bytes -> -1). + cudaMemset(d_leader, 0xFF, total_threads * sizeof(int)); + cudaMemset(d_rx, 0xFF, total_threads * sizeof(int)); + + // ── Launch (single block, one warp per intended leader lane) ───────────── + elect_sync_verify_kernel<<<1, total_threads>>>(d_leader, d_rx); + cudaDeviceSynchronize(); + + // ── Copy results back ──────────────────────────────────────────────────── + cudaMemcpy(h_leader, d_leader, total_threads * sizeof(int), cudaMemcpyDeviceToHost); + cudaMemcpy(h_rx, d_rx, total_threads * sizeof(int), cudaMemcpyDeviceToHost); + + // ── Dump the raw buffers ───────────────────────────────────────────────── + printf("Leader buffer (one slot per thread, 500 == elected leader):\n"); + for (int w = 0; w < total_warps; ++w) { + printf("warp %d:", w); + for (int l = 0; l < WARP_SIZE; ++l) { + printf(" %d", h_leader[w * WARP_SIZE + l]); + } + printf("\n"); + } + printf("\n"); + + printf("rx buffer (one slot per thread, elected laneid; -1 == did not participate):\n"); + for (int w = 0; w < total_warps; ++w) { + printf("warp %d:", w); + for (int l = 0; l < WARP_SIZE; ++l) { + printf(" %d", h_rx[w * WARP_SIZE + l]); + } + printf("\n"); + } + printf("\n"); + + // ── Host-side verification ─────────────────────────────────────────────── + // Expected leader lane in warp w is lane w. Every participating lane + // (l >= w) must read rx == w; non-participating lanes (l < w) stay -1. + int failures = 0; + for (int w = 0; w < total_warps; ++w) { + int expected_leader = w; + int warp_ok = 1; + for (int l = 0; l < WARP_SIZE; ++l) { + int idx = w * WARP_SIZE + l; + + // Check 1: predicate result via the leader marker. + int v = h_leader[idx]; + int expected_v = (l == expected_leader) ? 500 : 0; + if (v != expected_v) { + printf(" [FAIL] warp %d lane %d: leader marker got %d, expected %d\n", + w, l, v, expected_v); + warp_ok = 0; + failures++; + } + + // Check 2: rx register holds the elected laneid for participants. + int rx = h_rx[idx]; + int expected_rx = (l >= expected_leader) ? expected_leader : -1; + if (rx != expected_rx) { + printf(" [FAIL] warp %d lane %d: rx got %d, expected %d\n", + w, l, rx, expected_rx); + warp_ok = 0; + failures++; + } + } + if (warp_ok) { + printf(" [PASS] warp %d: leader is lane %d (value 500), " + "all participants read rx == %d\n", + w, expected_leader, expected_leader); + } + } + + printf("\n==> %s\n", failures == 0 ? "ALL CHECKS PASSED" : "SOME CHECKS FAILED"); + + // ── Cleanup ────────────────────────────────────────────────────────────── + cudaFree(d_leader); + cudaFree(d_rx); + free(h_leader); + free(h_rx); + + return failures == 0 ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/Makefile index 3b7aa310c..2aeee0f5f 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/Makefile @@ -1,12 +1,30 @@ -SRC = mbarrier.cu +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ -EXE = mbarrier +# Build all mbarrier microbenchmarks in this directory. +ALL_BENCHES := mbarrier mbarrier_reduction mbarrier_reduction2 mbarrier_test -ARCH?=sm_80 sm_90a sm_100a sm_101 sm_120 -# Unset the CUDA_CPPFLAGS which is set based on CUDA version +ARCH ?= sm_80 sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. CUDA_CPPFLAGS= -# Generate code for both sm_XXX and compute_XXX (SASS and PTX) +# Generate both SASS and PTX. NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 -include ../../../common/common.mk +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_cluster.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_cluster.cu new file mode 100644 index 000000000..3d57292cd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_cluster.cu @@ -0,0 +1,657 @@ +// mbarrier_cluster.cu +// +// Aggressive microbenchmark for mbarrier on sm_90 (Hopper). +// Covers every PTX form used by CUTLASS 4.8 and the known simulator +// correctness gaps, with one dedicated test per issue. +// +// PTX forms exercised (matching CUTLASS cute/arch/copy_sm90_desc.hpp and +// cutlass/arch/barrier.h exactly): +// +// mbarrier.init.shared::cta.b64 [bar], count +// mbarrier.arrive.expect_tx.shared::cta.b64 _, [bar], bytes (no token) +// mbarrier.arrive.shared::cta.b64 _, [bar] (no token, CUTLASS arrive_barrier) +// mbarrier.arrive.shared::cta.b64 tok, [bar] (token form) +// mbarrier.arrive.shared::cta.b64 tok, [bar], count (token + count) +// mbarrier.arrive.shared::cluster.b64 _, [remote_bar] (cross-CTA, no token) +// mbarrier.try_wait.parity.shared::cta.b64 P, [bar], phase_bit (CUTLASS wait form) +// mbarrier.try_wait.shared.b64 P, [bar], token (token wait form) +// mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 [bar], tx +// mbarrier.inval.shared.b64 [bar] +// +// Test index → output slot: +// h[0] TEST 1: mbarrier.init.shared::cta / arrive(token) / try_wait(token) — baseline +// h[1] TEST 2: mbarrier.try_wait.parity — sim crash test (SYNCS_MAX_ENUM assert) +// h[2] TEST 3: arrive.expect_tx + try_wait.parity (CUTLASS TMA wait pattern) +// h[3] TEST 4: arrive(no-token) _, [bar] — CUTLASS arrive_barrier form +// h[4] TEST 5: arrive.shared::cluster (remote) + try_wait.parity — CUTLASS cluster arrive +// h[5] TEST 6: Multi-phase ping-pong using try_wait.parity (looping parity bit) +// h[6] TEST 7: count=0 observe-only arrive must not decrement pending +// h[7] TEST 8: mbarrier.inval then re-init on same address +// h[8] TEST 9: expect_tx + complete_tx + all-thread arrive (combined barrier) +// +// All outputs: 1 = PASS, -1 = FAIL, 0 = did not run (from cudaMemset). +// +// Compile: +// nvcc -arch=sm_90 -std=c++17 mbarrier_cluster.cu -o mbarrier_cluster +// +// PTX audit: +// cuobjdump -ptx mbarrier_cluster | grep mbarrier + +#include +#include +#include + +namespace cg = cooperative_groups; + +// --------------------------------------------------------------------------- +// Inline PTX helpers — each one maps to exactly one CUTLASS or PTX form +// --------------------------------------------------------------------------- + +// 32-bit shared-memory address from a generic pointer (cvta.to.shared) +__device__ static inline uint32_t smem_u32(const void* ptr) { + uint32_t a; + asm("{ .reg .u64 tmp; cvta.to.shared.u64 tmp, %1; cvt.u32.u64 %0, tmp; }" + : "=r"(a) : "l"((uint64_t)ptr)); + return a; +} + +// mbarrier.init.shared::cta.b64 [bar], count +// CUTLASS: initialize_barrier() (copy_sm90_desc.hpp:69) +__device__ static inline void mbar_init(uint64_t* bar, uint32_t count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" + :: "r"(smem_u32(bar)), "r"(count) : "memory"); +} + +// mbarrier.inval.shared.b64 [bar] +__device__ static inline void mbar_inval(uint64_t* bar) { + asm volatile("mbarrier.inval.shared.b64 [%0];" :: "r"(smem_u32(bar)) : "memory"); +} + +// mbarrier.arrive.shared::cta.b64 tok, [bar], count — token form with count +// PTX ISA §9.7.12.4: decrements pending by count, returns prior-phase token +__device__ static inline uint64_t mbar_arrive_token(uint64_t* bar, uint32_t count = 1) { + uint64_t tok; + asm volatile("mbarrier.arrive.shared::cta.b64 %0, [%1], %2;" + : "=l"(tok) : "r"(smem_u32(bar)), "r"(count) : "memory"); + return tok; +} + +// mbarrier.arrive.shared::cta.b64 _, [bar] +// CUTLASS: arrive_barrier() (copy_sm90_desc.hpp:122) — no token returned +// Matches CUTLASS exactly: no volatile, no memory clobber (token discarded) +__device__ static inline void mbar_arrive_no_token(uint64_t* bar) { + asm("{ .reg .b64 state; mbarrier.arrive.shared::cta.b64 state, [%0]; }" + :: "r"(smem_u32(bar))); +} + +// mbarrier.arrive.expect_tx.shared::cta.b64 _, [bar], bytes +// CUTLASS: set_barrier_transaction_bytes() (copy_sm90_desc.hpp:83) +// Adds tx_bytes to tx_count; decrements pending by 1; no token returned. +__device__ static inline void mbar_arrive_expect_tx(uint64_t* bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" + :: "r"(smem_u32(bar)), "r"(bytes) : "memory"); +} + +// mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 [bar], tx +// Called after simulated TMA data arrives; decrements tx_count +__device__ static inline void mbar_complete_tx(uint64_t* bar, uint32_t tx) { + asm volatile("mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 [%0], %1;" + :: "r"(smem_u32(bar)), "r"(tx) : "memory"); +} + +// mbarrier.try_wait.parity.shared::cta.b64 P, [bar], phase_bit, ticks +// Matches CUTLASS ClusterBarrier::wait() in barrier.h exactly: +// 3-operand form with timeout ticks (0x989680); loops on bra until P=true. +// PTX ISA §9.7.12.6: returns P=true when phase%2 == phase_bit. +// Unique-label helper: two levels of macro indirection so __COUNTER__ expands +// before stringification, producing labels like LAB_WAIT_42 / DONE_42. +#define MBAR_WAIT_STR2(x) #x +#define MBAR_WAIT_STR(x) MBAR_WAIT_STR2(x) +#define MBAR_WAIT_PARITY_IMPL(sa, pb, tk, id) \ + asm volatile( \ + "{\n\t" \ + ".reg .pred P1;\n\t" \ + "LAB_WAIT_" MBAR_WAIT_STR(id) ":\n\t" \ + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1, %2;\n\t" \ + "@P1 bra DONE_" MBAR_WAIT_STR(id) ";\n\t" \ + "bra LAB_WAIT_" MBAR_WAIT_STR(id) ";\n\t" \ + "DONE_" MBAR_WAIT_STR(id) ":\n\t" \ + "}" \ + :: "r"(sa), "r"(pb), "r"(tk) : "memory") +// __LINE__ is unique per call site in the source file and survives inlining. +#define MBAR_WAIT_PARITY(sa, pb, tk) MBAR_WAIT_PARITY_IMPL(sa, pb, tk, __LINE__) + +// mbar_wait_parity intentionally removed — use mbar_spin_wait_parity macro instead, +// which captures __LINE__ at the call site to guarantee unique PTX labels per use. + +// Non-blocking one-shot parity test — returns true if barrier is already done +__device__ static inline bool mbar_test_wait_parity(uint64_t* bar, uint32_t phase_bit) { + uint32_t done; + asm volatile( + "{ .reg .pred P;\n\t" + "mbarrier.try_wait.parity.shared::cta.b64 P, [%0], %1;\n\t" + "selp.u32 %2, 1, 0, P; }" + : "=r"(done) : "r"(smem_u32(bar)), "r"(phase_bit) : "memory"); + return (bool)done; +} + +// mbarrier.try_wait.shared.b64 P, [bar], token — token form (used in tests 1, 7) +__device__ static inline bool mbar_try_wait_token(uint64_t* bar, uint64_t tok) { + uint32_t done; + asm volatile( + "{ .reg .pred P;\n\t" + "mbarrier.try_wait.shared.b64 P, [%1], %2;\n\t" + "selp.u32 %0, 1, 0, P; }" + : "=r"(done) : "r"(smem_u32(bar)), "l"(tok) : "memory"); + return (bool)done; +} + +// mbarrier.arrive.shared::cluster.b64 _, [remote_bar] +// CUTLASS: ClusterBarrier::arrive(smem_ptr, cta_id) (barrier.h:489) +// remote_bar is a 32-bit shared-memory address in another CTA's space +// obtained via mapa.shared::cluster.u32. +__device__ static inline void mbar_arrive_cluster_remote32(uint32_t remote_smem_addr) { + asm volatile( + "mbarrier.arrive.shared::cluster.b64 _, [%0];" + :: "r"(remote_smem_addr) : "memory"); +} + +// mapa.shared::cluster.u32 dst, src, cta_rank +// Maps a 32-bit shared-memory address in this CTA's space to the same +// offset in another CTA's shared memory, within the cluster. +// This is the CUTLASS barrier.h form (u32), distinct from mapa.u64. +__device__ static inline uint32_t mapa32(const void* local_ptr, uint32_t cta_rank) { + uint32_t dst; + asm("mapa.shared::cluster.u32 %0, %1, %2;" + : "=r"(dst) : "r"(smem_u32(local_ptr)), "r"(cta_rank)); + return dst; +} + +// Spin-wait helpers +__device__ static inline void mbar_spin_wait_token(uint64_t* bar, uint64_t tok) { + while (!mbar_try_wait_token(bar, tok)) __nanosleep(20); +} +// mbar_spin_wait_parity must be a macro so __LINE__ captures the call site, +// giving each inline PTX block a unique label. Two calls in the same kernel +// with the same function would otherwise generate duplicate LAB_WAIT labels +// that the gpgpu-sim PTX parser aliases to the first occurrence. +#define mbar_spin_wait_parity(bar, phase_bit) do { \ + uint32_t _sa = smem_u32(bar); \ + uint32_t _pb = (phase_bit); \ + uint32_t _tk = 0x989680; \ + MBAR_WAIT_PARITY(_sa, _pb, _tk); \ +} while(0) + +// ============================================================ +// TEST 1: mbarrier.init.shared::cta + arrive(token) + try_wait.token +// PTX: mbarrier.init.shared::cta.b64 / .arrive.shared::cta.b64 tok / .try_wait.shared.b64 +// +// Baseline: all 32 threads arrive with count=1, wait on 64-bit token. +// Verifies the fundamental init→arrive→wait cycle works. +// h[0] = 1 if sum of smem[0..31] == 496 after barrier completes. +// ============================================================ +__global__ void test1_baseline_token_wait(int* out) { + __shared__ uint64_t bar; + __shared__ int smem[32]; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + smem[threadIdx.x] = threadIdx.x; + __threadfence_block(); + + uint64_t tok = mbar_arrive_token(&bar); + mbar_spin_wait_token(&bar, tok); + + if (threadIdx.x == 0) { + int sum = 0; + for (int i = 0; i < 32; i++) sum += smem[i]; + out[0] = (sum == 496) ? 1 : -1; + } +} + +// ============================================================ +// TEST 2: mbarrier.try_wait.parity — the CUTLASS wait form +// PTX: mbarrier.try_wait.parity.shared::cta.b64 P, [bar], phase_bit +// +// CUTLASS uses parity-form exclusively (copy_sm90_desc.hpp wait_barrier and +// barrier.h ClusterBarrier::try_wait). This takes a plain u32 phase_bit (0 or 1) +// instead of a 64-bit token. The simulator must route PARITY_OPTION to +// SYNCS_TRY_WAIT, not to SYNCS_MAX_ENUM_NO_USED. +// +// SIMULATOR ISSUE: ptx_ir.cc PARITY_OPTION falls to default → SYNCS_MAX_ENUM_NO_USED +// → shader.cc assert(false) crash. Fix: add PARITY_OPTION → SYNCS_TRY_WAIT. +// PTX ISA §9.7.12.6: try_wait.parity completes when phase%2 == phase_bit. +// h[1] = 1 if barrier completes and smem sum is correct. +// ============================================================ +__global__ void test2_try_wait_parity(int* out) { + __shared__ uint64_t bar; + __shared__ int smem[32]; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + smem[threadIdx.x] = threadIdx.x; + __threadfence_block(); + + // All threads arrive without token — CUTLASS arrive_barrier form + mbar_arrive_no_token(&bar); + + // Wait using parity form: current phase=0, parity=0 → phase_bit=0 (CUTLASS convention) + mbar_spin_wait_parity(&bar, 0); + + if (threadIdx.x == 0) { + int sum = 0; + for (int i = 0; i < 32; i++) sum += smem[i]; + out[1] = (sum == 496) ? 1 : -1; + } +} + +// ============================================================ +// TEST 3: arrive.expect_tx + complete_tx + try_wait.parity +// PTX: mbarrier.arrive.expect_tx.shared::cta.b64 _, [bar], bytes +// mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 [bar], bytes +// mbarrier.try_wait.parity.shared::cta.b64 P, [bar], phase_bit +// +// This is the CUTLASS TMA producer/consumer pattern (copy_sm90_desc.hpp): +// 1. Consumer thread calls arrive.expect_tx (announces incoming async bytes, +// decrements pending by 1, adds bytes to tx_count) +// 2. Simulated TMA data arrives, producer calls complete_tx (decrements tx_count) +// 3. All threads wait with parity form +// Barrier completes only when BOTH pending==0 AND tx_count==0. +// PTX ISA §9.7.12.5: expect_tx adds to the expected transaction count. +// h[2] = 1 if data is correct after wait. +// ============================================================ +__global__ void test3_expect_tx_parity_wait(int* out) { + __shared__ uint64_t bar; + __shared__ int data[32]; + + // Init: 1 arrive expected (the expect_tx thread), plus tx bytes + if (threadIdx.x == 0) { + mbar_init(&bar, blockDim.x); + } + __syncthreads(); + + if (threadIdx.x == 0) { + // Announce 32*4=128 bytes of incoming async data and arrive (count=1) + mbar_arrive_expect_tx(&bar, 32 * sizeof(int)); + + // Simulate the async data arriving (in real use this is TMA) + for (int i = 0; i < 32; i++) data[i] = i + 100; + __threadfence_block(); + + // Signal tx completion + mbar_complete_tx(&bar, 32 * sizeof(int)); + } else { + // All other threads arrive without token + mbar_arrive_no_token(&bar); + } + + // All threads wait: current phase=0, parity=0 → phase_bit=0 (CUTLASS convention) + mbar_spin_wait_parity(&bar, 0); + + if (threadIdx.x == 0) { + int ok = 1; + for (int i = 0; i < 32; i++) if (data[i] != i + 100) { ok = 0; break; } + out[2] = ok ? 1 : -1; + } +} + +// ============================================================ +// TEST 4: arrive(no-token) _, [bar] — CUTLASS arrive_barrier form +// PTX: mbarrier.arrive.shared::cta.b64 tok, [bar] (token discarded) +// +// CUTLASS arrive_barrier() (copy_sm90_desc.hpp:122) issues this form with +// a .reg .b64 tok that is read but never stored — the destination is +// architecturally a register but the C++ wrapper ignores it. +// This verifies that arrive with implicit dst (vs explicit "_") is handled +// correctly by the simulator's dst.is_non_arch_reg() check. +// PTX ISA §9.7.12.4: the token is valid but the caller may discard it. +// h[3] = 1 if barrier completes (all 32 threads arrived, waited with parity). +// ============================================================ +__global__ void test4_arrive_no_token_form(int* out) { + __shared__ uint64_t bar; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + // All threads call arrive_no_token — token captured in reg but discarded + mbar_arrive_no_token(&bar); + + // current phase=0, parity=0 → phase_bit=0 + mbar_spin_wait_parity(&bar, 0); + + if (threadIdx.x == 0) out[3] = 1; +} + +// ============================================================ +// TEST 5: mbarrier.arrive.shared::cluster.b64 via mapa.shared::cluster.u32 +// PTX: mapa.shared::cluster.u32 remAddr, localAddr, cta_rank +// mbarrier.arrive.shared::cluster.b64 _, [remAddr] +// mbarrier.try_wait.parity.shared::cta.b64 P, [bar], phase_bit +// +// This is the CUTLASS ClusterBarrier::arrive(smem_ptr, cta_id) form (barrier.h:489). +// Uses 32-bit mapa (not 64-bit mapa.u64) to map the local shmem address to a +// remote CTA's shmem space. Thread 0 of each CTA signals the partner CTA's barrier. +// Each barrier is init'd for 1 arrival (from the partner's thread 0). +// Each CTA then waits with parity form for its own barrier to complete. +// +// SIMULATOR ISSUE: mapa.shared::cluster.u32 address encoding may differ from +// mapa.u64. The address passed to arrive.shared::cluster.b64 must be decoded +// from a 32-bit generic shared address, not a 64-bit one. +// h[4] = 1 (both CTAs write 1 to same slot) +// ============================================================ +__global__ void __cluster_dims__(2, 1, 1) +test5_cluster_arrive_mapa32(int* out) { + auto cluster = cg::this_cluster(); + unsigned my_rank = cluster.block_rank(); + unsigned partner = 1 - my_rank; + + __shared__ uint64_t bar; + + if (threadIdx.x == 0) mbar_init(&bar, 1); + cluster.sync(); + + if (threadIdx.x == 0) { + // Map partner's bar address using 32-bit mapa (CUTLASS barrier.h form) + uint32_t remote_addr = mapa32(&bar, partner); + mbar_arrive_cluster_remote32(remote_addr); + } + + // current phase=0, parity=0 → phase_bit=0 + if (threadIdx.x == 0) { + mbar_spin_wait_parity(&bar, 0); + out[4] = 1; // both CTAs write to same slot; that's fine, both write 1 + } +} + +// ============================================================ +// TEST 6: Two-phase parity alternation — arrive+try_wait.parity, phase 0 then phase 1 +// PTX: mbarrier.arrive.shared::cta.b64 / mbarrier.try_wait.parity.shared::cta.b64 +// +// Tests that parity form correctly handles two consecutive phases with alternating +// phase_bit (0 then 1), matching CUTLASS PipelineState::phase_ toggle (^= 1). +// +// Two separate barrier cycles in one kernel: +// Cycle A: init(count=32), all arrive, wait(phase_bit=0) — phase 0 in-progress +// Cycle B: re-use same bar (now at phase=1), all arrive, wait(phase_bit=1) — phase 1 +// +// Simulator note: each cycle is a single arrive→wait, so two-pass mode advances +// the barrier by exactly 1 per cycle. No parity ambiguity (phase 2 never reached +// within a single kernel execution, only phase 0→1 and 1→2). +// h[5] = 1 if both cycles complete with correct data. +// ============================================================ +__global__ void test6_looping_parity(int* out) { + __shared__ uint64_t bar; + __shared__ int smemA[32]; + __shared__ int smemB[32]; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + // Cycle A: phase 0 → phase_bit=0 + smemA[threadIdx.x] = threadIdx.x; + __threadfence_block(); + mbar_arrive_no_token(&bar); + mbar_spin_wait_parity(&bar, 0); + + // Cycle B: phase 1 → phase_bit=1 + smemB[threadIdx.x] = threadIdx.x + 100; + __threadfence_block(); + mbar_arrive_no_token(&bar); + mbar_spin_wait_parity(&bar, 1); + + if (threadIdx.x == 0) { + int sumA = 0, sumB = 0; + for (int i = 0; i < 32; i++) { sumA += smemA[i]; sumB += smemB[i]; } + int ok = (sumA == 496) && (sumB == 496 + 32*100); + out[5] = ok ? 1 : -1; + } +} + +// ============================================================ +// TEST 7: count=0 observe-only arrive must not decrement pending +// PTX: mbarrier.arrive.shared::cta.b64 tok, [bar], 0 (count=0) +// +// PTX ISA §9.7.12.4: "If count is 0, the operation acts as an observe and +// does not modify the pending count of the mbarrier object." +// Thread 0 does the real arrive (count=1); all others call count=0 to get +// a token. A simulator that subtracts 0 from a uint32 pending count and then +// calls try_phase_transition() could spuriously fire the transition or +// underflow the counter (wrapping below 0 as unsigned). +// h[6] = 1 if all threads see the barrier complete correctly. +// ============================================================ +__global__ void test7_count0_observe(int* out) { + __shared__ uint64_t bar; + __shared__ int flag; + + if (threadIdx.x == 0) { + mbar_init(&bar, 1); // only 1 real arrival expected + flag = 0; + } + __syncthreads(); + + uint64_t tok; + if (threadIdx.x == 0) { + flag = 42; + __threadfence_block(); + tok = mbar_arrive_token(&bar, 1); // real arrive: pending 1→0 → transition + } else { + tok = mbar_arrive_token(&bar, 0); // observe-only: no pending change + } + + mbar_spin_wait_token(&bar, tok); + + if (threadIdx.x == 0) + out[6] = (flag == 42) ? 1 : -1; +} + +// ============================================================ +// TEST 8: mbarrier.inval then re-init on same address +// PTX: mbarrier.inval.shared.b64 / mbarrier.init.shared::cta.b64 (re-init) +// +// PTX ISA §9.7.12.9: after inval, the mbarrier object is in an undefined +// state. A subsequent init on the same address must reinitialize it cleanly. +// SIMULATOR ISSUE: the mbarrier_clear_for_cta mechanism clears on CTA exit, +// but inval within a kernel should also work. If the simulator doesn't remove +// the mbarrier on inval and a second init hits the "already exists" assert, +// this test exposes it. +// h[7] = 1 if re-init and second barrier cycle complete successfully. +// ============================================================ +__global__ void test8_inval_reinit(int* out) { + __shared__ uint64_t bar; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + // First cycle + uint64_t tok = mbar_arrive_token(&bar); + mbar_spin_wait_token(&bar, tok); + + if (threadIdx.x == 0) mbar_inval(&bar); + __syncthreads(); + + // Re-init same address + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + // Second cycle — parity form (phase resets to 0 after re-init, so phase_bit=0) + mbar_arrive_no_token(&bar); + mbar_spin_wait_parity(&bar, 0); + + if (threadIdx.x == 0) out[7] = 1; +} + +// ============================================================ +// TEST 9: expect_tx + complete_tx + all-thread cluster arrive + parity wait +// PTX: mbarrier.arrive.expect_tx.shared::cta.b64 +// mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 +// mbarrier.arrive.shared::cta.b64 (all threads, no token) +// mbarrier.try_wait.parity.shared::cta.b64 +// +// This is the full CUTLASS TMA warp-specialized producer pattern: +// - Thread 0 calls expect_tx (arrives + announces TX bytes) +// - Thread 0 simulates data arriving then calls complete_tx +// - All 32 threads call arrive (so the barrier needs 32 arrives total, +// but thread 0's arrive was already done inside expect_tx) +// - All wait with parity form +// +// Note: expect_tx decrements pending by 1 AND adds tx_count. So init must +// count the total thread arrivals including the implicit one from expect_tx. +// Init with blockDim.x (32): expect_tx contributes 1 arrive (from thread 0), +// remaining 31 threads contribute 1 each = 32 total → transition fires when +// tx_count also reaches 0. +// h[8] = 1 if data is correct after wait. +// ============================================================ +__global__ void __cluster_dims__(2, 1, 1) +test9_full_tma_pattern(int* out) { + auto cluster = cg::this_cluster(); + unsigned my_rank = cluster.block_rank(); + + __shared__ uint64_t bar; + __shared__ int data[32]; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + cluster.sync(); + + if (my_rank == 0) { + if (threadIdx.x == 0) { + // arrive.expect_tx: arrives (pending 32→31) + announces 128 bytes tx + mbar_arrive_expect_tx(&bar, 32 * sizeof(int)); + + // Simulate data arriving asynchronously + for (int i = 0; i < 32; i++) data[i] = i + 300; + __threadfence_block(); + + // complete_tx: tx_count 128→0 + mbar_complete_tx(&bar, 32 * sizeof(int)); + } else { + // Remaining 31 threads arrive (no token) + mbar_arrive_no_token(&bar); + } + } else { + // CTA1: all 32 threads just arrive normally + mbar_arrive_no_token(&bar); + } + + // current phase=0, parity=0 → phase_bit=0 + mbar_spin_wait_parity(&bar, 0); + + if (my_rank == 0 && threadIdx.x == 0) { + int ok = 1; + for (int i = 0; i < 32; i++) if (data[i] != i + 300) { ok = 0; break; } + out[8] = ok ? 1 : -1; + } +} + +// ============================================================ +// main +// ============================================================ +int main() { + const int N = 9; + int h[N] = {}; + int* d; + cudaMalloc(&d, N * sizeof(int)); + + // Zero all slots once upfront — each test writes only its own slot(s) + cudaMemset(d, 0, N * sizeof(int)); + + // 2-CTA cluster config + cudaLaunchConfig_t cfg2 = {}; + cudaLaunchAttribute attr2[1]; + attr2[0].id = cudaLaunchAttributeClusterDimension; + attr2[0].val.clusterDim = {2, 1, 1}; + cfg2.gridDim = {2, 1, 1}; + cfg2.blockDim = {32, 1, 1}; + cfg2.attrs = attr2; + cfg2.numAttrs = 1; + + auto sync = [&]() { + cudaDeviceSynchronize(); + cudaMemcpy(h, d, N * sizeof(int), cudaMemcpyDeviceToHost); + }; + auto pf = [](int v) { return v == 1 ? "PASS" : (v == -1 ? "FAIL" : "SKIP"); }; + + printf("=== mbarrier Cluster Microbenchmark (sm_90 / Hopper) ===\n"); + printf("PTX forms covered:\n"); + printf(" mbarrier.init.shared::cta.b64\n"); + printf(" mbarrier.arrive.shared::cta.b64 tok/_ (token and no-token forms)\n"); + printf(" mbarrier.arrive.expect_tx.shared::cta.b64 _\n"); + printf(" mbarrier.arrive.shared::cluster.b64 _ (via mapa.shared::cluster.u32)\n"); + printf(" mbarrier.try_wait.parity.shared::cta.b64 (CUTLASS wait form)\n"); + printf(" mbarrier.try_wait.shared.b64 (token wait form)\n"); + printf(" mbarrier.complete_tx.relaxed.cluster.shared::cta.b64\n"); + printf(" mbarrier.inval.shared.b64\n\n"); + + // TEST 1 — h[0] + printf("TEST 1 [init/arrive-token/try_wait-token baseline]... "); + fflush(stdout); + test1_baseline_token_wait<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[0])); + + // TEST 2 — h[1] + printf("TEST 2 [try_wait.parity - CUTLASS wait form]... "); + fflush(stdout); + test2_try_wait_parity<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[1])); + + // TEST 3 — h[2] + printf("TEST 3 [arrive.expect_tx + complete_tx + try_wait.parity]... "); + fflush(stdout); + test3_expect_tx_parity_wait<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[2])); + + // TEST 4 — h[3] + printf("TEST 4 [arrive(no-token/_, [bar]) - CUTLASS arrive_barrier form]... "); + fflush(stdout); + test4_arrive_no_token_form<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[3])); + + // TEST 5 — h[4] (2-CTA cluster) + printf("TEST 5 [arrive.shared::cluster via mapa.shared::cluster.u32]... "); + fflush(stdout); + cudaLaunchKernelEx(&cfg2, test5_cluster_arrive_mapa32, d); + sync(); + printf("%s\n", pf(h[4])); + + // TEST 6 — h[5] + printf("TEST 6 [looping parity: 4 phase transitions, alternating phase_bit]... "); + fflush(stdout); + test6_looping_parity<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[5])); + + // TEST 7 — h[6] + printf("TEST 7 [count=0 observe-only arrive must not decrement pending]... "); + fflush(stdout); + test7_count0_observe<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[6])); + + // TEST 8 — h[7] + printf("TEST 8 [mbarrier.inval then re-init on same address]... "); + fflush(stdout); + test8_inval_reinit<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[7])); + + // TEST 9 — h[8] (2-CTA cluster) + printf("TEST 9 [full CUTLASS TMA pattern: expect_tx+complete_tx+arrive+parity-wait]... "); + fflush(stdout); + cudaLaunchKernelEx(&cfg2, test9_full_tma_pattern, d); + sync(); + printf("%s\n", pf(h[8])); + + printf("\n=== Results: [%d,%d,%d,%d,%d,%d,%d,%d,%d] ===\n", + h[0],h[1],h[2],h[3],h[4],h[5],h[6],h[7],h[8]); + bool all = true; + for (int i = 0; i < N; i++) all = all && (h[i] == 1); + printf("Overall: %s\n", all ? "PASS" : "PARTIAL/FAIL"); + + cudaFree(d); + return 0; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu new file mode 100644 index 000000000..5ee7702d7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu @@ -0,0 +1,124 @@ +#include +#include +#include + +#define BLOCK_SIZE 256 + +// -------------------------------------------------------- +// KERNEL 1: Reference Implementation (__syncthreads) +// -------------------------------------------------------- +__global__ void reduction_reference(int* in, int* out) { + __shared__ int s_data[BLOCK_SIZE]; + + // Load data into shared memory + int tid = threadIdx.x; + s_data[tid] = in[tid]; + __syncthreads(); // Standard barrier + + // Standard tree reduction + + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + s_data[tid] += s_data[tid + stride]; + } + __syncthreads(); // Barrier essential for correctness + } + + // Write result + if (tid == 0) *out = s_data[0]; +} + +// -------------------------------------------------------- +// KERNEL 2: mbarrier Implementation +// -------------------------------------------------------- +__global__ void reduction_mbarrier(int* in, int* out) { + __shared__ int s_data[BLOCK_SIZE]; + + // 1. Declare mbarrier in shared memory + __shared__ cuda::barrier bar; + + // 2. Initialize barrier (Thread 0 only) + if (threadIdx.x == 0) { + init(&bar, blockDim.x); + } + __syncthreads(); // Wait for init to finish + + // Load data + int tid = threadIdx.x; + s_data[tid] = in[tid]; + + // Synchronize after load + auto token = bar.arrive(); + + // CHANGE: Use standard wait(). + // This generates the 'mbarrier.test_wait' loop automatically. + bar.wait(std::move(token)); + + // Standard tree reduction using mbarrier + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + s_data[tid] += s_data[tid + stride]; + } + + // 3. BARRIER SYNCHRONIZATION + token = bar.arrive(); + bar.wait(std::move(token)); + } + + if (tid == 0) *out = s_data[0]; +} + +int main() { + const int N = BLOCK_SIZE; + const int BYTES = N * sizeof(int); + + // Host data + int* h_in = (int*)malloc(BYTES); + int h_ref_out = 0; + int h_mbar_out = 0; + + // Initialize input (Array of 1s, so Sum should be 256) + for (int i = 0; i < N; i++) h_in[i] = 1; + + // Device data + int *d_in, *d_ref_out, *d_mbar_out; + cudaMalloc(&d_in, BYTES); + cudaMalloc(&d_ref_out, sizeof(int)); + cudaMalloc(&d_mbar_out, sizeof(int)); + cudaMemcpy(d_in, h_in, BYTES, cudaMemcpyHostToDevice); + + // ---------------------------------------------------- + // Run Reference Kernel + // ---------------------------------------------------- + reduction_reference<<<1, BLOCK_SIZE>>>(d_in, d_ref_out); + cudaMemcpy(&h_ref_out, d_ref_out, sizeof(int), cudaMemcpyDeviceToHost); + + // ---------------------------------------------------- + // Run mbarrier Kernel + // ---------------------------------------------------- + reduction_mbarrier<<<1, BLOCK_SIZE>>>(d_in, d_mbar_out); + cudaMemcpy(&h_mbar_out, d_mbar_out, sizeof(int), cudaMemcpyDeviceToHost); + + // ---------------------------------------------------- + // Verify + // ---------------------------------------------------- + printf("Reference Sum: %d\n", h_ref_out); + printf("mbarrier Sum: %d\n", h_mbar_out); + + bool pass = (h_ref_out == h_mbar_out && h_ref_out == N); + if (pass) { + printf("SUCCESS: mbarrier matches reference output!\n"); + } else { + printf("FAILURE: Results mismatch.\n"); + printf("Likely Cause: Threads raced ahead without waiting.\n"); + } + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + // Cleanup + free(h_in); + cudaFree(d_in); + cudaFree(d_ref_out); + cudaFree(d_mbar_out); + + return pass ? 0 : 1; +} \ No newline at end of file diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu new file mode 100644 index 000000000..7dbe47617 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu @@ -0,0 +1,146 @@ +#include +#include +#include + +#define BLOCK_SIZE 256 + +// Helper to burn time and force a race condition +__device__ void sabotage_delay(int cycles) { + long long start = clock64(); + while (clock64() - start < cycles) { + // Busy wait to hold up the thread + } +} + +// -------------------------------------------------------- +// KERNEL 1: Reference Implementation (__syncthreads) +// -------------------------------------------------------- +__global__ void reduction_reference(int* in, int* out) { + __shared__ int s_data[BLOCK_SIZE]; + + // Load data into shared memory + int tid = threadIdx.x; + + // SABOTAGE: Delay writers here too to prove __syncthreads works + if (tid >= BLOCK_SIZE / 2) { + sabotage_delay(50000); + } + + s_data[tid] = in[tid]; + __syncthreads(); // Standard barrier + + // Standard tree reduction + + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + s_data[tid] += s_data[tid + stride]; + } + __syncthreads(); // Barrier essential for correctness + } + + // Write result + if (tid == 0) *out = s_data[0]; +} + +// -------------------------------------------------------- +// KERNEL 2: mbarrier Implementation +// -------------------------------------------------------- +__global__ void reduction_mbarrier(int* in, int* out) { + __shared__ int s_data[BLOCK_SIZE]; + + // 1. Declare mbarrier in shared memory + __shared__ cuda::barrier bar; + + // 2. Initialize barrier (Thread 0 only) + if (threadIdx.x == 0) { + init(&bar, blockDim.x); + } + __syncthreads(); // Wait for init to finish + + // Load data + int tid = threadIdx.x; + + // --- THE SABOTAGE --- + // We delay the threads that provide data (High IDs). + // The threads that read data (Low IDs) will rush ahead. + if (tid >= BLOCK_SIZE / 2) { + sabotage_delay(50000); // Delay 500 cycles + } + + // Write Data (Late) + s_data[tid] = in[tid]; + + // THE TEST: + // If barrier is broken, Low IDs reach here, see High IDs aren't done, + // ignore it, and read garbage data below. + auto token = bar.arrive(); + bar.wait(std::move(token)); + + // Standard tree reduction using mbarrier + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + s_data[tid] += s_data[tid + stride]; + } + + // 3. BARRIER SYNCHRONIZATION == __syncthreads + token = bar.arrive(); + bar.wait(std::move(token)); + } + + if (tid == 0) *out = s_data[0]; +} + +int main() { + const int N = BLOCK_SIZE; + const int BYTES = N * sizeof(int); + + // Host data + int* h_in = (int*)malloc(BYTES); + int h_ref_out = 0; + int h_mbar_out = 0; + + // Initialize input (Array of 1s, so Sum should be 256) + for (int i = 0; i < N; i++) h_in[i] = 1; + + // Device data + int *d_in, *d_ref_out, *d_mbar_out; + cudaMalloc(&d_in, BYTES); + cudaMalloc(&d_ref_out, sizeof(int)); + cudaMalloc(&d_mbar_out, sizeof(int)); + cudaMemcpy(d_in, h_in, BYTES, cudaMemcpyHostToDevice); + + // ---------------------------------------------------- + // Run Reference Kernel + // ---------------------------------------------------- + reduction_reference<<<1, BLOCK_SIZE>>>(d_in, d_ref_out); + cudaMemcpy(&h_ref_out, d_ref_out, sizeof(int), cudaMemcpyDeviceToHost); + + // ---------------------------------------------------- + // Run mbarrier Kernel + // ---------------------------------------------------- + reduction_mbarrier<<<1, BLOCK_SIZE>>>(d_in, d_mbar_out); + cudaMemcpy(&h_mbar_out, d_mbar_out, sizeof(int), cudaMemcpyDeviceToHost); + + // ---------------------------------------------------- + // Verify + // ---------------------------------------------------- + printf("Reference Sum: %d\n", h_ref_out); + printf("mbarrier Sum: %d\n", h_mbar_out); + + bool pass = (h_ref_out == h_mbar_out && h_ref_out == N); + if (pass) { + printf("SUCCESS: mbarrier matches reference output!\n"); + } else { + printf("FAILURE: Results mismatch.\n"); + printf("Likely Cause: Threads raced ahead without waiting.\n"); + } + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + // Cleanup + free(h_in); + cudaFree(d_in); + cudaFree(d_ref_out); + cudaFree(d_mbar_out); + + return pass ? 0 : 1; +} \ No newline at end of file diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu new file mode 100644 index 000000000..0bbf9ade0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu @@ -0,0 +1,42 @@ +#include +#include // Standard C++ header +#include + +__global__ void mbarrier_kernel(int* out) { + // 1. Declare barrier in Shared Memory + __shared__ cuda::barrier bar; + + // 2. Initialize (Thread 0 only) + if (threadIdx.x == 0) { + init(&bar, blockDim.x); + } + __syncthreads(); + + // 3. Arrive and Wait + // This generates the 'mbarrier.arrive' and 'mbarrier.try_wait' + // instructions automatically. + cuda::barrier::arrival_token token = bar.arrive(); + bar.wait(std::move(token)); + + // Success check + if (threadIdx.x == 0) { + *out = 1; + } +} + +int main() { + int* d_out; + int h_out = 0; + cudaMalloc(&d_out, sizeof(int)); + cudaMemset(d_out, 0, sizeof(int)); + + mbarrier_kernel<<<1, 32>>>(d_out); + + cudaMemcpy(&h_out, d_out, sizeof(int), cudaMemcpyDeviceToHost); + + bool pass = (h_out == 1); + if (pass) printf("mbarrier test passed (Standard C++).\n"); + else printf("mbarrier test failed.\n"); + + return pass ? 0 : 1; +} \ No newline at end of file diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/Makefile index f3c8bcd00..7c8b9909f 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/Makefile @@ -1,13 +1,33 @@ -SRC = tma_bulk.cu +# TMA Bulk microbenchmarks +# Build all benchmarks in this folder when invoked from the top-level +# GPU_Microbenchmark make recursion. -EXE = tma_bulk +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ -# TMA is supported on SM_90a and above -ARCH?=sm_90a sm_100a sm_101 sm_120 -# Unset the CUDA_CPPFLAGS which is set based on CUDA version -# but TMA is only supported on SM_90a and above -CUDA_CPPFLAGS= -# Generate code for both sm_XXX and compute_XXX (SASS and PTX) +# All bulk microbenchmarks in this directory. +ALL_BENCHES := tma_bulk ublkcp_g_s ublkcp_s_g ublkpf ublkred_g_s + +# TMA is supported on SM_90a and above. +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Keep same codegen style used across GPU_Microbenchmark. NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 -include ../../../common/common.mk +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +# Compile .cu -> .a so PTX is preserved, then link with shared cudart. +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/tma_bulk.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/tma_bulk.cu index fc885b515..7190422c8 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/tma_bulk.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/tma_bulk.cu @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_g_s.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_g_s.cu new file mode 100644 index 000000000..f4086c4ab --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_g_s.cu @@ -0,0 +1,158 @@ +/** + * UBLKCP_G_S Unit Test: TMA Bulk Copy Shared -> Global + * ===================================================== + * Tests that cp.async.bulk correctly stores data from shared to global memory. + * + * Strategy: + * 1. Global memory is poisoned with 0xDEADBEEF + * 2. Kernel generates pattern (0, 1, 2, ...) in shared memory + * 3. TMA stores shared -> global + * PASS: Global contains (0, 1, 2, ...) + * FAIL: Global still contains poison (TMA store was a NOP) + * + * Usage: ./ublkcp_g_s [-n ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, \ + "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90 or newer for TMA bulk operations."); +#endif + +static constexpr size_t buf_len = 1024; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UBLKCP_G_S(int32_t *data, int run_iters) +{ + __shared__ alignas(16) int32_t smem_data[buf_len]; + + size_t offset = blockIdx.x * blockDim.x; + + // Generate the test pattern (0, 1, 2, ...) in shared memory + for (int i = threadIdx.x; i < buf_len; i += blockDim.x) { + smem_data[i] = offset + i; + } + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + // TMA bulk store: shared -> global + for (int iter = 0; iter < run_iters; iter++) { + if (threadIdx.x == 0) { + ptx::cp_async_bulk( + ptx::space_global, + ptx::space_shared, + data + offset, smem_data, sizeof(smem_data)); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(int32_t *h_result, int n) +{ + int poison_count = 0; + int mismatch_count = 0; + + for (int i = 0; i < n; i++) { + if (h_result[i] == POISON) { + if (poison_count == 0) + printf(" NOP detected at index %d: got 0x%08X, expected %d\n", + i, (unsigned)h_result[i], i); + poison_count++; + } else if (h_result[i] != i) { + if (mismatch_count == 0) + printf(" Mismatch at index %d: got %d, expected %d\n", + i, h_result[i], i); + mismatch_count++; + } + } + + if (poison_count > 0) { + printf(" FAIL: %d/%d elements still contain poison (TMA store was NOP)\n", + poison_count, n); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%d elements have wrong values\n", mismatch_count, n); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + int n = 1024; + int run_iters = 128; + int opt; + while ((opt = getopt(argc, argv, "n:i:")) != -1) { + switch (opt) { + case 'n': n = atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-n ] [-i ]\n", argv[0]); + return 1; + } + } + + size_t bytes = n * sizeof(int32_t); + uint32_t blockSize = 1024; + uint32_t gridSize = (uint32_t)ceil((float)n / blockSize); + + // Host buffer — poisoned + int32_t *h_result = (int32_t *)malloc(bytes); + for (int i = 0; i < n; i++) { + h_result[i] = POISON; + } + + // Device buffer + int32_t *d_a; + cudaMalloc(&d_a, bytes); + cudaMemcpy(d_a, h_result, bytes, cudaMemcpyHostToDevice); + + // Run + CUDA_SAFECALL((test_UBLKCP_G_S<<>>(d_a, run_iters))); + cudaDeviceSynchronize(); + + // Copy back + cudaMemcpy(h_result, d_a, bytes, cudaMemcpyDeviceToHost); + + // Verify + printf("=== UBLKCP_G_S (TMA Bulk Store: Shared -> Global) ===\n"); + bool pass = verify(h_result, n); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_result); + cudaFree(d_a); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_s_g.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_s_g.cu new file mode 100644 index 000000000..1dbd13d07 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_s_g.cu @@ -0,0 +1,175 @@ +/** + * UBLKCP_S_G Unit Test: TMA Bulk Copy Global -> Shared + * ===================================================== + * Tests that cp.async.bulk correctly loads data from global to shared memory. + * + * Strategy: + * 1. Source global (d_src) initialized with pattern (0, 1, 2, ...) + * 2. Destination global (d_dst) poisoned with 0xDEADBEEF + * 3. TMA loads d_src -> shared memory + * 4. Scalar (trusted) copy: shared -> d_dst + * PASS: d_dst matches d_src + * FAIL: d_dst still contains poison (TMA load was a NOP) + * + * Usage: ./ublkcp_s_g [-n ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, \ + "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +using barrier = cuda::barrier; +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90 or newer for TMA bulk operations."); +#endif + +static constexpr size_t buf_len = 1024; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UBLKCP_S_G(int32_t *d_src, int32_t *d_dst, int run_iters) +{ + __shared__ alignas(16) int32_t smem_data[buf_len]; + + size_t offset = blockIdx.x * blockDim.x; + + #pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + if (threadIdx.x == 0) { + init(&bar, blockDim.x); + ptx::fence_proxy_async(ptx::space_shared); + } + __syncthreads(); + + // TMA bulk load: global -> shared + for (int iter = 0; iter < run_iters; iter++) { + if (threadIdx.x == 0) { + cuda::memcpy_async( + smem_data, + d_src + offset, + cuda::aligned_size_t<16>(sizeof(smem_data)), + bar); + } + barrier::arrival_token token = bar.arrive(); + bar.wait(std::move(token)); + } + + // Trusted scalar copy: shared -> d_dst (does NOT use TMA) + __syncthreads(); + for (int i = threadIdx.x; i < buf_len; i += blockDim.x) { + d_dst[offset + i] = smem_data[i]; + } +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(int32_t *h_src, int32_t *h_dst, int n) +{ + int poison_count = 0; + int mismatch_count = 0; + + for (int i = 0; i < n; i++) { + if (h_dst[i] == POISON) { + if (poison_count == 0) + printf(" NOP detected at index %d: got 0x%08X, expected %d\n", + i, (unsigned)h_dst[i], h_src[i]); + poison_count++; + } else if (h_dst[i] != h_src[i]) { + if (mismatch_count == 0) + printf(" Mismatch at index %d: got %d, expected %d\n", + i, h_dst[i], h_src[i]); + mismatch_count++; + } + } + + if (poison_count > 0) { + printf(" FAIL: %d/%d elements still contain poison (TMA load was NOP)\n", + poison_count, n); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%d elements have wrong values\n", mismatch_count, n); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + int n = 1024; + int run_iters = 128; + int opt; + while ((opt = getopt(argc, argv, "n:i:")) != -1) { + switch (opt) { + case 'n': n = atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-n ] [-i ]\n", argv[0]); + return 1; + } + } + + size_t bytes = n * sizeof(int32_t); + uint32_t blockSize = 1024; + uint32_t gridSize = (uint32_t)ceil((float)n / blockSize); + + // Host buffers + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + // Initialize source with pattern, destination with poison + for (int i = 0; i < n; i++) { + h_src[i] = i; + h_dst[i] = POISON; + } + + // Device buffers + int32_t *d_src, *d_dst; + cudaMalloc(&d_src, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_src, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Run + CUDA_SAFECALL((test_UBLKCP_S_G<<>>(d_src, d_dst, run_iters))); + cudaDeviceSynchronize(); + + // Copy back + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + // Verify + printf("=== UBLKCP_S_G (TMA Bulk Load: Global -> Shared) ===\n"); + bool pass = verify(h_src, h_dst, n); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_src); + free(h_dst); + cudaFree(d_src); + cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu new file mode 100644 index 000000000..002bcb4f0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu @@ -0,0 +1,245 @@ +/** + * UBLKPF Unit Test: TMA Bulk Prefetch (cp.async.bulk.prefetch.L2.global) + * ====================================================================== + * Tests that cp.async.bulk.prefetch.L2.global correctly issues a prefetch + * without corrupting memory contents. + * + * The PTX ISA only exposes one variant of 1D bulk prefetch: + * cp.async.bulk.prefetch.L2.global [srcMem], size; + * + * Verification Strategy: + * 1. Global memory is initialized with a known pattern (0, 1, 2, ...) + * 2. Kernel issues cp.async.bulk.prefetch.L2.global for each block's region + * 3. After the prefetch, kernel reads the data back via standard loads + * and writes to a destination buffer + * 4. Host verifies destination matches original source pattern + * + * PASS: Prefetch executed without error, data integrity maintained + * FAIL: Data is corrupted after prefetch (should never happen on correct HW) + * + * Usage: ./ublkpf [-n ] [-i ] [-s ] + * + * -n : number of int32 elements (default: 1024) + * -i : number of prefetch iterations in-kernel (default: 1) + * -s : bytes to prefetch per block (default: buf_len * 4) + * clamped to a multiple of 16, max buf_len * 4 + */ + +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, \ + "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90 or newer for TMA bulk operations."); +#endif + +static constexpr size_t buf_len = 1024; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel: prefetch + readback verification +// +// Thread 0 issues the bulk prefetch, then all threads cooperatively +// read the (now hopefully L2-resident) data via standard loads and +// write it to a separate destination buffer. If the prefetch corrupted +// memory or caused any fault, the readback will expose it. +// ============================================================================ +__global__ void test_UBLKPF(const int32_t *d_src, int32_t *d_dst, + int run_iters, uint32_t prefetch_bytes) +{ + size_t offset = blockIdx.x * blockDim.x; + + // Prefetch phase + if (threadIdx.x == 0) { + uint64_t addr = (uint64_t)(d_src + offset); + for (int iter = 0; iter < run_iters; iter++) { + asm volatile( + "cp.async.bulk.prefetch.L2.global" + " [%0], %1;" + : + : "l"(addr), + "r"(prefetch_bytes) + : "memory"); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } + + __syncthreads(); + + // Readback phase: standard loads to verify data integrity + uint32_t elems = prefetch_bytes / sizeof(int32_t); + for (uint32_t i = threadIdx.x; i < elems; i += blockDim.x) { + d_dst[offset + i] = d_src[offset + i]; + } +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_src, const int32_t *h_dst, int n) +{ + int poison_count = 0; + int mismatch_count = 0; + + for (int i = 0; i < n; i++) { + if (h_dst[i] == POISON) { + if (poison_count == 0) + printf(" Poison still present at index %d: got 0x%08X, expected %d\n", + i, (unsigned)h_dst[i], h_src[i]); + poison_count++; + } else if (h_dst[i] != h_src[i]) { + if (mismatch_count == 0) + printf(" Mismatch at index %d: got %d, expected %d\n", + i, h_dst[i], h_src[i]); + mismatch_count++; + } + } + + if (poison_count > 0) { + printf(" FAIL: %d/%d elements still contain poison (readback did not execute)\n", + poison_count, n); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%d elements have wrong values (data corrupted after prefetch)\n", + mismatch_count, n); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + int n = 1024; + int run_iters = 1; + int user_prefetch_bytes = -1; + + int opt; + while ((opt = getopt(argc, argv, "n:i:s:")) != -1) { + switch (opt) { + case 'n': + n = atoi(optarg); + break; + case 'i': + run_iters = atoi(optarg); + break; + case 's': + user_prefetch_bytes = atoi(optarg); + break; + default: + fprintf(stderr, + "Usage: %s [-n ] [-i ] [-s ]\n", + argv[0]); + fprintf(stderr, " -n : number of int32 elements (default: 1024)\n"); + fprintf(stderr, " -i : number of prefetch iterations (default: 1)\n"); + fprintf(stderr, " -s : bytes to prefetch per block (default: %zu)\n", + buf_len * sizeof(int32_t)); + fprintf(stderr, " must be a multiple of 16, max %zu\n", + buf_len * sizeof(int32_t)); + return 1; + } + } + + uint32_t max_prefetch = (uint32_t)(buf_len * sizeof(int32_t)); + uint32_t prefetch_bytes; + if (user_prefetch_bytes < 0) { + prefetch_bytes = max_prefetch; + } else { + prefetch_bytes = (uint32_t)user_prefetch_bytes; + prefetch_bytes = (prefetch_bytes + 15) & ~15u; + if (prefetch_bytes > max_prefetch) + prefetch_bytes = max_prefetch; + if (prefetch_bytes == 0) + prefetch_bytes = 16; + } + + size_t bytes = (size_t)n * sizeof(int32_t); + uint32_t blockSize = 1024; + uint32_t gridSize = (uint32_t)ceil((float)n / blockSize); + + // Host buffers + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + for (int i = 0; i < n; i++) { + h_src[i] = i; + h_dst[i] = POISON; + } + + // Device buffers + int32_t *d_src, *d_dst; + cudaMalloc(&d_src, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_src, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Run + CUDA_SAFECALL((test_UBLKPF<<>>(d_src, d_dst, run_iters, prefetch_bytes))); + cudaDeviceSynchronize(); + + // Copy back + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + // Verify + uint32_t verified_elems = prefetch_bytes / sizeof(int32_t); + if ((int)verified_elems > n) verified_elems = n; + + printf("=== UBLKPF (TMA Bulk Prefetch: cp.async.bulk.prefetch.L2.global) ===\n"); + printf(" Prefetch size: %u bytes (%u int32 elements)\n", + prefetch_bytes, prefetch_bytes / 4); + printf(" Iterations: %d\n", run_iters); + + bool pass = verify(h_src, h_dst, (int)verified_elems); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + // Dump results to file for easy expected-vs-actual comparison + char filename[128]; + sprintf(filename, "tma_bulk_test_UBLKPF_%d.txt", n); + FILE *f = fopen(filename, "w"); + if (f) { + fprintf(f, "# UBLKPF Test Results (TMA Bulk Prefetch + Readback)\n"); + fprintf(f, "# Prefetch size: %u bytes (%u int32 elements)\n", + prefetch_bytes, prefetch_bytes / 4); + fprintf(f, "# Iterations: %d\n", run_iters); + fprintf(f, "# Format: index, got (hex), expected (hex)\n"); + for (int i = 0; i < (int)verified_elems; i++) { + const char *status = (h_dst[i] == h_src[i]) ? "OK" : + (h_dst[i] == POISON) ? "POISON" : "MISMATCH"; + fprintf(f, "%4d: 0x%08X (expected 0x%08X) [%s]\n", + i, (unsigned)h_dst[i], (unsigned)h_src[i], status); + if ((i + 1) % 512 == 0) fprintf(f, "\n"); + } + fclose(f); + printf("Values dumped to %s\n", filename); + } else { + fprintf(stderr, "Warning: could not open %s for writing\n", filename); + } + + free(h_src); + free(h_dst); + cudaFree(d_src); + cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu new file mode 100644 index 000000000..2a721d195 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu @@ -0,0 +1,362 @@ +/** + * UBLKRED_G_S Unit Test: TMA Bulk Reduction Global <- Shared + * =========================================================== + * Tests that cp.reduce.async.bulk correctly reduces data from shared + * into global memory for several reduction operators. + * + * Strategy (per element i): + * - Global memory is initialized to an operator-specific BASE value. + * - Shared memory is initialized with the pattern (0, 1, 2, ...). + * - TMA performs a bulk reduction: global[i] = reduce(global[i], shared[i]). + * - Host computes the expected result and compares element-wise. + * + * Supported reduction kinds: + * - add : global += shared + * - min : global = min(global, shared) + * - max : global = max(global, shared) + * - and : global &= shared + * - or : global |= shared + * - xor : global ^= shared + * + * Usage: + * ./ublkred_g_s [-n ] [-i ] [-r ] + * + * -n : number of elements (default: 1024) + * -i : number of reduction iterations in-kernel (default: 1) + * -r : one of {add,min,max,and,or,xor,all} (default: add) + */ + +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, \ + "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90 or newer for TMA bulk operations."); +#endif + +static constexpr size_t buf_len = 1024; + +// Base values used for each reduction kind. +static constexpr int32_t BASE_ADD = 100; +static constexpr int32_t BASE_MIN = 1000000; +static constexpr int32_t BASE_MAX = -1000000; +static constexpr int32_t BASE_AND = ~0; +static constexpr int32_t BASE_OR = 0; +static constexpr int32_t BASE_XOR = 0xAAAAAAAA; + +enum class ReductionKind : int { + ADD = 0, + MIN, + MAX, + AND, + OR, + XOR +}; + +static const char* reduction_kind_name(ReductionKind kind) +{ + switch (kind) { + case ReductionKind::ADD: return "ADD"; + case ReductionKind::MIN: return "MIN"; + case ReductionKind::MAX: return "MAX"; + case ReductionKind::AND: return "AND"; + case ReductionKind::OR: return "OR"; + case ReductionKind::XOR: return "XOR"; + default: return "UNKNOWN"; + } +} + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UBLKRED_G_S(int32_t *data, int run_iters, ReductionKind kind) +{ + // Shared memory buffer. The destination shared memory buffer of + // a bulk operation should be 16-byte aligned. + __shared__ alignas(16) int32_t smem_data[buf_len]; + + size_t offset = blockIdx.x * blockDim.x; + + // Generate the test pattern (0, 1, 2, ...) in shared memory + for (int i = threadIdx.x; i < (int)buf_len; i += blockDim.x) { + smem_data[i] = offset + i; + } + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + for (int iter = 0; iter < run_iters; iter++) { + if (threadIdx.x == 0) { + switch (kind) { + case ReductionKind::ADD: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_add, + data + offset, smem_data, sizeof(smem_data)); + break; + case ReductionKind::MIN: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_min, + data + offset, smem_data, sizeof(smem_data)); + break; + case ReductionKind::MAX: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_max, + data + offset, smem_data, sizeof(smem_data)); + break; + case ReductionKind::AND: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_and_op, + data + offset, smem_data, sizeof(smem_data)); + break; + case ReductionKind::OR: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_or_op, + data + offset, smem_data, sizeof(smem_data)); + break; + case ReductionKind::XOR: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_xor_op, + data + offset, smem_data, sizeof(smem_data)); + break; + } + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Host-side helpers +// ============================================================================ +static int32_t base_for_kind(ReductionKind kind) +{ + switch (kind) { + case ReductionKind::ADD: return BASE_ADD; + case ReductionKind::MIN: return BASE_MIN; + case ReductionKind::MAX: return BASE_MAX; + case ReductionKind::AND: return BASE_AND; + case ReductionKind::OR: return BASE_OR; + case ReductionKind::XOR: return BASE_XOR; + default: return 0; + } +} + +static void init_global(int32_t *h_data, int n, ReductionKind kind) +{ + int32_t base = base_for_kind(kind); + for (int i = 0; i < n; i++) { + h_data[i] = base; + } +} + +static int32_t expected_value_for_index(int idx, ReductionKind kind, int run_iters) +{ + const int32_t i = (int32_t)idx; + + switch (kind) { + case ReductionKind::ADD: + // After run_iters iterations: base + run_iters * i + return BASE_ADD + (int32_t)(run_iters * i); + + case ReductionKind::MIN: + // min(BASE_MIN, i) + return (i < BASE_MIN) ? i : BASE_MIN; + + case ReductionKind::MAX: + // max(BASE_MAX, i) + return (i > BASE_MAX) ? i : BASE_MAX; + + case ReductionKind::AND: + // BASE_AND & i (idempotent across iterations) + return (BASE_AND & i); + + case ReductionKind::OR: + // BASE_OR | i (idempotent across iterations) + return (BASE_OR | i); + + case ReductionKind::XOR: + // XOR toggles each iteration + if ((run_iters & 1) == 0) { + return BASE_XOR; + } else { + return BASE_XOR ^ i; + } + } + return 0; +} + +static bool verify_reduction(int32_t *h_result, int n, ReductionKind kind, int run_iters) +{ + int mismatch_count = 0; + int first_mismatch_idx = -1; + int32_t first_got = 0; + int32_t first_expected = 0; + + for (int idx = 0; idx < n; idx++) { + int32_t expected = expected_value_for_index(idx, kind, run_iters); + if (h_result[idx] != expected) { + mismatch_count++; + if (first_mismatch_idx < 0) { + first_mismatch_idx = idx; + first_got = h_result[idx]; + first_expected = expected; + } + } + } + + if (mismatch_count > 0) { + printf(" First mismatch at index %d: got %d, expected %d\n", + first_mismatch_idx, first_got, first_expected); + printf(" FAIL: %d/%d elements have wrong values\n", mismatch_count, n); + return false; + } + + return true; +} + +static bool run_single_reduction(ReductionKind kind, int n, int run_iters) +{ + size_t bytes = (size_t)n * sizeof(int32_t); + uint32_t blockSize = 1024; + uint32_t gridSize = (uint32_t)ceil((float)n / blockSize); + + int32_t *h_data = (int32_t *)malloc(bytes); + if (!h_data) { + fprintf(stderr, "Host allocation failed\n"); + return false; + } + + int32_t *d_a = nullptr; + cudaMalloc(&d_a, bytes); + + // Initialize global memory on host and copy to device + init_global(h_data, n, kind); + cudaMemcpy(d_a, h_data, bytes, cudaMemcpyHostToDevice); + + // Run kernel + CUDA_SAFECALL((test_UBLKRED_G_S<<>>(d_a, run_iters, kind))); + cudaDeviceSynchronize(); + + // Copy result back + cudaMemcpy(h_data, d_a, bytes, cudaMemcpyDeviceToHost); + + // Verify + printf("=== UBLKRED_G_S (TMA Bulk Reduction: %s) ===\n", reduction_kind_name(kind)); + bool pass = verify_reduction(h_data, n, kind, run_iters); + printf("RESULT: %s\n\n", pass ? "PASSED" : "FAILED"); + + free(h_data); + cudaFree(d_a); + + return pass; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + int n = 1024; + int run_iters = 1; + const char *reduction_str = "add"; + bool test_all = false; + + int opt; + while ((opt = getopt(argc, argv, "n:i:r:")) != -1) { + switch (opt) { + case 'n': + n = atoi(optarg); + break; + case 'i': + run_iters = atoi(optarg); + break; + case 'r': + reduction_str = optarg; + break; + default: + fprintf(stderr, + "Usage: %s [-n ] [-i ] [-r ]\n", + argv[0]); + fprintf(stderr, + " -r where is one of: add, min, max, and, or, xor, all\n"); + return 1; + } + } + + ReductionKind selected = ReductionKind::ADD; + + if (strcmp(reduction_str, "add") == 0) { + selected = ReductionKind::ADD; + } else if (strcmp(reduction_str, "min") == 0) { + selected = ReductionKind::MIN; + } else if (strcmp(reduction_str, "max") == 0) { + selected = ReductionKind::MAX; + } else if (strcmp(reduction_str, "and") == 0) { + selected = ReductionKind::AND; + } else if (strcmp(reduction_str, "or") == 0) { + selected = ReductionKind::OR; + } else if (strcmp(reduction_str, "xor") == 0) { + selected = ReductionKind::XOR; + } else if (strcmp(reduction_str, "all") == 0) { + test_all = true; + } else { + fprintf(stderr, + "Invalid reduction type '%s'. Expected one of: add, min, max, and, or, xor, all\n", + reduction_str); + return 1; + } + + bool overall_pass = true; + + if (test_all) { + ReductionKind kinds[] = { + ReductionKind::ADD, + ReductionKind::MIN, + ReductionKind::MAX, + ReductionKind::AND, + ReductionKind::OR, + ReductionKind::XOR + }; + const int num_kinds = (int)(sizeof(kinds) / sizeof(kinds[0])); + + for (int i = 0; i < num_kinds; i++) { + bool pass = run_single_reduction(kinds[i], n, run_iters); + overall_pass = overall_pass && pass; + } + } else { + overall_pass = run_single_reduction(selected, n, run_iters); + } + + return overall_pass ? 0 : 1; +} + diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/Makefile index b24b4dac9..9fe44a16c 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/Makefile @@ -1,14 +1,35 @@ -SRC = tma_tensor.cu +# TMA Tensor microbenchmarks +# Builds the monolithic tma_tensor (manual -o flag) plus per-variant standalone +# binaries (utmapf, utmaldg, utmastg, utmaredg) that the CI runner discovers +# and gates automatically. -EXE = tma_tensor +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ -# TMA is supported on SM_90a and above -ARCH?=sm_90a sm_100a sm_101 sm_120 -# Unset the CUDA_CPPFLAGS which is set based on CUDA version -# but TMA is only supported on SM_90a and above -CUDA_CPPFLAGS= +ALL_BENCHES := tma_tensor utmapf utmaldg utmastg utmaredg \ + utmapf_3d utmaldg_3d utmastg_3d utmaredg_3d \ + utmaldg_3d_tx -# Generate code for both sm_XXX and compute_XXX (SASS and PTX) +# TMA tensor requires Hopper-class (sm_90a) or newer. +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Generate both SASS and PTX for each target architecture. NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 -include ../../../common/common.mk +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +# Two-stage build: .cu -> .a (preserves PTX in binary) -> executable +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/tma_tensor.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/tma_tensor.cu index d9f7879d7..4e6bbedc6 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/tma_tensor.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/tma_tensor.cu @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include using barrier = cuda::barrier; namespace ptx = cuda::ptx; diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu new file mode 100644 index 000000000..55dce1054 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu @@ -0,0 +1,209 @@ +/** + * UTMALDG Unit Test: TMA Tensor Load (Global -> Shared) + * ====================================================== + * Tests that cp.async.bulk.tensor correctly loads a 2D tile from global + * memory into shared memory using a CUtensorMap descriptor. + * + * Strategy: + * 1. Global source matrix initialized with unique values (1, 2, 3, ...) + * 2. Destination buffer (d_dst) poisoned with 0xDEADBEEF + * 3. Kernel: thread (0,0) issues TMA tensor load via mbarrier; all threads wait + * 4. Trusted scalar copy: each thread writes smem_buffer[ty][tx] -> d_dst + * PASS: d_dst matches original source for in-bounds elements + * FAIL_NOP: d_dst still contains poison (TMA load did nothing) + * FAIL_MISMATCH: d_dst has wrong values (descriptor or load produced bad data) + * + * Usage: ./utmaldg [-w ] [-h ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; +using barrier = cuda::barrier; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t SMEM_W = 32; +static constexpr uint32_t SMEM_H = 32; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMALDG(const __grid_constant__ CUtensorMap tensor_map, + int32_t *d_dst, uint32_t width_stride, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[SMEM_H][SMEM_W]; +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + + // bar is intentionally left uninitialized. The functional simulator executes + // TMA loads synchronously so the mbarrier completion path is never taken. + // Calling init() would register the mbarrier in the sim's tracking table + // and trigger a duplicate-registration assertion across multiple CTAs. + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + ptx::cp_async_bulk_tensor( + ptx::space_cluster, ptx::space_global, + &smem_buffer, &tensor_map, {x, y}, + cuda::device::barrier_native_handle(bar)); + } + __syncthreads(); + } + + // Trusted scalar copy: verifies TMA populated smem correctly + uint32_t gx = (uint32_t)x + threadIdx.x; + uint32_t gy = (uint32_t)y + threadIdx.y; + d_dst[gy * width_stride + gx] = smem_buffer[threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_dst, const int32_t *h_src, + uint64_t height, uint64_t width, + uint64_t height_stride, uint64_t width_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t r = 0; r < height; r++) { + for (uint64_t c = 0; c < width; c++) { + int32_t got = h_dst[r * width_stride + c]; + int32_t exp = h_src[r * width_stride + c]; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu]: got 0x%08X, expected %d\n", + r, c, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu]: got %d, expected %d\n", + r, c, got, exp); + mismatch_count++; + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t width = 64, height = 64; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "w:h:i:")) != -1) { + switch (opt) { + case 'w': width = (uint64_t)atoi(optarg); break; + case 'h': height = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-w ] [-h ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t width_stride = ((width + SMEM_W - 1) / SMEM_W) * SMEM_W; + uint64_t height_stride = ((height + SMEM_H - 1) / SMEM_H) * SMEM_H; + size_t bytes = height_stride * width_stride * sizeof(int32_t); + + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + // Source: unique values in-bounds, 0 for padding + int32_t val = 1; + for (uint64_t r = 0; r < height_stride; r++) + for (uint64_t c = 0; c < width_stride; c++) + h_src[r * width_stride + c] = (r < height && c < width) ? val++ : 0; + + // Destination: poisoned + for (size_t i = 0; i < height_stride * width_stride; i++) + h_dst[i] = POISON; + + int32_t *d_mat, *d_dst; + cudaMalloc(&d_mat, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_mat, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Build tensor descriptor + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 2; + uint64_t size[rank] = {width, height}; + uint64_t stride[1] = {width_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {SMEM_W, SMEM_H}; + uint32_t elem_stride[rank] = {1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(width_stride / SMEM_W, height_stride / SMEM_H); + dim3 block(SMEM_W, SMEM_H); + CUDA_SAFECALL((test_UTMALDG<<>>( + tensor_map, d_dst, (uint32_t)width_stride, run_iters))); + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMALDG (TMA Tensor Load: Global -> Shared) ===\n"); + bool pass = verify(h_dst, h_src, height, width, height_stride, width_stride); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_src); free(h_dst); + cudaFree(d_mat); cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu new file mode 100644 index 000000000..62c0972f2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu @@ -0,0 +1,229 @@ +/** + * UTMALDG_3D Unit Test: TMA Tensor Load, rank-3 (Global -> Shared) + * ================================================================= + * Tests cp.async.bulk.tensor.3d.shared::cluster.global with a rank-3 + * CUtensorMap descriptor, exercising the 3D coordinate path through the + * simulator's cp.async.bulk.tensor handler and tensormap decoder. + * + * Tensor layout: W x H x D (dim0=innermost, dim2=outermost) + * globalDim = {W, H, D} + * boxDim = {TILE_W, TILE_H, TILE_D} + * globalStrides[0] = W_stride * sizeof(elem) (bytes per H-row) + * globalStrides[1] = W_stride * H_stride * sizeof(elem) (bytes per D-plane) + * + * Strategy: + * 1. Global source tensor initialized with unique values (1, 2, 3, ...) + * 2. Destination buffer (d_dst) poisoned with 0xDEADBEEF + * 3. Kernel: thread (0,0,0) issues 3D TMA load; __syncthreads() as fence + * 4. Trusted scalar copy: each thread writes its smem element to d_dst + * PASS: d_dst matches original source for in-bounds elements + * FAIL_NOP: d_dst still contains poison + * FAIL_MISMATCH: d_dst has wrong values + * + * Usage: ./utmaldg_3d [-W ] [-H ] [-D ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; +using barrier = cuda::barrier; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t TILE_W = 32; +static constexpr uint32_t TILE_H = 4; +static constexpr uint32_t TILE_D = 2; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMALDG_3D(const __grid_constant__ CUtensorMap tensor_map, + int32_t *d_dst, + uint32_t w_stride, uint32_t h_stride, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[TILE_D][TILE_H][TILE_W]; +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + int z = (int)(blockDim.z * blockIdx.z); + + // bar is left uninitialized: functional sim executes TMA synchronously; + // calling init() triggers a multi-CTA mbarrier duplicate-registration assertion. + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + ptx::cp_async_bulk_tensor( + ptx::space_cluster, ptx::space_global, + &smem_buffer, &tensor_map, {x, y, z}, + cuda::device::barrier_native_handle(bar)); + } + __syncthreads(); + } + + // Trusted scalar copy: verify TMA load populated smem correctly + uint32_t gx = (uint32_t)x + threadIdx.x; + uint32_t gy = (uint32_t)y + threadIdx.y; + uint32_t gz = (uint32_t)z + threadIdx.z; + d_dst[gz * h_stride * w_stride + gy * w_stride + gx] = + smem_buffer[threadIdx.z][threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_dst, const int32_t *h_src, + uint64_t depth, uint64_t height, uint64_t width, + uint64_t h_stride, uint64_t w_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t z = 0; z < depth; z++) { + for (uint64_t y = 0; y < height; y++) { + for (uint64_t x = 0; x < width; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + int32_t got = h_dst[idx]; + int32_t exp = h_src[idx]; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu][%lu]: got 0x%08X, expected %d\n", + z, y, x, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu][%lu]: got %d, expected %d\n", + z, y, x, got, exp); + mismatch_count++; + } + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, depth * height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, depth * height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t W = 64, H = 8, D = 4; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "W:H:D:i:")) != -1) { + switch (opt) { + case 'W': W = (uint64_t)atoi(optarg); break; + case 'H': H = (uint64_t)atoi(optarg); break; + case 'D': D = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, + "Usage: %s [-W ] [-H ] [-D ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t w_stride = ((W + TILE_W - 1) / TILE_W) * TILE_W; + uint64_t h_stride = ((H + TILE_H - 1) / TILE_H) * TILE_H; + uint64_t d_stride = ((D + TILE_D - 1) / TILE_D) * TILE_D; + size_t bytes = d_stride * h_stride * w_stride * sizeof(int32_t); + + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + // Source: unique values for in-bounds, 0 for padding + int32_t val = 1; + for (uint64_t z = 0; z < d_stride; z++) + for (uint64_t y = 0; y < h_stride; y++) + for (uint64_t x = 0; x < w_stride; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + h_src[idx] = (z < D && y < H && x < W) ? val++ : 0; + } + + for (size_t i = 0; i < d_stride * h_stride * w_stride; i++) + h_dst[i] = POISON; + + int32_t *d_mat, *d_dst; + cudaMalloc(&d_mat, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_mat, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Build rank-3 tensor descriptor + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 3; + uint64_t size[rank] = {W, H, D}; + uint64_t stride[rank - 1] = {w_stride * sizeof(int32_t), + w_stride * h_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {TILE_W, TILE_H, TILE_D}; + uint32_t elem_stride[rank] = {1, 1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(w_stride / TILE_W, h_stride / TILE_H, d_stride / TILE_D); + dim3 block(TILE_W, TILE_H, TILE_D); + CUDA_SAFECALL((test_UTMALDG_3D<<>>( + tensor_map, d_dst, + (uint32_t)w_stride, (uint32_t)h_stride, run_iters))); + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMALDG_3D (TMA Tensor Load rank-3: Global -> Shared) ===\n"); + bool pass = verify(h_dst, h_src, D, H, W, h_stride, w_stride); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_src); free(h_dst); + cudaFree(d_mat); cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu new file mode 100644 index 000000000..966379266 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu @@ -0,0 +1,220 @@ +/** + * UTMALDG_3D_TX: TMA Tensor Load rank-3 with mbarrier::complete_tx + L2::cache_hint + * ==================================================================================== + * Exercises: + * cp.async.bulk.tensor.3d.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint + * + * This is the exact instruction emitted by CUTLASS's SM90_TMA_LOAD_3D::copy() + * (cute/arch/copy_sm90_tma.hpp) and used by example 48_hopper_warp_specialized_gemm. + * + * Unlike utmaldg_3d.cu (which avoids mbarrier by using __syncthreads), this ubench + * exercises the full mbarrier completion path: + * 1. mbarrier.init.shared::cta.b64 — initialize mbarrier, thread_count=1 + * 2. mbarrier.arrive.expect_tx.shared::cta.b64 — thread 0 arrives + declares expected bytes + * 3. TMA load with .mbarrier::complete_tx::bytes.L2::cache_hint — hw signals mbar on done + * 4. mbarrier.try_wait.parity.shared::cta.b64 — spin until mbarrier complete + * + * Single-CTA launch: avoids the GPGPU-Sim assertion + * "A mbarrier should only exist once in local/remote shmem" + * which fires when multiple CTAs each call mbarrier.init at the same relative smem offset. + * + * PASS: smem_buffer was correctly populated by TMA; scalar readback matches source. + * FAIL: smem_buffer contains poison or wrong values. + * TIMEOUT: mbarrier wait never returns (TMA completion signaling not implemented in sim). + * + * Usage: ./utmaldg_3d_tx + */ + +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t TILE_W = 32; +static constexpr uint32_t TILE_H = 4; +static constexpr uint32_t TILE_D = 2; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMALDG_3D_TX(const __grid_constant__ CUtensorMap tensor_map, + int32_t *d_dst) +{ + __shared__ alignas(128) int32_t smem_buffer[TILE_D][TILE_H][TILE_W]; + // Raw uint64_t mbarrier — same as CUTLASS SM90_TMA_LOAD_3D uses. + // Not cuda::barrier: we replicate the exact CUTLASS pattern with raw PTX. + __shared__ uint64_t mbar; + + const int x = 0, y = 0, z = 0; // single tile at origin (single-CTA launch) + + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + uint32_t smem_buf_addr = static_cast(__cvta_generic_to_shared(&smem_buffer)); + uint32_t smem_mbar_addr = static_cast(__cvta_generic_to_shared(&mbar)); + + // 1. Init mbarrier: 1 expected arrival (thread 0 only, producer-warp pattern) + asm volatile( + "mbarrier.init.shared::cta.b64 [%0], %1;" + :: "r"(smem_mbar_addr), "r"(1) + : "memory"); + + // 2. arrive.expect_tx: thread 0 provides its 1 arrival + declares tx byte count. + // mbarrier is now complete when TMA delivers sizeof(smem_buffer) bytes. + asm volatile( + "mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" + :: "r"(smem_mbar_addr), "r"((uint32_t)sizeof(smem_buffer)) + : "memory"); + + // 3. TMA load: 3D, shared::cluster dst, global src, mbarrier::complete_tx, L2::cache_hint. + // cache_hint=1 is a valid evict-first hint; treated as NOP by GPGPU-Sim. + // Argument order matches CUTLASS SM90_TMA_LOAD_3D (copy_sm90_tma.hpp): + // [smem_dst], [tmap, {crd0, crd1, crd2}], [mbar], cache_hint + uint64_t cache_hint = 1ULL; + asm volatile( + "cp.async.bulk.tensor.3d.shared::cluster.global" + ".mbarrier::complete_tx::bytes.L2::cache_hint" + " [%0], [%1, {%2, %3, %4}], [%5], %6;" + :: "r"(smem_buf_addr), + "l"((uint64_t)&tensor_map), + "r"(x), "r"(y), "r"(z), + "r"(smem_mbar_addr), + "l"(cache_hint) + : "memory"); + + // 4. Spin-wait on mbarrier phase 0 (matches CUTLASS wait_barrier pattern). + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra DONE;\n" + "bra LAB_WAIT;\n" + "DONE:\n" + "}\n" + :: "r"(static_cast(__cvta_generic_to_shared(&mbar))), + "r"(0) // phase 0 (initial phase) + : "memory"); + } + __syncthreads(); + + // Scalar readback: verify TMA populated smem_buffer correctly + d_dst[threadIdx.z * TILE_H * TILE_W + threadIdx.y * TILE_W + threadIdx.x] = + smem_buffer[threadIdx.z][threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_dst, const int32_t *h_src, uint32_t n) +{ + int poison_count = 0, mismatch_count = 0; + for (uint32_t i = 0; i < n; i++) { + int32_t got = h_dst[i]; + int32_t exp = h_src[i]; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%u]: got 0x%08X, expected %d\n", + i, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%u]: got %d, expected %d\n", i, got, exp); + mismatch_count++; + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%u elements contain poison (TMA NOP or mbarrier wait skipped)\n", + poison_count, n); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%u elements have wrong values\n", mismatch_count, n); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int, char **) +{ + constexpr uint32_t N = TILE_W * TILE_H * TILE_D; // 256 elements, one tile + size_t bytes = N * sizeof(int32_t); + + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + for (uint32_t i = 0; i < N; i++) h_src[i] = (int32_t)(i + 1); + for (uint32_t i = 0; i < N; i++) h_dst[i] = POISON; + + int32_t *d_src, *d_dst; + cudaMalloc(&d_src, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_src, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Rank-3 descriptor: tensor exactly one tile in size. + // globalDim = {TILE_W, TILE_H, TILE_D} + // globalStrides (bytes between rows): stride[0]=TILE_W*4, stride[1]=TILE_W*TILE_H*4 + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 3; + uint64_t size[rank] = {TILE_W, TILE_H, TILE_D}; + uint64_t stride[rank - 1] = {TILE_W * sizeof(int32_t), + TILE_W * TILE_H * sizeof(int32_t)}; + uint32_t box_size[rank] = {TILE_W, TILE_H, TILE_D}; + uint32_t elem_stride[rank] = {1, 1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_src, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + // Single CTA: grid=(1,1,1), block=(TILE_W, TILE_H, TILE_D) = 256 threads. + // This avoids the GPGPU-Sim mbarrier duplicate-registration assertion + // that fires when multiple CTAs each init a mbarrier at the same relative smem offset. + dim3 grid(1, 1, 1); + dim3 block(TILE_W, TILE_H, TILE_D); + CUDA_SAFECALL((test_UTMALDG_3D_TX<<>>(tensor_map, d_dst))); + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMALDG_3D_TX (TMA rank-3 load: mbarrier::complete_tx + L2::cache_hint) ===\n"); + bool pass = verify(h_dst, h_src, N); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_src); free(h_dst); + cudaFree(d_src); cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu new file mode 100644 index 000000000..4bd35aa78 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu @@ -0,0 +1,200 @@ +/** + * UTMAPF Unit Test: TMA Tensor Prefetch (L2 hint) + * ================================================ + * Tests that cp.async.bulk.prefetch.tensor issues without error and does not + * corrupt data. The prefetch is a performance hint that warms the L2 cache; + * correctness is verified by a subsequent scalar readback. + * + * Strategy: + * 1. Source matrix (d_mat) initialized with unique values (1, 2, 3, ...) + * 2. Destination buffer (d_dst) poisoned with 0xDEADBEEF + * 3. Kernel: thread (0,0) issues TMA tensor prefetch for this block's tile + * 4. Scalar readback: each thread copies d_mat[gy*stride+gx] -> d_dst + * PASS: d_dst matches original source (prefetch did not corrupt data) + * FAIL_NOP: d_dst still contains poison (readback did not execute) + * FAIL_MISMATCH: d_dst has wrong values (unexpected corruption) + * + * Usage: ./utmapf [-w ] [-h ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t SMEM_W = 32; +static constexpr uint32_t SMEM_H = 32; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMAPF(const __grid_constant__ CUtensorMap tensor_map, + const int32_t *d_mat, int32_t *d_dst, + uint32_t width_stride, int run_iters) +{ + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + + if (threadIdx.x == 0 && threadIdx.y == 0) { + for (int i = 0; i < run_iters; i++) { + asm volatile( + "cp.async.bulk.prefetch.tensor.2d.L2.global.tile" + " [%0, {%1, %2}];" + : : "l"(&tensor_map), "r"(x), "r"(y) : "memory"); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } + __syncthreads(); + + // Scalar readback: verifies prefetch didn't corrupt the source + uint32_t gx = (uint32_t)x + threadIdx.x; + uint32_t gy = (uint32_t)y + threadIdx.y; + d_dst[gy * width_stride + gx] = d_mat[gy * width_stride + gx]; +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_dst, const int32_t *h_src, + uint64_t height, uint64_t width, + uint64_t height_stride, uint64_t width_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t r = 0; r < height; r++) { + for (uint64_t c = 0; c < width; c++) { + int32_t got = h_dst[r * width_stride + c]; + int32_t exp = h_src[r * width_stride + c]; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu]: got 0x%08X, expected %d\n", + r, c, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu]: got %d, expected %d\n", + r, c, got, exp); + mismatch_count++; + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t width = 64, height = 64; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "w:h:i:")) != -1) { + switch (opt) { + case 'w': width = (uint64_t)atoi(optarg); break; + case 'h': height = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-w ] [-h ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t width_stride = ((width + SMEM_W - 1) / SMEM_W) * SMEM_W; + uint64_t height_stride = ((height + SMEM_H - 1) / SMEM_H) * SMEM_H; + size_t bytes = height_stride * width_stride * sizeof(int32_t); + + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + // Source: unique values in-bounds, 0 for padding + int32_t val = 1; + for (uint64_t r = 0; r < height_stride; r++) + for (uint64_t c = 0; c < width_stride; c++) + h_src[r * width_stride + c] = (r < height && c < width) ? val++ : 0; + + // Destination: poisoned + for (size_t i = 0; i < height_stride * width_stride; i++) + h_dst[i] = POISON; + + int32_t *d_mat, *d_dst; + cudaMalloc(&d_mat, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_mat, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Build tensor descriptor (needed to issue prefetch; descriptor encodes the tensor layout) + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 2; + uint64_t size[rank] = {width, height}; + uint64_t stride[1] = {width_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {SMEM_W, SMEM_H}; + uint32_t elem_stride[rank] = {1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(width_stride / SMEM_W, height_stride / SMEM_H); + dim3 block(SMEM_W, SMEM_H); + CUDA_SAFECALL((test_UTMAPF<<>>( + tensor_map, d_mat, d_dst, (uint32_t)width_stride, run_iters))); + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMAPF (TMA Tensor Prefetch + Scalar Readback) ===\n"); + bool pass = verify(h_dst, h_src, height, width, height_stride, width_stride); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_src); free(h_dst); + cudaFree(d_mat); cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu new file mode 100644 index 000000000..20b1302dc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu @@ -0,0 +1,212 @@ +/** + * UTMAPF_3D Unit Test: TMA Tensor Prefetch, rank-3 (L2 hint) + * ============================================================ + * Tests cp.async.bulk.prefetch.tensor.3d.L2.global.tile with a rank-3 + * CUtensorMap descriptor. Prefetch is a performance hint; correctness is + * verified by a subsequent scalar readback. + * + * Strategy: + * 1. Source tensor (d_mat) initialized with unique values (1, 2, 3, ...) + * 2. Destination buffer (d_dst) poisoned with 0xDEADBEEF + * 3. Thread (0,0,0) issues 3D TMA prefetch for this block's tile + * 4. Scalar readback: each thread copies d_mat -> d_dst + * PASS: d_dst matches original source (no corruption) + * FAIL_NOP: d_dst still contains poison (readback did not execute) + * + * Usage: ./utmapf_3d [-W ] [-H ] [-D ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t TILE_W = 32; +static constexpr uint32_t TILE_H = 4; +static constexpr uint32_t TILE_D = 2; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMAPF_3D(const __grid_constant__ CUtensorMap tensor_map, + const int32_t *d_mat, int32_t *d_dst, + uint32_t w_stride, uint32_t h_stride, + int run_iters) +{ + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + int z = (int)(blockDim.z * blockIdx.z); + + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + for (int i = 0; i < run_iters; i++) { + asm volatile( + "cp.async.bulk.prefetch.tensor.3d.L2.global.tile" + " [%0, {%1, %2, %3}];" + : : "l"(&tensor_map), "r"(x), "r"(y), "r"(z) : "memory"); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } + __syncthreads(); + + // Scalar readback: verify prefetch didn't corrupt the source + uint32_t gx = (uint32_t)x + threadIdx.x; + uint32_t gy = (uint32_t)y + threadIdx.y; + uint32_t gz = (uint32_t)z + threadIdx.z; + uint32_t idx = gz * h_stride * w_stride + gy * w_stride + gx; + d_dst[idx] = d_mat[idx]; +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_dst, const int32_t *h_src, + uint64_t depth, uint64_t height, uint64_t width, + uint64_t h_stride, uint64_t w_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t z = 0; z < depth; z++) { + for (uint64_t y = 0; y < height; y++) { + for (uint64_t x = 0; x < width; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + int32_t got = h_dst[idx]; + int32_t exp = h_src[idx]; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu][%lu]: got 0x%08X, expected %d\n", + z, y, x, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu][%lu]: got %d, expected %d\n", + z, y, x, got, exp); + mismatch_count++; + } + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, depth * height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, depth * height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t W = 64, H = 8, D = 4; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "W:H:D:i:")) != -1) { + switch (opt) { + case 'W': W = (uint64_t)atoi(optarg); break; + case 'H': H = (uint64_t)atoi(optarg); break; + case 'D': D = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, + "Usage: %s [-W ] [-H ] [-D ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t w_stride = ((W + TILE_W - 1) / TILE_W) * TILE_W; + uint64_t h_stride = ((H + TILE_H - 1) / TILE_H) * TILE_H; + uint64_t d_stride = ((D + TILE_D - 1) / TILE_D) * TILE_D; + size_t bytes = d_stride * h_stride * w_stride * sizeof(int32_t); + + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + int32_t val = 1; + for (uint64_t z = 0; z < d_stride; z++) + for (uint64_t y = 0; y < h_stride; y++) + for (uint64_t x = 0; x < w_stride; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + h_src[idx] = (z < D && y < H && x < W) ? val++ : 0; + } + + for (size_t i = 0; i < d_stride * h_stride * w_stride; i++) + h_dst[i] = POISON; + + int32_t *d_mat, *d_dst; + cudaMalloc(&d_mat, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_mat, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 3; + uint64_t size[rank] = {W, H, D}; + uint64_t stride[rank - 1] = {w_stride * sizeof(int32_t), + w_stride * h_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {TILE_W, TILE_H, TILE_D}; + uint32_t elem_stride[rank] = {1, 1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(w_stride / TILE_W, h_stride / TILE_H, d_stride / TILE_D); + dim3 block(TILE_W, TILE_H, TILE_D); + CUDA_SAFECALL((test_UTMAPF_3D<<>>( + tensor_map, d_mat, d_dst, + (uint32_t)w_stride, (uint32_t)h_stride, run_iters))); + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMAPF_3D (TMA Tensor Prefetch rank-3 + Scalar Readback) ===\n"); + bool pass = verify(h_dst, h_src, D, H, W, h_stride, w_stride); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_src); free(h_dst); + cudaFree(d_mat); cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu new file mode 100644 index 000000000..1dc3ed50e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu @@ -0,0 +1,194 @@ +/** + * UTMAREDG Unit Test: TMA Tensor Reduce (Shared -> Global, MAX) + * ============================================================== + * Tests that cp.reduce.async.bulk.tensor correctly applies an atomic MAX + * reduction from shared memory into global memory via a CUtensorMap descriptor. + * + * Strategy: + * 1. Global matrix (d_mat) initialized to BASE_VALUE=100 + * 2. Kernel: each thread writes a pattern into shared memory. + * Pattern at global (gx, gy): 1 + gx + gy * width_stride + * 3. Thread (0,0) issues TMA tensor reduce MAX: shared -> global + * PASS: d_mat[r][c] == max(BASE_VALUE, 1 + c + r*width_stride) + * FAIL_NOP: d_mat still contains BASE_VALUE for all elements that should + * have been updated (TMA reduce did nothing) + * FAIL_MISMATCH: d_mat has values that differ from the expected max + * + * Usage: ./utmaredg [-w ] [-h ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t SMEM_W = 32; +static constexpr uint32_t SMEM_H = 32; +static constexpr int32_t BASE_VALUE = 100; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMAREDG(const __grid_constant__ CUtensorMap tensor_map, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[SMEM_H][SMEM_W]; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + + // Each thread fills smem with a globally-unique pattern + int gx = x + (int)threadIdx.x; + int gy = y + (int)threadIdx.y; + smem_buffer[threadIdx.y][threadIdx.x] = + 1 + gx + gy * (int)(blockDim.x * gridDim.x); + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + ptx::cp_reduce_async_bulk_tensor( + ptx::space_global, ptx::space_shared, + ptx::op_max, &tensor_map, {x, y}, &smem_buffer); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_out, + uint64_t height, uint64_t width, uint64_t width_stride) +{ + int nop_count = 0, mismatch_count = 0; + for (uint64_t r = 0; r < height; r++) { + for (uint64_t c = 0; c < width; c++) { + int32_t got = h_out[r * width_stride + c]; + int32_t pattern = 1 + (int32_t)c + (int32_t)r * (int32_t)width_stride; + int32_t exp = (pattern > BASE_VALUE) ? pattern : BASE_VALUE; + // NOP: element should have been raised above BASE_VALUE but wasn't + bool should_update = (pattern > BASE_VALUE); + if (should_update && got == BASE_VALUE) { + if (nop_count == 0) + printf(" NOP at [%lu][%lu]: got %d, expected %d (pattern=%d)\n", + r, c, got, exp, pattern); + nop_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu]: got %d, expected %d\n", + r, c, got, exp); + mismatch_count++; + } + } + } + if (nop_count > 0) { + printf(" FAIL: %d/%lu elements not updated (TMA reduce was NOP)\n", + nop_count, height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t width = 64, height = 64; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "w:h:i:")) != -1) { + switch (opt) { + case 'w': width = (uint64_t)atoi(optarg); break; + case 'h': height = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-w ] [-h ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t width_stride = ((width + SMEM_W - 1) / SMEM_W) * SMEM_W; + uint64_t height_stride = ((height + SMEM_H - 1) / SMEM_H) * SMEM_H; + size_t bytes = height_stride * width_stride * sizeof(int32_t); + + // Host buffer: all BASE_VALUE; TMA reduce MAX will raise values where pattern > BASE_VALUE + int32_t *h_buf = (int32_t *)malloc(bytes); + for (size_t i = 0; i < height_stride * width_stride; i++) + h_buf[i] = BASE_VALUE; + + int32_t *d_mat; + cudaMalloc(&d_mat, bytes); + cudaMemcpy(d_mat, h_buf, bytes, cudaMemcpyHostToDevice); + + // Build tensor descriptor + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 2; + uint64_t size[rank] = {width, height}; + uint64_t stride[1] = {width_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {SMEM_W, SMEM_H}; + uint32_t elem_stride[rank] = {1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(width_stride / SMEM_W, height_stride / SMEM_H); + dim3 block(SMEM_W, SMEM_H); + CUDA_SAFECALL((test_UTMAREDG<<>>(tensor_map, run_iters))); + cudaMemcpy(h_buf, d_mat, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMAREDG (TMA Tensor Reduce MAX: Shared -> Global) ===\n"); + bool pass = verify(h_buf, height, width, width_stride); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_buf); + cudaFree(d_mat); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu new file mode 100644 index 000000000..c459b6129 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu @@ -0,0 +1,201 @@ +/** + * UTMAREDG_3D Unit Test: TMA Tensor Reduce MAX, rank-3 (Shared -> Global) + * ========================================================================= + * Tests cp.reduce.async.bulk.tensor.3d.global.shared::cta.op::max with a + * rank-3 CUtensorMap descriptor. + * + * Strategy: + * 1. d_mat initialized to BASE_VALUE=100 for all elements + * 2. Each thread writes pattern (1 + linear_index) into smem + * 3. Thread (0,0,0) issues 3D TMA reduce MAX: global = max(global, shared) + * PASS: d_mat[idx] == max(100, 1 + idx) for in-bounds elements + * FAIL_NOP: elements that should increase above 100 still equal 100 + * FAIL_MISMATCH: wrong values + * + * Usage: ./utmaredg_3d [-W ] [-H ] [-D ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t TILE_W = 32; +static constexpr uint32_t TILE_H = 4; +static constexpr uint32_t TILE_D = 2; +static constexpr int32_t BASE_VALUE = 100; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMAREDG_3D(const __grid_constant__ CUtensorMap tensor_map, + uint32_t w_stride, uint32_t h_stride, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[TILE_D][TILE_H][TILE_W]; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + int z = (int)(blockDim.z * blockIdx.z); + + int gx = x + (int)threadIdx.x; + int gy = y + (int)threadIdx.y; + int gz = z + (int)threadIdx.z; + smem_buffer[threadIdx.z][threadIdx.y][threadIdx.x] = + 1 + gx + gy * (int)w_stride + gz * (int)(h_stride * w_stride); + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + ptx::cp_reduce_async_bulk_tensor( + ptx::space_global, ptx::space_shared, + ptx::op_max, &tensor_map, {x, y, z}, &smem_buffer); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_out, + uint64_t depth, uint64_t height, uint64_t width, + uint64_t h_stride, uint64_t w_stride) +{ + int nop_count = 0, mismatch_count = 0; + for (uint64_t z = 0; z < depth; z++) { + for (uint64_t y = 0; y < height; y++) { + for (uint64_t x = 0; x < width; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + int32_t got = h_out[idx]; + int32_t pattern = 1 + (int32_t)idx; + int32_t exp = (pattern > BASE_VALUE) ? pattern : BASE_VALUE; + bool should_update = (pattern > BASE_VALUE); + if (should_update && got == BASE_VALUE) { + if (nop_count == 0) + printf(" NOP at [%lu][%lu][%lu]: got %d, expected %d\n", + z, y, x, got, exp); + nop_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu][%lu]: got %d, expected %d\n", + z, y, x, got, exp); + mismatch_count++; + } + } + } + } + if (nop_count > 0) { + printf(" FAIL: %d/%lu elements not updated (TMA reduce was NOP)\n", + nop_count, depth * height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, depth * height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t W = 64, H = 8, D = 4; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "W:H:D:i:")) != -1) { + switch (opt) { + case 'W': W = (uint64_t)atoi(optarg); break; + case 'H': H = (uint64_t)atoi(optarg); break; + case 'D': D = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, + "Usage: %s [-W ] [-H ] [-D ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t w_stride = ((W + TILE_W - 1) / TILE_W) * TILE_W; + uint64_t h_stride = ((H + TILE_H - 1) / TILE_H) * TILE_H; + uint64_t d_stride = ((D + TILE_D - 1) / TILE_D) * TILE_D; + size_t bytes = d_stride * h_stride * w_stride * sizeof(int32_t); + + int32_t *h_buf = (int32_t *)malloc(bytes); + for (size_t i = 0; i < d_stride * h_stride * w_stride; i++) + h_buf[i] = BASE_VALUE; + + int32_t *d_mat; + cudaMalloc(&d_mat, bytes); + cudaMemcpy(d_mat, h_buf, bytes, cudaMemcpyHostToDevice); + + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 3; + uint64_t size[rank] = {W, H, D}; + uint64_t stride[rank - 1] = {w_stride * sizeof(int32_t), + w_stride * h_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {TILE_W, TILE_H, TILE_D}; + uint32_t elem_stride[rank] = {1, 1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(w_stride / TILE_W, h_stride / TILE_H, d_stride / TILE_D); + dim3 block(TILE_W, TILE_H, TILE_D); + CUDA_SAFECALL((test_UTMAREDG_3D<<>>( + tensor_map, (uint32_t)w_stride, (uint32_t)h_stride, run_iters))); + cudaMemcpy(h_buf, d_mat, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMAREDG_3D (TMA Tensor Reduce MAX rank-3: Shared -> Global) ===\n"); + bool pass = verify(h_buf, D, H, W, h_stride, w_stride); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_buf); + cudaFree(d_mat); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu new file mode 100644 index 000000000..7253b1ef6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu @@ -0,0 +1,191 @@ +/** + * UTMASTG Unit Test: TMA Tensor Store (Shared -> Global) + * ======================================================= + * Tests that cp.async.bulk.tensor correctly stores a 2D tile from shared + * memory into global memory using a CUtensorMap descriptor. + * + * Strategy: + * 1. Global destination matrix (d_mat) poisoned with 0xDEADBEEF + * 2. Kernel: each thread writes a unique pattern into shared memory + * 3. Thread (0,0) issues TMA tensor store: shared -> global + * PASS: d_mat contains expected pattern for in-bounds elements + * FAIL_NOP: d_mat still contains poison (TMA store did nothing) + * FAIL_MISMATCH: d_mat has wrong values (descriptor or store produced bad data) + * + * Expected pattern at global position (gx, gy): 1 + gx + gy * width_stride + * + * Usage: ./utmastg [-w ] [-h ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t SMEM_W = 32; +static constexpr uint32_t SMEM_H = 32; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMASTG(const __grid_constant__ CUtensorMap tensor_map, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[SMEM_H][SMEM_W]; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + + // Each thread fills smem with a globally-unique pattern + int gx = x + (int)threadIdx.x; + int gy = y + (int)threadIdx.y; + smem_buffer[threadIdx.y][threadIdx.x] = + 1 + gx + gy * (int)(blockDim.x * gridDim.x); + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + ptx::cp_async_bulk_tensor( + ptx::space_global, ptx::space_shared, + &tensor_map, {x, y}, &smem_buffer); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_out, + uint64_t height, uint64_t width, uint64_t width_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t r = 0; r < height; r++) { + for (uint64_t c = 0; c < width; c++) { + int32_t got = h_out[r * width_stride + c]; + int32_t exp = 1 + (int32_t)c + (int32_t)r * (int32_t)width_stride; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu]: got 0x%08X, expected %d\n", + r, c, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu]: got %d, expected %d\n", + r, c, got, exp); + mismatch_count++; + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t width = 64, height = 64; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "w:h:i:")) != -1) { + switch (opt) { + case 'w': width = (uint64_t)atoi(optarg); break; + case 'h': height = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-w ] [-h ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t width_stride = ((width + SMEM_W - 1) / SMEM_W) * SMEM_W; + uint64_t height_stride = ((height + SMEM_H - 1) / SMEM_H) * SMEM_H; + size_t bytes = height_stride * width_stride * sizeof(int32_t); + + // Host buffer: poisoned (TMA store must overwrite) + int32_t *h_buf = (int32_t *)malloc(bytes); + for (size_t i = 0; i < height_stride * width_stride; i++) + h_buf[i] = POISON; + + int32_t *d_mat; + cudaMalloc(&d_mat, bytes); + cudaMemcpy(d_mat, h_buf, bytes, cudaMemcpyHostToDevice); + + // Build tensor descriptor + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 2; + uint64_t size[rank] = {width, height}; + uint64_t stride[1] = {width_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {SMEM_W, SMEM_H}; + uint32_t elem_stride[rank] = {1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(width_stride / SMEM_W, height_stride / SMEM_H); + dim3 block(SMEM_W, SMEM_H); + CUDA_SAFECALL((test_UTMASTG<<>>(tensor_map, run_iters))); + cudaMemcpy(h_buf, d_mat, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMASTG (TMA Tensor Store: Shared -> Global) ===\n"); + bool pass = verify(h_buf, height, width, width_stride); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_buf); + cudaFree(d_mat); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu new file mode 100644 index 000000000..784a2bf2c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu @@ -0,0 +1,205 @@ +/** + * UTMASTG_3D Unit Test: TMA Tensor Store, rank-3 (Shared -> Global) + * ================================================================== + * Tests cp.async.bulk.tensor.3d.global.shared::cta with a rank-3 + * CUtensorMap descriptor, exercising the 3D coordinate store path. + * + * Tensor layout: W x H x D (dim0=innermost) + * globalStrides[0] = W_stride * sizeof(elem) + * globalStrides[1] = W_stride * H_stride * sizeof(elem) + * + * Strategy: + * 1. Global destination (d_mat) poisoned with 0xDEADBEEF + * 2. Each thread writes a globally-unique pattern into smem + * pattern at (gx, gy, gz): 1 + gx + gy*W_stride + gz*H_stride*W_stride + * (equals 1 + linear_index for this layout) + * 3. Thread (0,0,0) issues 3D TMA store: shared -> global + * PASS: d_mat[idx] == 1 + idx for in-bounds elements + * FAIL_NOP: d_mat still contains poison + * FAIL_MISMATCH: wrong values + * + * Usage: ./utmastg_3d [-W ] [-H ] [-D ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t TILE_W = 32; +static constexpr uint32_t TILE_H = 4; +static constexpr uint32_t TILE_D = 2; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMASTG_3D(const __grid_constant__ CUtensorMap tensor_map, + uint32_t w_stride, uint32_t h_stride, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[TILE_D][TILE_H][TILE_W]; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + int z = (int)(blockDim.z * blockIdx.z); + + int gx = x + (int)threadIdx.x; + int gy = y + (int)threadIdx.y; + int gz = z + (int)threadIdx.z; + smem_buffer[threadIdx.z][threadIdx.y][threadIdx.x] = + 1 + gx + gy * (int)w_stride + gz * (int)(h_stride * w_stride); + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + ptx::cp_async_bulk_tensor( + ptx::space_global, ptx::space_shared, + &tensor_map, {x, y, z}, &smem_buffer); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_out, + uint64_t depth, uint64_t height, uint64_t width, + uint64_t h_stride, uint64_t w_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t z = 0; z < depth; z++) { + for (uint64_t y = 0; y < height; y++) { + for (uint64_t x = 0; x < width; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + int32_t got = h_out[idx]; + int32_t exp = 1 + (int32_t)idx; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu][%lu]: got 0x%08X, expected %d\n", + z, y, x, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu][%lu]: got %d, expected %d\n", + z, y, x, got, exp); + mismatch_count++; + } + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, depth * height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, depth * height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t W = 64, H = 8, D = 4; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "W:H:D:i:")) != -1) { + switch (opt) { + case 'W': W = (uint64_t)atoi(optarg); break; + case 'H': H = (uint64_t)atoi(optarg); break; + case 'D': D = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, + "Usage: %s [-W ] [-H ] [-D ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t w_stride = ((W + TILE_W - 1) / TILE_W) * TILE_W; + uint64_t h_stride = ((H + TILE_H - 1) / TILE_H) * TILE_H; + uint64_t d_stride = ((D + TILE_D - 1) / TILE_D) * TILE_D; + size_t bytes = d_stride * h_stride * w_stride * sizeof(int32_t); + + int32_t *h_buf = (int32_t *)malloc(bytes); + for (size_t i = 0; i < d_stride * h_stride * w_stride; i++) + h_buf[i] = POISON; + + int32_t *d_mat; + cudaMalloc(&d_mat, bytes); + cudaMemcpy(d_mat, h_buf, bytes, cudaMemcpyHostToDevice); + + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 3; + uint64_t size[rank] = {W, H, D}; + uint64_t stride[rank - 1] = {w_stride * sizeof(int32_t), + w_stride * h_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {TILE_W, TILE_H, TILE_D}; + uint32_t elem_stride[rank] = {1, 1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(w_stride / TILE_W, h_stride / TILE_H, d_stride / TILE_D); + dim3 block(TILE_W, TILE_H, TILE_D); + CUDA_SAFECALL((test_UTMASTG_3D<<>>( + tensor_map, (uint32_t)w_stride, (uint32_t)h_stride, run_iters))); + cudaMemcpy(h_buf, d_mat, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMASTG_3D (TMA Tensor Store rank-3: Shared -> Global) ===\n"); + bool pass = verify(h_buf, D, H, W, h_stride, w_stride); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); + + free(h_buf); + cudaFree(d_mat); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/Makefile new file mode 100644 index 000000000..f0e572d31 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/Makefile @@ -0,0 +1,28 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# TMA tensor multicast microbenchmarks. +ALL_BENCHES := tma_tensor_multicast tma_tensor_multicast_mask tma_tensor_multicast_4cta + +# TMA multicast requires Hopper-class architectures. +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Generate both SASS and PTX. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast.cu new file mode 100644 index 000000000..6a476e605 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast.cu @@ -0,0 +1,403 @@ +/* + * TMA Tensor Load with Multicast Microbenchmark + * ============================================== + * Tests: cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster + * + * This benchmark verifies TMA tensor load with multicast to multiple CTAs + * within a cluster. Each cluster of CLUSTER_DIM_X CTAs loads a single 2D tile + * from global memory. CTA 0 in each cluster issues the multicast TMA + * instruction, which delivers the tile data to ALL CTAs' shared memories + * simultaneously. Each CTA then copies its shared memory to a unique region + * of a destination buffer for host-side verification. + * + * Usage: ./tma_tensor_multicast [-w ] [-h ] [-i ] + * + * Verification Strategy: + * ---------------------- + * - d_mat (global tensor): initialized with unique values (1, 2, 3, ...) + * - d_dst (verification buffer): poisoned with POISON_VALUE (0xDEADBEEF) + * - Each cluster of 2 CTAs loads the SAME tile via multicast TMA + * - Each CTA copies its smem to a unique d_dst region (standard stores) + * - Verification checks: + * 1. Correctness: every block's d_dst region matches the expected tile + * 2. Multicast: CTAs within the same cluster have identical data + * - PASS: all blocks correct AND cluster-mates identical + * - FAIL_NOP: d_dst still contains poison (TMA had no effect) + * - FAIL_MISMATCH: values differ from expected + * - FAIL_MULTICAST: CTAs in same cluster received different data + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +using barrier = cuda::barrier; +namespace cg = cooperative_groups; +namespace ptx = cuda::ptx; + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) \ + { \ + fprintf( \ + stderr, \ + "Cuda error in function '%s' file '%s' in line %i : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +#define GMEM_WIDTH 128 +#define GMEM_HEIGHT 64 +#define SMEM_WIDTH 32 +#define SMEM_HEIGHT 32 +#define CLUSTER_DIM_X 2 +#define DEFAULT_RUN_ITERS 1 +#define POISON_VALUE 0xDEADBEEF + +// ============================================================================ +// Kernel: TMA tensor load with multicast::cluster +// +// Grid layout (example with defaults): +// grid_dim = (num_tiles_x * CLUSTER_DIM_X, num_tiles_y) +// = (4*2, 2) = (8, 2) = 16 blocks total +// Cluster (i,j) contains blocks (2*i, j) and (2*i+1, j) +// Both blocks in a cluster load the SAME tile from global memory +// +// Each cluster loads tile at: +// tile_x = cluster_idx_x * SMEM_WIDTH +// tile_y = cluster_idx_y * SMEM_HEIGHT +// +// CTA with cluster_rank==0 issues the multicast TMA instruction. +// All CTAs in the cluster receive the data in their shared memory. +// Each CTA writes its smem to d_dst at a unique offset for verification. +// ============================================================================ + +__global__ void __cluster_dims__(CLUSTER_DIM_X, 1, 1) +test_tma_multicast_kernel( + const __grid_constant__ CUtensorMap tensor_map, + int *d_dst, + int run_iters) +{ + __shared__ alignas(128) int smem_buffer[SMEM_HEIGHT][SMEM_WIDTH]; + +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + cg::cluster_group cluster = cg::this_cluster(); + unsigned int cluster_rank = cluster.block_rank(); + + unsigned int cluster_idx_x = blockIdx.x / CLUSTER_DIM_X; + int tile_x = cluster_idx_x * SMEM_WIDTH; + int tile_y = blockIdx.y * SMEM_HEIGHT; + + if (threadIdx.x == 0 && threadIdx.y == 0) { + init(&bar, blockDim.x * blockDim.y); + ptx::fence_proxy_async(ptx::space_shared); + } + __syncthreads(); + + if (blockIdx.x == 0 && blockIdx.y == 0 && + threadIdx.x == 0 && threadIdx.y == 0) { + printf("TensorMap address: %p\n", &tensor_map); + printf("Cluster rank: %u, tile: (%d, %d)\n", cluster_rank, tile_x, tile_y); + } + + for (int iter = 0; iter < run_iters; iter++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + // Thread 0: arrive with expected transaction bytes + auto token = cuda::device::barrier_arrive_tx(bar, 1, sizeof(smem_buffer)); + + if (cluster_rank == 0) { + uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem_buffer)); + uint32_t mbar_addr = static_cast(__cvta_generic_to_shared(&bar)); + uint16_t mcast_mask = static_cast((1u << CLUSTER_DIM_X) - 1u); + + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster" + " [%0], [%1, {%2, %3}], [%4], %5;" + : + : "r"(smem_addr), "l"(&tensor_map), "r"(tile_x), "r"(tile_y), + "r"(mbar_addr), "h"(mcast_mask) + : "memory" + ); + } + + bar.wait(std::move(token)); + } else { + bar.arrive_and_wait(); + } + } + + // Each block writes its smem to a unique region of d_dst + unsigned int block_flat_idx = blockIdx.x + blockIdx.y * gridDim.x; + int dst_offset = block_flat_idx * SMEM_HEIGHT * SMEM_WIDTH; + d_dst[dst_offset + threadIdx.y * SMEM_WIDTH + threadIdx.x] = + smem_buffer[threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Driver API helper +// ============================================================================ + +PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult driver_status; + void *cuTensorMapEncodeTiled_ptr = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion( + "cuTensorMapEncodeTiled", &cuTensorMapEncodeTiled_ptr, + 12000, cudaEnableDefault, &driver_status)); + assert(driver_status == cudaDriverEntryPointSuccess); + return reinterpret_cast(cuTensorMapEncodeTiled_ptr); +} + +// ============================================================================ +// Verification +// ============================================================================ + +enum verify_result_t { + VERIFY_PASS, + VERIFY_FAIL_NOP, + VERIFY_FAIL_MISMATCH, + VERIFY_FAIL_MULTICAST +}; + +static const char* verify_result_str(verify_result_t r) { + switch (r) { + case VERIFY_PASS: return "PASS"; + case VERIFY_FAIL_NOP: return "FAIL (NOP - TMA had no effect)"; + case VERIFY_FAIL_MISMATCH: return "FAIL (MISMATCH - wrong values)"; + case VERIFY_FAIL_MULTICAST: return "FAIL (MULTICAST - CTAs in cluster got different data)"; + default: return "UNKNOWN"; + } +} + +int main(int argc, char *argv[]) +{ + uint64_t width = GMEM_WIDTH; + uint64_t height = GMEM_HEIGHT; + int run_iters = DEFAULT_RUN_ITERS; + + int opt; + while ((opt = getopt(argc, argv, "w:h:i:")) != -1) { + switch (opt) { + case 'w': width = uint64_t(atoi(optarg)); break; + case 'h': height = uint64_t(atoi(optarg)); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-w ] [-h ] [-i ]\n", argv[0]); + fprintf(stderr, " Tile: %dx%d, Cluster: %d CTAs\n", + SMEM_WIDTH, SMEM_HEIGHT, CLUSTER_DIM_X); + return 1; + } + } + + uint64_t height_stride = ((height + (SMEM_HEIGHT - 1)) / SMEM_HEIGHT) * SMEM_HEIGHT; + uint64_t width_stride = ((width + (SMEM_WIDTH - 1)) / SMEM_WIDTH) * SMEM_WIDTH; + + printf("=== TMA Tensor Multicast Benchmark ===\n"); + printf("Global tensor: %lu x %lu (stride: %lu x %lu)\n", + width, height, width_stride, height_stride); + printf("Tile size: %d x %d\n", SMEM_WIDTH, SMEM_HEIGHT); + printf("Cluster: %d CTAs in x\n", CLUSTER_DIM_X); + printf("Iterations: %d\n", run_iters); + + // ---- Allocate and initialize source tensor ---- + size_t mat_bytes = height_stride * width_stride * sizeof(int); + int *mat = (int *)malloc(mat_bytes); + int *d_mat; + cudaMalloc(&d_mat, mat_bytes); + + uint64_t val = 1; + for (uint64_t r = 0; r < height_stride; r++) { + for (uint64_t c = 0; c < width_stride; c++) { + mat[r * width_stride + c] = (r < height && c < width) ? (int)(val++) : 0; + } + } + cudaMemcpy(d_mat, mat, mat_bytes, cudaMemcpyHostToDevice); + + // ---- Create tensor map ---- + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 2; + uint64_t size[rank] = {width, height}; + uint64_t stride[rank - 1] = {width_stride * sizeof(int)}; + uint32_t box_size[rank] = {SMEM_WIDTH, SMEM_HEIGHT}; + uint32_t elem_stride[rank] = {1, 1}; + + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + CUresult res = cuTensorMapEncodeTiled( + &tensor_map, + CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, + d_mat, + size, + stride, + box_size, + elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + if (res != CUDA_SUCCESS) { + fprintf(stderr, "cuTensorMapEncodeTiled failed: %d\n", res); + return 1; + } + + // ---- Grid / block configuration ---- + // Each cluster handles one tile; cluster has CLUSTER_DIM_X blocks + uint32_t num_tiles_x = width_stride / SMEM_WIDTH; + uint32_t num_tiles_y = height_stride / SMEM_HEIGHT; + dim3 grid_dim(num_tiles_x * CLUSTER_DIM_X, num_tiles_y); + dim3 block_dim(SMEM_WIDTH, SMEM_HEIGHT); + uint32_t total_blocks = grid_dim.x * grid_dim.y; + + printf("Grid: (%u, %u), Block: (%u, %u)\n", + grid_dim.x, grid_dim.y, block_dim.x, block_dim.y); + printf("Clusters: %u x %u = %u total (%u blocks/cluster)\n", + num_tiles_x, num_tiles_y, num_tiles_x * num_tiles_y, CLUSTER_DIM_X); + printf("Total blocks: %u\n\n", total_blocks); + + // ---- Allocate and poison destination buffer ---- + size_t dst_elements = (size_t)total_blocks * SMEM_HEIGHT * SMEM_WIDTH; + size_t dst_bytes = dst_elements * sizeof(int); + int *h_dst = (int *)malloc(dst_bytes); + int *d_dst; + cudaMalloc(&d_dst, dst_bytes); + + for (size_t idx = 0; idx < dst_elements; idx++) + h_dst[idx] = (int)POISON_VALUE; + cudaMemcpy(d_dst, h_dst, dst_bytes, cudaMemcpyHostToDevice); + + // ---- Launch kernel ---- + CUDA_SAFECALL((test_tma_multicast_kernel<<>>( + tensor_map, d_dst, run_iters))); + CUDA_SAFECALL(cudaDeviceSynchronize()); + CUDA_SAFECALL(cudaMemcpy(h_dst, d_dst, dst_bytes, cudaMemcpyDeviceToHost)); + + // ---- Verify correctness ---- + int poison_count = 0; + int mismatch_count = 0; + + for (uint32_t bx = 0; bx < grid_dim.x; bx++) { + for (uint32_t by = 0; by < grid_dim.y; by++) { + uint32_t cluster_idx_x = bx / CLUSTER_DIM_X; + int tile_x = cluster_idx_x * SMEM_WIDTH; + int tile_y = by * SMEM_HEIGHT; + + size_t dst_offset = (size_t)(bx + by * grid_dim.x) * SMEM_HEIGHT * SMEM_WIDTH; + + for (int r = 0; r < SMEM_HEIGHT; r++) { + for (int c = 0; c < SMEM_WIDTH; c++) { + int got = h_dst[dst_offset + r * SMEM_WIDTH + c]; + int expected = mat[(tile_y + r) * width_stride + (tile_x + c)]; + + if (got == (int)POISON_VALUE && expected != (int)POISON_VALUE) { + if (poison_count == 0) + printf(" Poison at block(%u,%u)[%d][%d]: " + "got 0x%08X, expected 0x%08X\n", + bx, by, r, c, (unsigned)got, (unsigned)expected); + poison_count++; + } else if (got != expected) { + if (mismatch_count == 0) + printf(" Mismatch at block(%u,%u)[%d][%d]: " + "got 0x%08X, expected 0x%08X\n", + bx, by, r, c, (unsigned)got, (unsigned)expected); + mismatch_count++; + } + } + } + } + } + + // ---- Verify multicast: CTAs in same cluster must have identical data ---- + int multicast_fail = 0; + + for (uint32_t by = 0; by < grid_dim.y; by++) { + for (uint32_t ci = 0; ci < num_tiles_x; ci++) { + uint32_t bx0 = ci * CLUSTER_DIM_X; + size_t off0 = (size_t)(bx0 + by * grid_dim.x) * SMEM_HEIGHT * SMEM_WIDTH; + + for (uint32_t rank = 1; rank < CLUSTER_DIM_X; rank++) { + size_t off1 = (size_t)(bx0 + rank + by * grid_dim.x) + * SMEM_HEIGHT * SMEM_WIDTH; + for (int i = 0; i < SMEM_HEIGHT * SMEM_WIDTH; i++) { + if (h_dst[off0 + i] != h_dst[off1 + i]) { + if (multicast_fail == 0) + printf(" Multicast mismatch: cluster(%u,%u) " + "rank0=0x%08X rank%u=0x%08X at elem %d\n", + ci, by, + (unsigned)h_dst[off0 + i], rank, + (unsigned)h_dst[off1 + i], i); + multicast_fail++; + } + } + } + } + } + + // ---- Report results ---- + verify_result_t result = VERIFY_PASS; + if (poison_count > 0) result = VERIFY_FAIL_NOP; + else if (mismatch_count > 0) result = VERIFY_FAIL_MISMATCH; + else if (multicast_fail > 0) result = VERIFY_FAIL_MULTICAST; + + printf("\n"); + printf("================================================================================\n"); + printf("TEST: TMA Tensor Load with Multicast (Global -> Shared::Cluster)\n"); + printf(" Instruction: cp.async.bulk.tensor.2d.shared::cluster.global\n"); + printf(" .mbarrier::complete_tx::bytes.multicast::cluster\n"); + printf("================================================================================\n"); + printf("RESULT: %s\n", verify_result_str(result)); + if (result == VERIFY_PASS) { + printf(" All %u blocks received correct tile data\n", total_blocks); + printf(" Multicast verified: CTAs in each cluster have identical data\n"); + } + if (poison_count > 0) + printf(" %d elements still contain poison\n", poison_count); + if (mismatch_count > 0) + printf(" %d elements have wrong values\n", mismatch_count); + if (multicast_fail > 0) + printf(" %d elements differ between CTAs in same cluster\n", multicast_fail); + printf("================================================================================\n\n"); + + // ---- Dump raw output for debugging ---- + char filename[128]; + sprintf(filename, "tma_multicast_test_%lu_%lu.txt", height, width); + FILE *f = fopen(filename, "w"); + fprintf(f, "# TMA Multicast Test Output\n"); + fprintf(f, "# Format: block(bx,by) [row][col] = value (hex)\n\n"); + for (uint32_t bx = 0; bx < grid_dim.x; bx++) { + for (uint32_t by = 0; by < grid_dim.y; by++) { + size_t dst_offset = (size_t)(bx + by * grid_dim.x) * SMEM_HEIGHT * SMEM_WIDTH; + fprintf(f, "--- block(%u,%u) cluster_rank=%u tile=(%d,%d) ---\n", + bx, by, bx % CLUSTER_DIM_X, + (int)(bx / CLUSTER_DIM_X) * SMEM_WIDTH, + (int)(by * SMEM_HEIGHT)); + for (int r = 0; r < SMEM_HEIGHT; r++) { + for (int c = 0; c < SMEM_WIDTH; c++) { + fprintf(f, "0x%08x ", (unsigned)h_dst[dst_offset + r * SMEM_WIDTH + c]); + } + fprintf(f, "\n"); + } + fprintf(f, "\n"); + } + } + fclose(f); + printf("Values dumped to %s\n", filename); + + // ---- Cleanup ---- + free(mat); + free(h_dst); + cudaFree(d_mat); + cudaFree(d_dst); + + return (result == VERIFY_PASS) ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_4cta.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_4cta.cu new file mode 100644 index 000000000..32fd4db84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_4cta.cu @@ -0,0 +1,341 @@ +/* + * TMA Tensor Multicast 4-CTA Cluster Test + * ======================================== + * Tests: cp.async.bulk.tensor.2d.shared::cluster.global + * .mbarrier::complete_tx::bytes.multicast::cluster + * + * Minimal configuration: 1 cluster, 4 CTAs, 1 tile (32x32 ints). + * The multicast mask is a runtime parameter (4-bit), enabling targeted + * testing of wider fan-out multicast patterns. + * + * Usage: ./tma_tensor_multicast_4cta [-m ] + * -m 1 (0b0001): only rank 0 + * -m 2 (0b0010): only rank 1 + * -m 4 (0b0100): only rank 2 + * -m 8 (0b1000): only rank 3 (farthest from issuer) + * -m 5 (0b0101): ranks 0 and 2 (alternating) + * -m 10 (0b1010): ranks 1 and 3 (non-issuing CTAs only) + * -m 14 (0b1110): all except issuer + * -m 15 (0b1111): broadcast to all 4 + * + * Verification Strategy (same as 2-CTA variant): + * ----------------------------------------------- + * - Shared memory poisoned with 0xDEADBEEF before TMA + * - CTAs IN mask: mbarrier expects TMA bytes, waits for delivery + * - CTAs NOT in mask: mbarrier expects 0 bytes, completes immediately + * - In-mask CTAs should have tile data; not-in-mask should have POISON + * + * Notable 4-CTA test patterns: + * mask=0b1000 (only rank 3): Data must traverse 3 hops from issuer. + * mask=0b1110 (all except issuer): Issuer sends but keeps nothing. + * mask=0b0101 (alternating): Non-contiguous subset receives. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +using barrier = cuda::barrier; +namespace cg = cooperative_groups; +namespace ptx = cuda::ptx; + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) \ + { \ + fprintf( \ + stderr, \ + "Cuda error in function '%s' file '%s' in line %i : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +#define TILE_DIM 32 +#define CLUSTER_DIM_X 4 +#define POISON_VALUE 0xDEADBEEF + +// ============================================================================ +// Kernel +// +// Layout: grid=(4,1), block=(32,32), cluster_dims=(4,1,1) +// -> 1 cluster containing CTA ranks 0, 1, 2, 3 +// -> All CTAs address the same tile at origin (0,0) +// ============================================================================ + +__global__ void __cluster_dims__(CLUSTER_DIM_X, 1, 1) +test_tma_multicast_4cta_kernel( + const __grid_constant__ CUtensorMap tensor_map, + int *d_dst, + uint16_t mcast_mask) +{ + __shared__ alignas(128) int smem[TILE_DIM][TILE_DIM]; + +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + cg::cluster_group cluster = cg::this_cluster(); + unsigned int rank = cluster.block_rank(); + bool in_mask = (mcast_mask >> rank) & 1; + + smem[threadIdx.y][threadIdx.x] = (int)POISON_VALUE; + + if (threadIdx.x == 0 && threadIdx.y == 0) { + init(&bar, blockDim.x * blockDim.y); + ptx::fence_proxy_async(ptx::space_shared); + } + __syncthreads(); + + if (threadIdx.x == 0 && threadIdx.y == 0) { + printf("[rank %u] in_mask=%d, blockIdx=(%u,%u)\n", + rank, (int)in_mask, blockIdx.x, blockIdx.y); + } + + if (threadIdx.x == 0 && threadIdx.y == 0) { + barrier::arrival_token token; + if (in_mask) { + token = cuda::device::barrier_arrive_tx(bar, 1, sizeof(smem)); + } else { + token = bar.arrive(); + } + + if (rank == 0) { + uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem)); + uint32_t mbar_addr = static_cast(__cvta_generic_to_shared(&bar)); + + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global" + ".mbarrier::complete_tx::bytes.multicast::cluster" + " [%0], [%1, {%2, %3}], [%4], %5;" + : + : "r"(smem_addr), "l"(&tensor_map), + "r"(0), "r"(0), + "r"(mbar_addr), "h"(mcast_mask) + : "memory" + ); + } + + bar.wait(std::move(token)); + } else { + bar.arrive_and_wait(); + } + +__syncthreads(); + + int dst_base = rank * TILE_DIM * TILE_DIM; + d_dst[dst_base + threadIdx.y * TILE_DIM + threadIdx.x] = + smem[threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Driver API helper +// ============================================================================ + +PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult driver_status; + void *cuTensorMapEncodeTiled_ptr = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion( + "cuTensorMapEncodeTiled", &cuTensorMapEncodeTiled_ptr, + 12000, cudaEnableDefault, &driver_status)); + assert(driver_status == cudaDriverEntryPointSuccess); + return reinterpret_cast(cuTensorMapEncodeTiled_ptr); +} + +// ============================================================================ +// Main +// ============================================================================ + +int main(int argc, char *argv[]) +{ + uint16_t mcast_mask = 0xF; + + int opt; + while ((opt = getopt(argc, argv, "m:")) != -1) { + switch (opt) { + case 'm': mcast_mask = (uint16_t)atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-m ]\n", argv[0]); + fprintf(stderr, " 4-bit mask for 4 CTAs in cluster:\n"); + fprintf(stderr, " -m 1 (0b0001): only rank 0\n"); + fprintf(stderr, " -m 2 (0b0010): only rank 1\n"); + fprintf(stderr, " -m 4 (0b0100): only rank 2\n"); + fprintf(stderr, " -m 8 (0b1000): only rank 3\n"); + fprintf(stderr, " -m 5 (0b0101): ranks 0,2\n"); + fprintf(stderr, " -m 10 (0b1010): ranks 1,3\n"); + fprintf(stderr, " -m 14 (0b1110): all except issuer\n"); + fprintf(stderr, " -m 15 (0b1111): broadcast all\n"); + return 1; + } + } + + mcast_mask &= (1u << CLUSTER_DIM_X) - 1; + if (mcast_mask == 0) { + fprintf(stderr, "Error: mask=0 is invalid (no destination CTA)\n"); + return 1; + } + + printf("=== TMA Multicast 4-CTA Cluster Test ===\n"); + printf("Cluster: %d CTAs, Tile: %dx%d ints\n", CLUSTER_DIM_X, TILE_DIM, TILE_DIM); + printf("Multicast mask: 0x%X (0b", mcast_mask); + for (int i = CLUSTER_DIM_X - 1; i >= 0; i--) + printf("%d", (mcast_mask >> i) & 1); + printf(")\n"); + for (int i = 0; i < CLUSTER_DIM_X; i++) + printf(" Rank %d: %s\n", i, + (mcast_mask >> i) & 1 ? "RECEIVES data" : "does NOT receive (expects poison)"); + printf("\n"); + + // ---- Source tensor: 1 tile ---- + const uint64_t width = TILE_DIM; + const uint64_t height = TILE_DIM; + size_t mat_bytes = width * height * sizeof(int); + int *mat = (int *)malloc(mat_bytes); + int *d_mat; + cudaMalloc(&d_mat, mat_bytes); + + for (uint64_t i = 0; i < width * height; i++) + mat[i] = (int)(i + 1); + cudaMemcpy(d_mat, mat, mat_bytes, cudaMemcpyHostToDevice); + + // ---- Tensor map ---- + CUtensorMap tensor_map{}; + constexpr uint32_t tma_rank = 2; + uint64_t size[tma_rank] = {width, height}; + uint64_t stride[tma_rank - 1] = {width * sizeof(int)}; + uint32_t box_size[tma_rank] = {TILE_DIM, TILE_DIM}; + uint32_t elem_stride[tma_rank] = {1, 1}; + + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + CUresult res = cuTensorMapEncodeTiled( + &tensor_map, + CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + tma_rank, + d_mat, + size, + stride, + box_size, + elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + if (res != CUDA_SUCCESS) { + fprintf(stderr, "cuTensorMapEncodeTiled failed: %d\n", res); + return 1; + } + + // ---- Destination buffer: 4 regions, each TILE_DIM*TILE_DIM ---- + size_t dst_elements = (size_t)CLUSTER_DIM_X * TILE_DIM * TILE_DIM; + size_t dst_bytes = dst_elements * sizeof(int); + int *h_dst = (int *)malloc(dst_bytes); + int *d_dst; + cudaMalloc(&d_dst, dst_bytes); + + for (size_t i = 0; i < dst_elements; i++) + h_dst[i] = (int)POISON_VALUE; + cudaMemcpy(d_dst, h_dst, dst_bytes, cudaMemcpyHostToDevice); + + // ---- Launch: 1 cluster = 4 blocks ---- + dim3 grid(CLUSTER_DIM_X, 1); + dim3 block(TILE_DIM, TILE_DIM); + + printf("Grid: (%u, %u), Block: (%u, %u)\n", grid.x, grid.y, block.x, block.y); + printf("Launching 1 cluster of %d CTAs...\n\n", CLUSTER_DIM_X); + + CUDA_SAFECALL((test_tma_multicast_4cta_kernel<<>>( + tensor_map, d_dst, mcast_mask))); + CUDA_SAFECALL(cudaDeviceSynchronize()); + CUDA_SAFECALL(cudaMemcpy(h_dst, d_dst, dst_bytes, cudaMemcpyDeviceToHost)); + + // ---- Verify each rank ---- + int all_pass = 1; + + for (int r = 0; r < CLUSTER_DIM_X; r++) { + bool in_mask = (mcast_mask >> r) & 1; + int base = r * TILE_DIM * TILE_DIM; + int poison_count = 0, mismatch_count = 0, correct_count = 0; + + for (int i = 0; i < TILE_DIM * TILE_DIM; i++) { + int got = h_dst[base + i]; + int expected = in_mask ? mat[i] : (int)POISON_VALUE; + + if (got == expected) { + correct_count++; + } else if (got == (int)POISON_VALUE && in_mask) { + if (poison_count == 0) + printf(" Rank %d: elem %d still POISON (got 0x%08X, expected 0x%08X)\n", + r, i, (unsigned)got, (unsigned)expected); + poison_count++; + } else { + if (mismatch_count == 0) + printf(" Rank %d: elem %d MISMATCH (got 0x%08X, expected 0x%08X)\n", + r, i, (unsigned)got, (unsigned)expected); + mismatch_count++; + } + } + + int total = TILE_DIM * TILE_DIM; + if (correct_count == total) { + if (in_mask) + printf(" Rank %d: PASS - received correct tile data (%d/%d)\n", + r, correct_count, total); + else + printf(" Rank %d: PASS - correctly still has poison (%d/%d)\n", + r, correct_count, total); + } else { + all_pass = 0; + if (in_mask && poison_count > 0) + printf(" Rank %d: FAIL - NOP detected, %d/%d elements still poison\n", + r, poison_count, total); + else + printf(" Rank %d: FAIL - %d/%d elements wrong (%d poison, %d mismatch)\n", + r, poison_count + mismatch_count, total, poison_count, mismatch_count); + } + } + + printf("\n"); + printf("================================================================================\n"); + printf("TEST: TMA Tensor Multicast 4-CTA Cluster (mask=0x%X)\n", mcast_mask); + printf(" Instruction: cp.async.bulk.tensor.2d.shared::cluster.global\n"); + printf(" .mbarrier::complete_tx::bytes.multicast::cluster\n"); + printf("================================================================================\n"); + printf("RESULT: %s\n", all_pass ? "**PASS**" : "**FAIL**"); + printf("================================================================================\n\n"); + + // ---- Dump for debugging ---- + char filename[128]; + sprintf(filename, "tma_multicast_4cta_mask_0x%X.txt", mcast_mask); + FILE *f = fopen(filename, "w"); + fprintf(f, "# Mask=0x%X Cluster=%d CTAs Tile=%dx%d\n\n", + mcast_mask, CLUSTER_DIM_X, TILE_DIM, TILE_DIM); + for (int r = 0; r < CLUSTER_DIM_X; r++) { + bool in_mask = (mcast_mask >> r) & 1; + fprintf(f, "--- Rank %d (%s) ---\n", r, + in_mask ? "in mask" : "NOT in mask"); + int base = r * TILE_DIM * TILE_DIM; + for (int row = 0; row < TILE_DIM; row++) { + for (int col = 0; col < TILE_DIM; col++) + fprintf(f, "0x%08x ", (unsigned)h_dst[base + row * TILE_DIM + col]); + fprintf(f, "\n"); + } + fprintf(f, "\n"); + } + fclose(f); + printf("Values dumped to %s\n", filename); + + free(mat); + free(h_dst); + cudaFree(d_mat); + cudaFree(d_dst); + + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_mask.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_mask.cu new file mode 100644 index 000000000..a1fd9fa65 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_mask.cu @@ -0,0 +1,344 @@ +/* + * TMA Tensor Multicast Mask Test + * ============================== + * Tests: cp.async.bulk.tensor.2d.shared::cluster.global + * .mbarrier::complete_tx::bytes.multicast::cluster + * + * Minimal configuration: 1 cluster, 2 CTAs, 1 tile (32x32 ints). + * The multicast mask is a runtime parameter, enabling targeted testing + * of which CTAs receive data. + * + * Usage: ./tma_tensor_multicast_mask [-m ] + * -m 1 (0b01): only rank 0 receives data + * -m 2 (0b10): only rank 1 receives data + * -m 3 (0b11): both ranks receive (broadcast) + * + * Verification Strategy: + * ---------------------- + * - Shared memory is initialized to POISON (0xDEADBEEF) before TMA + * - CTA 0 issues multicast TMA with the given mask + * - CTAs IN the mask: mbarrier expects TMA bytes, waits for delivery + * - CTAs NOT in the mask: mbarrier expects 0 bytes, completes immediately + * - After TMA: + * - In-mask CTAs: smem should contain tile data -> PASS + * - Not-in-mask CTAs: smem should still be POISON -> PASS + * - Each CTA copies smem to its own d_dst region for host-side checking + * + * Key mask=0b10 case: + * Rank 0 ISSUES the instruction but does NOT receive data. + * Rank 1 does NOT issue the instruction but DOES receive data. + * This tests that the simulator correctly routes data to a remote CTA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +using barrier = cuda::barrier; +namespace cg = cooperative_groups; +namespace ptx = cuda::ptx; + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) \ + { \ + fprintf( \ + stderr, \ + "Cuda error in function '%s' file '%s' in line %i : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +#define TILE_DIM 32 +#define CLUSTER_DIM_X 2 +#define POISON_VALUE 0xDEADBEEF + +// ============================================================================ +// Kernel +// +// Layout: grid=(2,1), block=(32,32), cluster_dims=(2,1,1) +// -> 1 cluster containing CTA rank 0 and CTA rank 1 +// -> Both CTAs address the same tile at origin (0,0) +// +// Flow per CTA: +// 1. Poison smem with POISON_VALUE +// 2. If in_mask: barrier_arrive_tx with sizeof(smem) expected bytes +// If not in_mask: barrier arrive with 0 expected bytes +// 3. Rank 0 issues multicast TMA (regardless of whether rank 0 is in mask) +// 4. All threads wait on barrier +// 5. Copy smem -> d_dst for host verification +// ============================================================================ + +__global__ void __cluster_dims__(CLUSTER_DIM_X, 1, 1) +test_tma_multicast_mask_kernel( + const __grid_constant__ CUtensorMap tensor_map, + int *d_dst, + uint16_t mcast_mask) +{ + __shared__ alignas(128) int smem[TILE_DIM][TILE_DIM]; + +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + cg::cluster_group cluster = cg::this_cluster(); + unsigned int rank = cluster.block_rank(); + bool in_mask = (mcast_mask >> rank) & 1; + + // Step 1: Poison smem so we can detect whether TMA wrote anything + smem[threadIdx.y][threadIdx.x] = (int)POISON_VALUE; + + if (threadIdx.x == 0 && threadIdx.y == 0) { + init(&bar, blockDim.x * blockDim.y); + ptx::fence_proxy_async(ptx::space_shared); + } + __syncthreads(); + + if (threadIdx.x == 0 && threadIdx.y == 0) { + printf("[rank %u] in_mask=%d, blockIdx=(%u,%u)\n", + rank, (int)in_mask, blockIdx.x, blockIdx.y); + } + + // Step 2 & 3: Arrive on barrier and optionally issue TMA + if (threadIdx.x == 0 && threadIdx.y == 0) { + barrier::arrival_token token; + if (in_mask) { + token = cuda::device::barrier_arrive_tx(bar, 1, sizeof(smem)); + } else { + token = bar.arrive(); + } + + if (rank == 0) { + uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem)); + uint32_t mbar_addr = static_cast(__cvta_generic_to_shared(&bar)); + + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global" + ".mbarrier::complete_tx::bytes.multicast::cluster" + " [%0], [%1, {%2, %3}], [%4], %5;" + : + : "r"(smem_addr), "l"(&tensor_map), + "r"(0), "r"(0), + "r"(mbar_addr), "h"(mcast_mask) + : "memory" + ); + } + + bar.wait(std::move(token)); + } else { + bar.arrive_and_wait(); + } + + __syncthreads(); + + // Step 4: Copy smem -> d_dst for host verification + int dst_base = rank * TILE_DIM * TILE_DIM; + d_dst[dst_base + threadIdx.y * TILE_DIM + threadIdx.x] = + smem[threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Driver API helper +// ============================================================================ + +PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult driver_status; + void *cuTensorMapEncodeTiled_ptr = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion( + "cuTensorMapEncodeTiled", &cuTensorMapEncodeTiled_ptr, + 12000, cudaEnableDefault, &driver_status)); + assert(driver_status == cudaDriverEntryPointSuccess); + return reinterpret_cast(cuTensorMapEncodeTiled_ptr); +} + +// ============================================================================ +// Main +// ============================================================================ + +int main(int argc, char *argv[]) +{ + uint16_t mcast_mask = 0x3; + + int opt; + while ((opt = getopt(argc, argv, "m:")) != -1) { + switch (opt) { + case 'm': mcast_mask = (uint16_t)atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-m ]\n", argv[0]); + fprintf(stderr, " -m 1 (0b01): only rank 0 receives\n"); + fprintf(stderr, " -m 2 (0b10): only rank 1 receives\n"); + fprintf(stderr, " -m 3 (0b11): both ranks receive\n"); + return 1; + } + } + + mcast_mask &= (1u << CLUSTER_DIM_X) - 1; + if (mcast_mask == 0) { + fprintf(stderr, "Error: mask=0 is invalid (no destination CTA)\n"); + return 1; + } + + printf("=== TMA Multicast Mask Test ===\n"); + printf("Cluster: %d CTAs, Tile: %dx%d ints\n", CLUSTER_DIM_X, TILE_DIM, TILE_DIM); + printf("Multicast mask: 0x%X (0b", mcast_mask); + for (int i = CLUSTER_DIM_X - 1; i >= 0; i--) + printf("%d", (mcast_mask >> i) & 1); + printf(")\n"); + for (int i = 0; i < CLUSTER_DIM_X; i++) + printf(" Rank %d: %s\n", i, + (mcast_mask >> i) & 1 ? "RECEIVES data" : "does NOT receive (expects poison)"); + printf("\n"); + + // ---- Source tensor: 1 tile, values 1..TILE_DIM*TILE_DIM ---- + const uint64_t width = TILE_DIM; + const uint64_t height = TILE_DIM; + size_t mat_bytes = width * height * sizeof(int); + int *mat = (int *)malloc(mat_bytes); + int *d_mat; + cudaMalloc(&d_mat, mat_bytes); + + for (uint64_t i = 0; i < width * height; i++) + mat[i] = (int)(i + 1); + cudaMemcpy(d_mat, mat, mat_bytes, cudaMemcpyHostToDevice); + + // ---- Tensor map ---- + CUtensorMap tensor_map{}; + constexpr uint32_t tma_rank = 2; + uint64_t size[tma_rank] = {width, height}; + uint64_t stride[tma_rank - 1] = {width * sizeof(int)}; + uint32_t box_size[tma_rank] = {TILE_DIM, TILE_DIM}; + uint32_t elem_stride[tma_rank] = {1, 1}; + + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + CUresult res = cuTensorMapEncodeTiled( + &tensor_map, + CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + tma_rank, + d_mat, + size, + stride, + box_size, + elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + if (res != CUDA_SUCCESS) { + fprintf(stderr, "cuTensorMapEncodeTiled failed: %d\n", res); + return 1; + } + + // ---- Destination buffer: CLUSTER_DIM_X regions, each TILE_DIM*TILE_DIM ---- + size_t dst_elements = (size_t)CLUSTER_DIM_X * TILE_DIM * TILE_DIM; + size_t dst_bytes = dst_elements * sizeof(int); + int *h_dst = (int *)malloc(dst_bytes); + int *d_dst; + cudaMalloc(&d_dst, dst_bytes); + + for (size_t i = 0; i < dst_elements; i++) + h_dst[i] = (int)POISON_VALUE; + cudaMemcpy(d_dst, h_dst, dst_bytes, cudaMemcpyHostToDevice); + + // ---- Launch: 1 cluster = CLUSTER_DIM_X blocks ---- + dim3 grid(CLUSTER_DIM_X, 1); + dim3 block(TILE_DIM, TILE_DIM); + + printf("Grid: (%u, %u), Block: (%u, %u)\n", grid.x, grid.y, block.x, block.y); + printf("Launching 1 cluster of %d CTAs...\n\n", CLUSTER_DIM_X); + + CUDA_SAFECALL((test_tma_multicast_mask_kernel<<>>( + tensor_map, d_dst, mcast_mask))); + CUDA_SAFECALL(cudaDeviceSynchronize()); + CUDA_SAFECALL(cudaMemcpy(h_dst, d_dst, dst_bytes, cudaMemcpyDeviceToHost)); + + // ---- Verify each rank ---- + int all_pass = 1; + + for (int r = 0; r < CLUSTER_DIM_X; r++) { + bool in_mask = (mcast_mask >> r) & 1; + int base = r * TILE_DIM * TILE_DIM; + int poison_count = 0, mismatch_count = 0, correct_count = 0; + + for (int i = 0; i < TILE_DIM * TILE_DIM; i++) { + int got = h_dst[base + i]; + int expected = in_mask ? mat[i] : (int)POISON_VALUE; + + if (got == expected) { + correct_count++; + } else if (got == (int)POISON_VALUE && in_mask) { + if (poison_count == 0) + printf(" Rank %d: elem %d still POISON (got 0x%08X, expected 0x%08X)\n", + r, i, (unsigned)got, (unsigned)expected); + poison_count++; + } else { + if (mismatch_count == 0) + printf(" Rank %d: elem %d MISMATCH (got 0x%08X, expected 0x%08X)\n", + r, i, (unsigned)got, (unsigned)expected); + mismatch_count++; + } + } + + int total = TILE_DIM * TILE_DIM; + if (correct_count == total) { + if (in_mask) + printf(" Rank %d: PASS - received correct tile data (%d/%d)\n", + r, correct_count, total); + else + printf(" Rank %d: PASS - correctly still has poison (%d/%d)\n", + r, correct_count, total); + } else { + all_pass = 0; + if (in_mask && poison_count > 0) + printf(" Rank %d: FAIL - NOP detected, %d/%d elements still poison\n", + r, poison_count, total); + else + printf(" Rank %d: FAIL - %d/%d elements wrong (%d poison, %d mismatch)\n", + r, poison_count + mismatch_count, total, poison_count, mismatch_count); + } + } + + printf("\n"); + printf("================================================================================\n"); + printf("TEST: TMA Tensor Multicast Mask (mask=0x%X)\n", mcast_mask); + printf(" Instruction: cp.async.bulk.tensor.2d.shared::cluster.global\n"); + printf(" .mbarrier::complete_tx::bytes.multicast::cluster\n"); + printf("================================================================================\n"); + printf("RESULT: %s\n", all_pass ? "**PASS**" : "**FAIL**"); + printf("================================================================================\n\n"); + + // ---- Dump for debugging ---- + char filename[128]; + sprintf(filename, "tma_multicast_mask_0x%X.txt", mcast_mask); + FILE *f = fopen(filename, "w"); + fprintf(f, "# Mask=0x%X Cluster=%d CTAs Tile=%dx%d\n\n", mcast_mask, CLUSTER_DIM_X, TILE_DIM, TILE_DIM); + for (int r = 0; r < CLUSTER_DIM_X; r++) { + bool in_mask = (mcast_mask >> r) & 1; + fprintf(f, "--- Rank %d (%s) ---\n", r, + in_mask ? "in mask" : "NOT in mask"); + int base = r * TILE_DIM * TILE_DIM; + for (int row = 0; row < TILE_DIM; row++) { + for (int col = 0; col < TILE_DIM; col++) + fprintf(f, "0x%08x ", (unsigned)h_dst[base + row * TILE_DIM + col]); + fprintf(f, "\n"); + } + fprintf(f, "\n"); + } + fclose(f); + printf("Values dumped to %s\n", filename); + + free(mat); + free(h_dst); + cudaFree(d_mat); + cudaFree(d_dst); + + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/Makefile new file mode 100644 index 000000000..ebdcfbbd0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/Makefile @@ -0,0 +1,29 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# fence.mbarrier_init.release.cluster status-probe microbenchmark. +ALL_BENCHES := ufence_mbarrier_init + +# fence.mbarrier_init and mbarrier.init.shared::cta are Hopper+ PTX (SM_90a and above). +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS and PTX. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu new file mode 100644 index 000000000..87134b1de --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu @@ -0,0 +1,122 @@ +#include +#include +#include + +// Status probe for `fence.mbarrier_init.release.cluster` in the GPGPU-Sim +// functional model. +// +// This Hopper (sm_90+) fence is emitted by CUTLASS example 48 right after +// `mbarrier.init`, to make the barrier initialization visible cluster-wide +// (release, .cluster scope) before any CTA arrives on it. Example 48 launches +// with ClusterShape = 1x1x1 — a SINGLE CTA per cluster. In pure functional +// simulation, shared-memory/mbarrier effects are instantaneous and there is +// only one CTA, so the cluster-scope release fence has nothing to order across +// CTAs: it is functionally a no-op. The simulator already treats it as one +// (lexer maps `fence.mbarrier_init...` -> NOP_OP), and this ubench verifies that +// a single-CTA kernel containing the exact instruction both parses and produces +// correct results under pure functional mode. +// +// The barrier is made semantically load-bearing so a broken fence/barrier would +// be caught: every thread publishes payload[tid], then (after the barrier) reads +// its NEIGHBOR's slot. A correct neighbor read requires every producer to have +// executed past the fence. Expected: out[t] == ((t+1) % N) * 2. +// +// The surrounding mbarrier arrive/try_wait path mirrors how CUTLASS uses the +// barrier the fence is initializing; its result is informational (kept live via +// the `info` buffer) and is not part of the pass criterion, which relies on the +// pure-functional-exact __syncthreads() ordering. + +#define N 32 // one warp / one CTA + +__global__ void ufence_mbarrier_init_kernel(unsigned int *out, unsigned int *info) { + __shared__ unsigned long long bar; + __shared__ unsigned int payload[N]; + const unsigned int t = threadIdx.x; + + // CUTLASS-48 addresses the mbarrier with a 32-bit .shared-window offset + // (e.g. `mbarrier.init.shared::cta.b64 [%r141], ...`), not a 64-bit generic + // pointer. Mirror that: the simulator's mbarrier tracking keys barriers by + // this shared offset, so init/arrive/try_wait must all use the same form. + const unsigned int bar_s = (unsigned int)__cvta_generic_to_shared(&bar); + + // 1. Initialize the mbarrier (thread 0): expect all N threads to arrive. + if (t == 0) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" + : : "r"(bar_s), "r"(N) : "memory"); + } + + // 2. INSTRUCTION UNDER TEST — exact CUTLASS-48 spelling and position: + // release, cluster-scope fence ordering the mbarrier init before arrives. + asm volatile("fence.mbarrier_init.release.cluster;" : : : "memory"); + + __syncthreads(); + + // 3. Producer: publish this thread's slot. + payload[t] = t * 2u; + + // 4. Exercise the mbarrier arrive/wait path (as CUTLASS does around the + // barrier this fence initializes), using CUTLASS-48's exact forms + // (`mbarrier.arrive.shared::cta.b64 _, [%r];` — token sunk to `_`). + // Informational only. + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];" + : : "r"(bar_s) : "memory"); + + unsigned int phase = 0u; + unsigned int done = 0u; + asm volatile( + "{\n\t" + ".reg .pred p;\n\t" + "mbarrier.try_wait.parity.shared::cta.b64 p, [%1], %2;\n\t" + "selp.u32 %0, 1, 0, p;\n\t" + "}" + : "=r"(done) : "r"(bar_s), "r"(phase) : "memory"); + + // 5. Barrier-ordered consumer: neighbor read requires every producer to have + // run past the fence. __syncthreads() is the functional ordering the + // pure-functional simulator models exactly. + __syncthreads(); + out[t] = payload[(t + 1u) % N]; + info[t] = done; // keep try_wait live; not part of the pass criterion. +} + +int main() { + // Unbuffered stdout so the verdict survives even if a later stage aborts + // hard inside the simulator. + setvbuf(stdout, NULL, _IONBF, 0); + + const int n = N; + unsigned int *d_out = NULL, *d_info = NULL; + unsigned int h_out[N], h_info[N]; + cudaMalloc(&d_out, n * sizeof(unsigned int)); + cudaMalloc(&d_info, n * sizeof(unsigned int)); + cudaMemset(d_out, 0xFF, n * sizeof(unsigned int)); + cudaMemset(d_info, 0xFF, n * sizeof(unsigned int)); + + // Single-CTA launch — matches CUTLASS-48's per-cluster CTA count (1). + ufence_mbarrier_init_kernel<<<1, N>>>(d_out, d_info); + cudaError_t err = cudaDeviceSynchronize(); + + int fails = 0; + if (err != cudaSuccess) { + printf("launch/sync error: %s\n", cudaGetErrorString(err)); + fails++; + } else { + cudaMemcpy(h_out, d_out, n * sizeof(unsigned int), cudaMemcpyDeviceToHost); + cudaMemcpy(h_info, d_info, n * sizeof(unsigned int), cudaMemcpyDeviceToHost); + + printf("neighbor-exchange after fence.mbarrier_init.release.cluster:\n"); + for (int i = 0; i < n; ++i) { + unsigned int expected = (unsigned int)(((i + 1) % N) * 2); + const char *mark = (h_out[i] == expected) ? "ok" : "MISMATCH"; + if (h_out[i] != expected) fails++; + printf(" t %2d: out = %3u (expected %3u) [%s] try_wait=%u\n", + i, h_out[i], expected, mark, h_info[i]); + } + } + + cudaFree(d_out); + cudaFree(d_info); + + printf("RESULT: %s\n", fails == 0 ? "PASSED" : "FAILED"); + return fails == 0 ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/.gitignore b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/.gitignore new file mode 100644 index 000000000..08f514ebc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/.gitignore @@ -0,0 +1 @@ +include/ diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/Makefile new file mode 100644 index 000000000..3272e6dad --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/Makefile @@ -0,0 +1,31 @@ +# Source files split for parallel compilation +# Use wildcard to automatically include all size-specific breakdown files +SRC = lat_gmma.cu $(wildcard kernels/lat_gmma_*.cu) + +EXE = lat_gmma + +# Add include path for CUTLASS. +# Prefer the bundled include/ directory so no external setup is needed. +INCLUDE += -I./include -I$(GPUAPPS_ROOT)/src/cuda/cutlass-bench/include -I./ + +# GMMA is only supported in sm_90a +ARCH?=sm_90a +# Unset the CUDA_CPPFLAGS which is set based on CUDA version +CUDA_CPPFLAGS= +# Generate code for both sm_90a and compute_90a (SASS and PTX) +HOPPER_CUDA_CPPFLAGS=$(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) + +# CUTLASS cute library requires C++17 +NVCC_FLAGS := $(HOPPER_CUDA_CPPFLAGS) -std=c++17 + +include ../../../common/common.mk + +# A test executable for checking the library and simulator debugging +TEST_SRC = lat_gmma_test.cu +TEST_EXE = lat_gmma_test +test: $(TEST_SRC) + $(CC) $(NVCC_FLAGS) $^ -o $(TEST_EXE) $(INCLUDE) -lcudart --cudart shared + cp $(TEST_EXE) $(BIN_DIR) + +# Append the test to the release target +release: .EXTRA_PREREQS = test \ No newline at end of file diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x128x32.cu new file mode 100644 index 000000000..ec65d5eff --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x128x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 128, 32, "MMA_64x128x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x16x32.cu new file mode 100644 index 000000000..0da1c52bd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x16x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 16, 32, "MMA_64x16x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x192x32.cu new file mode 100644 index 000000000..b227b645d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x192x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 192, 32, "MMA_64x192x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x256x32.cu new file mode 100644 index 000000000..a885466d9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x256x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 256, 32, "MMA_64x256x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x32x32.cu new file mode 100644 index 000000000..cbb28cea9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x32x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 32, 32, "MMA_64x32x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x64x32.cu new file mode 100644 index 000000000..c9687a66c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x64x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 64, 32, "MMA_64x64x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x8x32.cu new file mode 100644 index 000000000..9b445f92a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x8x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 8, 32, "MMA_64x8x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x96x32.cu new file mode 100644 index 000000000..fc4180513 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x96x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 96, 32, "MMA_64x96x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x128x32.cu new file mode 100644 index 000000000..a02254201 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x128x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 128, 32, "MMA_64x128x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x16x32.cu new file mode 100644 index 000000000..68b9546e7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x16x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 16, 32, "MMA_64x16x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x192x32.cu new file mode 100644 index 000000000..c0f3993b2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x192x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 192, 32, "MMA_64x192x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x256x32.cu new file mode 100644 index 000000000..297053fe7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x256x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 256, 32, "MMA_64x256x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x32x32.cu new file mode 100644 index 000000000..0126fcc9a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x32x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 32, 32, "MMA_64x32x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x64x32.cu new file mode 100644 index 000000000..744a42dfb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x64x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 64, 32, "MMA_64x64x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x8x32.cu new file mode 100644 index 000000000..bfca52f2a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x8x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 8, 32, "MMA_64x8x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x96x32.cu new file mode 100644 index 000000000..d4a580d6f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x96x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 96, 32, "MMA_64x96x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x128x32.cu new file mode 100644 index 000000000..9319c9ee6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x128x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 128, 32, "MMA_64x128x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x16x32.cu new file mode 100644 index 000000000..a1602c766 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x16x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 16, 32, "MMA_64x16x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x192x32.cu new file mode 100644 index 000000000..2db60e978 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x192x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 192, 32, "MMA_64x192x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x256x32.cu new file mode 100644 index 000000000..22e2df853 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x256x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 256, 32, "MMA_64x256x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x32x32.cu new file mode 100644 index 000000000..ec0fdbdef --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x32x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 32, 32, "MMA_64x32x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x64x32.cu new file mode 100644 index 000000000..1101feb0f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x64x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 64, 32, "MMA_64x64x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x8x32.cu new file mode 100644 index 000000000..755c81a33 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x8x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 8, 32, "MMA_64x8x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x96x32.cu new file mode 100644 index 000000000..350d8f5c4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x96x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 96, 32, "MMA_64x96x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x128x32.cu new file mode 100644 index 000000000..d466a9795 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x128x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 128, 32, "MMA_64x128x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x16x32.cu new file mode 100644 index 000000000..44d9835ad --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x16x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 16, 32, "MMA_64x16x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x192x32.cu new file mode 100644 index 000000000..31db9558a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x192x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 192, 32, "MMA_64x192x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x256x32.cu new file mode 100644 index 000000000..3ddacc999 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x256x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 256, 32, "MMA_64x256x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x32x32.cu new file mode 100644 index 000000000..25d80681d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x32x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 32, 32, "MMA_64x32x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x64x32.cu new file mode 100644 index 000000000..696c59694 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x64x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 64, 32, "MMA_64x64x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x8x32.cu new file mode 100644 index 000000000..1c2c62651 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x8x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 8, 32, "MMA_64x8x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x96x32.cu new file mode 100644 index 000000000..c4b93b17f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x96x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 96, 32, "MMA_64x96x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x128x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x128x16.cu new file mode 100644 index 000000000..38d284502 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x128x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x128x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 128, 16, "MMA_64x128x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x16x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x16x16.cu new file mode 100644 index 000000000..40e841ad0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x16x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x16x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 16, 16, "MMA_64x16x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x192x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x192x16.cu new file mode 100644 index 000000000..2200355a9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x192x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x192x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 192, 16, "MMA_64x192x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x256x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x256x16.cu new file mode 100644 index 000000000..057ee4222 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x256x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x256x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 256, 16, "MMA_64x256x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x32x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x32x16.cu new file mode 100644 index 000000000..252e4cde7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x32x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x32x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 32, 16, "MMA_64x32x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x64x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x64x16.cu new file mode 100644 index 000000000..d3d8715bd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x64x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x64x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 64, 16, "MMA_64x64x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x8x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x8x16.cu new file mode 100644 index 000000000..df7d161ba --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x8x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x8x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 8, 16, "MMA_64x8x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x96x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x96x16.cu new file mode 100644 index 000000000..92dc99dc5 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x96x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x96x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 96, 16, "MMA_64x96x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x128x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x128x16.cu new file mode 100644 index 000000000..58c068112 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x128x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x128x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 128, 16, "MMA_64x128x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x16x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x16x16.cu new file mode 100644 index 000000000..63b5adc61 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x16x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x16x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 16, 16, "MMA_64x16x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x192x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x192x16.cu new file mode 100644 index 000000000..1275b3a16 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x192x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x192x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 192, 16, "MMA_64x192x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x256x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x256x16.cu new file mode 100644 index 000000000..e3aabaa75 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x256x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x256x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 256, 16, "MMA_64x256x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x32x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x32x16.cu new file mode 100644 index 000000000..0d9e07f78 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x32x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x32x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 32, 16, "MMA_64x32x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x64x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x64x16.cu new file mode 100644 index 000000000..14f6fcd8c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x64x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x64x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 64, 16, "MMA_64x64x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x8x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x8x16.cu new file mode 100644 index 000000000..1b0d0e6d2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x8x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x8x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 8, 16, "MMA_64x8x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x96x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x96x16.cu new file mode 100644 index 000000000..62a4331de --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x96x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x96x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 96, 16, "MMA_64x96x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x128x32.cu new file mode 100644 index 000000000..71ea82971 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x128x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 128, 32, "MMA_64x128x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x16x32.cu new file mode 100644 index 000000000..6ef3e404c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x16x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 16, 32, "MMA_64x16x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x192x32.cu new file mode 100644 index 000000000..ad1cd2296 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x192x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 192, 32, "MMA_64x192x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x256x32.cu new file mode 100644 index 000000000..000a40d72 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x256x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 256, 32, "MMA_64x256x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x32x32.cu new file mode 100644 index 000000000..522366660 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x32x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 32, 32, "MMA_64x32x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x64x32.cu new file mode 100644 index 000000000..1394b2a40 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x64x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 64, 32, "MMA_64x64x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x8x32.cu new file mode 100644 index 000000000..f39d1456c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x8x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 8, 32, "MMA_64x8x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x96x32.cu new file mode 100644 index 000000000..fe7242b99 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x96x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 96, 32, "MMA_64x96x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x128x32.cu new file mode 100644 index 000000000..dc46a76b6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x128x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 128, 32, "MMA_64x128x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x16x32.cu new file mode 100644 index 000000000..14c139710 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x16x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 16, 32, "MMA_64x16x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x192x32.cu new file mode 100644 index 000000000..3678c5414 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x192x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 192, 32, "MMA_64x192x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x256x32.cu new file mode 100644 index 000000000..39f479742 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x256x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 256, 32, "MMA_64x256x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x32x32.cu new file mode 100644 index 000000000..7607963c4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x32x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 32, 32, "MMA_64x32x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x64x32.cu new file mode 100644 index 000000000..e20b25a04 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x64x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 64, 32, "MMA_64x64x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x8x32.cu new file mode 100644 index 000000000..8b2f9c8d6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x8x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 8, 32, "MMA_64x8x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x96x32.cu new file mode 100644 index 000000000..a8fb10753 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x96x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 96, 32, "MMA_64x96x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x128x32.cu new file mode 100644 index 000000000..279aa4d37 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x128x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 128, 32, "MMA_64x128x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x16x32.cu new file mode 100644 index 000000000..0a5aaf41a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x16x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 16, 32, "MMA_64x16x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x192x32.cu new file mode 100644 index 000000000..5eac44263 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x192x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 192, 32, "MMA_64x192x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x256x32.cu new file mode 100644 index 000000000..47bc6a3ef --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x256x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 256, 32, "MMA_64x256x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x32x32.cu new file mode 100644 index 000000000..4f0d5856a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x32x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 32, 32, "MMA_64x32x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x64x32.cu new file mode 100644 index 000000000..d8326f3ac --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x64x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 64, 32, "MMA_64x64x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x8x32.cu new file mode 100644 index 000000000..fbec6e6f1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x8x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 8, 32, "MMA_64x8x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x96x32.cu new file mode 100644 index 000000000..1ab9ef53d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x96x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 96, 32, "MMA_64x96x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x128x32.cu new file mode 100644 index 000000000..b41d328e1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x128x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 128, 32, "MMA_64x128x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x16x32.cu new file mode 100644 index 000000000..414ea343e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x16x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 16, 32, "MMA_64x16x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x192x32.cu new file mode 100644 index 000000000..53d23cdc9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x192x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 192, 32, "MMA_64x192x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x256x32.cu new file mode 100644 index 000000000..7055d9edc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x256x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 256, 32, "MMA_64x256x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x32x32.cu new file mode 100644 index 000000000..b34f08491 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x32x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 32, 32, "MMA_64x32x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x64x32.cu new file mode 100644 index 000000000..0f1c26687 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x64x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 64, 32, "MMA_64x64x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x8x32.cu new file mode 100644 index 000000000..1450e5f7e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x8x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 8, 32, "MMA_64x8x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x96x32.cu new file mode 100644 index 000000000..6077058b7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x96x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 96, 32, "MMA_64x96x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x128x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x128x16.cu new file mode 100644 index 000000000..9eaa49ab0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x128x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x128x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 128, 16, "MMA_64x128x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x16x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x16x16.cu new file mode 100644 index 000000000..183878140 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x16x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x16x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 16, 16, "MMA_64x16x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x192x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x192x16.cu new file mode 100644 index 000000000..ea30da004 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x192x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x192x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 192, 16, "MMA_64x192x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x256x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x256x16.cu new file mode 100644 index 000000000..5e82afe2b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x256x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x256x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 256, 16, "MMA_64x256x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x32x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x32x16.cu new file mode 100644 index 000000000..24f208beb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x32x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x32x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 32, 16, "MMA_64x32x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x64x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x64x16.cu new file mode 100644 index 000000000..68a4b96cb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x64x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x64x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 64, 16, "MMA_64x64x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x8x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x8x16.cu new file mode 100644 index 000000000..2f3dab3a4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x8x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x8x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 8, 16, "MMA_64x8x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x96x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x96x16.cu new file mode 100644 index 000000000..2b2e8722e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x96x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x96x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 96, 16, "MMA_64x96x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x128x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x128x8.cu new file mode 100644 index 000000000..158088161 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x128x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x128x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 128, 8, "MMA_64x128x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x16x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x16x8.cu new file mode 100644 index 000000000..fe1492be2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x16x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x16x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 16, 8, "MMA_64x16x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x192x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x192x8.cu new file mode 100644 index 000000000..463e9905b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x192x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x192x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 192, 8, "MMA_64x192x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x256x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x256x8.cu new file mode 100644 index 000000000..53d323884 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x256x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x256x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 256, 8, "MMA_64x256x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x32x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x32x8.cu new file mode 100644 index 000000000..a59c41861 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x32x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x32x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 32, 8, "MMA_64x32x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x64x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x64x8.cu new file mode 100644 index 000000000..ab5d032ca --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x64x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x64x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 64, 8, "MMA_64x64x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x8x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x8x8.cu new file mode 100644 index 000000000..97eb9b4b6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x8x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x8x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 8, 8, "MMA_64x8x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x96x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x96x8.cu new file mode 100644 index 000000000..fca3734c2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x96x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x96x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 96, 8, "MMA_64x96x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x128x32.cu new file mode 100644 index 000000000..c41a660a4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x128x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 128, 32, "MMA_64x128x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x16x32.cu new file mode 100644 index 000000000..9c3895b45 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x16x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 16, 32, "MMA_64x16x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x192x32.cu new file mode 100644 index 000000000..f4b60f169 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x192x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 192, 32, "MMA_64x192x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x256x32.cu new file mode 100644 index 000000000..c6522cb86 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x256x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 256, 32, "MMA_64x256x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x32x32.cu new file mode 100644 index 000000000..468abc59b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x32x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 32, 32, "MMA_64x32x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x64x32.cu new file mode 100644 index 000000000..9786b4c45 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x64x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 64, 32, "MMA_64x64x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x8x32.cu new file mode 100644 index 000000000..823375478 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x8x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 8, 32, "MMA_64x8x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x96x32.cu new file mode 100644 index 000000000..0104423e3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x96x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 96, 32, "MMA_64x96x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x128x32.cu new file mode 100644 index 000000000..6d8210349 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x128x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 128, 32, "MMA_64x128x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x16x32.cu new file mode 100644 index 000000000..8622cacfb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x16x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 16, 32, "MMA_64x16x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x192x32.cu new file mode 100644 index 000000000..a4feb009a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x192x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 192, 32, "MMA_64x192x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x256x32.cu new file mode 100644 index 000000000..e9d817de8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x256x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 256, 32, "MMA_64x256x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x32x32.cu new file mode 100644 index 000000000..65ff0d122 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x32x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 32, 32, "MMA_64x32x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x64x32.cu new file mode 100644 index 000000000..48938cac9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x64x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 64, 32, "MMA_64x64x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x8x32.cu new file mode 100644 index 000000000..55d601042 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x8x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 8, 32, "MMA_64x8x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x96x32.cu new file mode 100644 index 000000000..38925b524 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x96x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 96, 32, "MMA_64x96x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x128x32.cu new file mode 100644 index 000000000..70d09f174 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x128x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 128, 32, "MMA_64x128x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x16x32.cu new file mode 100644 index 000000000..f260d2d33 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x16x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 16, 32, "MMA_64x16x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x192x32.cu new file mode 100644 index 000000000..93f12eede --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x192x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 192, 32, "MMA_64x192x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x256x32.cu new file mode 100644 index 000000000..e25b2a8b8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x256x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 256, 32, "MMA_64x256x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x32x32.cu new file mode 100644 index 000000000..81329112b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x32x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 32, 32, "MMA_64x32x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x64x32.cu new file mode 100644 index 000000000..7ad6c9175 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x64x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 64, 32, "MMA_64x64x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x8x32.cu new file mode 100644 index 000000000..e6e4fc35a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x8x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 8, 32, "MMA_64x8x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x96x32.cu new file mode 100644 index 000000000..c35ec92ce --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x96x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 96, 32, "MMA_64x96x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x128x32.cu new file mode 100644 index 000000000..326a3f4dd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x128x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 128, 32, "MMA_64x128x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x16x32.cu new file mode 100644 index 000000000..46192dee8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x16x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 16, 32, "MMA_64x16x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x192x32.cu new file mode 100644 index 000000000..d26787621 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x192x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 192, 32, "MMA_64x192x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x256x32.cu new file mode 100644 index 000000000..9a5962ddd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x256x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 256, 32, "MMA_64x256x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x32x32.cu new file mode 100644 index 000000000..bbca73ed8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x32x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 32, 32, "MMA_64x32x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x64x32.cu new file mode 100644 index 000000000..9914d67a8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x64x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 64, 32, "MMA_64x64x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x8x32.cu new file mode 100644 index 000000000..f2044b708 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x8x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 8, 32, "MMA_64x8x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x96x32.cu new file mode 100644 index 000000000..d0d84c06b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x96x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 96, 32, "MMA_64x96x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.cu new file mode 100644 index 000000000..bcde5cbd0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.cu @@ -0,0 +1,22 @@ +#include +#include +#include "lat_gmma.h" +#include "../../../hw_def/hw_def.h" + +FILE* g_lat_gmma_outfile = NULL; + +int main(int argc, char *argv[]) +{ + initializeDeviceProp(0, argc, argv); + + g_lat_gmma_outfile = fopen("lat_gmma_results.csv", "w"); + if (g_lat_gmma_outfile) + fprintf(g_lat_gmma_outfile, "config,cycles_per_mma\n"); + + run_all_wgmma_latency_tests(); + + if (g_lat_gmma_outfile) + fclose(g_lat_gmma_outfile); + + return 0; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.h b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.h new file mode 100644 index 000000000..ccb2ca7d7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.h @@ -0,0 +1,385 @@ +/*************************************************************************************************** + * GMMA Latency Microbenchmark for SM90 (Hopper Architecture) + * + * This file provides a comprehensive sweep over all valid WGMMA (Warpgroup Matrix Multiply- + * Accumulate) operations supported by the NVIDIA Hopper architecture (SM90). + * + * The sweep covers: + * + * 1. F32 Accumulator (ElementC = float): + * - FP16 x FP16 -> FP32 + * - BF16 x BF16 -> FP32 + * - TF32 x TF32 -> FP32 + * - E4M3 x E4M3 -> FP32 + * - E4M3 x E5M2 -> FP32 + * - E5M2 x E4M3 -> FP32 + * - E5M2 x E5M2 -> FP32 + * + * 2. F16 Accumulator (ElementC = half_t): + * - FP16 x FP16 -> FP16 + * - E4M3 x E4M3 -> FP16 + * - E4M3 x E5M2 -> FP16 + * - E5M2 x E4M3 -> FP16 + * - E5M2 x E5M2 -> FP16 + * + * 3. INT32 Accumulator (ElementC = int32_t): + * - INT8 x INT8 -> INT32 + * - INT8 x UINT8 -> INT32 + * - UINT8 x INT8 -> INT32 + * - UINT8 x UINT8 -> INT32 + * + * For each data type combination, the sweep tests multiple tile shapes: + * - 64x8, 64x16, 64x32, 64x64, 64x96, 64x128, 64x192, 64x256 + * + * Usage: + * Call run_all_wgmma_latency_tests() to run the comprehensive sweep. + * + * NOTE: This file has been refactored for parallel compilation. Test implementations + * are split across lat_gmma_f32.cu, lat_gmma_f16.cu, and lat_gmma_int32.cu. + * + **************************************************************************************************/ + +#ifndef LAT_GMMA_DEF_H +#define LAT_GMMA_DEF_H + +#include +#include +#include +#include + + +// Function declarations for test suites +// These are defined in separate .cu files for parallel compilation + +// F32 accumulator tests - TF32 x TF32 -> F32 +void run_f32tf32tf32tf32_64x8x8_test(); +void run_f32tf32tf32tf32_64x16x8_test(); +void run_f32tf32tf32tf32_64x32x8_test(); +void run_f32tf32tf32tf32_64x64x8_test(); +void run_f32tf32tf32tf32_64x96x8_test(); +void run_f32tf32tf32tf32_64x128x8_test(); +void run_f32tf32tf32tf32_64x192x8_test(); +void run_f32tf32tf32tf32_64x256x8_test(); + +// F32 accumulator tests - E4M3 x E4M3 -> F32 +void run_f32e4m3e4m3e4m3_64x8x32_test(); +void run_f32e4m3e4m3e4m3_64x16x32_test(); +void run_f32e4m3e4m3e4m3_64x32x32_test(); +void run_f32e4m3e4m3e4m3_64x64x32_test(); +void run_f32e4m3e4m3e4m3_64x96x32_test(); +void run_f32e4m3e4m3e4m3_64x128x32_test(); +void run_f32e4m3e4m3e4m3_64x192x32_test(); +void run_f32e4m3e4m3e4m3_64x256x32_test(); + +// F32 accumulator tests - E4M3 x E5M2 -> F32 +void run_f32e4m3e5m2e4m3_64x8x32_test(); +void run_f32e4m3e5m2e4m3_64x16x32_test(); +void run_f32e4m3e5m2e4m3_64x32x32_test(); +void run_f32e4m3e5m2e4m3_64x64x32_test(); +void run_f32e4m3e5m2e4m3_64x96x32_test(); +void run_f32e4m3e5m2e4m3_64x128x32_test(); +void run_f32e4m3e5m2e4m3_64x192x32_test(); +void run_f32e4m3e5m2e4m3_64x256x32_test(); + +// F32 accumulator tests - E5M2 x E4M3 -> F32 +void run_f32e5m2e4m3e5m2_64x8x32_test(); +void run_f32e5m2e4m3e5m2_64x16x32_test(); +void run_f32e5m2e4m3e5m2_64x32x32_test(); +void run_f32e5m2e4m3e5m2_64x64x32_test(); +void run_f32e5m2e4m3e5m2_64x96x32_test(); +void run_f32e5m2e4m3e5m2_64x128x32_test(); +void run_f32e5m2e4m3e5m2_64x192x32_test(); +void run_f32e5m2e4m3e5m2_64x256x32_test(); + +// F32 accumulator tests - E5M2 x E5M2 -> F32 +void run_f32e5m2e5m2e5m2_64x8x32_test(); +void run_f32e5m2e5m2e5m2_64x16x32_test(); +void run_f32e5m2e5m2e5m2_64x32x32_test(); +void run_f32e5m2e5m2e5m2_64x64x32_test(); +void run_f32e5m2e5m2e5m2_64x96x32_test(); +void run_f32e5m2e5m2e5m2_64x128x32_test(); +void run_f32e5m2e5m2e5m2_64x192x32_test(); +void run_f32e5m2e5m2e5m2_64x256x32_test(); + +// INT32 accumulator tests - INT8 x INT8 -> INT32 +void run_int32s8s8s8_64x8x32_test(); +void run_int32s8s8s8_64x16x32_test(); +void run_int32s8s8s8_64x32x32_test(); +void run_int32s8s8s8_64x64x32_test(); +void run_int32s8s8s8_64x96x32_test(); +void run_int32s8s8s8_64x128x32_test(); +void run_int32s8s8s8_64x192x32_test(); +void run_int32s8s8s8_64x256x32_test(); + +// INT32 accumulator tests - INT8 x UINT8 -> INT32 +void run_int32s8u8s8_64x8x32_test(); +void run_int32s8u8s8_64x16x32_test(); +void run_int32s8u8s8_64x32x32_test(); +void run_int32s8u8s8_64x64x32_test(); +void run_int32s8u8s8_64x96x32_test(); +void run_int32s8u8s8_64x128x32_test(); +void run_int32s8u8s8_64x192x32_test(); +void run_int32s8u8s8_64x256x32_test(); + +// INT32 accumulator tests - UINT8 x INT8 -> INT32 +void run_int32u8s8u8_64x8x32_test(); +void run_int32u8s8u8_64x16x32_test(); +void run_int32u8s8u8_64x32x32_test(); +void run_int32u8s8u8_64x64x32_test(); +void run_int32u8s8u8_64x96x32_test(); +void run_int32u8s8u8_64x128x32_test(); +void run_int32u8s8u8_64x192x32_test(); +void run_int32u8s8u8_64x256x32_test(); + +// INT32 accumulator tests - UINT8 x UINT8 -> INT32 +void run_int32u8u8u8_64x8x32_test(); +void run_int32u8u8u8_64x16x32_test(); +void run_int32u8u8u8_64x32x32_test(); +void run_int32u8u8u8_64x64x32_test(); +void run_int32u8u8u8_64x96x32_test(); +void run_int32u8u8u8_64x128x32_test(); +void run_int32u8u8u8_64x192x32_test(); +void run_int32u8u8u8_64x256x32_test(); + +// F16 accumulator tests (defined in lat_gmma_f16.cu) +// F32 accumulator tests - FP16 x FP16 -> F32 +void run_f32f16f16_64x8x16_test(); +void run_f32f16f16_64x16x16_test(); +void run_f32f16f16_64x32x16_test(); +void run_f32f16f16_64x64x16_test(); +void run_f32f16f16_64x96x16_test(); +void run_f32f16f16_64x128x16_test(); +void run_f32f16f16_64x192x16_test(); +void run_f32f16f16_64x256x16_test(); + +// F32 accumulator tests - BF16 x BF16 -> F32 +void run_f32bf16bf16_64x8x16_test(); +void run_f32bf16bf16_64x16x16_test(); +void run_f32bf16bf16_64x32x16_test(); +void run_f32bf16bf16_64x64x16_test(); +void run_f32bf16bf16_64x96x16_test(); +void run_f32bf16bf16_64x128x16_test(); +void run_f32bf16bf16_64x192x16_test(); +void run_f32bf16bf16_64x256x16_test(); + +// F16 accumulator tests - FP16 x FP16 -> F16 +void run_f16f16f16_64x8x16_test(); +void run_f16f16f16_64x16x16_test(); +void run_f16f16f16_64x32x16_test(); +void run_f16f16f16_64x64x16_test(); +void run_f16f16f16_64x96x16_test(); +void run_f16f16f16_64x128x16_test(); +void run_f16f16f16_64x192x16_test(); +void run_f16f16f16_64x256x16_test(); + +// F16 accumulator tests - E4M3 x E4M3 -> F16 +void run_f16e4m3e4m3_64x8x32_test(); +void run_f16e4m3e4m3_64x16x32_test(); +void run_f16e4m3e4m3_64x32x32_test(); +void run_f16e4m3e4m3_64x64x32_test(); +void run_f16e4m3e4m3_64x96x32_test(); +void run_f16e4m3e4m3_64x128x32_test(); +void run_f16e4m3e4m3_64x192x32_test(); +void run_f16e4m3e4m3_64x256x32_test(); + +// F16 accumulator tests - E4M3 x E5M2 -> F16 +void run_f16e4m3e5m2_64x8x32_test(); +void run_f16e4m3e5m2_64x16x32_test(); +void run_f16e4m3e5m2_64x32x32_test(); +void run_f16e4m3e5m2_64x64x32_test(); +void run_f16e4m3e5m2_64x96x32_test(); +void run_f16e4m3e5m2_64x128x32_test(); +void run_f16e4m3e5m2_64x192x32_test(); +void run_f16e4m3e5m2_64x256x32_test(); + +// F16 accumulator tests - E5M2 x E4M3 -> F16 +void run_f16e5m2e4m3_64x8x32_test(); +void run_f16e5m2e4m3_64x16x32_test(); +void run_f16e5m2e4m3_64x32x32_test(); +void run_f16e5m2e4m3_64x64x32_test(); +void run_f16e5m2e4m3_64x96x32_test(); +void run_f16e5m2e4m3_64x128x32_test(); +void run_f16e5m2e4m3_64x192x32_test(); +void run_f16e5m2e4m3_64x256x32_test(); + +// F16 accumulator tests - E5M2 x E5M2 -> F16 +void run_f16e5m2e5m2_64x8x32_test(); +void run_f16e5m2e5m2_64x16x32_test(); +void run_f16e5m2e5m2_64x32x32_test(); +void run_f16e5m2e5m2_64x64x32_test(); +void run_f16e5m2e5m2_64x96x32_test(); +void run_f16e5m2e5m2_64x128x32_test(); +void run_f16e5m2e5m2_64x192x32_test(); +void run_f16e5m2e5m2_64x256x32_test(); + +void run_f16accumulator_tests() { + run_f16f16f16_64x8x16_test(); + run_f16f16f16_64x16x16_test(); + run_f16f16f16_64x32x16_test(); + run_f16f16f16_64x64x16_test(); + run_f16f16f16_64x96x16_test(); + run_f16f16f16_64x128x16_test(); + run_f16f16f16_64x192x16_test(); + run_f16f16f16_64x256x16_test(); + run_f16e4m3e4m3_64x8x32_test(); + run_f16e4m3e4m3_64x16x32_test(); + run_f16e4m3e4m3_64x32x32_test(); + run_f16e4m3e4m3_64x64x32_test(); + run_f16e4m3e4m3_64x96x32_test(); + run_f16e4m3e4m3_64x128x32_test(); + run_f16e4m3e4m3_64x192x32_test(); + run_f16e4m3e4m3_64x256x32_test(); + run_f16e4m3e5m2_64x8x32_test(); + run_f16e4m3e5m2_64x16x32_test(); + run_f16e4m3e5m2_64x32x32_test(); + run_f16e4m3e5m2_64x64x32_test(); + run_f16e4m3e5m2_64x96x32_test(); + run_f16e4m3e5m2_64x128x32_test(); + run_f16e4m3e5m2_64x192x32_test(); + run_f16e4m3e5m2_64x256x32_test(); + run_f16e5m2e4m3_64x8x32_test(); + run_f16e5m2e4m3_64x16x32_test(); + run_f16e5m2e4m3_64x32x32_test(); + run_f16e5m2e4m3_64x64x32_test(); + run_f16e5m2e4m3_64x96x32_test(); + run_f16e5m2e4m3_64x128x32_test(); + run_f16e5m2e4m3_64x192x32_test(); + run_f16e5m2e4m3_64x256x32_test(); + run_f16e5m2e5m2_64x8x32_test(); + run_f16e5m2e5m2_64x16x32_test(); + run_f16e5m2e5m2_64x32x32_test(); + run_f16e5m2e5m2_64x64x32_test(); + run_f16e5m2e5m2_64x96x32_test(); + run_f16e5m2e5m2_64x128x32_test(); + run_f16e5m2e5m2_64x192x32_test(); + run_f16e5m2e5m2_64x256x32_test(); +} + +void run_f32accumulator_tests() { + run_f32tf32tf32tf32_64x8x8_test(); + run_f32tf32tf32tf32_64x16x8_test(); + run_f32tf32tf32tf32_64x32x8_test(); + run_f32tf32tf32tf32_64x64x8_test(); + run_f32tf32tf32tf32_64x96x8_test(); + run_f32tf32tf32tf32_64x128x8_test(); + run_f32tf32tf32tf32_64x192x8_test(); + run_f32tf32tf32tf32_64x256x8_test(); + run_f32f16f16_64x8x16_test(); + run_f32f16f16_64x16x16_test(); + run_f32f16f16_64x32x16_test(); + run_f32f16f16_64x64x16_test(); + run_f32f16f16_64x96x16_test(); + run_f32f16f16_64x128x16_test(); + run_f32f16f16_64x192x16_test(); + run_f32f16f16_64x256x16_test(); + run_f32bf16bf16_64x8x16_test(); + run_f32bf16bf16_64x16x16_test(); + run_f32bf16bf16_64x32x16_test(); + run_f32bf16bf16_64x64x16_test(); + run_f32bf16bf16_64x96x16_test(); + run_f32bf16bf16_64x128x16_test(); + run_f32bf16bf16_64x192x16_test(); + run_f32bf16bf16_64x256x16_test(); + run_f32e4m3e4m3e4m3_64x8x32_test(); + run_f32e4m3e4m3e4m3_64x16x32_test(); + run_f32e4m3e4m3e4m3_64x32x32_test(); + run_f32e4m3e4m3e4m3_64x64x32_test(); + run_f32e4m3e4m3e4m3_64x96x32_test(); + run_f32e4m3e4m3e4m3_64x128x32_test(); + run_f32e4m3e4m3e4m3_64x192x32_test(); + run_f32e4m3e4m3e4m3_64x256x32_test(); + run_f32e4m3e5m2e4m3_64x8x32_test(); + run_f32e4m3e5m2e4m3_64x16x32_test(); + run_f32e4m3e5m2e4m3_64x32x32_test(); + run_f32e4m3e5m2e4m3_64x64x32_test(); + run_f32e4m3e5m2e4m3_64x96x32_test(); + run_f32e4m3e5m2e4m3_64x128x32_test(); + run_f32e4m3e5m2e4m3_64x192x32_test(); + run_f32e4m3e5m2e4m3_64x256x32_test(); + run_f32e5m2e4m3e5m2_64x8x32_test(); + run_f32e5m2e4m3e5m2_64x16x32_test(); + run_f32e5m2e4m3e5m2_64x32x32_test(); + run_f32e5m2e4m3e5m2_64x64x32_test(); + run_f32e5m2e4m3e5m2_64x96x32_test(); + run_f32e5m2e4m3e5m2_64x128x32_test(); + run_f32e5m2e4m3e5m2_64x192x32_test(); + run_f32e5m2e4m3e5m2_64x256x32_test(); + run_f32e5m2e5m2e5m2_64x8x32_test(); + run_f32e5m2e5m2e5m2_64x16x32_test(); + run_f32e5m2e5m2e5m2_64x32x32_test(); + run_f32e5m2e5m2e5m2_64x64x32_test(); + run_f32e5m2e5m2e5m2_64x96x32_test(); + run_f32e5m2e5m2e5m2_64x128x32_test(); + run_f32e5m2e5m2e5m2_64x192x32_test(); + run_f32e5m2e5m2e5m2_64x256x32_test(); +} + +void run_int32accumulator_tests() { + run_int32s8s8s8_64x8x32_test(); + run_int32s8s8s8_64x16x32_test(); + run_int32s8s8s8_64x32x32_test(); + run_int32s8s8s8_64x64x32_test(); + run_int32s8s8s8_64x96x32_test(); + run_int32s8s8s8_64x128x32_test(); + run_int32s8s8s8_64x192x32_test(); + run_int32s8s8s8_64x256x32_test(); + run_int32s8u8s8_64x8x32_test(); + run_int32s8u8s8_64x16x32_test(); + run_int32s8u8s8_64x32x32_test(); + run_int32s8u8s8_64x64x32_test(); + run_int32s8u8s8_64x96x32_test(); + run_int32s8u8s8_64x128x32_test(); + run_int32s8u8s8_64x192x32_test(); + run_int32s8u8s8_64x256x32_test(); + run_int32u8s8u8_64x8x32_test(); + run_int32u8s8u8_64x16x32_test(); + run_int32u8s8u8_64x32x32_test(); + run_int32u8s8u8_64x64x32_test(); + run_int32u8s8u8_64x96x32_test(); + run_int32u8s8u8_64x128x32_test(); + run_int32u8s8u8_64x192x32_test(); + run_int32u8s8u8_64x256x32_test(); + run_int32u8u8u8_64x8x32_test(); + run_int32u8u8u8_64x16x32_test(); + run_int32u8u8u8_64x32x32_test(); + run_int32u8u8u8_64x64x32_test(); + run_int32u8u8u8_64x96x32_test(); + run_int32u8u8u8_64x128x32_test(); + run_int32u8u8u8_64x192x32_test(); + run_int32u8u8u8_64x256x32_test(); +} + +// ============================================================================ +// Main Test Function - Run All Configurations +// ============================================================================ + +inline void run_all_wgmma_latency_tests() { + printf("\n"); + printf("================================================================================\n"); + printf(" SM90 GMMA Latency Comprehensive Sweep\n"); + printf("================================================================================\n"); + printf("\n"); + + // Run F32 accumulator tests + run_f32accumulator_tests(); + + // Run F16 accumulator tests + run_f16accumulator_tests(); + + // Run INT32 accumulator tests + run_int32accumulator_tests(); + + printf("================================================================================\n"); + printf(" Sweep Complete\n"); + printf("================================================================================\n"); + printf("\n"); +} + +// Legacy function signatures for compatibility +float gmma_latency_ss() { + printf("Running comprehensive WGMMA latency tests...\n"); + run_all_wgmma_latency_tests(); + return 0.0f; +} + +#endif diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_common.h b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_common.h new file mode 100644 index 000000000..53c70f0f4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_common.h @@ -0,0 +1,210 @@ +/*************************************************************************************************** + * GMMA Latency Microbenchmark - Common Definitions + * + * This header contains shared kernel templates and helper macros used by all GMMA latency tests. + * + **************************************************************************************************/ + +#ifndef LAT_GMMA_COMMON_H +#define LAT_GMMA_COMMON_H + +#include +#include +#include +#include +#include +#include + +// Output CSV file; defined in lat_gmma.cu, NULL disables file output. +extern FILE* g_lat_gmma_outfile; + +#include "cute/arch/util.hpp" +#include "../../../hw_def/hw_def.h" + +// CUTLASS cute library headers +#include +#include "cutlass/numeric_types.h" +#include +#include +#include +#include +#include +#include + +using namespace cute; + +// ============================================================================ +// Base Kernel Template +// ============================================================================ + +template< + class ElementA, + class ElementB, + class ElementC, + class TileShape_MNK, + class RepeatTimes = cute::Int<1024> +> +__global__ void wgmma_latency_kernel(uint32_t *startClk, uint32_t *stopClk, uint32_t *checksum) { + int thread_idx = threadIdx.x + blockDim.x * threadIdx.y + threadIdx.z * blockDim.x * blockDim.y ; + int warp_group_idx = __shfl_sync(0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0); + + static constexpr GMMA::Major GmmaMajorA = cute::GMMA::Major::K; + static constexpr GMMA::Major GmmaMajorB = cute::GMMA::Major::K; + + // Create the GMMA operation + auto gmma_op = cute::GMMA::ss_op_selector< + ElementA, ElementB, ElementC, TileShape_MNK, GmmaMajorA, GmmaMajorB>(); + using MMA_Op = decltype(gmma_op); + + // Create the TiledMma based on element types and tile shape + using TiledMma = decltype(cute::make_tiled_mma(gmma_op)); + using MMA_Traits = typename TiledMma::Traits; + TiledMma tiled_mma; + MMA_Traits traits; + + // Create the fragment A, B, C + // Define the smem layouts using GMMA layout helpers for K-major + // Using Layout_K_INTER_Atom which has minimal swizzling (Swizzle<0,4,3> = identity) + // Layout_K_INTER_Atom_Bits has shape (8, 128 bits) = (8, 4) for 32-bit elements + constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape(GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape(GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int{}))); + + // Allocate shared memory with proper size (using type aliases for constexpr evaluation) + // There will be warnings as CUDA cannot determine if the sharedmem size is static + // But for this ubench case, since we are only measuring GMMA instruction, + // we can ignore the warnings on shmem allocation. + // CUTLASS bypass this by passing the shmem size during kernel launch via + // sizeof(typename GemmKernel::SharedStorage), but this means we have to + // rewrite this function as part of a Class to access the SharedStorage type + // defined with the M,N,K shape. + __shared__ ElementA smem_A[cosize_v]; + __shared__ ElementB smem_B[cosize_v]; + + // Create the layout objects for tensor construction + SmemLayoutA sA_layout{}; + SmemLayoutB sB_layout{}; + + + // Create the tensors with GMMA-compatible layouts + Tensor sA = make_tensor(make_smem_ptr(smem_A), sA_layout); // (BLK_M, BLK_K, PIPE) + Tensor sB = make_tensor(make_smem_ptr(smem_B), sB_layout); // (BLK_N, BLK_K, PIPE) + constexpr int MmaWarpGroups = size(TiledMma{}) / cutlass::NumThreadsPerWarpGroup; + Layout warp_group_thread_layout = make_layout(Int{}, + Int{}); + auto thread_mma = tiled_mma.get_slice(warp_group_thread_layout(warp_group_idx)); + + Tensor tCsA = thread_mma.partition_A(sA); // (MMA,MMA_M,MMA_K,PIPE) + Tensor tCsB = thread_mma.partition_B(sB); // (MMA,MMA_N,MMA_K,PIPE) + + // Allocate "fragments/descriptors" + Tensor tCrA = thread_mma.make_fragment_A(tCsA); // (MMA,MMA_M,MMA_K,PIPE) + Tensor tCrB = thread_mma.make_fragment_B(tCsB); // (MMA,MMA_N,MMA_K,PIPE) + + // Get fragment registers for accumulator with MN size + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape_MNK{})); + + __syncthreads(); + + // Start timing (only thread 0) + uint32_t start = 0; + if (thread_idx == 0) { + asm volatile("mov.u32 %0, %%clock;" : "=r"(start) :: "memory"); + } + __syncthreads(); + + // Fence accumulator operands + warpgroup_fence_operand(accum); + + // Arrive and execute WGMMA + warpgroup_arrive(); + constexpr int repeat_times = static_value(); + + #pragma unroll + for (int j = 0; j < repeat_times; j++) { + // Call the fma method + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + } + // Wait for WGMMA to complete + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + __syncthreads(); + + // Stop timing + uint32_t stop = 0; + if (thread_idx == 0) { + asm volatile("mov.u32 %0, %%clock;" : "=r"(stop) :: "memory"); + } + + // Write results + if (thread_idx == 0) { + startClk[blockIdx.x] = start; + stopClk[blockIdx.x] = stop; + + // Compute checksum to prevent optimization + uint32_t sum = reinterpret_cast(accum.data())[0]; + // Simple checksum over accumulator + checksum[blockIdx.x] = sum; + } +} + +// ============================================================================ +// Host Function Template +// ============================================================================ + +template> +float run_wgmma_latency_test_typed() { + // Allocate device memory + uint32_t *startClk_g, *stopClk_g, *checksum_g; + gpuErrchk(cudaMalloc(&startClk_g, sizeof(uint32_t))); + gpuErrchk(cudaMalloc(&stopClk_g, sizeof(uint32_t))); + gpuErrchk(cudaMalloc(&checksum_g, sizeof(uint32_t))); + + // Launch kernel with 128 threads (warpgroup size) + dim3 grid(1); + dim3 block(128); + wgmma_latency_kernel<<>>(startClk_g, stopClk_g, checksum_g); + + gpuErrchk(cudaPeekAtLastError()); + gpuErrchk(cudaDeviceSynchronize()); + + // Copy results back + uint32_t startClk, stopClk, checksum; + gpuErrchk(cudaMemcpy(&startClk, startClk_g, sizeof(uint32_t), cudaMemcpyDeviceToHost)); + gpuErrchk(cudaMemcpy(&stopClk, stopClk_g, sizeof(uint32_t), cudaMemcpyDeviceToHost)); + gpuErrchk(cudaMemcpy(&checksum, checksum_g, sizeof(uint32_t), cudaMemcpyDeviceToHost)); + + // Calculate latency + constexpr int repeat_times = static_value(); + float latency = ((float)(stopClk - startClk)) / ((float)repeat_times); + + // Cleanup + cudaFree(startClk_g); + cudaFree(stopClk_g); + cudaFree(checksum_g); + + return latency; +} + +// ============================================================================ +// Helper Macro for Testing +// ============================================================================ + +#define TEST_MMA_CONFIG(EA, EB, EC, M, N, K, DESC) \ + do { \ + try { \ + using TileShape = decltype(make_shape(Int{}, Int{}, Int{})); \ + float lat = run_wgmma_latency_test_typed(); \ + printf("%-50s: %6.2f cycles\n", DESC, lat); \ + if (g_lat_gmma_outfile) fprintf(g_lat_gmma_outfile, "%s,%.2f\n", DESC, lat); \ + } catch (...) { \ + printf("%-50s: FAILED\n", DESC); \ + if (g_lat_gmma_outfile) fprintf(g_lat_gmma_outfile, "%s,FAILED\n", DESC); \ + } \ + } while(0) + +#endif // LAT_GMMA_COMMON_H diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_test.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_test.cu new file mode 100644 index 000000000..a216f65a8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_test.cu @@ -0,0 +1,17 @@ +#include +#include "lat_gmma_common.h" + +int main() +{ + // A simple test kernel to check if the library is working + try { + using TileShape = decltype(make_shape(cute::Int<64>{}, cute::Int<256>{}, cute::Int<16>{})); + // Repeat once for the test + using RepeatTimes = cute::Int<1>; + float lat = run_wgmma_latency_test_typed(); + printf("MMA_64x256x16_F32F16F16_SS: %6.2f cycles\n", lat); + } catch (...) { + printf("MMA_64x256x16_F32F16F16_SS: FAILED\n"); + } + return 0; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile new file mode 100644 index 000000000..09798c3d7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile @@ -0,0 +1,40 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ +CUTLASS_ROOT ?= $(BASE_DIR)/../../../../cutlass-bench + +# WGMMA microbenchmarks. +# wgmma_no_tma / wgmma_tf32_128 use CuTe (CUTLASS) headers; wgmma_verify is +# standalone. Only build the CuTe benches when the headers are present +# (cutlass-bench submodule initialized), so CI can still build wgmma_verify. +ALL_BENCHES := wgmma_verify +ifneq ($(wildcard $(CUTLASS_ROOT)/include/cute/tensor.hpp),) +ALL_BENCHES += wgmma_no_tma wgmma_tf32_128 +endif + +# WGMMA requires Hopper (SM90a). +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 \ + -I$(CUTLASS_ROOT)/include \ + -I$(CUTLASS_ROOT)/tools/util/include \ + -DCUTLASS_ARCH_MMA_SM90_SUPPORTED + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu new file mode 100644 index 000000000..1a49b5039 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu @@ -0,0 +1,302 @@ +// ============================================================================= +// wgmma_no_tma.cu +// +// Simplified Hopper WGMMA GEMM copied from CuTe tutorial wgmma_sm90.cu with: +// - TMA removed (smem filled by single-threaded loop from thread 0) +// - mbarrier removed (sync uses __syncthreads only) +// - cp_async_fence/wait removed (unimplemented in GPGPU-Sim; not needed with thread-0 fill) +// - warp specialization removed (single role, all 128 threads do load+mma) +// - F32 accumulator (like CUTLASS example 48) +// - N=16 tile (smallest WGMMA shape; fast to simulate in GPGPU-Sim) +// +// The key test: cute::gemm() issues wgmma.mma_async PTX through the compiler, +// not via hand-written asm(). This verifies GPGPU-Sim handles compiled WGMMA. +// +// GEMM shape: C[M×N] = A[M×K] * B^T[N×K] (NT layout, column-major) +// MMA tile : m64n16k16 (accumulated over bK=64 → 4 wgmma calls per tile) +// Global : M=64, N=16, K=64 +// +// Build (SM90a required): +// nvcc -gencode=arch=compute_90a,code=sm_90a \ +// -gencode=arch=compute_90a,code=compute_90a \ +// -I${CUTLASS_ROOT}/include -I${CUTLASS_ROOT}/tools/util/include \ +// -std=c++17 -O2 -o wgmma_no_tma wgmma_no_tma.cu -lcudart +// +// Run on real H100: +// ./wgmma_no_tma +// Run under GPGPU-Sim (functional): +// PTX_SIM_USE_PTX_FILE=1 ./wgmma_no_tma +// ============================================================================= + +#include +#include +#include +#include + +#include +#include + +#include + +#include "cutlass/cluster_launch.hpp" +#include "cutlass/util/print_error.hpp" +#include "cutlass/util/GPU_Clock.hpp" +#include "cutlass/util/helper_cuda.hpp" + +using namespace cute; + +// ============================================================================ +// Shared memory layout struct (same as wgmma_sm90.cu) +// ============================================================================ + +template +struct SharedStorage { + alignas(128) cute::ArrayEngine> A; + alignas(128) cute::ArrayEngine> B; +}; + +// ============================================================================ +// Kernel (adapted from gemm_device in wgmma_sm90.cu) +// +// Changes vs. the original: +// 1. No pipeline prefetch — single stage (bP=1). +// 2. Inner loop: thread-0 fill → __syncthreads → wgmma → __syncthreads. +// cp_async_fence/wait removed (unimplemented in GPGPU-Sim). +// 3. warpgroup_fence_operand / warpgroup_arrive / warpgroup_commit_batch / +// warpgroup_wait stay exactly as in wgmma_sm90.cu. +// ============================================================================ + +template +__global__ static +__launch_bounds__(decltype(size(TiledMma{}))::value) +void gemm_device(ProblemShape shape_MNK, CtaTiler cta_tiler, + TA const* A, AStride dA, ASmemLayout /*sA_layout*/, TiledCopyA copy_a, + TB const* B, BStride dB, BSmemLayout /*sB_layout*/, TiledCopyB copy_b, + TC * C, CStride dC, TiledMma mma) +{ + CUTE_STATIC_ASSERT_V(rank(shape_MNK) == Int<3>{}); + CUTE_STATIC_ASSERT_V(rank(cta_tiler) == Int<3>{}); + CUTE_STATIC_ASSERT_V(size(copy_a) == size(mma)); + CUTE_STATIC_ASSERT_V(size(copy_b) == size(mma)); + static_assert(is_static::value); + static_assert(is_static::value); + + // ---- Full and tiled global tensors ---------------------------------------- + Tensor mA = make_tensor(make_gmem_ptr(A), select<0,2>(shape_MNK), dA); // (M,K) + Tensor mB = make_tensor(make_gmem_ptr(B), select<1,2>(shape_MNK), dB); // (N,K) + Tensor mC = make_tensor(make_gmem_ptr(C), select<0,1>(shape_MNK), dC); // (M,N) + + auto cta_coord = make_coord(blockIdx.x, blockIdx.y, _); + Tensor gA = local_tile(mA, cta_tiler, cta_coord, Step<_1, X,_1>{}); // (BLK_M,BLK_K,k) + Tensor gB = local_tile(mB, cta_tiler, cta_coord, Step< X,_1,_1>{}); // (BLK_N,BLK_K,k) + Tensor gC = local_tile(mC, cta_tiler, cta_coord, Step<_1,_1, X>{}); // (BLK_M,BLK_N) + + // ---- Shared memory -------------------------------------------------------- + extern __shared__ char shared_memory[]; + using SS = SharedStorage; + SS& smem = *reinterpret_cast(shared_memory); + Tensor sA = make_tensor(make_smem_ptr(smem.A.begin()), ASmemLayout{}); // (BLK_M,BLK_K,1) + Tensor sB = make_tensor(make_smem_ptr(smem.B.begin()), BSmemLayout{}); // (BLK_N,BLK_K,1) + + // copy_a / copy_b not used — replaced by single-threaded fill below. + (void)copy_a; (void)copy_b; + + // ---- MMA partitioning (smem → regs → accum) ------------------------------ + ThrMMA thr_mma = mma.get_slice(threadIdx.x); + Tensor tCsA = thr_mma.partition_A(sA); // (MMA,MMA_M,MMA_K,1) + Tensor tCsB = thr_mma.partition_B(sB); // (MMA,MMA_N,MMA_K,1) + Tensor tCgC = thr_mma.partition_C(gC); // (MMA,MMA_M,MMA_N) + + Tensor tCrA = thr_mma.make_fragment_A(tCsA); + Tensor tCrB = thr_mma.make_fragment_B(tCsB); + Tensor tCrC = thr_mma.make_fragment_C(tCgC); + clear(tCrC); + + // ---- K-tile loop (no pipelining — single smem stage) --------------------- + auto K_TILE_MAX = size<2>(gA); // gA shape = (BLK_M, BLK_K, num_k_tiles) + + CUTE_NO_UNROLL + for (int k_tile = 0; k_tile < K_TILE_MAX; ++k_tile) { + // Single-threaded smem fill: thread 0 writes each element directly through + // CuTe's swizzled layout indexing — no TiledCopy, no intra-warp conflicts. + if (threadIdx.x == 0) { + for (int m = 0; m < size<0>(sA); ++m) + for (int k = 0; k < size<1>(sA); ++k) + sA(m, k, 0) = gA(m, k, k_tile); + for (int n = 0; n < size<0>(sB); ++n) + for (int k = 0; k < size<1>(sB); ++k) + sB(n, k, 0) = gB(n, k, k_tile); + } + __syncthreads(); + + // WGMMA: accumulate D += A * B (generated as wgmma.mma_async PTX by compiler) + warpgroup_fence_operand(tCrC); + warpgroup_arrive(); + cute::gemm(mma, tCrA(_,_,_,0), tCrB(_,_,_,0), tCrC); // issues 4 wgmma PTX calls + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(tCrC); + + __syncthreads(); + } + + // ---- Epilogue: store accumulators to C ----------------------------------- + axpby(TC(1), tCrC, TC(0), tCgC); +} + +// ============================================================================ +// Host launcher (NT layout: A col-major M×K, B col-major N×K, C col-major M×N) +// Adapted from gemm_nt() in wgmma_sm90.cu, with MMA atom changed to +// SM90_64x16x16_F32F16F16_SS (N=16 tile, F32 accumulator). +// ============================================================================ + +void gemm_nt(int m, int n, int k, + float alpha, + cute::half_t const* A, int ldA, // col-major M×K + cute::half_t const* B, int ldB, // col-major N×K (B^T is K×N) + float beta, + float * C, int ldC, // col-major M×N + cudaStream_t stream = 0) +{ + using TA = cute::half_t; + using TB = cute::half_t; + using TC = float; + + auto M = int(m), N = int(n), K = int(k); + auto prob_shape = make_shape(M, N, K); // (M, N, K) + + // NT strides: A col-major (stride-1 in M), B col-major (stride-1 in N) + auto dA = make_stride(Int<1>{}, ldA); // (dM=1, dK=ldA) + auto dB = make_stride(Int<1>{}, ldB); // (dN=1, dK=ldB) + auto dC = make_stride(Int<1>{}, ldC); // (dM=1, dN=ldC) + + // CTA tile sizes + auto bM = Int<64>{}; + auto bN = Int<16>{}; + auto bK = Int<64>{}; // bK/16 = 4 wgmma calls per K-tile + auto bP = Int< 1>{}; // single pipeline stage (no TMA double-buffer) + auto cta_tiler = make_shape(bM, bN, bK); + + // Smem layouts (MN-major = col-major). + // Atom widths for half_t: SW128→64, SW64→32, SW32→16. + // bM=64 fits SW128 (64), bN=16 fits SW32 (16). + auto sA = tile_to_shape(GMMA::Layout_MN_SW128_Atom{}, make_shape(bM, bK, bP)); + auto sB = tile_to_shape(GMMA::Layout_MN_SW32_Atom{}, make_shape(bN, bK, bP)); + + // Copy atoms: original SM80_CP_ASYNC (not called — smem fill is done by thread 0 loop). + // cp_async_fence/wait removed (unimplemented in GPGPU-Sim; not needed with thread-0 fill). + TiledCopy copyA = make_tiled_copy( + Copy_Atom, TA>{}, + Layout>{}, // 16×8 threads, m-major + Layout>{}); // 8×1 values per thread + TiledCopy copyB = make_tiled_copy( + Copy_Atom, TB>{}, + Layout>{}, + Layout>{}); + + // WGMMA tiled MMA: m64n16k16 F32←F16×F16 (MN-major smem for both A and B) + TiledMMA tiled_mma = make_tiled_mma( + SM90_64x16x16_F32F16F16_SS{}); + + // Shared memory size + int smem_size = int(sizeof(SharedStorage)); + + dim3 dimBlock(size(tiled_mma)); // 128 threads (1 warpgroup) + dim3 dimCluster(1, 1, 1); + dim3 dimGrid(ceil_div(m, int(bM)), ceil_div(n, int(bN))); + + auto kernel_ptr = reinterpret_cast( + &gemm_device); + + CUTE_CHECK_ERROR(cudaFuncSetAttribute( + kernel_ptr, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + + cutlass::ClusterLaunchParams params = {dimGrid, dimBlock, dimCluster, smem_size, stream}; + cutlass::Status status = cutlass::launch_kernel_on_cluster( + params, kernel_ptr, + prob_shape, cta_tiler, + A, dA, sA, copyA, + B, dB, sB, copyB, + C, dC, tiled_mma); + + CUTE_CHECK_LAST(); + if (status != cutlass::Status::kSuccess) + fprintf(stderr, "launch_kernel_on_cluster failed\n"); +} + +// ============================================================================ +// main: allocate, run, verify +// ============================================================================ + +int main() { + cudaDeviceProp props; + int dev; + CUTE_CHECK_ERROR(cudaGetDevice(&dev)); + CUTE_CHECK_ERROR(cudaGetDeviceProperties(&props, dev)); + printf("wgmma_no_tma — %s (SM %d.%d)\n", props.name, props.major, props.minor); + + // Problem size: keep small for GPGPU-Sim + constexpr int M = 64, N = 16, K = 64; + printf("GEMM: C[%dx%d] = A[%dx%d] * B^T[%dx%d] (NT col-major, FP16→FP32)\n", + M, N, M, K, N, K); + printf("Tile: m64n16k64 (4 wgmma.m64n16k16 calls via cute::gemm per K-tile)\n"); + printf("------------------------------------------------------------\n"); + + // Host allocation (col-major) + thrust::host_vector h_A(M * K), h_B(N * K); + thrust::host_vector h_C(M * N, 0.f); + + // Fill with small integer values (avoids FP16 rounding in reference) + for (int j = 0; j < M*K; ++j) h_A[j] = cute::half_t(float((j % 5) - 2)); + for (int j = 0; j < N*K; ++j) h_B[j] = cute::half_t(float((j % 3) - 1)); // {-1,0,1} + + // Device + thrust::device_vector d_A = h_A; + thrust::device_vector d_B = h_B; + thrust::device_vector d_C = h_C; + + // Run WGMMA kernel + gemm_nt(M, N, K, 1.0f, + d_A.data().get(), M, // A: ldA = M (col-major M×K) + d_B.data().get(), N, // B: ldB = N (col-major N×K) + 0.0f, + d_C.data().get(), M); // C: ldC = M (col-major M×N) + + CUTE_CHECK_ERROR(cudaDeviceSynchronize()); + h_C = d_C; + + // ---- CPU reference ------------------------------------------------------- + // C_ref[m][n] = sum_k A[m][k] * B[n][k] + // Col-major indexing: A[m + k*M], B[n + k*N], C[m + n*M] + std::vector ref(M * N, 0.0); + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) + for (int kk = 0; kk < K; ++kk) + ref[m + n*M] += double(float(h_A[m + kk*M])) * double(float(h_B[n + kk*N])); + + // ---- Check --------------------------------------------------------------- + double max_err = 0.0; + int max_m = 0, max_n = 0; + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) { + double e = std::fabs(double(h_C[m + n*M]) - ref[m + n*M]); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + + double ref_mag = std::fabs(ref[max_m + max_n*M]); + double tol = (ref_mag < 1.0) ? 1e-3 : ref_mag * 1e-2; + bool pass = (max_err <= tol); + + printf("Result: %s max_err=%.4e (at C[%d][%d] ref=%.4f sim=%.4f)\n", + pass ? "PASS" : "FAIL", max_err, max_m, max_n, + float(ref[max_m + max_n*M]), h_C[max_m + max_n*M]); + + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_tf32_128.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_tf32_128.cu new file mode 100644 index 000000000..6501babda --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_tf32_128.cu @@ -0,0 +1,258 @@ +// ============================================================================= +// wgmma_tf32_128.cu +// +// Functional-correctness microbenchmark for the Hopper WGMMA F32+=TF32*TF32 +// atom at its native N=128 tile size — the shape/dtype combo CUTLASS example +// 48 (Hopper warp-specialized GEMM) uses for its TF32 kernel: +// +// wgmma.mma_async.sync.aligned.m64n128k8.f32.tf32.tf32 +// wgmma.fence.sync.aligned +// wgmma.commit_group.sync.aligned +// wgmma.wait_group.sync.aligned +// +// wgmma_verify.cu hand-writes wgmma PTX with an explicit output-operand list, +// which is only practical up to N=16 (8 D-registers/thread). The m64n128k8 +// atom has 64 D-registers/thread, so this benchmark instead goes through +// CUTLASS/CuTe's MMA_64x128x8_F32TF32TF32_SS_TN atom +// (cute/arch/mma_sm90_gmma.hpp) — the compiler emits the real instruction +// with all 64 accumulator registers wired up, and CuTe's partition_C() +// handles the fragment-to-global-(m,n) mapping so no hand-derived layout +// formula is needed (unlike wgmma_verify.cu's d_frag_pos()). +// +// GEMM shape: D[M×N] = A[M×K] * B^T[N×K] (TN / K-major smem, single tile, +// exactly the atom's native shape — one wgmma call, no K-loop) +// M=64 N=128 K=8 +// +// Smem fill is single-threaded (thread 0 only), same workaround as +// wgmma_no_tma.cu: GPGPU-Sim's sequential execution resolves intra-warp smem +// write ordering differently than real H100 concurrent execution, so a +// TiledCopy-based fill can produce spurious conflicts. +// +// Numeric note: GPGPU-Sim's tf32 functional model currently reads operands +// as full-precision float32 (no mantissa truncation on read — see +// quantize_to_type()/element_to_double() in instructions.cc). Real hardware +// truncates to 10 mantissa bits inside the tensor core. To stay correct +// against both, this test follows wgmma_verify.cu's convention: store raw +// (untruncated) float32 bits in the tf32 smem buffers, but compute the host +// reference from tf32-truncated inputs, and use the same loose tolerance +// (1e-3 absolute / 1% relative) that already absorbs the difference. +// +// Usage: +// ./wgmma_tf32_128 – real H100 +// PTX_SIM_USE_PTX_FILE=1 ./wgmma_tf32_128 – GPGPU-Sim functional sim +// ============================================================================= + +#include +#include +#include +#include +#include + +#include + +#include "cutlass/util/print_error.hpp" +#include "cutlass/util/helper_cuda.hpp" + +using namespace cute; + +static constexpr int M = 64, N = 128, K = 8; + +// --------------------------------------------------------------------------- +// Round float to the nearest tf32-representable value (10 mantissa bits). +// Matches wgmma_verify.cu's quant_tf32(); used only for the host reference. +// --------------------------------------------------------------------------- +static double quant_tf32(double f) { + float fp = (float)f; + uint32_t b; memcpy(&b, &fp, 4); + b &= 0xFFFFE000u; + memcpy(&fp, &b, 4); + return (double)fp; +} + +// ============================================================================ +// Shared memory (K-major, no swizzle — GMMA::Layout_K_INTER_Atom) +// ============================================================================ +template +struct SharedStorage { + alignas(128) cute::ArrayEngine> A; + alignas(128) cute::ArrayEngine> B; +}; + +// ============================================================================ +// Kernel: single-tile GEMM — exactly one wgmma.m64n128k8 call. +// ============================================================================ +template +__global__ static +__launch_bounds__(decltype(size(TiledMma{}))::value) +void wgmma_tf32_128_kernel(tfloat32_t const* A_g, tfloat32_t const* B_g, + float* C_g, uint32_t* clk_g, + TiledMma mma, ASmemLayout sA_layout, BSmemLayout sB_layout) +{ + extern __shared__ char shared_memory[]; + using SS = SharedStorage; + SS& smem = *reinterpret_cast(shared_memory); + Tensor sA = make_tensor(make_smem_ptr(smem.A.begin()), sA_layout); // (M,K) + Tensor sB = make_tensor(make_smem_ptr(smem.B.begin()), sB_layout); // (N,K) + + Tensor gA = make_tensor(make_gmem_ptr(A_g), make_shape(Int{}, Int{}), + make_stride(Int{}, Int<1>{})); // row-major (M,K) + Tensor gB = make_tensor(make_gmem_ptr(B_g), make_shape(Int{}, Int{}), + make_stride(Int{}, Int<1>{})); // row-major (N,K) == B^T + Tensor gC = make_tensor(make_gmem_ptr(C_g), make_shape(Int{}, Int{}), + make_stride(Int{}, Int<1>{})); // row-major (M,N) + + const int tid = threadIdx.x; + if (tid == 0) { uint32_t c; asm volatile("mov.u32 %0,%%clock;\n" : "=r"(c)); clk_g[0] = c; } + + // Single-threaded smem fill (see file header for why). + if (tid == 0) { + for (int m = 0; m < M; ++m) for (int k = 0; k < K; ++k) sA(m, k) = gA(m, k); + for (int n = 0; n < N; ++n) for (int k = 0; k < K; ++k) sB(n, k) = gB(n, k); + } + __syncthreads(); + if (tid == 0) { uint32_t c; asm volatile("mov.u32 %0,%%clock;\n" : "=r"(c)); clk_g[1] = c; } + + ThrMMA thr_mma = mma.get_slice(tid); + Tensor tCsA = thr_mma.partition_A(sA); + Tensor tCsB = thr_mma.partition_B(sB); + Tensor tCgC = thr_mma.partition_C(gC); + + Tensor tCrA = thr_mma.make_fragment_A(tCsA); + Tensor tCrB = thr_mma.make_fragment_B(tCsB); + Tensor tCrC = thr_mma.make_fragment_C(tCgC); + clear(tCrC); + + warpgroup_fence_operand(tCrC); + warpgroup_arrive(); // wgmma.fence.sync.aligned + cute::gemm(mma, tCrA, tCrB, tCrC); // wgmma.mma_async.sync.aligned.m64n128k8.f32.tf32.tf32 + warpgroup_commit_batch(); // wgmma.commit_group.sync.aligned + warpgroup_wait<0>(); // wgmma.wait_group.sync.aligned 0 + warpgroup_fence_operand(tCrC); + if (tid == 0) { uint32_t c; asm volatile("mov.u32 %0,%%clock;\n" : "=r"(c)); clk_g[2] = c; } + + axpby(1.0f, tCrC, 0.0f, tCgC); +} + +// ============================================================================ +// Host launcher +// ============================================================================ +static void run_kernel(tfloat32_t const* A, tfloat32_t const* B, float* C, uint32_t* clk, + cudaStream_t stream = 0) { + auto sA_layout = tile_to_shape(GMMA::Layout_K_INTER_Atom{}, make_shape(Int{}, Int{})); + auto sB_layout = tile_to_shape(GMMA::Layout_K_INTER_Atom{}, make_shape(Int{}, Int{})); + + TiledMMA tiled_mma = make_tiled_mma(SM90_64x128x8_F32TF32TF32_SS_TN<>{}); + + using SS = SharedStorage; + int smem_size = int(sizeof(SS)); + + dim3 block(size(tiled_mma)); // 128 threads = 1 warpgroup + dim3 grid(1, 1, 1); + + auto kernel_ptr = &wgmma_tf32_128_kernel; + CUTE_CHECK_ERROR(cudaFuncSetAttribute( + kernel_ptr, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + + kernel_ptr<<>>(A, B, C, clk, tiled_mma, sA_layout, sB_layout); + CUTE_CHECK_LAST(); +} + +// ============================================================================ +// Test runner: fill A/B, launch, compare against a tf32-truncated CPU +// reference, print PASS/FAIL like wgmma_verify.cu. +// ============================================================================ +static bool run_test(const char* name, const float* A_host, const float* B_host) { + std::vector A_d(M * K), B_d(N * K); + std::vector A_ref(M * K), B_ref(N * K); + for (int i = 0; i < M * K; ++i) { + uint32_t bits; memcpy(&bits, &A_host[i], 4); + A_d[i] = tfloat32_t::bitcast(bits); // raw float32 bits (no truncation) + A_ref[i] = quant_tf32(A_host[i]); // tf32-truncated reference input + } + for (int i = 0; i < N * K; ++i) { + uint32_t bits; memcpy(&bits, &B_host[i], 4); + B_d[i] = tfloat32_t::bitcast(bits); + B_ref[i] = quant_tf32(B_host[i]); + } + + tfloat32_t *d_A, *d_B; + float *d_C; + uint32_t *d_clk; + cudaMalloc(&d_A, M * K * sizeof(tfloat32_t)); + cudaMalloc(&d_B, N * K * sizeof(tfloat32_t)); + cudaMalloc(&d_C, M * N * sizeof(float)); + cudaMalloc(&d_clk, 3 * sizeof(uint32_t)); + cudaMemcpy(d_A, A_d.data(), M * K * sizeof(tfloat32_t), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, B_d.data(), N * K * sizeof(tfloat32_t), cudaMemcpyHostToDevice); + + run_kernel(d_A, d_B, d_C, d_clk); + cudaError_t err = cudaDeviceSynchronize(); + if (err != cudaSuccess) { + printf("[%s] CUDA error: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_C); cudaFree(d_clk); + return false; + } + + std::vector D_sim(M * N); + uint32_t clk[3]; + cudaMemcpy(D_sim.data(), d_C, M * N * sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk, d_clk, 3 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_C); cudaFree(d_clk); + + // CPU reference: D[m][n] = sum_k A_ref[m][k] * B_ref[n][k] + double max_err = 0; int max_m = 0, max_n = 0; + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + double s = 0.0; + for (int k = 0; k < K; ++k) s += A_ref[m * K + k] * B_ref[n * K + k]; + double e = fabs((double)D_sim[m * N + n] - s); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + } + double ref_check = 0.0; + for (int k = 0; k < K; ++k) ref_check += A_ref[max_m * K + k] * B_ref[max_n * K + k]; + double ref_mag = fabs(ref_check); + double tol = (ref_mag < 1.0) ? 1e-3 : ref_mag * 1e-2; + bool pass = (max_err <= tol); + + printf("[%-24s] %s max_err=%.4e fill=%u wgmma=%u cycles\n", + name, pass ? "PASS" : "FAIL", max_err, clk[1] - clk[0], clk[2] - clk[1]); + if (!pass) + printf(" Worst: D[%d][%d] sim=%.6f ref=%.6f\n", + max_m, max_n, D_sim[max_m * N + max_n], (float)ref_check); + return pass; +} + +// ============================================================================ +// main +// ============================================================================ +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_tf32_128 — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("Tests wgmma.mma_async.sync.aligned.m64n128k8.f32.tf32.tf32 (CUTLASS SM90_64x128x8_F32TF32TF32_SS_TN)\n"); + printf("plus wgmma.fence/commit_group/wait_group.sync.aligned\n"); + printf("================================================================\n"); + + bool all_pass = true; + + // all-ones: D[m][n] = K = 8 (trivially verifiable) + { + static float A[M][K], B[N][K]; + for (int m = 0; m < M; ++m) for (int k = 0; k < K; ++k) A[m][k] = 1.f; + for (int n = 0; n < N; ++n) for (int k = 0; k < K; ++k) B[n][k] = 1.f; + all_pass &= run_test("tf32 m64n128k8 all-ones", (float*)A, (float*)B); + } + + // sequential: small deterministic values vs. tf32-truncated double reference + { + static float A[M][K], B[N][K]; + for (int m = 0; m < M; ++m) for (int k = 0; k < K; ++k) A[m][k] = (m * K + k + 1) * 0.1f; + for (int n = 0; n < N; ++n) for (int k = 0; k < K; ++k) B[n][k] = (k * N + n + 1) * 0.1f; + all_pass &= run_test("tf32 m64n128k8 sequential", (float*)A, (float*)B); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu new file mode 100644 index 000000000..c9db9b67f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu @@ -0,0 +1,980 @@ +// ============================================================================= +// wgmma_verify.cu (v2) +// +// Functional-correctness and timing microbenchmark for wgmma.mma_async (SM90a). +// +// Type / shape combinations tested (all m64n16kK): +// f16 k16 D=f32 swizzle modes 0, 1(128B), 2(64B), 3(32B) +// bf16 k16 D=f32 +// tf32 k8 D=f32 +// e4m3 k32 D=f32 +// e5m2 k32 D=f32 +// s8 k32 D=s32 +// u8 k32 D=s32 +// b1 k256 D=s32 (GPGPU-Sim functional sim only; +// real H100 needs bit-packed smem, not 1 byte/bit) +// +// Smem layout (K-major, no swizzle unless SW!=0): +// T = 16 / E (elements per 128-bit column) +// off = (stride%T + leading%8 * T)*E + (stride/T)*SBO + (leading/8)*LBO +// SBO = 128 always. LBO_A = M*E*8, LBO_B = N*E*8. +// All shapes satisfy M*K*E = 2048 and K*N*E = 512. +// +// Two test cases per type: +// all-ones – A=B=1 → D[m][n] = K (trivially verifiable) +// sequential – small values compared against a host double-precision reference +// +// Usage: +// ./wgmma_verify – real H100 +// PTX_SIM_MODE_FUNC=1 ./wgmma_verify – GPGPU-Sim functional sim +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; // 8 D-registers per thread + +// For all standard types: M*K*E = 2048, K*N*E = 512 (fixed by WGMMA design) +static constexpr int SMEM_A = 2048; +static constexpr int SMEM_B = 512; +static constexpr int SBO = 128; // stride byte offset (always 128 for K-major) + +// b1 (sim convention): 1 byte per bit, K=256 "elements" +static constexpr int B1_K = 256; +static constexpr int B1_SMEM_A = M * B1_K; // 16384 +static constexpr int B1_SMEM_B = B1_K * N; // 4096 + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }\n" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +// K-major smem byte offset (before swizzle). +// For A (M×K): leading=k, stride=m. For B (K×N): leading=k, stride=n. +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int E, int LBO) { + int T = 16 / E; + return (stride % T + (leading % 8) * T) * E + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +// XOR-swizzle (self-inverse) applied to a byte offset. +__host__ __device__ __forceinline__ +int apply_sw(int off, int sw) { + if (!sw) return off; + int s = (sw == 1) ? 128 : (sw == 2) ? 64 : 32; + int b = off / s, i = off % s; + return b * s + (i ^ ((b & 1) ? (s >> 1) : 0)); +} + +__host__ __device__ __forceinline__ +int smem_off_sw(int leading, int stride, int E, int LBO, int sw) { + return apply_sw(smem_off(leading, stride, E, LBO), sw); +} + +// --------------------------------------------------------------------------- +// WGMMA protocol macros +// --------------------------------------------------------------------------- +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") +#define READ_CLK(c) asm volatile("mov.u32 %0,%%clock;\n" : "=r"(c)) + +// =========================================================================== +// Kernels +// =========================================================================== + +// --------------------------------------------------------------------------- +// F16 m64n16k16 D=f32 template on swizzle mode SW (0–3) +// --------------------------------------------------------------------------- +template +__global__ void kernel_f16(const half* A_g, const half* B_g, + float* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 16, E = 2, LBO_A = M*E*8, LBO_B = N*E*8; // 1024, 256 + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off_sw(k, m, E, LBO_A, SW)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off_sw(k, n, E, LBO_B, SW)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, SW); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, SW); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// BF16 m64n16k16 D=f32 +// --------------------------------------------------------------------------- +__global__ void kernel_bf16(const __nv_bfloat16* A_g, const __nv_bfloat16* B_g, + float* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 16, E = 2, LBO_A = M*E*8, LBO_B = N*E*8; + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(__nv_bfloat16*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(__nv_bfloat16*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.bf16.bf16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// TF32 m64n16k8 D=f32 (no trans parameters) +// --------------------------------------------------------------------------- +__global__ void kernel_tf32(const float* A_g, const float* B_g, + float* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 8, E = 4, LBO_A = M*E*8, LBO_B = N*E*8; // 2048, 512 + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(float*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(float*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k8.f32.tf32.tf32 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// FP8 E4M3 m64n16k32 D=f32 (no trans) +// A_g / B_g carry raw fp8 bytes (uint8_t). +// --------------------------------------------------------------------------- +__global__ void kernel_e4m3(const uint8_t* A_g, const uint8_t* B_g, + float* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; // 512, 128 + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(uint8_t*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(uint8_t*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e4m3.e4m3 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// FP8 E5M2 m64n16k32 D=f32 (no trans) +// --------------------------------------------------------------------------- +__global__ void kernel_e5m2(const uint8_t* A_g, const uint8_t* B_g, + float* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(uint8_t*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(uint8_t*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e5m2.e5m2 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// S8 m64n16k32 D=s32 (no scale, no trans) +// --------------------------------------------------------------------------- +__global__ void kernel_s8(const int8_t* A_g, const int8_t* B_g, + int32_t* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(int8_t*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(int8_t*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + int32_t d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.s32.s8.s8 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p; }\n" + : "+r"(d0),"+r"(d1),"+r"(d2),"+r"(d3),"+r"(d4),"+r"(d5),"+r"(d6),"+r"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// U8 m64n16k32 D=s32 +// --------------------------------------------------------------------------- +__global__ void kernel_u8(const uint8_t* A_g, const uint8_t* B_g, + int32_t* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(uint8_t*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(uint8_t*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + int32_t d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.s32.u8.u8 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p; }\n" + : "+r"(d0),"+r"(d1),"+r"(d2),"+r"(d3),"+r"(d4),"+r"(d5),"+r"(d6),"+r"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// B1 m64n16k256 D=s32 +// +// WARNING — GPGPU-Sim functional sim only. +// The sim reads each "bit" as a separate byte (1 byte per bit element, using +// only the LSB). Real H100 hardware expects bits packed 8-per-byte (256 bits +// = 32 bytes per row of A). This kernel uses 1 byte/bit, so the smem +// buffers are 256× larger than on real hardware. +// --------------------------------------------------------------------------- +__global__ void kernel_b1(const uint8_t* A_g, const uint8_t* B_g, + int32_t* D_g, uint32_t* clk_g) { + __shared__ __align__(128) uint8_t smA[B1_SMEM_A]; // 16384 bytes + __shared__ __align__(128) uint8_t smB[B1_SMEM_B]; // 4096 bytes + const int tid = threadIdx.x; + constexpr int K = B1_K, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; // 512, 128 + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + smA[smem_off(k, m, E, LBO_A)] = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + smB[smem_off(k, n, E, LBO_B)] = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + int32_t d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k256.s32.b1.b1.and.popc " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p; }\n" + : "+r"(d0),"+r"(d1),"+r"(d2),"+r"(d3),"+r"(d4),"+r"(d5),"+r"(d6),"+r"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Host: fragment mapping and matrix reassembly +// =========================================================================== + +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = lane/4 + (k/2)*8 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static void reassemble_D_f32(const float* flat, float D[M][N]) { + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) D[m][n] = 0.f; + for (int T = 0; T < WGSIZE; ++T) + for (int e = 0; e < D_ELEMS; ++e) { + int row, col; d_frag_pos(T, e, &row, &col); + if (row < M && col < N) D[row][col] = flat[T*D_ELEMS + e]; + } +} + +static void reassemble_D_s32(const int32_t* flat, int32_t D[M][N]) { + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) D[m][n] = 0; + for (int T = 0; T < WGSIZE; ++T) + for (int e = 0; e < D_ELEMS; ++e) { + int row, col; d_frag_pos(T, e, &row, &col); + if (row < M && col < N) D[row][col] = flat[T*D_ELEMS + e]; + } +} + +// =========================================================================== +// Host: result checking and printing +// =========================================================================== + +static bool check_f32(const char* name, + float D_sim[M][N], const double* D_ref_flat, + uint32_t c0, uint32_t c1, uint32_t c2) { + double max_err = 0; int max_m = 0, max_n = 0; + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) { + double e = fabs((double)D_sim[m][n] - D_ref_flat[m*N+n]); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + double ref_mag = fabs(D_ref_flat[max_m*N+max_n]); + double tol = (ref_mag < 1.0) ? 1e-3 : ref_mag * 1e-2; + bool pass = (max_err <= tol); + printf("[%-40s] %s max_err=%.4e fill=%u wgmma=%u cycles\n", + name, pass ? "PASS" : "FAIL", max_err, c1-c0, c2-c1); + if (!pass) + printf(" Worst: D[%d][%d] sim=%.6f ref=%.6f\n", + max_m, max_n, D_sim[max_m][max_n], (float)D_ref_flat[max_m*N+max_n]); + return pass; +} + +static bool check_s32(const char* name, + int32_t D_sim[M][N], const int64_t* D_ref_flat, + uint32_t c0, uint32_t c1, uint32_t c2) { + int64_t max_err = 0; int max_m = 0, max_n = 0; + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) { + int64_t e = llabs((int64_t)D_sim[m][n] - D_ref_flat[m*N+n]); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + bool pass = (max_err == 0); + printf("[%-40s] %s max_err=%lld fill=%u wgmma=%u cycles\n", + name, pass ? "PASS" : "FAIL", (long long)max_err, c1-c0, c2-c1); + if (!pass) + printf(" Worst: D[%d][%d] sim=%d ref=%lld\n", + max_m, max_n, D_sim[max_m][max_n], (long long)D_ref_flat[max_m*N+max_n]); + return pass; +} + +// =========================================================================== +// Generic test runner helpers +// =========================================================================== + +// Run a f32-accumulator test. +// K — K dimension +// bytes_a/b — device memory sizes for A and B +// h_A / h_B — host pointers to device-formatted input data +// A_dbl/B_dbl — double arrays (M*K and K*N) for the reference matmul; +// must already reflect type quantization (e.g. bf16-rounded) +// launch — lambda: (d_A, d_B, d_D, d_clk) → void +static bool run_f32(const char* name, int K, + size_t bytes_a, size_t bytes_b, + const void* h_A, const void* h_B, + const double* A_dbl, const double* B_dbl, + std::function launch) { + void *d_A, *d_B; + float *d_D; + uint32_t *d_clk; + cudaMalloc(&d_A, bytes_a); + cudaMalloc(&d_B, bytes_b); + cudaMalloc(&d_D, WGSIZE * D_ELEMS * sizeof(float)); + cudaMalloc(&d_clk, 3 * sizeof(uint32_t)); + cudaMemcpy(d_A, h_A, bytes_a, cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B, bytes_b, cudaMemcpyHostToDevice); + + launch(d_A, d_B, d_D, d_clk); + cudaError_t err = cudaDeviceSynchronize(); + if (err != cudaSuccess) { + printf("[%s] CUDA error: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); cudaFree(d_clk); + return false; + } + + std::vector flat(WGSIZE * D_ELEMS); + uint32_t clk[3]; + cudaMemcpy(flat.data(), d_D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk, d_clk, 3*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); cudaFree(d_clk); + + static float D_sim[M][N]; + static double D_ref[M*N]; + reassemble_D_f32(flat.data(), D_sim); + + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) { + double s = 0.0; + for (int k = 0; k < K; ++k) s += A_dbl[m*K+k] * B_dbl[k*N+n]; + D_ref[m*N+n] = s; + } + + return check_f32(name, D_sim, D_ref, clk[0], clk[1], clk[2]); +} + +// Run a s32-accumulator test. +// use_and — if true, reference uses AND-accumulate (b1); else multiply-accumulate +static bool run_s32(const char* name, int K, + size_t bytes_a, size_t bytes_b, + const void* h_A, const void* h_B, + const int64_t* A_i64, const int64_t* B_i64, + bool use_and, + std::function launch) { + void *d_A, *d_B; + int32_t *d_D; + uint32_t *d_clk; + cudaMalloc(&d_A, bytes_a); + cudaMalloc(&d_B, bytes_b); + cudaMalloc(&d_D, WGSIZE * D_ELEMS * sizeof(int32_t)); + cudaMalloc(&d_clk, 3 * sizeof(uint32_t)); + cudaMemcpy(d_A, h_A, bytes_a, cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B, bytes_b, cudaMemcpyHostToDevice); + + launch(d_A, d_B, d_D, d_clk); + cudaError_t err = cudaDeviceSynchronize(); + if (err != cudaSuccess) { + printf("[%s] CUDA error: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); cudaFree(d_clk); + return false; + } + + std::vector flat(WGSIZE * D_ELEMS); + uint32_t clk[3]; + cudaMemcpy(flat.data(), d_D, WGSIZE*D_ELEMS*sizeof(int32_t), cudaMemcpyDeviceToHost); + cudaMemcpy(clk, d_clk, 3*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); cudaFree(d_clk); + + static int32_t D_sim[M][N]; + static int64_t D_ref[M*N]; + reassemble_D_s32(flat.data(), D_sim); + + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) { + int64_t s = 0; + for (int k = 0; k < K; ++k) { + int64_t a = A_i64[m*K+k], b = B_i64[k*N+n]; + s += use_and ? ((a & b) & 1) : (a * b); + } + D_ref[m*N+n] = s; + } + + return check_s32(name, D_sim, D_ref, clk[0], clk[1], clk[2]); +} + +// =========================================================================== +// Host-side type quantization helpers +// =========================================================================== + +// Round float to the nearest bf16-representable value. +static double quant_bf16(double f) { + return (double)__bfloat162float(__float2bfloat16((float)f)); +} + +// Round float to the nearest tf32-representable value (10 mantissa bits). +static double quant_tf32(double f) { + float fp = (float)f; + uint32_t b; memcpy(&b, &fp, 4); + b &= 0xFFFFE000u; // keep sign + 8 exp + top 10 mantissa bits + memcpy(&fp, &b, 4); + return (double)fp; +} + +// Round float to nearest fp8 e4m3 value. +static double quant_e4m3(double f) { + return (double)(float)(__nv_fp8_e4m3)((float)f); +} + +// Round float to nearest fp8 e5m2 value. +static double quant_e5m2(double f) { + return (double)(float)(__nv_fp8_e5m2)((float)f); +} + +// Convert float to raw fp8 e4m3 byte. +static uint8_t float_to_e4m3(float f) { + __nv_fp8_e4m3 v(f); + uint8_t b; memcpy(&b, &v, 1); + return b; +} + +// Convert float to raw fp8 e5m2 byte. +static uint8_t float_to_e5m2(float f) { + __nv_fp8_e5m2 v(f); + uint8_t b; memcpy(&b, &v, 1); + return b; +} + +// =========================================================================== +// Per-type test functions +// =========================================================================== + +// ---- F16 ---- +static bool test_f16(const char* name, int sw, + const float* A_host, const float* B_host) { + constexpr int K = 16; + std::vector A_h(M*K), B_h(K*N); + std::vector A_d(M*K), B_d(K*N); + for (int i = 0; i < M*K; ++i) { + A_h[i] = __float2half(A_host[i]); + A_d[i] = (double)__half2float(A_h[i]); + } + for (int i = 0; i < K*N; ++i) { + B_h[i] = __float2half(B_host[i]); + B_d[i] = (double)__half2float(B_h[i]); + } + auto launch = [sw](const void* dA, const void* dB, float* dD, uint32_t* dC) { + switch (sw) { + case 0: kernel_f16<0><<<1,WGSIZE>>>((const half*)dA,(const half*)dB,dD,dC); break; + case 1: kernel_f16<1><<<1,WGSIZE>>>((const half*)dA,(const half*)dB,dD,dC); break; + case 2: kernel_f16<2><<<1,WGSIZE>>>((const half*)dA,(const half*)dB,dD,dC); break; + case 3: kernel_f16<3><<<1,WGSIZE>>>((const half*)dA,(const half*)dB,dD,dC); break; + } + }; + return run_f32(name, K, M*K*sizeof(half), K*N*sizeof(half), + A_h.data(), B_h.data(), A_d.data(), B_d.data(), launch); +} + +// ---- BF16 ---- +static bool test_bf16(const char* name, + const float* A_host, const float* B_host) { + constexpr int K = 16; + std::vector<__nv_bfloat16> A_b(M*K), B_b(K*N); + std::vector A_d(M*K), B_d(K*N); + for (int i = 0; i < M*K; ++i) { + A_b[i] = __float2bfloat16(A_host[i]); + A_d[i] = quant_bf16(A_host[i]); + } + for (int i = 0; i < K*N; ++i) { + B_b[i] = __float2bfloat16(B_host[i]); + B_d[i] = quant_bf16(B_host[i]); + } + auto launch = [](const void* dA, const void* dB, float* dD, uint32_t* dC) { + kernel_bf16<<<1,WGSIZE>>>((const __nv_bfloat16*)dA,(const __nv_bfloat16*)dB,dD,dC); + }; + return run_f32(name, K, M*K*sizeof(__nv_bfloat16), K*N*sizeof(__nv_bfloat16), + A_b.data(), B_b.data(), A_d.data(), B_d.data(), launch); +} + +// ---- TF32 ---- +static bool test_tf32(const char* name, + const float* A_host, const float* B_host) { + constexpr int K = 8; + std::vector A_f(M*K), B_f(K*N); + std::vector A_d(M*K), B_d(K*N); + for (int i = 0; i < M*K; ++i) { + A_d[i] = quant_tf32(A_host[i]); + A_f[i] = (float)A_d[i]; + } + for (int i = 0; i < K*N; ++i) { + B_d[i] = quant_tf32(B_host[i]); + B_f[i] = (float)B_d[i]; + } + auto launch = [](const void* dA, const void* dB, float* dD, uint32_t* dC) { + kernel_tf32<<<1,WGSIZE>>>((const float*)dA,(const float*)dB,dD,dC); + }; + return run_f32(name, K, M*K*sizeof(float), K*N*sizeof(float), + A_f.data(), B_f.data(), A_d.data(), B_d.data(), launch); +} + +// ---- FP8 E4M3 ---- +static bool test_e4m3(const char* name, + const float* A_host, const float* B_host) { + constexpr int K = 32; + std::vector A_b(M*K), B_b(K*N); + std::vector A_d(M*K), B_d(K*N); + for (int i = 0; i < M*K; ++i) { + A_b[i] = float_to_e4m3(A_host[i]); + A_d[i] = quant_e4m3(A_host[i]); + } + for (int i = 0; i < K*N; ++i) { + B_b[i] = float_to_e4m3(B_host[i]); + B_d[i] = quant_e4m3(B_host[i]); + } + auto launch = [](const void* dA, const void* dB, float* dD, uint32_t* dC) { + kernel_e4m3<<<1,WGSIZE>>>((const uint8_t*)dA,(const uint8_t*)dB,dD,dC); + }; + return run_f32(name, K, M*K, K*N, + A_b.data(), B_b.data(), A_d.data(), B_d.data(), launch); +} + +// ---- FP8 E5M2 ---- +static bool test_e5m2(const char* name, + const float* A_host, const float* B_host) { + constexpr int K = 32; + std::vector A_b(M*K), B_b(K*N); + std::vector A_d(M*K), B_d(K*N); + for (int i = 0; i < M*K; ++i) { + A_b[i] = float_to_e5m2(A_host[i]); + A_d[i] = quant_e5m2(A_host[i]); + } + for (int i = 0; i < K*N; ++i) { + B_b[i] = float_to_e5m2(B_host[i]); + B_d[i] = quant_e5m2(B_host[i]); + } + auto launch = [](const void* dA, const void* dB, float* dD, uint32_t* dC) { + kernel_e5m2<<<1,WGSIZE>>>((const uint8_t*)dA,(const uint8_t*)dB,dD,dC); + }; + return run_f32(name, K, M*K, K*N, + A_b.data(), B_b.data(), A_d.data(), B_d.data(), launch); +} + +// ---- S8 ---- +static bool test_s8(const char* name, + const int8_t* A_host, const int8_t* B_host) { + constexpr int K = 32; + std::vector A_i(M*K), B_i(K*N); + for (int i = 0; i < M*K; ++i) A_i[i] = A_host[i]; + for (int i = 0; i < K*N; ++i) B_i[i] = B_host[i]; + auto launch = [](const void* dA, const void* dB, int32_t* dD, uint32_t* dC) { + kernel_s8<<<1,WGSIZE>>>((const int8_t*)dA,(const int8_t*)dB,dD,dC); + }; + return run_s32(name, K, M*K, K*N, + A_host, B_host, A_i.data(), B_i.data(), false, launch); +} + +// ---- U8 ---- +static bool test_u8(const char* name, + const uint8_t* A_host, const uint8_t* B_host) { + constexpr int K = 32; + std::vector A_i(M*K), B_i(K*N); + for (int i = 0; i < M*K; ++i) A_i[i] = A_host[i]; + for (int i = 0; i < K*N; ++i) B_i[i] = B_host[i]; + auto launch = [](const void* dA, const void* dB, int32_t* dD, uint32_t* dC) { + kernel_u8<<<1,WGSIZE>>>((const uint8_t*)dA,(const uint8_t*)dB,dD,dC); + }; + return run_s32(name, K, M*K, K*N, + A_host, B_host, A_i.data(), B_i.data(), false, launch); +} + +// ---- B1 (sim-only) ---- +static bool test_b1(const char* name, + const uint8_t* A_host, const uint8_t* B_host) { + constexpr int K = B1_K; + std::vector A_i(M*K), B_i(K*N); + for (int i = 0; i < M*K; ++i) A_i[i] = A_host[i] & 1; + for (int i = 0; i < K*N; ++i) B_i[i] = B_host[i] & 1; + auto launch = [](const void* dA, const void* dB, int32_t* dD, uint32_t* dC) { + kernel_b1<<<1,WGSIZE>>>((const uint8_t*)dA,(const uint8_t*)dB,dD,dC); + }; + return run_s32(name, K, (size_t)M*K, (size_t)K*N, + A_host, B_host, A_i.data(), B_i.data(), true, launch); +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_verify v2 — %s (SM %d.%d)\n", + prop.name, prop.major, prop.minor); + } + printf("================================================================\n"); + + bool all_pass = true; + + // ========================================================================= + // F16 m64n16k16 D=f32 (swizzle modes 0–3) + // ========================================================================= + printf("\n--- F16 m64n16k16 D=f32 ---\n"); + { + static float A[M][16], B[16][N]; + // all-ones + for (int m = 0; m < M; ++m) for (int k = 0; k < 16; ++k) A[m][k] = 1.f; + for (int k = 0; k < 16; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1.f; + for (int sw = 0; sw < 4; ++sw) { + char name[64]; snprintf(name, sizeof(name), "f16 all-ones swizzle=%d", sw); + all_pass &= test_f16(name, sw, (float*)A, (float*)B); + } + // sequential + for (int m = 0; m < M; ++m) + for (int k = 0; k < 16; ++k) A[m][k] = (m*16 + k + 1) * 0.1f; + for (int k = 0; k < 16; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (k*N + n + 1) * 0.1f; + for (int sw = 0; sw < 4; ++sw) { + char name[64]; snprintf(name, sizeof(name), "f16 sequential swizzle=%d", sw); + all_pass &= test_f16(name, sw, (float*)A, (float*)B); + } + } + + // ========================================================================= + // BF16 m64n16k16 D=f32 + // ========================================================================= + printf("\n--- BF16 m64n16k16 D=f32 ---\n"); + { + static float A[M][16], B[16][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 16; ++k) A[m][k] = 1.f; + for (int k = 0; k < 16; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1.f; + all_pass &= test_bf16("bf16 all-ones", (float*)A, (float*)B); + + for (int m = 0; m < M; ++m) + for (int k = 0; k < 16; ++k) A[m][k] = (m*16 + k + 1) * 0.1f; + for (int k = 0; k < 16; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (k*N + n + 1) * 0.1f; + all_pass &= test_bf16("bf16 sequential", (float*)A, (float*)B); + } + + // ========================================================================= + // TF32 m64n16k8 D=f32 + // ========================================================================= + printf("\n--- TF32 m64n16k8 D=f32 ---\n"); + { + static float A[M][8], B[8][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 8; ++k) A[m][k] = 1.f; + for (int k = 0; k < 8; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1.f; + all_pass &= test_tf32("tf32 all-ones", (float*)A, (float*)B); + + for (int m = 0; m < M; ++m) + for (int k = 0; k < 8; ++k) A[m][k] = (m*8 + k + 1) * 0.1f; + for (int k = 0; k < 8; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (k*N + n + 1) * 0.1f; + all_pass &= test_tf32("tf32 sequential", (float*)A, (float*)B); + } + + // ========================================================================= + // FP8 E4M3 m64n16k32 D=f32 + // Values {1,2,3,4} are exactly representable in e4m3; no quantisation error. + // ========================================================================= + printf("\n--- FP8 E4M3 m64n16k32 D=f32 ---\n"); + { + static float A[M][32], B[32][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 32; ++k) A[m][k] = 1.f; + for (int k = 0; k < 32; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1.f; + all_pass &= test_e4m3("e4m3 all-ones", (float*)A, (float*)B); + + for (int m = 0; m < M; ++m) + for (int k = 0; k < 32; ++k) A[m][k] = (float)(1 + (m+k) % 4); + for (int k = 0; k < 32; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (float)(1 + (k+n) % 4); + all_pass &= test_e4m3("e4m3 sequential", (float*)A, (float*)B); + } + + // ========================================================================= + // FP8 E5M2 m64n16k32 D=f32 + // ========================================================================= + printf("\n--- FP8 E5M2 m64n16k32 D=f32 ---\n"); + { + static float A[M][32], B[32][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 32; ++k) A[m][k] = 1.f; + for (int k = 0; k < 32; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1.f; + all_pass &= test_e5m2("e5m2 all-ones", (float*)A, (float*)B); + + for (int m = 0; m < M; ++m) + for (int k = 0; k < 32; ++k) A[m][k] = (float)(1 + (m+k) % 4); + for (int k = 0; k < 32; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (float)(1 + (k+n) % 4); + all_pass &= test_e5m2("e5m2 sequential", (float*)A, (float*)B); + } + + // ========================================================================= + // S8 m64n16k32 D=s32 + // ========================================================================= + printf("\n--- S8 m64n16k32 D=s32 ---\n"); + { + static int8_t A[M][32], B[32][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 32; ++k) A[m][k] = 1; + for (int k = 0; k < 32; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1; + all_pass &= test_s8("s8 all-ones", (int8_t*)A, (int8_t*)B); + + // Small signed values to avoid overflow: max sum = 32 * 5 * 5 = 800 < INT32_MAX + for (int m = 0; m < M; ++m) + for (int k = 0; k < 32; ++k) A[m][k] = (int8_t)(1 + (m+k) % 5); + for (int k = 0; k < 32; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (int8_t)(1 + (k+n) % 5); + all_pass &= test_s8("s8 sequential", (int8_t*)A, (int8_t*)B); + } + + // ========================================================================= + // U8 m64n16k32 D=s32 + // ========================================================================= + printf("\n--- U8 m64n16k32 D=s32 ---\n"); + { + static uint8_t A[M][32], B[32][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 32; ++k) A[m][k] = 1; + for (int k = 0; k < 32; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1; + all_pass &= test_u8("u8 all-ones", (uint8_t*)A, (uint8_t*)B); + + for (int m = 0; m < M; ++m) + for (int k = 0; k < 32; ++k) A[m][k] = (uint8_t)(1 + (m+k) % 5); + for (int k = 0; k < 32; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (uint8_t)(1 + (k+n) % 5); + all_pass &= test_u8("u8 sequential", (uint8_t*)A, (uint8_t*)B); + } + + // ========================================================================= + // B1 m64n16k256 D=s32 (GPGPU-Sim functional sim only) + // D[m][n] = popcount of positions k where A[m][k]=1 AND B[k][n]=1. + // ========================================================================= + printf("\n--- B1 m64n16k256 D=s32 (sim-only layout) ---\n"); + { + static uint8_t A[M][B1_K], B[B1_K][N]; + // all-ones: every bit = 1 → D[m][n] = K = 256 + for (int m = 0; m < M; ++m) for (int k = 0; k < B1_K; ++k) A[m][k] = 1; + for (int k = 0; k < B1_K; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1; + all_pass &= test_b1("b1 all-ones", (uint8_t*)A, (uint8_t*)B); + + // checkerboard: A[m][k] = k%2, B[k][n] = k%2 + // D[m][n] = #{k : k%2=1 AND k%2=1} = K/2 = 128 + for (int m = 0; m < M; ++m) for (int k = 0; k < B1_K; ++k) A[m][k] = k % 2; + for (int k = 0; k < B1_K; ++k) for (int n = 0; n < N; ++n) B[k][n] = k % 2; + all_pass &= test_b1("b1 checkerboard", (uint8_t*)A, (uint8_t*)B); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/Makefile new file mode 100644 index 000000000..01f6b40ea --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/Makefile @@ -0,0 +1,28 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := wgmma_dtypes + +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu new file mode 100644 index 000000000..6cce61abe --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu @@ -0,0 +1,190 @@ +// ============================================================================= +// wgmma_dtypes.cu +// +// Tests WGMMA with f16 accumulator: m64n16k16.f16.f16.f16. +// +// wgmma_verify already covers f16/bf16/tf32/fp8/int8 with f32/s32 accumulators. +// This file adds the f16←f16×f16 variant where D registers are f16x2 packed +// into .b32 (N/4 = 4 registers per thread for N=16, each holding 2 f16 values). +// +// Fragment layout for f16 accumulator is identical to f32 — same d_frag_pos +// formula — but each register slot holds 2 f16 values (low and high halfword). +// +// Two tests (all-ones style, exactly representable in f16): +// A=1, B=1 → D[m][n] = K = 16 +// A=2, B=3 → D[m][n] = 6*K = 96 +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int E = 2; // bytes per f16 +static constexpr int SBO = 128; +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int SMEM_A = M * K * E; // 2048 +static constexpr int SMEM_B = K * N * E; // 512 +static constexpr int D_REGS = N / 4; // 4 f16x2 registers per thread (N=16) + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +// --------------------------------------------------------------------------- +// Kernel: m64n16k16.f16.f16.f16 +// D is 4 f16x2 registers (uint32_t) per thread for N=16. +// --------------------------------------------------------------------------- +__global__ void kernel_f16f16(const half* A_g, const half* B_g, uint32_t* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + uint32_t d0=0, d1=0, d2=0, d3=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f16.f16.f16 " + "{%0,%1,%2,%3},%4,%5,p,1,1,0,0; }\n" + : "+r"(d0),"+r"(d1),"+r"(d2),"+r"(d3) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b = tid * D_REGS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; +} + +// --------------------------------------------------------------------------- +// Host helpers +// --------------------------------------------------------------------------- +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +// Unpack WGSIZE*D_REGS f16x2 registers into M×N float matrix. +// Thread T, register r → scalar elements e=2r (low half) and e=2r+1 (high half). +static void reassemble(const uint32_t* flat, float* D) { + for (int i = 0; i < M*N; i++) D[i] = 0.f; + for (int T = 0; T < WGSIZE; T++) { + for (int r = 0; r < D_REGS; r++) { + uint32_t packed = flat[T * D_REGS + r]; + for (int h = 0; h < 2; h++) { + int e = 2*r + h; + uint16_t bits = (h == 0) ? (uint16_t)(packed & 0xFFFFu) + : (uint16_t)(packed >> 16); + half val; + memcpy(&val, &bits, sizeof(val)); + int row, col; + d_frag_pos(T, e, &row, &col); + if (row < M && col < N) + D[row * N + col] = __half2float(val); + } + } + } +} + +static bool check_uniform(const char* name, const float* D, float expected) { + float max_err = 0.f; + int worst = 0; + for (int i = 0; i < M*N; i++) { + float e = fabsf(D[i] - expected); + if (e > max_err) { max_err = e; worst = i; } + } + float tol = fabsf(expected) * 5e-3f + 0.5f; // generous for f16 precision + bool pass = (max_err <= tol); + printf("[%-40s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + worst/N, worst%N, D[worst], expected); + return pass; +} + +static bool run(const char* name, float a_val, float b_val, float expected) { + std::vector h_A(M*K, __float2half(a_val)); + std::vector h_B(K*N, __float2half(b_val)); + half* d_A; + half* d_B; + uint32_t* d_D; + cudaMalloc(&d_A, h_A.size() * sizeof(half)); + cudaMalloc(&d_B, h_B.size() * sizeof(half)); + cudaMalloc(&d_D, WGSIZE * D_REGS * sizeof(uint32_t)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + kernel_f16f16<<<1, WGSIZE>>>(d_A, d_B, d_D); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE * D_REGS); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + std::vector D(M * N); + reassemble(flat.data(), D.data()); + return check_uniform(name, D.data(), expected); +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_dtypes — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + all_pass &= run("f16←f16×f16 m64n16k16 A=1 B=1", 1.f, 1.f, (float)K); + all_pass &= run("f16←f16×f16 m64n16k16 A=2 B=3", 2.f, 3.f, (float)(2*3*K)); + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/Makefile new file mode 100644 index 000000000..2f918a69f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/Makefile @@ -0,0 +1,28 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := wgmma_multik + +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu new file mode 100644 index 000000000..0940fc352 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu @@ -0,0 +1,670 @@ +// ============================================================================= +// wgmma_multik.cu +// +// Tests WGMMA D-register accumulation across multiple K-tiles. +// +// The critical property being tested: after each wgmma.mma_async with scaleD=1, +// the D registers hold the running sum. The simulator must preserve the D +// register file across wgmma commits and not reset it between iterations. +// +// Two tests using f16->f32 m64n16k16: +// +// static_4tiles - same smem (A=B=1) used for 4 back-to-back MMAs in one group. +// All 4 contribute K=16 each -> D[m][n] = 4*K = 64. +// No smem reload; one commit/wait covers all 4 ops. +// +// varying_4tiles - smem reloaded each tile with A[tile] = (tile+1), B = 1. +// tile 0: A=1 -> K*1 = 16 +// tile 1: A=2 -> K*2 = 32 +// tile 2: A=3 -> K*3 = 48 +// tile 3: A=4 -> K*4 = 64 +// Total: K*(1+2+3+4) = 160 +// Each tile is a separate commit/wait group with smem reload. +// +// Dumps for silicon-vs-sim comparison: +// smem_A / smem_B : raw smem content before WGMMA (verifies descriptor layout) +// D registers : final D values (verifies WGMMA computation) +// Per-tile D snap : intermediate D after each tile in varying_4tiles +// Clock timing : %clock delta around WGMMA ops +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include + +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; // 8 f32 registers per thread +static constexpr int E = 2; // bytes per f16 +static constexpr int SBO = 128; +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int SMEM_A = M * K * E; // 2048 +static constexpr int SMEM_B = K * N * E; // 512 + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +__device__ __forceinline__ void mma_f16( + uint64_t da, uint64_t db, + float& d0, float& d1, float& d2, float& d3, + float& d4, float& d5, float& d6, float& d7) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3), + "+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); +} + +// --------------------------------------------------------------------------- +// Kernel 1: 4 MMAs in one group, constant smem (A=B=1). +// D accumulates across all 4 ops -> expected = 4*K = 64. +// Dumps: smem A/B (before WGMMA), D registers (after), clock timing. +// --------------------------------------------------------------------------- +__global__ void kernel_static4( + const half* A_g, const half* B_g, + float* D_g, uint32_t* clk_out, + half* smA_raw, half* smB_raw) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + + // Dump raw smem content (physical byte order) before WGMMA + for (int i = tid; i < M*K; i += WGSIZE) + smA_raw[i] = *(const half*)(smA + i * E); + for (int i = tid; i < K*N; i += WGSIZE) + smB_raw[i] = *(const half*)(smB + i * E); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t_start = 0, t_stop = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_start) :: "memory"); + __syncthreads(); + + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // D += K (D was 0) + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // D += K + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // D += K + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // D += K + WGMMA_COMMIT; + WGMMA_WAIT; + WGMMA_FENCE; + + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_stop) :: "memory"); + if (tid == 0) { clk_out[0] = t_start; clk_out[1] = t_stop; } + + const int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// Kernel 2: 4 tiles with different A values, smem reloaded each tile. +// A_g layout: [4 * M * K], where A_g[tile*M*K + m*K + k] = (half)(tile+1). +// B_g: [K * N] all ones (shared across tiles). +// Expected D[m][n] = K*(1+2+3+4) = 160. +// +// Dumps: +// smA_raw/smB_raw : physical smem content of tile 0 (before first WGMMA) +// D_tiles : D register snapshot after each of the 4 tiles +// (4 * WGSIZE * D_ELEMS floats, tile-major) +// clk_out : clock_start/stop bracketing all 4 tiles +// --------------------------------------------------------------------------- +__global__ void kernel_varying4( + const half* A_g, const half* B_g, + float* D_g, uint32_t* clk_out, + half* smA_raw, half* smB_raw, + float* D_tiles) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + + // Load B once - same for all tiles. + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + + // Tile 0: A = 1.0 + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[0*M*K + m*K + k]; + } + __syncthreads(); + + // Dump tile 0 smem (physical byte order) before any WGMMA + for (int i = tid; i < M*K; i += WGSIZE) + smA_raw[i] = *(const half*)(smA + i * E); + for (int i = tid; i < K*N; i += WGSIZE) + smB_raw[i] = *(const half*)(smB + i * E); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t_start = 0, t_stop = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_start) :: "memory"); + __syncthreads(); + + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + + // Snapshot D after tile 0 (expected each element = K*1 = 16) + { const int b = tid * D_ELEMS, off = 0 * WGSIZE * D_ELEMS; + D_tiles[off+b]=d0; D_tiles[off+b+1]=d1; D_tiles[off+b+2]=d2; D_tiles[off+b+3]=d3; + D_tiles[off+b+4]=d4; D_tiles[off+b+5]=d5; D_tiles[off+b+6]=d6; D_tiles[off+b+7]=d7; } + + // Tile 1: A = 2.0 + WGMMA_FENCE; // post-wait: releases D, signals smem can be updated + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[1*M*K + m*K + k]; + } + __syncthreads(); + WGMMA_FENCE; // pre-MMA: signals new smem content visible to wgmma + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + + // Snapshot D after tile 1 (expected each element = K*(1+2) = 48) + { const int b = tid * D_ELEMS, off = 1 * WGSIZE * D_ELEMS; + D_tiles[off+b]=d0; D_tiles[off+b+1]=d1; D_tiles[off+b+2]=d2; D_tiles[off+b+3]=d3; + D_tiles[off+b+4]=d4; D_tiles[off+b+5]=d5; D_tiles[off+b+6]=d6; D_tiles[off+b+7]=d7; } + + // Tile 2: A = 3.0 + WGMMA_FENCE; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[2*M*K + m*K + k]; + } + __syncthreads(); + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + + // Snapshot D after tile 2 (expected each element = K*(1+2+3) = 96) + { const int b = tid * D_ELEMS, off = 2 * WGSIZE * D_ELEMS; + D_tiles[off+b]=d0; D_tiles[off+b+1]=d1; D_tiles[off+b+2]=d2; D_tiles[off+b+3]=d3; + D_tiles[off+b+4]=d4; D_tiles[off+b+5]=d5; D_tiles[off+b+6]=d6; D_tiles[off+b+7]=d7; } + + // Tile 3: A = 4.0 + WGMMA_FENCE; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[3*M*K + m*K + k]; + } + __syncthreads(); + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + WGMMA_FENCE; // final fence + + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_stop) :: "memory"); + if (tid == 0) { clk_out[0] = t_start; clk_out[1] = t_stop; } + + // Snapshot D after tile 3 (final, expected each element = K*(1+2+3+4) = 160) + { const int b = tid * D_ELEMS, off = 3 * WGSIZE * D_ELEMS; + D_tiles[off+b]=d0; D_tiles[off+b+1]=d1; D_tiles[off+b+2]=d2; D_tiles[off+b+3]=d3; + D_tiles[off+b+4]=d4; D_tiles[off+b+5]=d5; D_tiles[off+b+6]=d6; D_tiles[off+b+7]=d7; } + + const int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// Kernel 3: Fence overhead. +// Measures the cost of wgmma.fence.sync.aligned in isolation. +// N_FENCE consecutive FENCEs; no WGMMA ops. +// cycles_per_fence = (stop - start) / N_FENCE +// --------------------------------------------------------------------------- +static constexpr int N_FENCE = 256; + +__global__ void kernel_fence_overhead(uint32_t* clk_out) { + const int tid = threadIdx.x; + uint32_t t_start = 0, t_stop = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_start) :: "memory"); + __syncthreads(); + #pragma unroll + for (int i = 0; i < N_FENCE; i++) + WGMMA_FENCE; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_stop) :: "memory"); + if (tid == 0) { clk_out[0] = t_start; clk_out[1] = t_stop; } +} + +// --------------------------------------------------------------------------- +// Kernel 4: Commit+Wait barrier overhead (no MMA ops). +// Commits an empty wgmma group (no ops issued) and waits on it. +// This measures the pure synchronization barrier cost of COMMIT+WAIT. +// N_BARRIER iterations; cycles_per_barrier = (stop - start) / N_BARRIER. +// --------------------------------------------------------------------------- +static constexpr int N_BARRIER = 256; + +__global__ void kernel_commit_wait_overhead(uint32_t* clk_out) { + const int tid = threadIdx.x; + uint32_t t_start = 0, t_stop = 0; + WGMMA_FENCE; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_start) :: "memory"); + __syncthreads(); + #pragma unroll + for (int i = 0; i < N_BARRIER; i++) { + WGMMA_COMMIT; + WGMMA_WAIT; + } + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_stop) :: "memory"); + if (tid == 0) { clk_out[0] = t_start; clk_out[1] = t_stop; } +} + +// --------------------------------------------------------------------------- +// Kernel 5: Single-shape MMA throughput (m64n16k16 f32<-f16xf16). +// N_MMA back-to-back MMA ops in one commit group. +// Includes smem setup; clock brackets fence..commit_wait..fence. +// cycles_per_mma = (stop - start) / N_MMA (includes fence+commit+wait amortised). +// --------------------------------------------------------------------------- +static constexpr int N_MMA = 1024; + +__global__ void kernel_mma_throughput( + const half* A_g, const half* B_g, float* D_g, uint32_t* clk_out) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t_start = 0, t_stop = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_start) :: "memory"); + __syncthreads(); + + WGMMA_FENCE; + #pragma unroll 8 + for (int i = 0; i < N_MMA; i++) + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + WGMMA_FENCE; + + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_stop) :: "memory"); + if (tid == 0) { clk_out[0] = t_start; clk_out[1] = t_stop; } + + const int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Host helpers +// =========================================================================== +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static void reassemble(const float* flat, float D[M][N]) { + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) D[m][n] = 0.f; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, &row, &col); + if (row < M && col < N) + D[row][col] = flat[T * D_ELEMS + e]; + } +} + +static bool check_uniform(const char* name, float D[M][N], float expected, FILE* outfile = NULL) { + float max_err = 0.f; + int max_m = 0, max_n = 0; + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) { + float e = fabsf(D[m][n] - expected); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + float tol = fabsf(expected) * 1e-2f; + bool pass = (max_err <= tol); + printf("[%-36s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + max_m, max_n, D[max_m][max_n], expected); + if (outfile) + fprintf(outfile, "%s,%s,%.1f,%.4e\n", name, pass ? "PASS" : "FAIL", expected, max_err); + return pass; +} + +// Write a comprehensive dump file for silicon-vs-sim diff comparison. +// smA_raw/smB_raw: raw physical smem content (M*K and K*N fp16 values). +// flat : per-thread D registers (WGSIZE*D_ELEMS floats). +// D : reassembled D[M][N] matrix. +// D_tiles : per-tile D snapshots (num_tiles * WGSIZE*D_ELEMS), NULL if none. +// num_tiles : number of tile snapshots in D_tiles. +// t_start/stop : raw %clock values. +static void dump_results( + const char* filename, const char* test_name, + const half* smA_raw, const half* smB_raw, + const float* flat, float D[M][N], + const float* D_tiles, int num_tiles, + uint32_t t_start, uint32_t t_stop, + int num_mma_ops) +{ + FILE* f = fopen(filename, "w"); + if (!f) { fprintf(stderr, "cannot open %s\n", filename); return; } + + fprintf(f, "=== %s ===\n", test_name); + fprintf(f, "[timing]\n"); + fprintf(f, "clock_start=%u\n", t_start); + fprintf(f, "clock_stop=%u\n", t_stop); + fprintf(f, "total_cycles=%u\n", t_stop - t_start); + if (num_mma_ops > 0) + fprintf(f, "cycles_per_mma=%.2f\n", (float)(t_stop - t_start) / num_mma_ops); + + // Physical smem layout — what the WGMMA descriptor sees + fprintf(f, "\n[smem_A physical M=%d K=%d halfwords]\n", M, K); + for (int i = 0; i < M*K; i++) + fprintf(f, "%d %.6f\n", i, __half2float(smA_raw[i])); + + fprintf(f, "\n[smem_B physical K=%d N=%d halfwords]\n", K, N); + for (int i = 0; i < K*N; i++) + fprintf(f, "%d %.6f\n", i, __half2float(smB_raw[i])); + + // Logical smem layout via smem_off mapping (shows which (k,m)/(k,n) -> physical index) + fprintf(f, "\n[smem_A logical k m phys_idx value]\n"); + for (int k = 0; k < K; k++) + for (int m = 0; m < M; m++) { + int off = smem_off(k, m, E, LBO_A); + int idx = off / E; + fprintf(f, "%d %d %d %.6f\n", k, m, idx, __half2float(smA_raw[idx])); + } + + fprintf(f, "\n[smem_B logical k n phys_idx value]\n"); + for (int k = 0; k < K; k++) + for (int n = 0; n < N; n++) { + int off = smem_off(k, n, E, LBO_B); + int idx = off / E; + fprintf(f, "%d %d %d %.6f\n", k, n, idx, __half2float(smB_raw[idx])); + } + + // Per-tile D snapshots (if provided) + if (D_tiles && num_tiles > 0) { + static const float tile_expected[] = {16.f, 48.f, 96.f, 160.f}; + for (int t = 0; t < num_tiles; t++) { + fprintf(f, "\n[D_tile_%d thread elem value expected=%.1f]\n", + t, (t < 4) ? tile_expected[t] : 0.f); + const float* snap = D_tiles + t * WGSIZE * D_ELEMS; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) + fprintf(f, "%d %d %.6f\n", T, e, snap[T*D_ELEMS + e]); + } + } + + // Final D registers (per-thread, raw) + fprintf(f, "\n[D_registers thread elem value]\n"); + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) + fprintf(f, "%d %d %.6f\n", T, e, flat[T*D_ELEMS + e]); + + // Thread-to-matrix mapping + D matrix + fprintf(f, "\n[thread_to_matrix thread elem row col]\n"); + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, &row, &col); + fprintf(f, "%d %d %d %d\n", T, e, row, col); + } + + fprintf(f, "\n[D_matrix row col value]\n"); + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) + fprintf(f, "%d %d %.6f\n", m, n, D[m][n]); + + fclose(f); + printf(" -> dumped to %s\n", filename); +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_multik - %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + FILE* outfile = fopen("wgmma_multik_results.csv", "w"); + if (outfile) + fprintf(outfile, "test,status,expected,max_err\n"); + + bool all_pass = true; + static float D[M][N]; + + // --- Test 1: static_4tiles --- + printf("\n--- static_4tiles (4 MMAs, same smem A=B=1, expected D=64) ---\n"); + { + std::vector h_A(M*K, __float2half(1.f)); + std::vector h_B(K*N, __float2half(1.f)); + half *d_A, *d_B; float *d_D; uint32_t *d_clk; + half *d_smA_raw, *d_smB_raw; + cudaMalloc(&d_A, h_A.size()*sizeof(half)); + cudaMalloc(&d_B, h_B.size()*sizeof(half)); + cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + cudaMalloc(&d_clk, 2*sizeof(uint32_t)); + cudaMalloc(&d_smA_raw, M*K*sizeof(half)); + cudaMalloc(&d_smB_raw, K*N*sizeof(half)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + + kernel_static4<<<1, WGSIZE>>>(d_A, d_B, d_D, d_clk, d_smA_raw, d_smB_raw); + cudaDeviceSynchronize(); + + std::vector flat(WGSIZE*D_ELEMS); + std::vector clk(2); + std::vector smA_raw(M*K), smB_raw(K*N); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), d_clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaMemcpy(smA_raw.data(), d_smA_raw, M*K*sizeof(half), cudaMemcpyDeviceToHost); + cudaMemcpy(smB_raw.data(), d_smB_raw, K*N*sizeof(half), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + cudaFree(d_clk); cudaFree(d_smA_raw); cudaFree(d_smB_raw); + + reassemble(flat.data(), D); + all_pass &= check_uniform("static_4tiles", D, (float)(4 * K), outfile); + + printf(" clock: start=%u stop=%u cycles=%u (%.2f cyc/mma)\n", + clk[0], clk[1], clk[1]-clk[0], (float)(clk[1]-clk[0])/4.f); + + dump_results("wgmma_multik_static4.txt", "static_4tiles", + smA_raw.data(), smB_raw.data(), + flat.data(), D, + nullptr, 0, + clk[0], clk[1], 4); + } + + // --- Test 2: varying_4tiles --- + printf("\n--- varying_4tiles (A[tile]=tile+1, B=1, expected D=160) ---\n"); + { + std::vector h_A(4 * M * K); + for (int t = 0; t < 4; t++) + for (int i = 0; i < M*K; i++) + h_A[t*M*K + i] = __float2half((float)(t + 1)); + std::vector h_B(K*N, __float2half(1.f)); + half *d_A, *d_B; float *d_D; uint32_t *d_clk; + half *d_smA_raw, *d_smB_raw; + float *d_D_tiles; + cudaMalloc(&d_A, h_A.size()*sizeof(half)); + cudaMalloc(&d_B, h_B.size()*sizeof(half)); + cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + cudaMalloc(&d_clk, 2*sizeof(uint32_t)); + cudaMalloc(&d_smA_raw, M*K*sizeof(half)); + cudaMalloc(&d_smB_raw, K*N*sizeof(half)); + cudaMalloc(&d_D_tiles, 4*WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + + kernel_varying4<<<1, WGSIZE>>>(d_A, d_B, d_D, d_clk, d_smA_raw, d_smB_raw, d_D_tiles); + cudaDeviceSynchronize(); + + std::vector flat(WGSIZE*D_ELEMS); + std::vector clk(2); + std::vector smA_raw(M*K), smB_raw(K*N); + std::vector D_tiles(4*WGSIZE*D_ELEMS); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), d_clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaMemcpy(smA_raw.data(), d_smA_raw, M*K*sizeof(half), cudaMemcpyDeviceToHost); + cudaMemcpy(smB_raw.data(), d_smB_raw, K*N*sizeof(half), cudaMemcpyDeviceToHost); + cudaMemcpy(D_tiles.data(), d_D_tiles, D_tiles.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + cudaFree(d_clk); cudaFree(d_smA_raw); cudaFree(d_smB_raw); cudaFree(d_D_tiles); + + reassemble(flat.data(), D); + all_pass &= check_uniform("varying_4tiles", D, (float)(K * 10), outfile); + + printf(" clock: start=%u stop=%u cycles=%u (%.2f cyc/mma)\n", + clk[0], clk[1], clk[1]-clk[0], (float)(clk[1]-clk[0])/4.f); + + dump_results("wgmma_multik_varying4.txt", "varying_4tiles", + smA_raw.data(), smB_raw.data(), + flat.data(), D, + D_tiles.data(), 4, + clk[0], clk[1], 4); + } + + // --- Timing breakdown: fence overhead --- + printf("\n--- fence_overhead (N=%d WGMMA_FENCEs, no MMA) ---\n", N_FENCE); + { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2*sizeof(uint32_t)); + kernel_fence_overhead<<<1, WGSIZE>>>(d_clk); + cudaDeviceSynchronize(); + std::vector clk(2); + cudaMemcpy(clk.data(), d_clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + float cyc_per_fence = (float)(clk[1]-clk[0]) / N_FENCE; + printf(" clock: start=%u stop=%u total=%u cyc/fence=%.2f\n", + clk[0], clk[1], clk[1]-clk[0], cyc_per_fence); + if (outfile) + fprintf(outfile, "fence_overhead,TIMING,N/A,cyc_per_fence=%.2f\n", cyc_per_fence); + } + + // --- Timing breakdown: commit+wait barrier overhead (no MMA ops) --- + printf("\n--- commit_wait_overhead (N=%d empty COMMIT+WAIT, no MMA) ---\n", N_BARRIER); + { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2*sizeof(uint32_t)); + kernel_commit_wait_overhead<<<1, WGSIZE>>>(d_clk); + cudaDeviceSynchronize(); + std::vector clk(2); + cudaMemcpy(clk.data(), d_clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + float cyc_per_barrier = (float)(clk[1]-clk[0]) / N_BARRIER; + printf(" clock: start=%u stop=%u total=%u cyc/barrier=%.2f\n", + clk[0], clk[1], clk[1]-clk[0], cyc_per_barrier); + if (outfile) + fprintf(outfile, "commit_wait_overhead,TIMING,N/A,cyc_per_barrier=%.2f\n", cyc_per_barrier); + } + + // --- Timing: MMA throughput (m64n16k16 f32<-f16xf16, N=%d ops) --- + printf("\n--- mma_throughput_m64n16k16 (N=%d MMAs, A=B=1) ---\n", N_MMA); + { + std::vector h_A(M*K, __float2half(1.f)); + std::vector h_B(K*N, __float2half(1.f)); + half *d_A, *d_B; float *d_D; uint32_t *d_clk; + cudaMalloc(&d_A, h_A.size()*sizeof(half)); + cudaMalloc(&d_B, h_B.size()*sizeof(half)); + cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + cudaMalloc(&d_clk, 2*sizeof(uint32_t)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + kernel_mma_throughput<<<1, WGSIZE>>>(d_A, d_B, d_D, d_clk); + cudaDeviceSynchronize(); + std::vector clk(2); + cudaMemcpy(clk.data(), d_clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); cudaFree(d_clk); + float cyc_per_mma = (float)(clk[1]-clk[0]) / N_MMA; + printf(" clock: start=%u stop=%u total=%u cyc/mma=%.2f\n", + clk[0], clk[1], clk[1]-clk[0], cyc_per_mma); + if (outfile) + fprintf(outfile, "mma_throughput_m64n16k16,TIMING,N/A,cyc_per_mma=%.2f\n", cyc_per_mma); + } + + if (outfile) + fclose(outfile); + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/.gitignore b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/.gitignore new file mode 100644 index 000000000..08f514ebc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/.gitignore @@ -0,0 +1 @@ +include/ diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile new file mode 100644 index 000000000..a17977b56 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile @@ -0,0 +1,21 @@ +SRC = wgmma_rs_verify.cu +EXE = wgmma_rs_verify + +INCLUDE += -I../lat_gmma/include -I$(GPUAPPS_ROOT)/src/cuda/cutlass-bench/include -I./ + +ARCH ?= sm_90a +CUDA_CPPFLAGS= +HOPPER_CUDA_CPPFLAGS=$(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) + +NVCC_FLAGS := $(HOPPER_CUDA_CPPFLAGS) -std=c++17 + +# Needs CuTe/CUTLASS headers (bundled lat_gmma/include or the cutlass-bench +# submodule). Skip gracefully when absent (e.g. CI containers). +CUTE_HDR := $(or $(wildcard ../lat_gmma/include/cute/tensor.hpp), \ + $(wildcard $(GPUAPPS_ROOT)/src/cuda/cutlass-bench/include/cute/tensor.hpp)) +ifeq ($(CUTE_HDR),) +release clean: + @echo "$(EXE): SKIPPED (CuTe/CUTLASS headers not found)" +else +include ../../../common/common.mk +endif diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu new file mode 100644 index 000000000..331d65004 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu @@ -0,0 +1,481 @@ +// ============================================================================= +// wgmma_rs_verify.cu — RS WGMMA correctness + A-register layout dump +// +// RS WGMMA = "Register-Source" WGMMA: A operand loaded into registers via +// ldmatrix before the wgmma.mma_async instruction. B still uses smem descriptor. +// This is distinct from SS WGMMA (both A and B from smem descriptors). +// +// Purpose: everything the simulator needs to implement RS WGMMA. +// +// PHASE 1 — RS correctness, column test +// A=1, B[k][n]=n => D[m][n] = K*n +// Verifies that RS WGMMA produces correct D and that the D-register +// position formula (d_frag_pos) is the same as for SS WGMMA. +// Covers: F32F16, F16F16, F32TF32, F32BF16 × N=8,16,32,64,96,128,192,256 +// Output: wgmma_rs_col_N.txt +// +// PHASE 2 — RS correctness, row test +// A[m][k]=m, B=1 => D[m][n] = K*m +// Jointly verifies: (a) ldmatrix correctly loaded A from smem, and +// (b) RS WGMMA computed the right product. +// If any ldmatrix address or swizzle is wrong, A values are misread +// and D deviates from K*m. +// Same shapes as Phase 1. +// Output: wgmma_rs_row_N.txt +// +// PHASE 3 — A register layout dump (NO WGMMA) +// A[m][k] = m*K + k (unique per matrix element) +// Loads smem_A into rA via ldmatrix, then dumps rA directly to global +// memory WITHOUT issuing WGMMA. For each (thread T, register element e): +// decoded_m = round(val) / K +// decoded_k = round(val) % K +// This gives the full (T, e) -> (m, k) mapping for the A register fragment — +// exactly what the simulator needs to implement RS WGMMA functional mode. +// Covers: F32F16 N=16 (K=16), F32TF32 N=16 (K=8) +// BF16 uses same element size / ldmatrix as F16; its layout is identical. +// Output: wgmma_rs_areg_N.txt +// +// Build: make (see Makefile) +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cute/arch/util.hpp" +#include +#include "cutlass/numeric_types.h" +#include +#include +#include +#include +#include +#include +#include // SM75_U32x4_LDSM_N (ldmatrix) + +using namespace cute; + +static constexpr int WGSIZE = 128; + +// --------------------------------------------------------------------------- +// Kernel 1 & 2: RS correctness (col test and row test) +// COL_MODE=true : A=1, B[k][n]=n => D[m][n] = K*n +// COL_MODE=false : A[m][k]=m, B=1 => D[m][n] = K*m +// --------------------------------------------------------------------------- +template +__global__ void kernel_rs_verify(EC* D_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + static constexpr int M_val = (int)get<0>(TileShape{}); + static constexpr int N_val = (int)get<1>(TileShape{}); + static constexpr int K_val = (int)get<2>(TileShape{}); + + auto rs_op = GMMA::rs_op_selector(); + using TiledMma = decltype(make_tiled_mma(rs_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + + if constexpr (COL_MODE) { + // A=1 everywhere + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_A[i] = EA(1.0f); + // B[k][n] = n (sB logical is (N, K, PIPE)) + for (int i = tid; i < N_val * K_val; i += WGSIZE) { + int n = i % N_val, k = i / N_val; + sB(n, k, 0) = EB(float(n)); + } + } else { + // A[m][k] = m (row index) + for (int i = tid; i < M_val * K_val; i += WGSIZE) { + int m = i / K_val, k = i % K_val; + sA(m, k, 0) = EA(float(m)); + } + // B=1 everywhere + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(1.0f); + } + __syncthreads(); + + // --- Load A from smem into registers via ldmatrix --- + // make_tiled_copy_A derives the right thread-to-smem mapping from the + // RS TiledMma's A tile layout (get_layoutA_TV). + // SM75_U32x4_LDSM_N = ldmatrix.sync.aligned.m8n8.x4.shared.b16 + auto tiled_copy_A = make_tiled_copy_A( + Copy_Atom{}, tiled_mma); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + + // For RS: make_fragment_A returns a REGISTER tensor (not a smem view). + // Use single-stage partition so tCrA is rank-3 (MMA, MMA_M, MMA_K), + // matching rank-3 output of partition_S(sA(_,_,0)) after retile_D. + auto tCsA_mma = thread_mma.partition_A(sA); + auto tCrA = thread_mma.make_fragment_A(tCsA_mma(_,_,_,0)); // rank-3, stage 0 + + auto smem_thr_copy_A = tiled_copy_A.get_thread_slice(tid); + auto tCsA_copy = smem_thr_copy_A.partition_S(sA(_,_,0)); // smem source (stage 0) + auto tCrA_view = smem_thr_copy_A.retile_D(tCrA); // register dest view + copy(tiled_copy_A, tCsA_copy, tCrA_view); // execute ldmatrix + + // --- Issue RS WGMMA --- + auto tCsB = thread_mma.partition_B(sB); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + warpgroup_fence_operand(accum); + warpgroup_arrive(); + cute::gemm(tiled_mma, tCrA, tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + const int D_ELEMS = (int)size(accum); + EC* base = D_out + tid * D_ELEMS; + for (int e = 0; e < D_ELEMS; e++) + base[e] = accum(e); +} + +// --------------------------------------------------------------------------- +// Kernel 3: A register layout dump — no WGMMA issued +// Fill A[m][k] = m*K+k (unique per element), load via ldmatrix, +// dump raw rA register values to global memory. +// Host decodes: m = round(val)/K, k = round(val)%K. +// --------------------------------------------------------------------------- +template +__global__ void kernel_rs_areg_dump(float* A_out, int* A_reg_size_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + static constexpr int M_val = (int)get<0>(TileShape{}); + static constexpr int K_val = (int)get<2>(TileShape{}); + + auto rs_op = GMMA::rs_op_selector(); + using TiledMma = decltype(make_tiled_mma(rs_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + + // A[m][k] = m*K + k (unique per matrix element) + for (int i = tid; i < M_val * K_val; i += WGSIZE) { + int m = i / K_val, k = i % K_val; + sA(m, k, 0) = EA(float(m * K_val + k)); + } + // B unused but smem needs to be valid to avoid issues + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(0.0f); + __syncthreads(); + + auto tiled_copy_A = make_tiled_copy_A( + Copy_Atom{}, tiled_mma); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + + auto tCsA_mma = thread_mma.partition_A(sA); + auto tCrA = thread_mma.make_fragment_A(tCsA_mma(_,_,_,0)); // rank-3, stage 0 + + auto smem_thr_copy_A = tiled_copy_A.get_thread_slice(tid); + auto tCsA_copy = smem_thr_copy_A.partition_S(sA(_,_,0)); + auto tCrA_view = smem_thr_copy_A.retile_D(tCrA); + copy(tiled_copy_A, tCsA_copy, tCrA_view); // ldmatrix → registers + + // Dump: no WGMMA, just write register values to global memory + const int A_regs = (int)size(tCrA); + if (tid == 0) *A_reg_size_out = A_regs; + for (int e = 0; e < A_regs; e++) + A_out[tid * A_regs + e] = (float)tCrA(e); +} + +// --------------------------------------------------------------------------- +// Host helpers +// --------------------------------------------------------------------------- + +// D-register position: (thread T, accumulator element e) -> (row, col) +// Valid for F32/S32 accumulator at all N. F16 accumulator uses same formula. +static void d_frag_pos(int T, int e, int /*N*/, int* row, int* col) { + int warp = T / 32, lane = T % 32; + int g = e / 4, k = e % 4; + *row = (lane / 4) + (k / 2) * 8 + warp * 16; + *col = (lane % 4) * 2 + (k % 2) + g * 8; +} + +template +static void run_verify(const char* dtype_tag, const char* name, int N, int K) { + const int D_ELEMS = N / 2; + const int total = WGSIZE * D_ELEMS; + std::vector h_D(total, EC(0)); + EC* d_D; + cudaMalloc(&d_D, total * sizeof(EC)); + cudaMemset(d_D, 0, total * sizeof(EC)); + + kernel_rs_verify<<<1, WGSIZE>>>(d_D); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" RS-%-3s %-50s KERNEL ERROR: %s\n", + COL_MODE ? "Col" : "Row", name, cudaGetErrorString(err)); + cudaFree(d_D); return; + } + cudaDeviceSynchronize(); + cudaMemcpy(h_D.data(), d_D, total * sizeof(EC), cudaMemcpyDeviceToHost); + cudaFree(d_D); + + char fname[128]; + snprintf(fname, sizeof(fname), "wgmma_rs_%s_%sN%d.txt", + COL_MODE ? "col" : "row", dtype_tag, N); + FILE* f = fopen(fname, "w"); + if (f) { + if (COL_MODE) + fprintf(f, "# %s RS: A=1 B[k][n]=n expected D[m][n]=K*n (K=%d)\n" + "# thread elem row col expected actual match\n", name, K); + else + fprintf(f, "# %s RS: A[m][k]=m B=1 expected D[m][n]=K*m (K=%d)\n" + "# thread elem row col expected actual match\n", name, K); + } + + int errors = 0, checked = 0; + for (int T = 0; T < WGSIZE; T++) { + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, N, &row, &col); + if (row >= 64 || col >= N) continue; + float expected = COL_MODE ? (float)(K * col) : (float)(K * row); + float actual = (float)h_D[T * D_ELEMS + e]; + bool ok = (fabsf(actual - expected) < 0.5f); + if (!ok) errors++; + checked++; + if (f) + fprintf(f, "%d %d %d %d %.1f %.1f %s\n", + T, e, row, col, expected, actual, ok ? "OK" : "FAIL"); + } + } + if (f) { fprintf(f, "# errors=%d / %d checked\n", errors, checked); fclose(f); } + + printf(" RS-%-3s %-50s errors=%d/%d -> %s\n", + COL_MODE ? "Col" : "Row", name, errors, checked, + errors == 0 ? "PASS" : "FAIL"); +} + +// A register layout dump: loads smem_A via ldmatrix, writes raw values. +// Produces (T, e) -> (m, k) mapping for the simulator to reference. +template +static void run_areg_dump(const char* dtype_tag, const char* name, int K) { + // A_reg_size per thread is K*M/(128) / (sizeof_EA_in_regs) — query from kernel + // Allocate conservatively: max is K*64/128 = K/2 per thread (for 2-byte types) + // For F16 K=16: 8 elements per thread. For TF32 K=8: 4 per thread. + const int MAX_REGS = 16; + const int total = WGSIZE * MAX_REGS; + std::vector h_A(total, 0.0f); + float* d_A; + int* d_cnt; + cudaMalloc(&d_A, total * sizeof(float)); + cudaMalloc(&d_cnt, sizeof(int)); + cudaMemset(d_A, 0, total * sizeof(float)); + cudaMemset(d_cnt, 0, sizeof(int)); + + kernel_rs_areg_dump<<<1, WGSIZE>>>(d_A, d_cnt); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" RS-AregDump %-46s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_A); cudaFree(d_cnt); return; + } + cudaDeviceSynchronize(); + cudaMemcpy(h_A.data(), d_A, total * sizeof(float), cudaMemcpyDeviceToHost); + int h_cnt = 0; + cudaMemcpy(&h_cnt, d_cnt, sizeof(int), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_cnt); + + if (h_cnt == 0 || h_cnt > MAX_REGS) { + printf(" RS-AregDump %-46s unexpected A_reg_size=%d\n", name, h_cnt); + return; + } + + char fname[128]; + snprintf(fname, sizeof(fname), "wgmma_rs_areg_%sN%d.txt", dtype_tag, + (int)get<1>(TileShape{})); + FILE* f = fopen(fname, "w"); + if (f) { + fprintf(f, "# %s A[m][k]=m*K+k loaded via ldmatrix -> rA registers\n", name); + fprintf(f, "# A_reg_size per thread = %d\n", h_cnt); + fprintf(f, "# K=%d (so decoded_m = round(val)/%d, decoded_k = round(val)%%%d)\n", + K, K, K); + fprintf(f, "# thread elem raw_value decoded_m decoded_k\n"); + } + + int decode_errors = 0; + for (int T = 0; T < WGSIZE; T++) { + for (int e = 0; e < h_cnt; e++) { + float val = h_A[T * MAX_REGS + e]; + int iv = (int)roundf(val); + int m = iv / K; + int k = iv % K; + bool ok = (m >= 0 && m < 64 && k >= 0 && k < K); + if (!ok) decode_errors++; + if (f) + fprintf(f, "%3d %2d %8.3f %2d %2d%s\n", + T, e, val, ok ? m : -1, ok ? k : -1, + ok ? "" : " [DECODE_ERROR]"); + } + } + if (f) { + fprintf(f, "# decode_errors=%d / %d entries\n", + decode_errors, WGSIZE * h_cnt); + fclose(f); + } + + printf(" RS-AregDump %-46s A_reg_size=%d decode_errors=%d/%d -> %s\n", + name, h_cnt, decode_errors, WGSIZE * h_cnt, + decode_errors == 0 ? "OK" : "DECODE_ERROR"); +} + +// --------------------------------------------------------------------------- +// Convenience macros +// --------------------------------------------------------------------------- +#define SHAPE(M,N,K) decltype(make_shape(Int{}, Int{}, Int{})) + +#define RUN_COL(EA, EB, EC, M, N, K, TAG, LABEL) \ + run_verify(TAG, LABEL, N, K) + +#define RUN_ROW(EA, EB, EC, M, N, K, TAG, LABEL) \ + run_verify(TAG, LABEL, N, K) + +#define RUN_DUMP(EA, EB, EC, M, N, K, TAG, LABEL) \ + run_areg_dump(TAG, LABEL, K) + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_rs_verify - %s (SM %d.%d)\n\n", prop.name, prop.major, prop.minor); + + // ----------------------------------------------------------------------- + // Phase 1: RS correctness — column test (A=1, B[k][n]=n => D=K*n) + // Verifies RS WGMMA computes correct D and d_frag_pos formula matches SS. + // ----------------------------------------------------------------------- + printf("=== Phase 1: RS column test (A=1, B[k][n]=n => D[m][n]=K*n) ===\n"); + printf(" %-55s errors/checked\n", "shape"); + + printf("\n -- F32 accumulator, F16 inputs RS (K=16) --\n"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32F16", "F32F16_m64n8k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16", "F32F16_m64n16k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32F16", "F32F16_m64n32k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16", "F32F16_m64n64k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32F16", "F32F16_m64n96k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32F16", "F32F16_m64n128k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32F16", "F32F16_m64n192k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32F16", "F32F16_m64n256k16"); + + printf("\n -- F16 accumulator, F16 inputs RS (K=16) --\n"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 8, 16, "F16F16", "F16F16_m64n8k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 16, 16, "F16F16", "F16F16_m64n16k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 32, 16, "F16F16", "F16F16_m64n32k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 64, 16, "F16F16", "F16F16_m64n64k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 96, 16, "F16F16", "F16F16_m64n96k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 128, 16, "F16F16", "F16F16_m64n128k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 192, 16, "F16F16", "F16F16_m64n192k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 256, 16, "F16F16", "F16F16_m64n256k16"); + + printf("\n -- F32 accumulator, TF32 inputs RS (K=8) --\n"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 8, 8, "F32TF32", "F32TF32_m64n8k8"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "F32TF32", "F32TF32_m64n16k8"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 32, 8, "F32TF32", "F32TF32_m64n32k8"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 64, 8, "F32TF32", "F32TF32_m64n64k8"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 128, 8, "F32TF32", "F32TF32_m64n128k8"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 256, 8, "F32TF32", "F32TF32_m64n256k8"); + + printf("\n -- F32 accumulator, BF16 inputs RS (K=16) --\n"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 8, 16, "F32BF16", "F32BF16_m64n8k16"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 16, 16, "F32BF16", "F32BF16_m64n16k16"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 32, 16, "F32BF16", "F32BF16_m64n32k16"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 64, 16, "F32BF16", "F32BF16_m64n64k16"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 128, 16, "F32BF16", "F32BF16_m64n128k16"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 256, 16, "F32BF16", "F32BF16_m64n256k16"); + + // ----------------------------------------------------------------------- + // Phase 2: RS correctness — row test (A[m][k]=m, B=1 => D=K*m) + // Jointly verifies ldmatrix loaded A correctly AND RS WGMMA computed D. + // ----------------------------------------------------------------------- + printf("\n=== Phase 2: RS row test (A[m][k]=m, B=1 => D[m][n]=K*m) ===\n"); + printf(" %-55s errors/checked\n", "shape"); + + printf("\n -- F32 accumulator, F16 inputs RS (K=16) --\n"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32F16", "F32F16_m64n8k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16", "F32F16_m64n16k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32F16", "F32F16_m64n32k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16", "F32F16_m64n64k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32F16", "F32F16_m64n96k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32F16", "F32F16_m64n128k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32F16", "F32F16_m64n192k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32F16", "F32F16_m64n256k16"); + + printf("\n -- F16 accumulator, F16 inputs RS (K=16) --\n"); + RUN_ROW(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 16, 16, "F16F16", "F16F16_m64n16k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 64, 16, "F16F16", "F16F16_m64n64k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 128, 16, "F16F16", "F16F16_m64n128k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 256, 16, "F16F16", "F16F16_m64n256k16"); + + printf("\n -- F32 accumulator, TF32 inputs RS (K=8) --\n"); + RUN_ROW(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "F32TF32", "F32TF32_m64n16k8"); + RUN_ROW(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 64, 8, "F32TF32", "F32TF32_m64n64k8"); + RUN_ROW(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 128, 8, "F32TF32", "F32TF32_m64n128k8"); + + printf("\n -- F32 accumulator, BF16 inputs RS (K=16) --\n"); + RUN_ROW(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 16, 16, "F32BF16", "F32BF16_m64n16k16"); + RUN_ROW(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 64, 16, "F32BF16", "F32BF16_m64n64k16"); + RUN_ROW(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 256, 16, "F32BF16", "F32BF16_m64n256k16"); + + // ----------------------------------------------------------------------- + // Phase 3: A register layout dump — the key data for simulator implementation + // Shows which A[m][k] each (thread, register_element) holds after ldmatrix. + // BF16 uses same element size and ldmatrix layout as F16 (K=16, 2-byte); + // its mapping is implied by the F32F16 dump. + // ----------------------------------------------------------------------- + printf("\n=== Phase 3: A register layout dump (A[m][k]=m*K+k, no WGMMA) ===\n"); + printf(" Decodes (thread T, register element e) -> (row m, col k) in A tile.\n"); + printf(" This is the reference for simulator RS WGMMA functional implementation.\n\n"); + + // F16 K=16: primary case, covers F16 and BF16 (same 2-byte ldmatrix) + RUN_DUMP(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16", "F32F16_m64n16k16"); + RUN_DUMP(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16", "F32F16_m64n64k16"); + + // TF32 K=8: 4-byte elements, different ldmatrix pattern + RUN_DUMP(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "F32TF32", "F32TF32_m64n16k8"); + + printf("\n A register layout dumps -> wgmma_rs_areg_N.txt\n"); + printf(" Format: thread elem raw_value decoded_m decoded_k\n"); + + printf("\nDone.\n"); + return 0; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/Makefile new file mode 100644 index 000000000..244de21bb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/Makefile @@ -0,0 +1,29 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := wgmma_scale + +# WGMMA requires Hopper (SM90a). +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu new file mode 100644 index 000000000..12a9be3ae --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu @@ -0,0 +1,300 @@ +// ============================================================================= +// wgmma_scale.cu +// +// Tests WGMMA scale / accumulator-init parameters on SM90a. +// +// Two parameter axes: +// +// scaleD (predicate) — controls whether D is used as accumulator: +// scaleD=1 (true): D_out = D_in + A*B +// scaleD=0 (false): D_out = A*B (D_in is ignored) +// Applies to all WGMMA types. Tested here with f16. +// +// scaleA / scaleB — negate A or B operand (fp8 types only): +// scaleA=-1: D_out = (-A)*B = -(A*B) +// scaleB=-1: D_out = A*(-B) = -(A*B) +// both=-1: D_out = (-A)*(-B) = +(A*B) +// Tested with e4m3 m64n16k32. +// +// Tests: +// scaleD=0 (f16) : D pre-init to 50, expect D = K = 16 +// scaleD=1 (f16) : D pre-init to K, expect D = 2K = 32 +// e4m3 scaleA=-1 : all-ones A,B, expect D = -K32 = -32 +// e4m3 scaleB=-1 : all-ones A,B, expect D = -K32 = -32 +// e4m3 scaleA=-1 scaleB=-1: all-ones, expect D = +K32 = +32 +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; // 8 +static constexpr int SBO = 128; +static constexpr int E_F16 = 2; +static constexpr int E_FP8 = 1; +static constexpr int K_F16 = 16; +static constexpr int K_FP8 = 32; + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }\n" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +// =========================================================================== +// scaleD tests (f16) +// =========================================================================== + +// D pre-initialized to init_val; scaleD selects whether D_in is used. +// Template parameter SCALE_D: 1 = accumulate, 0 = ignore D_in. +template +__global__ void kernel_f16_scaled(const half* A_g, const half* B_g, + float init_val, float* D_g) { + constexpr int LBO_A = M * E_F16 * 8; + constexpr int LBO_B = N * E_F16 * 8; + __shared__ __align__(128) char smA[M*K_F16*E_F16], smB[K_F16*N*E_F16]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K_F16; i += WGSIZE) { int m=i/K_F16,k=i%K_F16; *(half*)(smA+smem_off(k,m,E_F16,LBO_A))=A_g[i]; } + for (int i = tid; i < K_F16*N; i += WGSIZE) { int k=i/N,n=i%N; *(half*)(smB+smem_off(k,n,E_F16,LBO_B))=B_g[i]; } + __syncthreads(); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + // Pre-initialize D to init_val to test whether scaleD=0 ignores it. + float d0=init_val,d1=init_val,d2=init_val,d3=init_val; + float d4=init_val,d5=init_val,d6=init_val,d7=init_val; + WGMMA_FENCE; + if constexpr (SCALE_D == 1) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" // p = true → accumulate + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } else { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,0,0;\n" // p = false → D_in ignored + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b = tid * D_ELEMS; + D_g[b]=d0;D_g[b+1]=d1;D_g[b+2]=d2;D_g[b+3]=d3; + D_g[b+4]=d4;D_g[b+5]=d5;D_g[b+6]=d6;D_g[b+7]=d7; +} + +// =========================================================================== +// scaleA / scaleB tests (fp8 e4m3) +// SA, SB: compile-time literal, must be 1 or -1 +// =========================================================================== +template +__global__ void kernel_e4m3_scale(const uint8_t* A_g, const uint8_t* B_g, + float* D_g) { + constexpr int LBO_A = M * E_FP8 * 8; + constexpr int LBO_B = N * E_FP8 * 8; + __shared__ __align__(128) char smA[M*K_FP8*E_FP8], smB[K_FP8*N*E_FP8]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K_FP8; i += WGSIZE) { int m=i/K_FP8,k=i%K_FP8; *(uint8_t*)(smA+smem_off(k,m,E_FP8,LBO_A))=A_g[i]; } + for (int i = tid; i < K_FP8*N; i += WGSIZE) { int k=i/N,n=i%N; *(uint8_t*)(smB+smem_off(k,n,E_FP8,LBO_B))=B_g[i]; } + __syncthreads(); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + // SA and SB are template parameters, so the compiler emits a constant + // literal in the PTX instruction (required by the ISA). + if constexpr (SA == 1 && SB == 1) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e4m3.e4m3 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } else if constexpr (SA == -1 && SB == 1) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e4m3.e4m3 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,-1,1; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } else if constexpr (SA == 1 && SB == -1) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e4m3.e4m3 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,-1; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } else { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e4m3.e4m3 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,-1,-1; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b = tid * D_ELEMS; + D_g[b]=d0;D_g[b+1]=d1;D_g[b+2]=d2;D_g[b+3]=d3; + D_g[b+4]=d4;D_g[b+5]=d5;D_g[b+6]=d6;D_g[b+7]=d7; +} + +// =========================================================================== +// Host helpers +// =========================================================================== +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static void reassemble(const float* flat, float D[M][N]) { + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) D[m][n] = 0.f; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; d_frag_pos(T, e, &row, &col); + if (row < M && col < N) D[row][col] = flat[T*D_ELEMS + e]; + } +} + +static bool check_uniform(const char* name, float D[M][N], float expected) { + float max_err = 0.f; int max_m = 0, max_n = 0; + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) { + float e = fabsf(D[m][n] - expected); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + float tol = (fabsf(expected) < 1.f) ? 1e-3f : fabsf(expected) * 1e-2f; + bool pass = (max_err <= tol); + printf("[%-42s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + max_m, max_n, D[max_m][max_n], expected); + return pass; +} + +// Convert float to raw fp8 e4m3 byte (same as wgmma_verify) +static uint8_t float_to_e4m3(float f) { + __nv_fp8_e4m3 v(f); uint8_t b; memcpy(&b, &v, 1); return b; +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_scale — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + static float D[M][N]; + + // ------------------------------------------------------------------------- + // scaleD tests (f16) + // ------------------------------------------------------------------------- + printf("\n--- scaleD (f16 m64n16k16) ---\n"); + { + std::vector hA(M*K_F16, __float2half(1.f)); + std::vector hB(K_F16*N, __float2half(1.f)); + half *dA, *dB; float *dD; + cudaMalloc(&dA, hA.size()*sizeof(half)); + cudaMalloc(&dB, hB.size()*sizeof(half)); + cudaMalloc(&dD, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(dA, hA.data(), hA.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(dB, hB.data(), hB.size()*sizeof(half), cudaMemcpyHostToDevice); + + // scaleD=0: D_in=50 is ignored → result = A*B = K_F16 = 16 + kernel_f16_scaled<0><<<1, WGSIZE>>>(dA, dB, 50.f, dD); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE*D_ELEMS); + cudaMemcpy(flat.data(), dD, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + reassemble(flat.data(), D); + all_pass &= check_uniform("f16 scaleD=0 (D_in=50 ignored)", D, (float)K_F16); + + // scaleD=1: D_in=K_F16=16, accumulates → result = 16+16 = 32 + kernel_f16_scaled<1><<<1, WGSIZE>>>(dA, dB, (float)K_F16, dD); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), dD, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + reassemble(flat.data(), D); + all_pass &= check_uniform("f16 scaleD=1 (D_in=K accumulated)", D, (float)(2*K_F16)); + + cudaFree(dA); cudaFree(dB); cudaFree(dD); + } + + // ------------------------------------------------------------------------- + // scaleA / scaleB tests (fp8 e4m3, all-ones A and B) + // K_FP8=32, all-ones: baseline D[m][n] = 32, negated = -32, double-neg = +32 + // ------------------------------------------------------------------------- + printf("\n--- scaleA/scaleB (e4m3 m64n16k32, A=B=1.0) ---\n"); + { + uint8_t raw = float_to_e4m3(1.f); + std::vector hA(M*K_FP8, raw), hB(K_FP8*N, raw); + uint8_t *dA, *dB; float *dD; + cudaMalloc(&dA, hA.size()); + cudaMalloc(&dB, hB.size()); + cudaMalloc(&dD, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(dA, hA.data(), hA.size(), cudaMemcpyHostToDevice); + cudaMemcpy(dB, hB.data(), hB.size(), cudaMemcpyHostToDevice); + std::vector flat(WGSIZE*D_ELEMS); + + auto run_fp8 = [&](auto kern, float expected, const char* name) { + kern<<<1, WGSIZE>>>(dA, dB, dD); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), dD, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + reassemble(flat.data(), D); + return check_uniform(name, D, expected); + }; + + all_pass &= run_fp8(kernel_e4m3_scale<1,1>, (float) K_FP8, "e4m3 scaleA=+1 scaleB=+1 (baseline)"); + all_pass &= run_fp8(kernel_e4m3_scale<-1,1>, (float)-K_FP8, "e4m3 scaleA=-1 scaleB=+1"); + all_pass &= run_fp8(kernel_e4m3_scale<1,-1>, (float)-K_FP8, "e4m3 scaleA=+1 scaleB=-1"); + all_pass &= run_fp8(kernel_e4m3_scale<-1,-1>, (float) K_FP8, "e4m3 scaleA=-1 scaleB=-1 (double-neg)"); + + cudaFree(dA); cudaFree(dB); cudaFree(dD); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/Makefile new file mode 100644 index 000000000..70c78db5d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/Makefile @@ -0,0 +1,29 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := wgmma_shapes + +# WGMMA requires Hopper (SM90a). +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu new file mode 100644 index 000000000..3915b1190 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu @@ -0,0 +1,248 @@ +// ============================================================================= +// wgmma_shapes.cu +// +// Tests f16 m64nNk16 → f32 for N = 16, 32, 64. +// +// Motivation: different N values produce different D-register counts per thread +// and different fragment layouts. wgmma_verify only covers N=16; this file +// stresses the larger shapes that exercise different simulator code paths. +// +// N=16: 8 D-regs/thread (baseline — same as wgmma_verify) +// N=32: 16 D-regs/thread +// N=64: 32 D-regs/thread +// +// N=128/256 follow the same fragment formula as N=64 (more regs, same layout +// logic); they are covered implicitly by wgmma_no_tma via CUTLASS. +// +// All tests use all-ones A and B. +// Expected D[m][n] = K = 16 for every (m, n) regardless of N. +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include + +static constexpr int M = 64; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int SBO = 128; +static constexpr int E = 2; // bytes per f16 + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }\n" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +// --------------------------------------------------------------------------- +// Kernel: f16 m64n16k16 (8 D-regs — baseline) +// --------------------------------------------------------------------------- +__global__ void kernel_n16(const half* A_g, const half* B_g, float* D_g) { + constexpr int N = 16, D_ELEMS = N/2; + constexpr int LBO_A = M*E*8, LBO_B = N*E*8; + __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } + for (int i = tid; i < K*N; i += WGSIZE) { int k=i/N,n=i%N; *(half*)(smB+smem_off(k,n,E,LBO_B))=B_g[i]; } + __syncthreads(); + uint64_t da=make_gmma_desc(smem_addr(smA),LBO_A,SBO); + uint64_t db=make_gmma_desc(smem_addr(smB),LBO_B,SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b=tid*D_ELEMS; + D_g[b]=d0;D_g[b+1]=d1;D_g[b+2]=d2;D_g[b+3]=d3; + D_g[b+4]=d4;D_g[b+5]=d5;D_g[b+6]=d6;D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// Kernel: f16 m64n32k16 (16 D-regs) +// --------------------------------------------------------------------------- +__global__ void kernel_n32(const half* A_g, const half* B_g, float* D_g) { + constexpr int N = 32, D_ELEMS = N/2; + constexpr int LBO_A = M*E*8, LBO_B = N*E*8; + __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } + for (int i = tid; i < K*N; i += WGSIZE) { int k=i/N,n=i%N; *(half*)(smB+smem_off(k,n,E,LBO_B))=B_g[i]; } + __syncthreads(); + uint64_t da=make_gmma_desc(smem_addr(smA),LBO_A,SBO); + uint64_t db=make_gmma_desc(smem_addr(smB),LBO_B,SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + float d8=0,d9=0,d10=0,d11=0,d12=0,d13=0,d14=0,d15=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n32k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15},%16,%17,p,1,1,0,0; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7), + "+f"(d8),"+f"(d9),"+f"(d10),"+f"(d11),"+f"(d12),"+f"(d13),"+f"(d14),"+f"(d15) + :"l"(da),"l"(db):"memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b=tid*D_ELEMS; + D_g[b]=d0;D_g[b+1]=d1;D_g[b+2]=d2;D_g[b+3]=d3; + D_g[b+4]=d4;D_g[b+5]=d5;D_g[b+6]=d6;D_g[b+7]=d7; + D_g[b+8]=d8;D_g[b+9]=d9;D_g[b+10]=d10;D_g[b+11]=d11; + D_g[b+12]=d12;D_g[b+13]=d13;D_g[b+14]=d14;D_g[b+15]=d15; +} + +// --------------------------------------------------------------------------- +// Kernel: f16 m64n64k16 (32 D-regs) +// --------------------------------------------------------------------------- +__global__ void kernel_n64(const half* A_g, const half* B_g, float* D_g) { + constexpr int N = 64, D_ELEMS = N/2; + constexpr int LBO_A = M*E*8, LBO_B = N*E*8; + __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } + for (int i = tid; i < K*N; i += WGSIZE) { int k=i/N,n=i%N; *(half*)(smB+smem_off(k,n,E,LBO_B))=B_g[i]; } + __syncthreads(); + uint64_t da=make_gmma_desc(smem_addr(smA),LBO_A,SBO); + uint64_t db=make_gmma_desc(smem_addr(smB),LBO_B,SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + float d8=0,d9=0,d10=0,d11=0,d12=0,d13=0,d14=0,d15=0; + float d16=0,d17=0,d18=0,d19=0,d20=0,d21=0,d22=0,d23=0; + float d24=0,d25=0,d26=0,d27=0,d28=0,d29=0,d30=0,d31=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n64k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15," + "%16,%17,%18,%19,%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31},%32,%33,p,1,1,0,0; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7), + "+f"(d8),"+f"(d9),"+f"(d10),"+f"(d11),"+f"(d12),"+f"(d13),"+f"(d14),"+f"(d15), + "+f"(d16),"+f"(d17),"+f"(d18),"+f"(d19),"+f"(d20),"+f"(d21),"+f"(d22),"+f"(d23), + "+f"(d24),"+f"(d25),"+f"(d26),"+f"(d27),"+f"(d28),"+f"(d29),"+f"(d30),"+f"(d31) + :"l"(da),"l"(db):"memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b=tid*D_ELEMS; + D_g[b]=d0;D_g[b+1]=d1;D_g[b+2]=d2;D_g[b+3]=d3; + D_g[b+4]=d4;D_g[b+5]=d5;D_g[b+6]=d6;D_g[b+7]=d7; + D_g[b+8]=d8;D_g[b+9]=d9;D_g[b+10]=d10;D_g[b+11]=d11; + D_g[b+12]=d12;D_g[b+13]=d13;D_g[b+14]=d14;D_g[b+15]=d15; + D_g[b+16]=d16;D_g[b+17]=d17;D_g[b+18]=d18;D_g[b+19]=d19; + D_g[b+20]=d20;D_g[b+21]=d21;D_g[b+22]=d22;D_g[b+23]=d23; + D_g[b+24]=d24;D_g[b+25]=d25;D_g[b+26]=d26;D_g[b+27]=d27; + D_g[b+28]=d28;D_g[b+29]=d29;D_g[b+30]=d30;D_g[b+31]=d31; +} + +// =========================================================================== +// Host helpers +// =========================================================================== + +// Fragment layout formula is the same for all N. +// e ranges 0..(N/2-1); s=e/4 indexes groups of 8 columns. +static void reassemble(const float* flat, float* D, int N) { + const int D_ELEMS = N / 2; + for (int i = 0; i < M*N; i++) D[i] = 0.f; + for (int T = 0; T < WGSIZE; T++) { + int warp = T/32, lane = T%32; + for (int e = 0; e < D_ELEMS; e++) { + int s = e/4, k = e%4; + int row = (lane/4)*2 + k/2 + warp*16; + int col = (lane%4)*2 + k%2 + s*8; + if (row < M && col < N) + D[row*N + col] = flat[T*D_ELEMS + e]; + } + } +} + +static bool check_uniform(const char* name, const float* D, int N, float expected) { + float max_err = 0.f; + int worst = 0; + for (int i = 0; i < M*N; i++) { + float e = fabsf(D[i] - expected); + if (e > max_err) { max_err = e; worst = i; } + } + float tol = (fabsf(expected) < 1.f) ? 1e-3f : fabsf(expected) * 1e-2f; + bool pass = (max_err <= tol); + printf("[%-38s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + worst/N, worst%N, D[worst], expected); + return pass; +} + +struct KernFn { void (*fn)(const half*, const half*, float*); int N; const char* name; }; + +static bool run(KernFn kf) { + const int D_ELEMS = kf.N / 2; + std::vector hA(M*K, __float2half(1.f)); + std::vector hB(K*kf.N, __float2half(1.f)); + half *dA, *dB; float *dD; + cudaMalloc(&dA, M*K*sizeof(half)); + cudaMalloc(&dB, K*kf.N*sizeof(half)); + cudaMalloc(&dD, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(dA, hA.data(), hA.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(dB, hB.data(), hB.size()*sizeof(half), cudaMemcpyHostToDevice); + kf.fn<<<1, WGSIZE>>>(dA, dB, dD); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE*D_ELEMS); + cudaMemcpy(flat.data(), dD, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(dA); cudaFree(dB); cudaFree(dD); + std::vector D(M*kf.N); + reassemble(flat.data(), D.data(), kf.N); + return check_uniform(kf.name, D.data(), kf.N, (float)K); +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_shapes — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + KernFn tests[] = { + { kernel_n16, 16, "f16 m64n16k16 (8 D-regs/thread)" }, + { kernel_n32, 32, "f16 m64n32k16 (16 D-regs/thread)" }, + { kernel_n64, 64, "f16 m64n64k16 (32 D-regs/thread)" }, + }; + for (auto& t : tests) all_pass &= run(t); + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/.gitignore b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/.gitignore new file mode 100644 index 000000000..08f514ebc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/.gitignore @@ -0,0 +1 @@ +include/ diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile new file mode 100644 index 000000000..5e54f2c2b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile @@ -0,0 +1,21 @@ +SRC = wgmma_sweep.cu +EXE = wgmma_sweep + +INCLUDE += -I../lat_gmma/include -I$(GPUAPPS_ROOT)/src/cuda/cutlass-bench/include -I./ + +ARCH ?= sm_90a +CUDA_CPPFLAGS= +HOPPER_CUDA_CPPFLAGS=$(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) + +NVCC_FLAGS := $(HOPPER_CUDA_CPPFLAGS) -std=c++17 + +# Needs CuTe/CUTLASS headers (bundled lat_gmma/include or the cutlass-bench +# submodule). Skip gracefully when absent (e.g. CI containers). +CUTE_HDR := $(or $(wildcard ../lat_gmma/include/cute/tensor.hpp), \ + $(wildcard $(GPUAPPS_ROOT)/src/cuda/cutlass-bench/include/cute/tensor.hpp)) +ifeq ($(CUTE_HDR),) +release clean: + @echo "$(EXE): SKIPPED (CuTe/CUTLASS headers not found)" +else +include ../../../common/common.mk +endif diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/wgmma_sweep.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/wgmma_sweep.cu new file mode 100644 index 000000000..c2f656eca --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/wgmma_sweep.cu @@ -0,0 +1,951 @@ +// ============================================================================= +// wgmma_sweep.cu — Combined WGMMA timing + D-fragment / smem layout sweep +// +// Fills ALL measurement gaps left by lat_gmma and wgmma_multik benchmarks. +// +// PHASE 0 — Shape-independent overhead (run once) +// kernel_fence_overhead: 256 FENCEs → cyc_per_fence +// kernel_commit_wait_overhead: 256 empty COMMIT+WAITs → cyc_per_barrier +// +// PHASE 1 — SS timing sweep (smem-A, smem-B) +// kernel_mma_throughput: 1024 back-to-back SS MMAs → cyc_per_mma +// Derived true latency: +// lat = cyc_per_mma*N_MMA - 2*cyc_fence - (N_MMA-1)*II - cyc_barrier +// Covers all 6 dtypes × 9 N values (N=8,16,32,64,96,128,192,224,256): +// F32F16, F32TF32, F32BF16, F32E4M3, F16F16, S32S8 +// Output: wgmma_sweep_timing.csv +// +// PHASE 1b — Pipelined 2-group overlap (SS, F32F16, all N) +// Issues N_MMA/2 MMAs + COMMIT + N_MMA/2 MMAs + COMMIT + WAIT<1> + WAIT<0>. +// gap_after_wait1 = clock(after WAIT<0>) - clock(after WAIT<1>). +// If gap ≈ 0: group 2 completed during group 1's latency tail → overlap confirmed. +// Output: wgmma_sweep_pipeline.csv +// +// PHASE 2 — D-fragment layout verification (all N) +// A=1, B[k][n]=n → D[m][n]=K*n. Checks d_frag_pos column formula. +// F32F16 (F32 accum) : N=8..256 — primary +// F16F16 (F16 accum) : N=8..256 — verifies F16 accumulator column mapping +// S32S8 (S32 accum) : N=8..256 — verifies int accumulator column mapping +// B[k][n] = n%64 (int8 overflow guard), expected K*(col%64) +// Output: wgmma_sweep_layout_colN.txt per shape +// +// PHASE 3 — smem_A layout verification (selected N and K values) +// A[m][k]=m, B[k][n]=1 → D[m][n]=K*m. Verifies smem_A row layout +// AND d_frag_pos row formula jointly. +// F16 K=16: N=16,32,64,128,256 (K-interleaved for 2-byte elements) +// TF32 K=8 : N=16,32,64,128 (K-interleaved for 4-byte elements) +// S8 K=32: N=16,32,64 (K-interleaved for 1-byte elements) +// Output: wgmma_sweep_layout_rowN.txt per shape +// +// PHASE 4 — RS timing sweep (register-A, smem-B) +// kernel_mma_throughput_rs: 1024 back-to-back RS MMAs → cyc_per_mma_rs +// A register fragment filled with 1s (values immaterial for timing). +// RS is supported for F32{F16,BF16,TF32} and F16F16 dtypes. +// Covers all 4 RS dtypes × 9 N values. +// Output: appended to wgmma_sweep_timing.csv (section marked RS) +// +// Build: make (see Makefile) +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cute/arch/util.hpp" +#include +#include "cutlass/numeric_types.h" +#include +#include +#include +#include +#include + +using namespace cute; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- +static constexpr int WGSIZE = 128; // one warpgroup +static constexpr int N_FENCE = 256; +static constexpr int N_BARRIER = 256; +static constexpr int N_MMA = 1024; +static constexpr int N_HALF = N_MMA / 2; + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT0 asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") +#define WGMMA_WAIT1 asm volatile("wgmma.wait_group.sync.aligned 1;\n" ::: "memory") + +// --------------------------------------------------------------------------- +// Kernel 1: Fence overhead (shape-independent) +// --------------------------------------------------------------------------- +__global__ void kernel_fence_overhead(uint32_t* clk_out) { + const int tid = threadIdx.x; + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t0) :: "memory"); + __syncthreads(); + #pragma unroll + for (int i = 0; i < N_FENCE; i++) WGMMA_FENCE; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t1) :: "memory"); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } +} + +// --------------------------------------------------------------------------- +// Kernel 2: Commit+wait overhead with EMPTY group (shape-independent) +// --------------------------------------------------------------------------- +__global__ void kernel_commit_wait_overhead(uint32_t* clk_out) { + const int tid = threadIdx.x; + uint32_t t0 = 0, t1 = 0; + WGMMA_FENCE; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t0) :: "memory"); + __syncthreads(); + #pragma unroll + for (int i = 0; i < N_BARRIER; i++) { WGMMA_COMMIT; WGMMA_WAIT0; } + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t1) :: "memory"); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } +} + +// --------------------------------------------------------------------------- +// Kernel 3: SS MMA throughput timing +// FENCE + N_MMA*MMA + COMMIT + WAIT + FENCE +// cyc_per_mma = total / N_MMA +// --------------------------------------------------------------------------- +template +__global__ void kernel_mma_throughput(uint32_t* clk_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + auto gmma_op = GMMA::ss_op_selector(); + using TiledMma = decltype(make_tiled_mma(gmma_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_A[i] = EA(1.0f); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(1.0f); + __syncthreads(); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + auto tCsA = thread_mma.partition_A(sA); + auto tCsB = thread_mma.partition_B(sB); + auto tCrA = thread_mma.make_fragment_A(tCsA); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t0) :: "memory"); + __syncthreads(); + + warpgroup_fence_operand(accum); + warpgroup_arrive(); + #pragma unroll 8 + for (int j = 0; j < N_MMA; j++) + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t1) :: "memory"); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + if (tid == 0 && accum(0) == -9999.f) printf("x"); // prevent DCE +} + +// --------------------------------------------------------------------------- +// Kernel 4: RS MMA throughput timing (register-A, smem-B) +// Same structure as kernel_mma_throughput but uses rs_op_selector. +// A register fragment is pre-filled with zeros (values immaterial for timing). +// --------------------------------------------------------------------------- +template +__global__ void kernel_mma_throughput_rs(uint32_t* clk_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + auto gmma_op = GMMA::rs_op_selector(); + using TiledMma = decltype(make_tiled_mma(gmma_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_A[i] = EA(1.0f); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(1.0f); + __syncthreads(); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + + // For RS: partition_A gives smem shape; make_fragment_A returns a REGISTER + // tensor (FrgTypeA is a value type for RS atoms, not a smem-pointer wrapper). + // clear() zeros the registers — values don't affect timing. + auto tCsA = thread_mma.partition_A(sA); + auto tCrA = thread_mma.make_fragment_A(tCsA); + clear(tCrA); + + auto tCsB = thread_mma.partition_B(sB); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t0) :: "memory"); + __syncthreads(); + + warpgroup_fence_operand(accum); + warpgroup_arrive(); + #pragma unroll 8 + for (int j = 0; j < N_MMA; j++) + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t1) :: "memory"); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + if (tid == 0 && accum(0) == -9999.f) printf("x"); +} + +// --------------------------------------------------------------------------- +// Kernel 5: Pipelined 2-group SS timing +// Issues two committed groups of N_HALF MMAs each: +// FENCE + N_HALF*MMA + COMMIT + FENCE + N_HALF*MMA + COMMIT +// + WAIT<1> [wait for group 1, group 2 still pending] +// + WAIT<0> [wait for group 2] +// Outputs 3 clock values: [t_before, t_after_wait1, t_after_wait0]. +// gap_after_wait1 = clk[2]-clk[1]: if small → group 2 completed during +// group 1's latency tail (true overlap). +// --------------------------------------------------------------------------- +template +__global__ void kernel_mma_pipelined(uint32_t* clk_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + auto gmma_op = GMMA::ss_op_selector(); + using TiledMma = decltype(make_tiled_mma(gmma_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_A[i] = EA(1.0f); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(1.0f); + __syncthreads(); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + auto tCsA = thread_mma.partition_A(sA); + auto tCsB = thread_mma.partition_B(sB); + auto tCrA = thread_mma.make_fragment_A(tCsA); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + uint32_t t0 = 0, t1 = 0, t2 = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t0) :: "memory"); + __syncthreads(); + + warpgroup_fence_operand(accum); + + // Group 1: N_HALF MMAs + warpgroup_arrive(); + #pragma unroll 8 + for (int j = 0; j < N_HALF; j++) + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + + // Group 2: N_HALF more MMAs (overlaps with group 1's latency tail) + warpgroup_arrive(); + #pragma unroll 8 + for (int j = 0; j < N_HALF; j++) + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + + // Wait for group 1 only (1 group still pending); group 2 may be done already + warpgroup_wait<1>(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t1) :: "memory"); + + // Wait for group 2 + warpgroup_wait<0>(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t2) :: "memory"); + + warpgroup_fence_operand(accum); + + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; clk_out[2] = t2; } + if (tid == 0 && accum(0) == -9999.f) printf("x"); +} + +// --------------------------------------------------------------------------- +// Kernel 6: D-fragment column layout verification +// A[m][k]=1, B[k][n]=(n%FILL_MOD if FILL_MOD>0 else n) => D[m][n]=K*(n or n%FILL_MOD) +// Verifies d_frag_pos column formula. FILL_MOD=64 for int8 B to avoid overflow. +// --------------------------------------------------------------------------- +template +__global__ void kernel_mma_layout_col(EC* D_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + static constexpr int N_val = (int)get<1>(TileShape{}); + static constexpr int K_val = (int)get<2>(TileShape{}); + + auto gmma_op = GMMA::ss_op_selector(); + using TiledMma = decltype(make_tiled_mma(gmma_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_A[i] = EA(1.0f); + for (int i = tid; i < N_val * K_val; i += WGSIZE) { + int n = i % N_val, k = i / N_val; + int n_fill = (FILL_MOD > 0) ? (n % FILL_MOD) : n; + sB(n, k, 0) = EB(float(n_fill)); + } + __syncthreads(); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + auto tCsA = thread_mma.partition_A(sA); + auto tCsB = thread_mma.partition_B(sB); + auto tCrA = thread_mma.make_fragment_A(tCsA); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + warpgroup_fence_operand(accum); + warpgroup_arrive(); + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + const int D_ELEMS = (int)size(accum); + EC* base = D_out + tid * D_ELEMS; + for (int e = 0; e < D_ELEMS; e++) + base[e] = accum(e); +} + +// --------------------------------------------------------------------------- +// Kernel 7: smem_A row layout verification +// A[m][k]=m, B[k][n]=1 => D[m][n]=K*m +// Verifies d_frag_pos row formula AND smem_A K-interleaved swizzled layout. +// --------------------------------------------------------------------------- +template +__global__ void kernel_mma_layout_row(EC* D_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + static constexpr int M_val = (int)get<0>(TileShape{}); + static constexpr int N_val = (int)get<1>(TileShape{}); + static constexpr int K_val = (int)get<2>(TileShape{}); + + auto gmma_op = GMMA::ss_op_selector(); + using TiledMma = decltype(make_tiled_mma(gmma_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + + for (int i = tid; i < M_val * K_val; i += WGSIZE) { + int m = i / K_val, k = i % K_val; + sA(m, k, 0) = EA(float(m)); + } + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(1.0f); + __syncthreads(); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + auto tCsA = thread_mma.partition_A(sA); + auto tCsB = thread_mma.partition_B(sB); + auto tCrA = thread_mma.make_fragment_A(tCsA); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + warpgroup_fence_operand(accum); + warpgroup_arrive(); + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + const int D_ELEMS = (int)size(accum); + EC* base = D_out + tid * D_ELEMS; + for (int e = 0; e < D_ELEMS; e++) + base[e] = accum(e); +} + +// --------------------------------------------------------------------------- +// Host helpers +// --------------------------------------------------------------------------- + +// d_frag_pos: maps (thread T, accumulator element e, tile width N) → (row, col) +// Valid for f32/s32 accumulator at all N. For f16 accumulator, the spatial +// mapping is identical (same warpgroup thread layout), just different precision. +// elems_per_group=4 for f32/s32 and for f16 (each f16 element is 1 per slot). +static void d_frag_pos(int T, int e, int /*N*/, int* row, int* col) { + int warp = T / 32, lane = T % 32; + int g = e / 4, k = e % 4; + *row = (lane / 4) * 2 + (k / 2) + warp * 16; + *col = (lane % 4) * 2 + (k % 2) + g * 8; +} + +static float measure_fence_overhead() { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2 * sizeof(uint32_t)); + kernel_fence_overhead<<<1, WGSIZE>>>(d_clk); + cudaDeviceSynchronize(); + uint32_t clk[2]; + cudaMemcpy(clk, d_clk, 2 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + return (float)(clk[1] - clk[0]) / N_FENCE; +} + +static float measure_barrier_overhead() { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2 * sizeof(uint32_t)); + kernel_commit_wait_overhead<<<1, WGSIZE>>>(d_clk); + cudaDeviceSynchronize(); + uint32_t clk[2]; + cudaMemcpy(clk, d_clk, 2 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + return (float)(clk[1] - clk[0]) / N_BARRIER; +} + +// Known II values from lat_gmma silicon run (H100, 2026-05-08). +// N=224 is interpolated (II=112); verify with this benchmark. +static int table_II(int N) { + switch (N) { + case 8: return 18; case 16: return 20; case 32: return 24; + case 64: return 32; case 96: return 48; case 128: return 64; + case 192: return 96; case 224: return 112; case 256: return 128; + default: return N / 2; + } +} + +// SS timing: calls kernel_mma_throughput, derives latency +template +static void run_timing(const char* name, int N, float cyc_fence, + float cyc_barrier, FILE* csv) { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2 * sizeof(uint32_t)); + kernel_mma_throughput<<<1, WGSIZE>>>(d_clk); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" %-60s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_clk); return; + } + cudaDeviceSynchronize(); + uint32_t clk[2]; + cudaMemcpy(clk, d_clk, 2 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + + float cyc_mma = (float)(clk[1] - clk[0]) / N_MMA; + int II = table_II(N); + float lat = cyc_mma * N_MMA - 2.0f * cyc_fence + - (float)(N_MMA - 1) * II - cyc_barrier; + printf(" %-60s cyc/mma=%7.3f II=%3d lat=%7.3f\n", name, cyc_mma, II, lat); + if (csv) + fprintf(csv, "%s,SS,%.3f,%.3f,%.3f,%d,%.3f\n", + name, cyc_fence, cyc_barrier, cyc_mma, II, lat); +} + +// RS timing: calls kernel_mma_throughput_rs, derives latency +template +static void run_timing_rs(const char* name, int N, float cyc_fence, + float cyc_barrier, FILE* csv) { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2 * sizeof(uint32_t)); + kernel_mma_throughput_rs<<<1, WGSIZE>>>(d_clk); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" RS %-57s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_clk); return; + } + cudaDeviceSynchronize(); + uint32_t clk[2]; + cudaMemcpy(clk, d_clk, 2 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + + float cyc_mma = (float)(clk[1] - clk[0]) / N_MMA; + int II = table_II(N); + float lat = cyc_mma * N_MMA - 2.0f * cyc_fence + - (float)(N_MMA - 1) * II - cyc_barrier; + printf(" RS %-57s cyc/mma=%7.3f II=%3d lat=%7.3f\n", name, cyc_mma, II, lat); + if (csv) + fprintf(csv, "%s,RS,%.3f,%.3f,%.3f,%d,%.3f\n", + name, cyc_fence, cyc_barrier, cyc_mma, II, lat); +} + +// Pipelined 2-group timing: reports total cyc/mma and gap between WAIT<1> and WAIT<0> +template +static void run_timing_pipelined(const char* name, int N, FILE* csv) { + uint32_t *d_clk; + cudaMalloc(&d_clk, 3 * sizeof(uint32_t)); + kernel_mma_pipelined<<<1, WGSIZE>>>(d_clk); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" Pipe %-55s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_clk); return; + } + cudaDeviceSynchronize(); + uint32_t clk[3]; + cudaMemcpy(clk, d_clk, 3 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + + float cyc_total = (float)(clk[2] - clk[0]) / N_MMA; + float gap_wait1 = (float)(clk[2] - clk[1]); // cycles from WAIT<1> to WAIT<0> + printf(" Pipe %-55s cyc/mma=%7.3f gap_after_wait1=%7.1f%s\n", + name, cyc_total, gap_wait1, gap_wait1 < 5.0f ? " [OVERLAP]" : ""); + if (csv) + fprintf(csv, "%s,%.3f,%.1f\n", name, cyc_total, gap_wait1); +} + +// Layout column verification: A=1, B[k][n]=n (or n%FILL_MOD) => D=K*n (or K*(n%FILL_MOD)) +template +static void run_layout_col(const char* tag, const char* name, int N, int K) { + const int D_ELEMS = N / 2; // universal: f32, f16, s32 accumulators all = N/2 + const int total = WGSIZE * D_ELEMS; + std::vector h_D(total, EC(0)); + EC* d_D; + cudaMalloc(&d_D, total * sizeof(EC)); + cudaMemset(d_D, 0, total * sizeof(EC)); + + kernel_mma_layout_col<<<1, WGSIZE>>>(d_D); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" ColLayout %-50s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_D); return; + } + cudaDeviceSynchronize(); + cudaMemcpy(h_D.data(), d_D, total * sizeof(EC), cudaMemcpyDeviceToHost); + cudaFree(d_D); + + char fname[128]; + snprintf(fname, sizeof(fname), "wgmma_sweep_layout_col%sN%d.txt", tag, N); + FILE* f = fopen(fname, "w"); + if (f) { + if (FILL_MOD > 0) + fprintf(f, "# %s A=1 B[k][n]=n%%64 expected D[m][n]=K*(n%%64) (K=%d)\n" + "# thread elem row col expected actual match\n", name, K); + else + fprintf(f, "# %s A=1 B[k][n]=n expected D[m][n]=K*n (K=%d)\n" + "# thread elem row col expected actual match\n", name, K); + } + + int errors = 0, checked = 0; + for (int T = 0; T < WGSIZE; T++) { + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, N, &row, &col); + if (row >= 64 || col >= N) continue; + int col_fill = (FILL_MOD > 0) ? (col % FILL_MOD) : col; + float expected = (float)(K * col_fill); + float actual = (float)h_D[T * D_ELEMS + e]; + bool ok = (fabsf(actual - expected) < 0.5f); + if (!ok) errors++; + checked++; + if (f) + fprintf(f, "%d %d %d %d %.1f %.1f %s\n", + T, e, row, col, expected, actual, ok ? "OK" : "FAIL"); + } + } + if (f) { fprintf(f, "# errors=%d / %d checked\n", errors, checked); fclose(f); } + printf(" ColLayout%-6s %-48s errors=%d/%d -> %s\n", + tag, name, errors, checked, errors == 0 ? "PASS" : "FAIL"); +} + +// Layout row verification: A[m][k]=m, B=1 => D=K*m +template +static void run_layout_row(const char* tag, const char* name, int N, int K) { + const int D_ELEMS = N / 2; + const int total = WGSIZE * D_ELEMS; + std::vector h_D(total, EC(0)); + EC* d_D; + cudaMalloc(&d_D, total * sizeof(EC)); + cudaMemset(d_D, 0, total * sizeof(EC)); + + kernel_mma_layout_row<<<1, WGSIZE>>>(d_D); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" RowLayout %-50s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_D); return; + } + cudaDeviceSynchronize(); + cudaMemcpy(h_D.data(), d_D, total * sizeof(EC), cudaMemcpyDeviceToHost); + cudaFree(d_D); + + char fname[128]; + snprintf(fname, sizeof(fname), "wgmma_sweep_layout_row%sN%d.txt", tag, N); + FILE* f = fopen(fname, "w"); + if (f) + fprintf(f, "# %s A[m][k]=m B=1 expected D[m][n]=K*m (K=%d)\n" + "# thread elem row col expected actual match\n", name, K); + + int errors = 0, checked = 0; + for (int T = 0; T < WGSIZE; T++) { + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, N, &row, &col); + if (row >= 64 || col >= N) continue; + float expected = (float)(K * row); + float actual = (float)h_D[T * D_ELEMS + e]; + bool ok = (fabsf(actual - expected) < 0.5f); + if (!ok) errors++; + checked++; + if (f) + fprintf(f, "%d %d %d %d %.1f %.1f %s\n", + T, e, row, col, expected, actual, ok ? "OK" : "FAIL"); + } + } + if (f) { fprintf(f, "# errors=%d / %d checked\n", errors, checked); fclose(f); } + printf(" RowLayout%-6s %-48s errors=%d/%d -> %s\n", + tag, name, errors, checked, errors == 0 ? "PASS" : "FAIL"); +} + +// --------------------------------------------------------------------------- +// Convenience macros +// --------------------------------------------------------------------------- +#define SHAPE(M,N,K) decltype(make_shape(Int{}, Int{}, Int{})) + +#define RUN_TIMING(EA, EB, EC, M, N, K, LABEL) \ + run_timing(LABEL, N, cyc_fence, cyc_barrier, timing_csv) + +#define RUN_TIMING_RS(EA, EB, EC, M, N, K, LABEL) \ + run_timing_rs(LABEL, N, cyc_fence, cyc_barrier, timing_csv) + +#define RUN_PIPELINED(EA, EB, EC, M, N, K, LABEL) \ + run_timing_pipelined(LABEL, N, pipeline_csv) + +#define RUN_COL_LAYOUT(EA, EB, EC, M, N, K, TAG, LABEL) \ + run_layout_col(TAG, LABEL, N, K) + +#define RUN_COL_LAYOUT_MOD(EA, EB, EC, M, N, K, MOD, TAG, LABEL) \ + run_layout_col(TAG, LABEL, N, K) + +#define RUN_ROW_LAYOUT(EA, EB, EC, M, N, K, TAG, LABEL) \ + run_layout_row(TAG, LABEL, N, K) + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_sweep - %s (SM %d.%d)\n\n", prop.name, prop.major, prop.minor); + + // ----------------------------------------------------------------------- + // Phase 0: Shape-independent overhead + // ----------------------------------------------------------------------- + printf("=== Phase 0: Overhead kernels (shape-independent) ===\n"); + float cyc_fence = measure_fence_overhead(); + float cyc_barrier = measure_barrier_overhead(); + printf(" cyc_per_fence = %.3f (256 x wgmma.fence.sync.aligned)\n", cyc_fence); + printf(" cyc_per_barrier = %.3f (256 x COMMIT+WAIT empty group)\n\n", cyc_barrier); + + FILE* timing_csv = fopen("wgmma_sweep_timing.csv", "w"); + if (timing_csv) + fprintf(timing_csv, "shape,mode,cyc_fence,cyc_barrier,cyc_mma,II_table,derived_lat\n"); + + // ----------------------------------------------------------------------- + // Phase 1: SS timing sweep — all dtypes × all N + // ----------------------------------------------------------------------- + printf("=== Phase 1: SS timing sweep ===\n"); + printf(" %-60s cyc/mma II derived_lat\n", "shape"); + + printf("\n -- F32 accumulator, F16 inputs (K=16) --\n"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32F16_m64n8k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16_m64n16k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32F16_m64n32k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16_m64n64k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32F16_m64n96k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32F16_m64n128k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32F16_m64n192k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 224, 16, "F32F16_m64n224k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32F16_m64n256k16"); + + printf("\n -- F32 accumulator, TF32 inputs (K=8) --\n"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 8, 8, "F32TF32_m64n8k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "F32TF32_m64n16k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 32, 8, "F32TF32_m64n32k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 64, 8, "F32TF32_m64n64k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 96, 8, "F32TF32_m64n96k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 128, 8, "F32TF32_m64n128k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 192, 8, "F32TF32_m64n192k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 224, 8, "F32TF32_m64n224k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 256, 8, "F32TF32_m64n256k8"); + + printf("\n -- F32 accumulator, BF16 inputs (K=16) --\n"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 8, 16, "F32BF16_m64n8k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 16, 16, "F32BF16_m64n16k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 32, 16, "F32BF16_m64n32k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 64, 16, "F32BF16_m64n64k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 96, 16, "F32BF16_m64n96k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 128, 16, "F32BF16_m64n128k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 192, 16, "F32BF16_m64n192k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 224, 16, "F32BF16_m64n224k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 256, 16, "F32BF16_m64n256k16"); + + printf("\n -- F32 accumulator, FP8-E4M3 inputs (K=32) --\n"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 8, 32, "F32E4M3_m64n8k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 16, 32, "F32E4M3_m64n16k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 32, 32, "F32E4M3_m64n32k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 64, 32, "F32E4M3_m64n64k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 96, 32, "F32E4M3_m64n96k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 128, 32, "F32E4M3_m64n128k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 192, 32, "F32E4M3_m64n192k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 224, 32, "F32E4M3_m64n224k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 256, 32, "F32E4M3_m64n256k32"); + + printf("\n -- F16 accumulator, F16 inputs (K=16) --\n"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 8, 16, "F16F16_m64n8k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 16, 16, "F16F16_m64n16k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 32, 16, "F16F16_m64n32k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 64, 16, "F16F16_m64n64k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 96, 16, "F16F16_m64n96k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 128, 16, "F16F16_m64n128k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 192, 16, "F16F16_m64n192k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 224, 16, "F16F16_m64n224k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 256, 16, "F16F16_m64n256k16"); + + printf("\n -- S32 accumulator, S8 inputs (K=32) --\n"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 8, 32, "S32S8_m64n8k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 16, 32, "S32S8_m64n16k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 32, 32, "S32S8_m64n32k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 64, 32, "S32S8_m64n64k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 96, 32, "S32S8_m64n96k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 128, 32, "S32S8_m64n128k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 192, 32, "S32S8_m64n192k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 224, 32, "S32S8_m64n224k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 256, 32, "S32S8_m64n256k32"); + + // ----------------------------------------------------------------------- + // Phase 1b: Pipelined 2-group overlap (F32F16, all N) + // ----------------------------------------------------------------------- + printf("\n=== Phase 1b: Pipelined 2-group overlap (F32F16) ===\n"); + printf(" gap_after_wait1 ≈ 0 means group 2 completed during group 1's latency tail.\n"); + printf(" %-60s cyc/mma gap_after_wait1\n", "shape"); + + FILE* pipeline_csv = fopen("wgmma_sweep_pipeline.csv", "w"); + if (pipeline_csv) + fprintf(pipeline_csv, "shape,cyc_per_mma,gap_after_wait1_cycles\n"); + + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32F16_m64n8k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16_m64n16k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32F16_m64n32k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16_m64n64k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32F16_m64n96k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32F16_m64n128k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32F16_m64n192k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 224, 16, "F32F16_m64n224k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32F16_m64n256k16"); + + if (pipeline_csv) fclose(pipeline_csv); + printf(" Pipelined results -> wgmma_sweep_pipeline.csv\n"); + + // ----------------------------------------------------------------------- + // Phase 4: RS timing sweep (before layout phases to group timing output) + // ----------------------------------------------------------------------- + printf("\n=== Phase 4: RS timing sweep (register-A, smem-B) ===\n"); + printf(" %-60s cyc/mma II derived_lat\n", "shape"); + + printf("\n -- F32 accumulator, F16 inputs RS (K=16) --\n"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32F16_m64n8k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16_m64n16k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32F16_m64n32k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16_m64n64k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32F16_m64n96k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32F16_m64n128k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32F16_m64n192k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 224, 16, "F32F16_m64n224k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32F16_m64n256k16"); + + printf("\n -- F32 accumulator, TF32 inputs RS (K=8) --\n"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 8, 8, "F32TF32_m64n8k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "F32TF32_m64n16k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 32, 8, "F32TF32_m64n32k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 64, 8, "F32TF32_m64n64k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 96, 8, "F32TF32_m64n96k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 128, 8, "F32TF32_m64n128k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 192, 8, "F32TF32_m64n192k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 224, 8, "F32TF32_m64n224k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 256, 8, "F32TF32_m64n256k8"); + + printf("\n -- F32 accumulator, BF16 inputs RS (K=16) --\n"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 8, 16, "F32BF16_m64n8k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 16, 16, "F32BF16_m64n16k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 32, 16, "F32BF16_m64n32k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 64, 16, "F32BF16_m64n64k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 96, 16, "F32BF16_m64n96k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 128, 16, "F32BF16_m64n128k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 192, 16, "F32BF16_m64n192k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 224, 16, "F32BF16_m64n224k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 256, 16, "F32BF16_m64n256k16"); + + printf("\n -- F16 accumulator, F16 inputs RS (K=16) --\n"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 8, 16, "F16F16_m64n8k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 16, 16, "F16F16_m64n16k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 32, 16, "F16F16_m64n32k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 64, 16, "F16F16_m64n64k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 96, 16, "F16F16_m64n96k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 128, 16, "F16F16_m64n128k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 192, 16, "F16F16_m64n192k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 224, 16, "F16F16_m64n224k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 256, 16, "F16F16_m64n256k16"); + + if (timing_csv) fclose(timing_csv); + printf("\n Timing results -> wgmma_sweep_timing.csv\n"); + + // ----------------------------------------------------------------------- + // Phase 2: D-fragment column layout — all N, three accumulators + // ----------------------------------------------------------------------- + printf("\n=== Phase 2: D-fragment column layout (B[k][n]=n => D=K*n) ===\n"); + printf(" Verifies d_frag_pos column formula for all N and three accumulator types.\n"); + printf(" S32S8 uses B[k][n]=n%%64 (int8 overflow guard), expected K*(col%%64).\n"); + + printf("\n -- F32 accumulator (F16 inputs) --\n"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32", "F32F16_m64n8k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32", "F32F16_m64n16k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32", "F32F16_m64n32k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32", "F32F16_m64n64k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32", "F32F16_m64n96k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32", "F32F16_m64n128k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32", "F32F16_m64n192k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 224, 16, "F32", "F32F16_m64n224k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32", "F32F16_m64n256k16"); + + printf("\n -- F16 accumulator (F16 inputs) — D_ELEMS=N/2, same d_frag_pos --\n"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 8, 16, "F16", "F16F16_m64n8k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 16, 16, "F16", "F16F16_m64n16k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 32, 16, "F16", "F16F16_m64n32k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 64, 16, "F16", "F16F16_m64n64k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 96, 16, "F16", "F16F16_m64n96k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 128, 16, "F16", "F16F16_m64n128k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 192, 16, "F16", "F16F16_m64n192k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 224, 16, "F16", "F16F16_m64n224k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 256, 16, "F16", "F16F16_m64n256k16"); + + printf("\n -- S32 accumulator (S8 inputs) — B[k][n]=n%%64, expected K*(col%%64) --\n"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 8, 32, 64, "S32", "S32S8_m64n8k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 16, 32, 64, "S32", "S32S8_m64n16k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 32, 32, 64, "S32", "S32S8_m64n32k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 64, 32, 64, "S32", "S32S8_m64n64k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 96, 32, 64, "S32", "S32S8_m64n96k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 128, 32, 64, "S32", "S32S8_m64n128k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 192, 32, 64, "S32", "S32S8_m64n192k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 224, 32, 64, "S32", "S32S8_m64n224k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 256, 32, 64, "S32", "S32S8_m64n256k32"); + + printf("\n Column layout dumps -> wgmma_sweep_layout_colN.txt\n"); + + // ----------------------------------------------------------------------- + // Phase 3: smem_A row layout — three K values + // A[m][k]=m, B=1 => D=K*m. Independently verifies both d_frag_pos row + // formula AND smem_A K-interleaved physical layout. + // ----------------------------------------------------------------------- + printf("\n=== Phase 3: smem_A row layout (A[m][k]=m, B=1 => D=K*m) ===\n"); + printf(" Verifies smem_A layout for different K (element size) values.\n"); + + printf("\n -- F16 inputs K=16 (2-byte elements, Swizzle<0,4,3>) --\n"); + RUN_ROW_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F16", "F32F16_m64n16k16"); + RUN_ROW_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F16", "F32F16_m64n32k16"); + RUN_ROW_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F16", "F32F16_m64n64k16"); + RUN_ROW_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F16", "F32F16_m64n128k16"); + RUN_ROW_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F16", "F32F16_m64n256k16"); + + printf("\n -- TF32 inputs K=8 (4-byte elements) — different K interleaving --\n"); + RUN_ROW_LAYOUT(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "TF32", "F32TF32_m64n16k8"); + RUN_ROW_LAYOUT(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 32, 8, "TF32", "F32TF32_m64n32k8"); + RUN_ROW_LAYOUT(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 64, 8, "TF32", "F32TF32_m64n64k8"); + RUN_ROW_LAYOUT(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 128, 8, "TF32", "F32TF32_m64n128k8"); + + printf("\n -- S8 inputs K=32 (1-byte elements) — different K interleaving --\n"); + RUN_ROW_LAYOUT(int8_t, int8_t, int32_t, 64, 16, 32, "S8", "S32S8_m64n16k32"); + RUN_ROW_LAYOUT(int8_t, int8_t, int32_t, 64, 32, 32, "S8", "S32S8_m64n32k32"); + RUN_ROW_LAYOUT(int8_t, int8_t, int32_t, 64, 64, 32, "S8", "S32S8_m64n64k32"); + + printf("\n Row layout dumps -> wgmma_sweep_layout_rowN.txt\n"); + + printf("\nDone.\n"); + return 0; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/Makefile new file mode 100644 index 000000000..ec21ffdfd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/Makefile @@ -0,0 +1,30 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# WGMMA sync primitive microbenchmarks. +ALL_BENCHES := wgmma_fence wgmma_sync + +# WGMMA requires Hopper (SM90a). +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu new file mode 100644 index 000000000..fb9fdfcda --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu @@ -0,0 +1,331 @@ +// ============================================================================= +// wgmma_fence.cu +// +// Tests wgmma.fence.sync.aligned semantics on SM90a. +// +// Three subtests using f16 m64n16k16 → f32: +// +// fence_single – one fence→MMA→commit→wait_group_0→fence cycle. +// Baseline: verifies MMA sees smem data written before fence. +// Expected D[m][n] = 16.0 (K ones × 1×1). +// +// fence_sequence – two back-to-back cycles in one kernel with different smem +// data. The second fence gates a fresh smem write, so the +// second MMA must produce a different result from the first. +// Expected D1[m][n] = 16.0 (A=1, B=1) +// Expected D2[m][n] = 64.0 (A=2, B=2, K × 2×2) +// +// fence_accum – two cycles accumulating into the same D registers. +// Verifies that D is not accidentally reset between cycles +// and that the post-wait fence properly releases D. +// Expected D[m][n] = 32.0 (2 × 16.0) +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; // 8 D-regs per thread +static constexpr int E = 2; // bytes per f16 +static constexpr int SBO = 128; +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int SMEM_A = M * K * E; // 2048 +static constexpr int SMEM_B = K * N * E; // 512 + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }\n" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +// --------------------------------------------------------------------------- +// WGMMA protocol macros +// --------------------------------------------------------------------------- +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +__device__ __forceinline__ void mma_f16( + uint64_t da, uint64_t db, + float& d0, float& d1, float& d2, float& d3, + float& d4, float& d5, float& d6, float& d7) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3), + "+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); +} + +// --------------------------------------------------------------------------- +// Kernel helpers: load A/B globals into K-major smem (no swizzle) +// --------------------------------------------------------------------------- +__device__ void load_smem(char* smA, char* smB, + const half* A_g, const half* B_g) { + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); +} + +// =========================================================================== +// Kernel 1: single fence→MMA→commit→wait_group_0→fence +// =========================================================================== +__global__ void kernel_fence_single(const half* A_g, const half* B_g, float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + load_smem(smA, smB, A_g, B_g); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + + const int b = threadIdx.x * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Kernel 2: two back-to-back fence→MMA→commit→wait→fence sequences. +// Sequence 1 uses A_g1, B_g1 (all 1.0). Sequence 2 uses A_g2, B_g2 (all 2.0). +// Outputs D_g1 and D_g2 separately for independent checking. +// =========================================================================== +__global__ void kernel_fence_sequence(const half* A_g1, const half* B_g1, + const half* A_g2, const half* B_g2, + float* D_g1, float* D_g2) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + + // Sequence 1 + load_smem(smA, smB, A_g1, B_g1); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + const int b = tid * D_ELEMS; + D_g1[b]=d0; D_g1[b+1]=d1; D_g1[b+2]=d2; D_g1[b+3]=d3; + D_g1[b+4]=d4; D_g1[b+5]=d5; D_g1[b+6]=d6; D_g1[b+7]=d7; + + // Overwrite smem with sequence-2 data, then re-fence + __syncthreads(); + load_smem(smA, smB, A_g2, B_g2); + float e0=0,e1=0,e2=0,e3=0,e4=0,e5=0,e6=0,e7=0; + WGMMA_FENCE; + mma_f16(da, db, e0,e1,e2,e3,e4,e5,e6,e7); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + D_g2[b]=e0; D_g2[b+1]=e1; D_g2[b+2]=e2; D_g2[b+3]=e3; + D_g2[b+4]=e4; D_g2[b+5]=e5; D_g2[b+6]=e6; D_g2[b+7]=e7; +} + +// =========================================================================== +// Kernel 3: two accumulating cycles using the same smem. +// The post-wait fence releases D so the second cycle can start immediately. +// Expected: D[m][n] = 32.0 (2 × K × 1 × 1). +// =========================================================================== +__global__ void kernel_fence_accum(const half* A_g, const half* B_g, float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + load_smem(smA, smB, A_g, B_g); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + // Cycle 1 + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; WGMMA_WAIT; + // Single fence serves as end-of-cycle-1 and start-of-cycle-2 + WGMMA_FENCE; + // Cycle 2: accumulate into same D registers + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + + const int b = threadIdx.x * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Host helpers +// =========================================================================== + +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static void reassemble_D(const float* flat, float D[M][N]) { + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) D[m][n] = 0.f; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, &row, &col); + if (row < M && col < N) D[row][col] = flat[T*D_ELEMS + e]; + } +} + +static bool check_uniform(const char* name, float D[M][N], float expected) { + float max_err = 0.f; + int max_m = 0, max_n = 0; + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) { + float e = fabsf(D[m][n] - expected); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + float tol = (fabsf(expected) < 1.f) ? 1e-3f : fabsf(expected) * 1e-2f; + bool pass = (max_err <= tol); + printf("[%-40s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + max_m, max_n, D[max_m][max_n], expected); + return pass; +} + +// Allocate device buffers, copy h_A/h_B, run kernel, retrieve result. +// Returns a flat WGSIZE*D_ELEMS float vector. +static std::vector run1(const std::vector& h_A, + const std::vector& h_B, + void (*kern)(const half*, const half*, float*)) { + half *d_A, *d_B; + float *d_D; + cudaMalloc(&d_A, h_A.size() * sizeof(half)); + cudaMalloc(&d_B, h_B.size() * sizeof(half)); + cudaMalloc(&d_D, WGSIZE * D_ELEMS * sizeof(float)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + kern<<<1, WGSIZE>>>(d_A, d_B, d_D); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE * D_ELEMS); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + return flat; +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_fence — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + + // Host matrices: all-ones (A1=B1=1.0), all-twos (A2=B2=2.0) + std::vector A1(M*K), B1(K*N), A2(M*K), B2(K*N); + for (auto& v : A1) v = __float2half(1.f); + for (auto& v : B1) v = __float2half(1.f); + for (auto& v : A2) v = __float2half(2.f); + for (auto& v : B2) v = __float2half(2.f); + + static float D[M][N]; + + // ------------------------------------------------------------------------- + // Test 1: fence_single + // ------------------------------------------------------------------------- + printf("\n--- fence_single ---\n"); + { + auto flat = run1(A1, B1, kernel_fence_single); + reassemble_D(flat.data(), D); + all_pass &= check_uniform("fence_single", D, (float)K); + } + + // ------------------------------------------------------------------------- + // Test 2: fence_sequence (two separate cycles, independent D) + // ------------------------------------------------------------------------- + printf("\n--- fence_sequence ---\n"); + { + half *dA1, *dB1, *dA2, *dB2; + float *dD1, *dD2; + cudaMalloc(&dA1, M*K*sizeof(half)); cudaMalloc(&dB1, K*N*sizeof(half)); + cudaMalloc(&dA2, M*K*sizeof(half)); cudaMalloc(&dB2, K*N*sizeof(half)); + cudaMalloc(&dD1, WGSIZE*D_ELEMS*sizeof(float)); + cudaMalloc(&dD2, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(dA1, A1.data(), M*K*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(dB1, B1.data(), K*N*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(dA2, A2.data(), M*K*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(dB2, B2.data(), K*N*sizeof(half), cudaMemcpyHostToDevice); + + kernel_fence_sequence<<<1, WGSIZE>>>(dA1, dB1, dA2, dB2, dD1, dD2); + cudaDeviceSynchronize(); + + std::vector flat1(WGSIZE*D_ELEMS), flat2(WGSIZE*D_ELEMS); + cudaMemcpy(flat1.data(), dD1, flat1.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(flat2.data(), dD2, flat2.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(dA1); cudaFree(dB1); cudaFree(dA2); cudaFree(dB2); + cudaFree(dD1); cudaFree(dD2); + + static float D2[M][N]; + reassemble_D(flat1.data(), D); + reassemble_D(flat2.data(), D2); + all_pass &= check_uniform("fence_sequence seq1 (A=1,B=1)", D, (float)K); + all_pass &= check_uniform("fence_sequence seq2 (A=2,B=2)", D2, (float)(K*4)); + } + + // ------------------------------------------------------------------------- + // Test 3: fence_accum (two cycles accumulate into same D) + // ------------------------------------------------------------------------- + printf("\n--- fence_accum ---\n"); + { + auto flat = run1(A1, B1, kernel_fence_accum); + reassemble_D(flat.data(), D); + all_pass &= check_uniform("fence_accum (2 × K)", D, (float)(2*K)); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu new file mode 100644 index 000000000..2163c9d07 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu @@ -0,0 +1,290 @@ +// ============================================================================= +// wgmma_sync.cu +// +// Tests wgmma.commit_group / wgmma.wait_group semantics on SM90a. +// +// Three subtests using f16 m64n16k16 → f32, all-ones inputs: +// +// one_group_two_mmas – two MMA ops in the same committed group. +// Verifies D accumulates both before commit. +// Expected D[m][n] = 32.0 +// +// two_groups_serial – two groups committed and waited one at a time. +// The inter-group fence ends group 0 and starts group 1. +// Expected D[m][n] = 32.0 (accumulated across both groups) +// +// two_groups_pipeline – two groups committed before any wait (software pipeline). +// Tests the group queue: both groups in flight simultaneously +// before a single wait_group 0 drains them. +// Expected D[m][n] = 32.0 +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; +static constexpr int E = 2; +static constexpr int SBO = 128; +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int SMEM_A = M * K * E; // 2048 +static constexpr int SMEM_B = K * N * E; // 512 + +// --------------------------------------------------------------------------- +// Device helpers (identical to wgmma_verify / wgmma_fence) +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }\n" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +__device__ __forceinline__ void mma_f16( + uint64_t da, uint64_t db, + float& d0, float& d1, float& d2, float& d3, + float& d4, float& d5, float& d6, float& d7) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3), + "+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); +} + +__device__ void load_smem(char* smA, char* smB, + const half* A_g, const half* B_g) { + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); +} + +// =========================================================================== +// Kernel 1: two MMA ops in one committed group +// fence → MMA_a → MMA_b → commit_group → wait_group_0 → fence +// Both MMAs accumulate into the same D registers before commit. +// =========================================================================== +__global__ void kernel_one_group_two_mmas(const half* A_g, const half* B_g, + float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + load_smem(smA, smB, A_g, B_g); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // contributes K = 16 + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // contributes K = 16 more + WGMMA_COMMIT; + WGMMA_WAIT; WGMMA_FENCE; + + const int b = threadIdx.x * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Kernel 2: two groups committed and waited serially +// fence → MMA_a → commit → wait → fence → MMA_b → commit → wait → fence +// The inter-group fence acts as both the post-wait fence (releasing D) +// and the pre-MMA fence (signalling smem ready for the next MMA). +// =========================================================================== +__global__ void kernel_two_groups_serial(const half* A_g, const half* B_g, + float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + load_smem(smA, smB, A_g, B_g); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + // Group 0 + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + // Inter-group fence: ends group 0 (D readable) and starts group 1 (smem ready) + WGMMA_FENCE; + // Group 1: accumulates into same D + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; WGMMA_FENCE; + + const int b = threadIdx.x * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Kernel 3: two groups pipelined (both committed before any wait) +// fence → MMA_a → commit_group_0 → MMA_b → commit_group_1 → wait_group_0 → fence +// No fence between the two groups: smem is unchanged so no new fence is needed. +// Tests the group queue: two in-flight groups drained by a single wait. +// =========================================================================== +__global__ void kernel_two_groups_pipeline(const half* A_g, const half* B_g, + float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + load_smem(smA, smB, A_g, B_g); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // into group 0 + WGMMA_COMMIT; // commit group 0 + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // into group 1 (no fence needed) + WGMMA_COMMIT; // commit group 1 + WGMMA_WAIT; WGMMA_FENCE; // drain both groups + + const int b = threadIdx.x * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Host helpers +// =========================================================================== + +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static void reassemble_D(const float* flat, float D[M][N]) { + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) D[m][n] = 0.f; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, &row, &col); + if (row < M && col < N) D[row][col] = flat[T*D_ELEMS + e]; + } +} + +static bool check_uniform(const char* name, float D[M][N], float expected) { + float max_err = 0.f; + int max_m = 0, max_n = 0; + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) { + float e = fabsf(D[m][n] - expected); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + float tol = (fabsf(expected) < 1.f) ? 1e-3f : fabsf(expected) * 1e-2f; + bool pass = (max_err <= tol); + printf("[%-40s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + max_m, max_n, D[max_m][max_n], expected); + return pass; +} + +static std::vector run1(const std::vector& h_A, + const std::vector& h_B, + void (*kern)(const half*, const half*, float*)) { + half *d_A, *d_B; + float *d_D; + cudaMalloc(&d_A, h_A.size()*sizeof(half)); + cudaMalloc(&d_B, h_B.size()*sizeof(half)); + cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + kern<<<1, WGSIZE>>>(d_A, d_B, d_D); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE*D_ELEMS); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + return flat; +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_sync — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + + std::vector A(M*K), B(K*N); + for (auto& v : A) v = __float2half(1.f); + for (auto& v : B) v = __float2half(1.f); + + static float D[M][N]; + const float expected = (float)(2 * K); // two accumulated MMAs × K ones + + printf("\n--- one_group_two_mmas ---\n"); + { + auto flat = run1(A, B, kernel_one_group_two_mmas); + reassemble_D(flat.data(), D); + all_pass &= check_uniform("one_group_two_mmas", D, expected); + } + + printf("\n--- two_groups_serial ---\n"); + { + auto flat = run1(A, B, kernel_two_groups_serial); + reassemble_D(flat.data(), D); + all_pass &= check_uniform("two_groups_serial", D, expected); + } + + printf("\n--- two_groups_pipeline ---\n"); + { + auto flat = run1(A, B, kernel_two_groups_pipeline); + reassemble_D(flat.data(), D); + all_pass &= check_uniform("two_groups_pipeline", D, expected); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/Makefile new file mode 100644 index 000000000..c7f70f9d7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/Makefile @@ -0,0 +1,28 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := wgmma_timing_edge + +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu new file mode 100644 index 000000000..c3b56eae4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu @@ -0,0 +1,687 @@ +// ============================================================================= +// wgmma_timing_edge.cu +// +// Silicon-validation microbenchmarks for WGMMA *timing-model edge cases* that +// lat_gmma / wgmma_multik do not cover. Every test self-checks functional +// correctness (exit 0 = all PASS) and reports %clock-based cycle counts to +// stdout + wgmma_timing_edge_results.csv for sim-vs-silicon comparison. +// +// Tests (one named kernel each, so Nsight Compute can profile them separately): +// +// 1. kernel_group_size - S MMAs in ONE commit group, S = 1..32. +// total(S) ~= LAT + (S-1)*II -> linear fit +// separates true completion latency (intercept) +// from initiation interval (slope). Directly +// tests commit_wgmma_group_with_ops()'s formula. +// 2. kernel_wait_partial - 4 MMAs in 4 separate groups; timed staged +// wait_group 3 -> 2 -> 1 -> 0. Tests the +// "all but latest N" drain semantics + timing. +// 3. kernel_serialize - 2 groups of 1 MMA vs 1 group of 2 MMAs. +// Tests inter-group serialization (model forces +// group completion >= previous group completion). +// 4. kernel_overlap - MMA + commit, then a dependent FMA chain of +// length X, then wait. Sweeping X shows how much +// independent ALU work hides under the async MMA +// (the knee = true async latency; cross-checks #1). +// 5. kernel_inflight - 16x (MMA;COMMIT) with a %clock stamp after each +// pair, no wait until the end. A knee in the +// per-iteration deltas reveals the hardware +// in-flight group limit (model: wgmma_max_pending). +// 6. kernel_swizzle - 32-MMA chain with swizzle mode 0/3/2/1 +// (none/32B/64B/128B). Model predicts identical +// timing; functional check validates the swizzled +// smem layout on silicon. +// 7. kernel_multiwg - 1 vs 2 warpgroups (128 vs 256 threads) each +// running a 64-MMA chain on private smem tiles. +// Tests tensor-core contention across warpgroups +// (model tracks warpgroups independently). +// +// All tests: m64n16k16 f32 <- f16 x f16, A = B = 1 so D element = K * #MMAs. +// +// Run on silicon: +// ./wgmma_timing_edge # all tests + CSV + exit code +// ncu --kernel-name 'regex:kernel_.*' \ +// --metrics gpu__time_duration.sum,sm__cycles_elapsed.avg,\ +// sm__pipe_tensor_op_hmma_cycles_active.avg \ +// ./wgmma_timing_edge # per-kernel timing cross-check +// nsys profile --stats=true ./wgmma_timing_edge # coarse launch timeline +// +// Run in GPGPU-Sim: +// functional: PTX_SIM_USE_PTX_FILE=1 ./wgmma_timing_edge (mode 1 config) +// timing: PTX_SIM_MODE_FUNC=0 ./wgmma_timing_edge +// +// Exit status: 0 = all functional checks PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; // 8 f32 registers per thread +static constexpr int E = 2; // bytes per f16 +static constexpr int SBO = 128; +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int SMEM_A = M * K * E; // 2048 +static constexpr int SMEM_B = K * N * E; // 512 + +// --------------------------------------------------------------------------- +// Device helpers (same conventions as wgmma_multik.cu / sim wgmma_layout.h) +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +// Byte offset of logical element (leading, stride) - matches sim wgmma_smem_offset. +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +// XOR swizzle - matches sim wgmma_apply_swizzle (mode 0=none,1=128B,2=64B,3=32B). +__host__ __device__ __forceinline__ +int apply_swizzle(int off, int sw) { + if (sw == 0) return off; + int stride = (sw == 1) ? 128 : (sw == 2) ? 64 : 32; + int block = off / stride; + int in_block = off % stride; + int xm = (block & 1) ? (stride >> 1) : 0; + return block * stride + (in_block ^ xm); +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT_N(n) asm volatile("wgmma.wait_group.sync.aligned " #n ";\n" ::: "memory") + +#define CLOCK(v) asm volatile("mov.u32 %0, %%clock;" : "=r"(v) :: "memory") + +__device__ __forceinline__ void mma_f16( + uint64_t da, uint64_t db, + float& d0, float& d1, float& d2, float& d3, + float& d4, float& d5, float& d6, float& d7) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3), + "+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); +} + +// Fill smem with the canonical (optionally swizzled) layout; A/B value = 1.0. +// lane_stride lets a 256-thread block fill two private tiles. +__device__ __forceinline__ void fill_tiles( + char* smA, char* smB, const half* A_g, const half* B_g, + int tid, int nthreads, int sw) +{ + for (int i = tid; i < M*K; i += nthreads) { + int m = i/K, k = i%K; + *(half*)(smA + apply_swizzle(smem_off(k, m, E, LBO_A), sw)) = A_g[i]; + } + for (int i = tid; i < K*N; i += nthreads) { + int k = i/N, n = i%N; + *(half*)(smB + apply_swizzle(smem_off(k, n, E, LBO_B), sw)) = B_g[i]; + } +} + +__device__ __forceinline__ void store_d( + float* D_g, int tid, + float d0, float d1, float d2, float d3, + float d4, float d5, float d6, float d7) { + const int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// 1. kernel_group_size: S MMAs in one commit group; time fence..wait0. +// total(S) = LAT + (S-1)*II -> host does the linear fit. +// --------------------------------------------------------------------------- +__global__ void kernel_group_size( + const half* A_g, const half* B_g, float* D_g, + uint32_t* clk_out, int S) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) CLOCK(t0); + __syncthreads(); + + WGMMA_FENCE; + for (int i = 0; i < S; i++) + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT_N(0); + WGMMA_FENCE; + + __syncthreads(); + if (tid == 0) CLOCK(t1); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 2. kernel_wait_partial: 4 single-MMA groups pending, then staged waits. +// clk stamps: [0]=start, [1]=after wait 3, [2]=after wait 2, +// [3]=after wait 1, [4]=after wait 0. +// --------------------------------------------------------------------------- +__global__ void kernel_wait_partial( + const half* A_g, const half* B_g, float* D_g, uint32_t* clk_out) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + uint32_t c0=0,c1=0,c2=0,c3=0,c4=0; + + __syncthreads(); + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; // group 0 + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; // group 1 + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; // group 2 + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; // group 3 + if (tid == 0) CLOCK(c0); + WGMMA_WAIT_N(3); // oldest group (0) must be done + if (tid == 0) CLOCK(c1); + WGMMA_WAIT_N(2); // groups 0,1 done + if (tid == 0) CLOCK(c2); + WGMMA_WAIT_N(1); // groups 0,1,2 done + if (tid == 0) CLOCK(c3); + WGMMA_WAIT_N(0); // all done + WGMMA_FENCE; + if (tid == 0) CLOCK(c4); + + if (tid == 0) { + clk_out[0]=c0; clk_out[1]=c1; clk_out[2]=c2; clk_out[3]=c3; clk_out[4]=c4; + } + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 3. kernel_serialize: mode 0 = 1 group of 2 MMAs; mode 1 = 2 groups of 1 MMA. +// Model serializes groups: mode1 total >= mode0 total expected on sim. +// --------------------------------------------------------------------------- +__global__ void kernel_serialize( + const half* A_g, const half* B_g, float* D_g, + uint32_t* clk_out, int mode) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) CLOCK(t0); + __syncthreads(); + + WGMMA_FENCE; + if (mode == 0) { + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + } else { + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; + } + WGMMA_WAIT_N(0); + WGMMA_FENCE; + + __syncthreads(); + if (tid == 0) CLOCK(t1); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 4. kernel_overlap: MMA+commit, dependent FMA chain of length X, then wait. +// do_mma / X let the host measure mma-only, fma-only and combined runs. +// acc result is checked so the chain can't be optimized away. +// --------------------------------------------------------------------------- +__global__ void kernel_overlap( + const half* A_g, const half* B_g, float* D_g, + float* acc_g, uint32_t* clk_out, int X, int do_mma) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + float acc = acc_g[tid]; // opaque start value (1.0f) + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) CLOCK(t0); + __syncthreads(); + + WGMMA_FENCE; + if (do_mma) { + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + } + // Dependent FMA chain, pinned between commit and wait by volatile asm. + asm volatile("" : "+f"(acc)); + #pragma unroll 1 + for (int i = 0; i < X; i++) + acc = __fmaf_rn(acc, 1.0f, 1.0f); // acc += 1 each step (dependent) + asm volatile("" : "+f"(acc)); + if (do_mma) { + WGMMA_WAIT_N(0); + WGMMA_FENCE; + } + + __syncthreads(); + if (tid == 0) CLOCK(t1); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + acc_g[tid] = acc; + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 5. kernel_inflight: NGRP x (MMA;COMMIT) with a clock stamp after each pair; +// wait 0 only at the end. Stall knee in the deltas = HW in-flight limit. +// --------------------------------------------------------------------------- +static constexpr int NGRP = 16; + +__global__ void kernel_inflight( + const half* A_g, const half* B_g, float* D_g, uint32_t* clk_out) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + uint32_t stamps[NGRP + 2]; + + __syncthreads(); + if (tid == 0) CLOCK(stamps[0]); + __syncthreads(); + + WGMMA_FENCE; + #pragma unroll + for (int g = 0; g < NGRP; g++) { + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + if (tid == 0) CLOCK(stamps[g + 1]); + } + WGMMA_WAIT_N(0); + WGMMA_FENCE; + __syncthreads(); + if (tid == 0) CLOCK(stamps[NGRP + 1]); + + if (tid == 0) + for (int i = 0; i < NGRP + 2; i++) clk_out[i] = stamps[i]; + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 6. kernel_swizzle: 32-MMA chain with swizzle mode sw (0/1/2/3). +// Functional check validates the swizzled fill; timing compares modes. +// --------------------------------------------------------------------------- +static constexpr int SW_MMAS = 32; + +__global__ void kernel_swizzle( + const half* A_g, const half* B_g, float* D_g, + uint32_t* clk_out, int sw) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, sw); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, (uint32_t)sw); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, (uint32_t)sw); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) CLOCK(t0); + __syncthreads(); + + WGMMA_FENCE; + for (int i = 0; i < SW_MMAS; i++) + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT_N(0); + WGMMA_FENCE; + + __syncthreads(); + if (tid == 0) CLOCK(t1); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 7. kernel_multiwg: blockDim = 128 or 256 (1 or 2 warpgroups). Each +// warpgroup runs a WG_MMAS chain on its OWN smem tiles. Per-warpgroup +// start/stop clocks -> contention shows as higher cyc/mma with 2 wgs. +// --------------------------------------------------------------------------- +static constexpr int WG_MMAS = 64; +static constexpr int MAX_WG = 2; + +__global__ void kernel_multiwg( + const half* A_g, const half* B_g, float* D_g, uint32_t* clk_out) +{ + __shared__ __align__(128) char smA[MAX_WG][SMEM_A], smB[MAX_WG][SMEM_B]; + const int tid = threadIdx.x; + const int wg = tid / WGSIZE; // warpgroup index in block + const int wtid = tid % WGSIZE; // thread index within warpgroup + + fill_tiles(smA[wg], smB[wg], A_g, B_g, wtid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA[wg]), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB[wg]), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); // common start line for all wgs + if (wtid == 0) CLOCK(t0); + + WGMMA_FENCE; + for (int i = 0; i < WG_MMAS; i++) + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT_N(0); + WGMMA_FENCE; + + if (wtid == 0) CLOCK(t1); + if (wtid == 0) { clk_out[2*wg] = t0; clk_out[2*wg + 1] = t1; } + store_d(D_g + wg * WGSIZE * D_ELEMS, wtid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// =========================================================================== +// Host side +// =========================================================================== +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static bool check_uniform_flat(const char* name, const float* flat, + float expected, FILE* csv, + const char* param, float cycles, + float cyc_per_mma) { + float max_err = 0.f; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, &row, &col); + if (row >= M || col >= N) continue; + float err = fabsf(flat[T*D_ELEMS + e] - expected); + if (err > max_err) max_err = err; + } + float tol = fabsf(expected) * 1e-2f + 1e-3f; + bool pass = (max_err <= tol); + printf("[%-28s %-12s] %s expected=%.1f max_err=%.3e cycles=%.0f", + name, param, pass ? "PASS" : "FAIL", expected, max_err, cycles); + if (cyc_per_mma > 0) printf(" cyc/mma=%.2f", cyc_per_mma); + printf("\n"); + if (csv) + fprintf(csv, "%s,%s,%s,%.1f,%.4e,%.0f,%.2f\n", + name, param, pass ? "PASS" : "FAIL", expected, max_err, + cycles, cyc_per_mma); + return pass; +} + +struct DevBufs { + half *A, *B; + float *D; + uint32_t *clk; + float *acc; +}; + +static DevBufs alloc_bufs(int d_mult = 1) { + DevBufs b; + std::vector h_A(M*K, __float2half(1.f)); + std::vector h_B(K*N, __float2half(1.f)); + cudaMalloc(&b.A, M*K*sizeof(half)); + cudaMalloc(&b.B, K*N*sizeof(half)); + cudaMalloc(&b.D, d_mult*WGSIZE*D_ELEMS*sizeof(float)); + cudaMalloc(&b.clk, 32*sizeof(uint32_t)); + cudaMalloc(&b.acc, WGSIZE*sizeof(float)); + cudaMemcpy(b.A, h_A.data(), M*K*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(b.B, h_B.data(), K*N*sizeof(half), cudaMemcpyHostToDevice); + return b; +} + +static void free_bufs(DevBufs& b) { + cudaFree(b.A); cudaFree(b.B); cudaFree(b.D); + cudaFree(b.clk); cudaFree(b.acc); +} + +int main(int argc, char** argv) { + const char* filter = (argc > 1) ? argv[1] : NULL; // substring test filter + auto run = [&](const char* name) { + return !filter || strstr(name, filter); + }; + + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_timing_edge - %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + FILE* csv = fopen("wgmma_timing_edge_results.csv", "w"); + if (csv) fprintf(csv, "test,param,status,expected,max_err,cycles,cycles_per_mma\n"); + + bool all_pass = true; + std::vector flat(2*WGSIZE*D_ELEMS); + std::vector clk(32); + + // --- 1. group-size sweep: LAT + (S-1)*II fit ------------------------- + if (run("group_size")) { + printf("\n--- group_size: S MMAs in one group (fit: total = LAT + (S-1)*II) ---\n"); + const int Ss[] = {1, 2, 4, 8, 16, 32}; + double sx=0, sy=0, sxx=0, sxy=0; int np = 0; + DevBufs b = alloc_bufs(); + for (int S : Ss) { + kernel_group_size<<<1, WGSIZE>>>(b.A, b.B, b.D, b.clk, S); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + float cyc = (float)(clk[1] - clk[0]); + char param[32]; snprintf(param, 32, "S=%d", S); + all_pass &= check_uniform_flat("group_size", flat.data(), + (float)(S*K), csv, param, cyc, cyc/S); + sx += S; sy += cyc; sxx += (double)S*S; sxy += (double)S*cyc; np++; + } + free_bufs(b); + double ii = (np*sxy - sx*sy) / (np*sxx - sx*sx); + double lat = (sy - ii*sx) / np + ii; // intercept at S=1... total(1)=LAT + printf(" linear fit: II (slope) = %.2f cyc/mma, LAT (total at S=1) = %.2f cyc\n", ii, lat); + if (csv) fprintf(csv, "group_size_fit,II=%.2f;LAT=%.2f,TIMING,,,,\n", ii, lat); + } + + // --- 2. staged partial waits ----------------------------------------- + if (run("wait_partial")) { + printf("\n--- wait_partial: 4 groups pending; wait 3 -> 2 -> 1 -> 0 ---\n"); + DevBufs b = alloc_bufs(); + kernel_wait_partial<<<1, WGSIZE>>>(b.A, b.B, b.D, b.clk); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 5*sizeof(uint32_t), cudaMemcpyDeviceToHost); + free_bufs(b); + float total = (float)(clk[4] - clk[0]); + all_pass &= check_uniform_flat("wait_partial", flat.data(), + (float)(4*K), csv, "4grp", total, 0); + printf(" wait3=%u wait2=+%u wait1=+%u wait0=+%u cycles\n", + clk[1]-clk[0], clk[2]-clk[1], clk[3]-clk[2], clk[4]-clk[3]); + if (csv) fprintf(csv, "wait_partial_stages,w3=%u;w2=%u;w1=%u;w0=%u,TIMING,,,%'.0f,\n", + clk[1]-clk[0], clk[2]-clk[1], clk[3]-clk[2], clk[4]-clk[3], total); + } + + // --- 3. group serialization ------------------------------------------ + if (run("serialize")) { + printf("\n--- serialize: 1 group x 2 MMAs vs 2 groups x 1 MMA ---\n"); + DevBufs b = alloc_bufs(); + float cyc_mode[2]; + for (int mode = 0; mode < 2; mode++) { + kernel_serialize<<<1, WGSIZE>>>(b.A, b.B, b.D, b.clk, mode); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cyc_mode[mode] = (float)(clk[1] - clk[0]); + all_pass &= check_uniform_flat("serialize", flat.data(), (float)(2*K), + csv, mode == 0 ? "1grpx2mma" : "2grpx1mma", + cyc_mode[mode], cyc_mode[mode]/2); + } + free_bufs(b); + printf(" delta (2grp - 1grp) = %.0f cycles\n", cyc_mode[1] - cyc_mode[0]); + } + + // --- 4. async overlap sweep ------------------------------------------ + if (run("overlap")) { + printf("\n--- overlap: MMA || dependent FMA chain of length X ---\n"); + DevBufs b = alloc_bufs(); + const int Xs[] = {0, 8, 16, 32, 64, 96, 128, 192, 256}; + std::vector h_acc(WGSIZE, 1.f), r_acc(WGSIZE); + for (int X : Xs) { + for (int do_mma = (X == 0 ? 1 : 0); do_mma < 2; do_mma++) { + cudaMemcpy(b.acc, h_acc.data(), WGSIZE*sizeof(float), cudaMemcpyHostToDevice); + kernel_overlap<<<1, WGSIZE>>>(b.A, b.B, b.D, b.acc, b.clk, X, do_mma); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(r_acc.data(), b.acc, WGSIZE*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + float cyc = (float)(clk[1] - clk[0]); + // functional: acc = 1 + X for every thread; D = K if do_mma + bool acc_ok = true; + for (int t = 0; t < WGSIZE; t++) + if (fabsf(r_acc[t] - (1.f + X)) > 1e-3f) acc_ok = false; + char param[40]; + snprintf(param, 40, "X=%d%s", X, do_mma ? "+mma" : ",fma_only"); + if (do_mma) { + all_pass &= check_uniform_flat("overlap", flat.data(), (float)K, + csv, param, cyc, 0) && acc_ok; + } else { + printf("[%-28s %-12s] %s cycles=%.0f (fma chain alone)\n", + "overlap", param, acc_ok ? "PASS" : "FAIL", cyc); + if (csv) fprintf(csv, "overlap,%s,%s,,,%0.f,\n", + param, acc_ok ? "PASS" : "FAIL", cyc); + all_pass &= acc_ok; + } + if (!acc_ok) printf(" FMA chain result wrong (X=%d)\n", X); + } + } + free_bufs(b); + printf(" (flat combined-total until X exceeds the MMA latency = async overlap works)\n"); + } + + // --- 5. in-flight group limit ----------------------------------------- + if (run("inflight")) { + printf("\n--- inflight: %d x (MMA;COMMIT) with per-commit clock stamps ---\n", NGRP); + DevBufs b = alloc_bufs(); + kernel_inflight<<<1, WGSIZE>>>(b.A, b.B, b.D, b.clk); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, (NGRP+2)*sizeof(uint32_t), cudaMemcpyDeviceToHost); + free_bufs(b); + float total = (float)(clk[NGRP+1] - clk[0]); + all_pass &= check_uniform_flat("inflight", flat.data(), (float)(NGRP*K), + csv, "16grp", total, total/NGRP); + printf(" per-(mma;commit) deltas: "); + for (int g = 1; g <= NGRP; g++) printf("%u ", clk[g] - clk[g-1]); + printf("\n drain after last commit: %u cycles\n", clk[NGRP+1] - clk[NGRP]); + if (csv) { + fprintf(csv, "inflight_deltas,"); + for (int g = 1; g <= NGRP; g++) + fprintf(csv, "%u%s", clk[g] - clk[g-1], g == NGRP ? "" : ";"); + fprintf(csv, ",TIMING,,,%0.f,\n", total); + } + } + + // --- 6. swizzle modes -------------------------------------------------- + if (run("swizzle")) { + printf("\n--- swizzle: %d-MMA chain, modes none/32B/64B/128B ---\n", SW_MMAS); + static const char* swname[] = {"none", "128B", "64B", "32B"}; + DevBufs b = alloc_bufs(); + for (int sw = 0; sw < 4; sw++) { + kernel_swizzle<<<1, WGSIZE>>>(b.A, b.B, b.D, b.clk, sw); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + float cyc = (float)(clk[1] - clk[0]); + char param[24]; snprintf(param, 24, "sw=%s", swname[sw]); + all_pass &= check_uniform_flat("swizzle", flat.data(), (float)(SW_MMAS*K), + csv, param, cyc, cyc/SW_MMAS); + } + free_bufs(b); + } + + // --- 7. multi-warpgroup contention -------------------------------------- + if (run("multiwg")) { + printf("\n--- multiwg: %d-MMA chain per warpgroup, 1 vs 2 warpgroups ---\n", WG_MMAS); + DevBufs b = alloc_bufs(MAX_WG); + for (int nwg = 1; nwg <= MAX_WG; nwg++) { + cudaMemset(b.D, 0, MAX_WG*WGSIZE*D_ELEMS*sizeof(float)); + kernel_multiwg<<<1, nwg*WGSIZE>>>(b.A, b.B, b.D, b.clk); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, nwg*WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 2*nwg*sizeof(uint32_t), cudaMemcpyDeviceToHost); + for (int wg = 0; wg < nwg; wg++) { + float cyc = (float)(clk[2*wg+1] - clk[2*wg]); + char param[24]; snprintf(param, 24, "nwg=%d;wg=%d", nwg, wg); + all_pass &= check_uniform_flat("multiwg", + flat.data() + wg*WGSIZE*D_ELEMS, + (float)(WG_MMAS*K), csv, param, + cyc, cyc/WG_MMAS); + } + } + free_bufs(b); + printf(" (2-wg cyc/mma vs 1-wg cyc/mma = tensor-core contention factor)\n"); + } + + if (csv) fclose(csv); + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); + return all_pass ? 0 : 1; +}