From 4cfe155392ea93437b1d0fa3113db51c2ff2a2c2 Mon Sep 17 00:00:00 2001 From: jyasuumi <144069912+jyasuumi@users.noreply.github.com> Date: Fri, 13 Feb 2026 12:07:40 -0500 Subject: [PATCH 01/35] cp.async.bulk g,s undergrad ubenches --- .../ubench/tma/ublkcp_g_s.cu | 158 ++++++++++++++++ .../ubench/tma/ublkcp_s_g.cu | 175 ++++++++++++++++++ 2 files changed, 333 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_g_s.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_s_g.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_g_s.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_g_s.cu new file mode 100644 index 000000000..60c000980 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_g_s.cu @@ -0,0 +1,158 @@ +/** + * UBLKCP_G_S Unit Test: TMA Bulk Copy Shared -> Global + * ===================================================== + * Tests that cp.async.bulk correctly stores data from shared to global memory. + * + * Strategy: + * 1. Global memory is poisoned with 0xDEADBEEF + * 2. Kernel generates pattern (0, 1, 2, ...) in shared memory + * 3. TMA stores shared -> global + * PASS: Global contains (0, 1, 2, ...) + * FAIL: Global still contains poison (TMA store was a NOP) + * + * Usage: ./ublkcp_g_s [-n ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, \ + "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90 or newer for TMA bulk operations."); +#endif + +static constexpr size_t buf_len = 1024; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UBLKCP_G_S(int32_t *data, int run_iters) +{ + __shared__ alignas(16) int32_t smem_data[buf_len]; + + size_t offset = blockIdx.x * blockDim.x; + + // Generate the test pattern (0, 1, 2, ...) in shared memory + for (int i = threadIdx.x; i < buf_len; i += blockDim.x) { + smem_data[i] = offset + i; + } + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + // TMA bulk store: shared -> global + for (int iter = 0; iter < run_iters; iter++) { + if (threadIdx.x == 0) { + ptx::cp_async_bulk( + ptx::space_global, + ptx::space_shared, + data + offset, smem_data, sizeof(smem_data)); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(int32_t *h_result, int n) +{ + int poison_count = 0; + int mismatch_count = 0; + + for (int i = 0; i < n; i++) { + if (h_result[i] == POISON) { + if (poison_count == 0) + printf(" NOP detected at index %d: got 0x%08X, expected %d\n", + i, (unsigned)h_result[i], i); + poison_count++; + } else if (h_result[i] != i) { + if (mismatch_count == 0) + printf(" Mismatch at index %d: got %d, expected %d\n", + i, h_result[i], i); + mismatch_count++; + } + } + + if (poison_count > 0) { + printf(" FAIL: %d/%d elements still contain poison (TMA store was NOP)\n", + poison_count, n); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%d elements have wrong values\n", mismatch_count, n); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + int n = 1024; + int run_iters = 128; + int opt; + while ((opt = getopt(argc, argv, "n:i:")) != -1) { + switch (opt) { + case 'n': n = atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-n ] [-i ]\n", argv[0]); + return 1; + } + } + + size_t bytes = n * sizeof(int32_t); + uint32_t blockSize = 1024; + uint32_t gridSize = (uint32_t)ceil((float)n / blockSize); + + // Host buffer — poisoned + int32_t *h_result = (int32_t *)malloc(bytes); + for (int i = 0; i < n; i++) { + h_result[i] = POISON; + } + + // Device buffer + int32_t *d_a; + cudaMalloc(&d_a, bytes); + cudaMemcpy(d_a, h_result, bytes, cudaMemcpyHostToDevice); + + // Run + CUDA_SAFECALL((test_UBLKCP_G_S<<>>(d_a, run_iters))); + cudaDeviceSynchronize(); + + // Copy back + cudaMemcpy(h_result, d_a, bytes, cudaMemcpyDeviceToHost); + + // Verify + printf("=== UBLKCP_G_S (TMA Bulk Store: Shared -> Global) ===\n"); + bool pass = verify(h_result, n); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_result); + cudaFree(d_a); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_s_g.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_s_g.cu new file mode 100644 index 000000000..143415d4f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_s_g.cu @@ -0,0 +1,175 @@ +/** + * UBLKCP_S_G Unit Test: TMA Bulk Copy Global -> Shared + * ===================================================== + * Tests that cp.async.bulk correctly loads data from global to shared memory. + * + * Strategy: + * 1. Source global (d_src) initialized with pattern (0, 1, 2, ...) + * 2. Destination global (d_dst) poisoned with 0xDEADBEEF + * 3. TMA loads d_src -> shared memory + * 4. Scalar (trusted) copy: shared -> d_dst + * PASS: d_dst matches d_src + * FAIL: d_dst still contains poison (TMA load was a NOP) + * + * Usage: ./ublkcp_s_g [-n ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, \ + "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +using barrier = cuda::barrier; +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90 or newer for TMA bulk operations."); +#endif + +static constexpr size_t buf_len = 1024; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UBLKCP_S_G(int32_t *d_src, int32_t *d_dst, int run_iters) +{ + __shared__ alignas(16) int32_t smem_data[buf_len]; + + size_t offset = blockIdx.x * blockDim.x; + + #pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + if (threadIdx.x == 0) { + init(&bar, blockDim.x); + ptx::fence_proxy_async(ptx::space_shared); + } + __syncthreads(); + + // TMA bulk load: global -> shared + for (int iter = 0; iter < run_iters; iter++) { + if (threadIdx.x == 0) { + cuda::memcpy_async( + smem_data, + d_src + offset, + cuda::aligned_size_t<16>(sizeof(smem_data)), + bar); + } + barrier::arrival_token token = bar.arrive(); + bar.wait(std::move(token)); + } + + // Trusted scalar copy: shared -> d_dst (does NOT use TMA) + __syncthreads(); + for (int i = threadIdx.x; i < buf_len; i += blockDim.x) { + d_dst[offset + i] = smem_data[i]; + } +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(int32_t *h_src, int32_t *h_dst, int n) +{ + int poison_count = 0; + int mismatch_count = 0; + + for (int i = 0; i < n; i++) { + if (h_dst[i] == POISON) { + if (poison_count == 0) + printf(" NOP detected at index %d: got 0x%08X, expected %d\n", + i, (unsigned)h_dst[i], h_src[i]); + poison_count++; + } else if (h_dst[i] != h_src[i]) { + if (mismatch_count == 0) + printf(" Mismatch at index %d: got %d, expected %d\n", + i, h_dst[i], h_src[i]); + mismatch_count++; + } + } + + if (poison_count > 0) { + printf(" FAIL: %d/%d elements still contain poison (TMA load was NOP)\n", + poison_count, n); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%d elements have wrong values\n", mismatch_count, n); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + int n = 1024; + int run_iters = 128; + int opt; + while ((opt = getopt(argc, argv, "n:i:")) != -1) { + switch (opt) { + case 'n': n = atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-n ] [-i ]\n", argv[0]); + return 1; + } + } + + size_t bytes = n * sizeof(int32_t); + uint32_t blockSize = 1024; + uint32_t gridSize = (uint32_t)ceil((float)n / blockSize); + + // Host buffers + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + // Initialize source with pattern, destination with poison + for (int i = 0; i < n; i++) { + h_src[i] = i; + h_dst[i] = POISON; + } + + // Device buffers + int32_t *d_src, *d_dst; + cudaMalloc(&d_src, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_src, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Run + CUDA_SAFECALL((test_UBLKCP_S_G<<>>(d_src, d_dst, run_iters))); + cudaDeviceSynchronize(); + + // Copy back + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + // Verify + printf("=== UBLKCP_S_G (TMA Bulk Load: Global -> Shared) ===\n"); + bool pass = verify(h_src, h_dst, n); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_src); + free(h_dst); + cudaFree(d_src); + cudaFree(d_dst); + return pass ? 0 : 1; +} From a7888159e728e9f9a1ba1d438f20802f0ecad400 Mon Sep 17 00:00:00 2001 From: Kai Ze Ee <136288731+definitelyka1ze@users.noreply.github.com> Date: Sun, 15 Feb 2026 20:18:16 +0000 Subject: [PATCH 02/35] mbarrier basic and reduction tests refer to presentation for motivation details --- .../ubench/tma/mbarrier/mbarrier_reduction.cu | 122 +++++++++++++++ .../tma/mbarrier/mbarrier_reduction2.cu | 144 ++++++++++++++++++ .../ubench/tma/mbarrier/mbarrier_test.cu | 41 +++++ 3 files changed, 307 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu new file mode 100644 index 000000000..dc7e14af0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu @@ -0,0 +1,122 @@ +#include +#include +#include + +#define BLOCK_SIZE 256 + +// -------------------------------------------------------- +// KERNEL 1: Reference Implementation (__syncthreads) +// -------------------------------------------------------- +__global__ void reduction_reference(int* in, int* out) { + __shared__ int s_data[BLOCK_SIZE]; + + // Load data into shared memory + int tid = threadIdx.x; + s_data[tid] = in[tid]; + __syncthreads(); // Standard barrier + + // Standard tree reduction + + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + s_data[tid] += s_data[tid + stride]; + } + __syncthreads(); // Barrier essential for correctness + } + + // Write result + if (tid == 0) *out = s_data[0]; +} + +// -------------------------------------------------------- +// KERNEL 2: mbarrier Implementation +// -------------------------------------------------------- +__global__ void reduction_mbarrier(int* in, int* out) { + __shared__ int s_data[BLOCK_SIZE]; + + // 1. Declare mbarrier in shared memory + __shared__ cuda::barrier bar; + + // 2. Initialize barrier (Thread 0 only) + if (threadIdx.x == 0) { + init(&bar, blockDim.x); + } + __syncthreads(); // Wait for init to finish + + // Load data + int tid = threadIdx.x; + s_data[tid] = in[tid]; + + // Synchronize after load + auto token = bar.arrive(); + + // CHANGE: Use standard wait(). + // This generates the 'mbarrier.test_wait' loop automatically. + bar.wait(std::move(token)); + + // Standard tree reduction using mbarrier + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + s_data[tid] += s_data[tid + stride]; + } + + // 3. BARRIER SYNCHRONIZATION + token = bar.arrive(); + bar.wait(std::move(token)); + } + + if (tid == 0) *out = s_data[0]; +} + +int main() { + const int N = BLOCK_SIZE; + const int BYTES = N * sizeof(int); + + // Host data + int* h_in = (int*)malloc(BYTES); + int h_ref_out = 0; + int h_mbar_out = 0; + + // Initialize input (Array of 1s, so Sum should be 256) + for (int i = 0; i < N; i++) h_in[i] = 1; + + // Device data + int *d_in, *d_ref_out, *d_mbar_out; + cudaMalloc(&d_in, BYTES); + cudaMalloc(&d_ref_out, sizeof(int)); + cudaMalloc(&d_mbar_out, sizeof(int)); + cudaMemcpy(d_in, h_in, BYTES, cudaMemcpyHostToDevice); + + // ---------------------------------------------------- + // Run Reference Kernel + // ---------------------------------------------------- + reduction_reference<<<1, BLOCK_SIZE>>>(d_in, d_ref_out); + cudaMemcpy(&h_ref_out, d_ref_out, sizeof(int), cudaMemcpyDeviceToHost); + + // ---------------------------------------------------- + // Run mbarrier Kernel + // ---------------------------------------------------- + reduction_mbarrier<<<1, BLOCK_SIZE>>>(d_in, d_mbar_out); + cudaMemcpy(&h_mbar_out, d_mbar_out, sizeof(int), cudaMemcpyDeviceToHost); + + // ---------------------------------------------------- + // Verify + // ---------------------------------------------------- + printf("Reference Sum: %d\n", h_ref_out); + printf("mbarrier Sum: %d\n", h_mbar_out); + + if (h_ref_out == h_mbar_out && h_ref_out == N) { + printf("SUCCESS: mbarrier matches reference output!\n"); + } else { + printf("FAILURE: Results mismatch.\n"); + printf("Likely Cause: Threads raced ahead without waiting.\n"); + } + + // Cleanup + free(h_in); + cudaFree(d_in); + cudaFree(d_ref_out); + cudaFree(d_mbar_out); + + return 0; +} \ No newline at end of file diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu new file mode 100644 index 000000000..f258b5633 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu @@ -0,0 +1,144 @@ +#include +#include +#include + +#define BLOCK_SIZE 256 + +// Helper to burn time and force a race condition +__device__ void sabotage_delay(int cycles) { + long long start = clock64(); + while (clock64() - start < cycles) { + // Busy wait to hold up the thread + } +} + +// -------------------------------------------------------- +// KERNEL 1: Reference Implementation (__syncthreads) +// -------------------------------------------------------- +__global__ void reduction_reference(int* in, int* out) { + __shared__ int s_data[BLOCK_SIZE]; + + // Load data into shared memory + int tid = threadIdx.x; + + // SABOTAGE: Delay writers here too to prove __syncthreads works + if (tid >= BLOCK_SIZE / 2) { + sabotage_delay(50000); + } + + s_data[tid] = in[tid]; + __syncthreads(); // Standard barrier + + // Standard tree reduction + + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + s_data[tid] += s_data[tid + stride]; + } + __syncthreads(); // Barrier essential for correctness + } + + // Write result + if (tid == 0) *out = s_data[0]; +} + +// -------------------------------------------------------- +// KERNEL 2: mbarrier Implementation +// -------------------------------------------------------- +__global__ void reduction_mbarrier(int* in, int* out) { + __shared__ int s_data[BLOCK_SIZE]; + + // 1. Declare mbarrier in shared memory + __shared__ cuda::barrier bar; + + // 2. Initialize barrier (Thread 0 only) + if (threadIdx.x == 0) { + init(&bar, blockDim.x); + } + __syncthreads(); // Wait for init to finish + + // Load data + int tid = threadIdx.x; + + // --- THE SABOTAGE --- + // We delay the threads that provide data (High IDs). + // The threads that read data (Low IDs) will rush ahead. + if (tid >= BLOCK_SIZE / 2) { + sabotage_delay(50000); // Delay 500 cycles + } + + // Write Data (Late) + s_data[tid] = in[tid]; + + // THE TEST: + // If barrier is broken, Low IDs reach here, see High IDs aren't done, + // ignore it, and read garbage data below. + auto token = bar.arrive(); + bar.wait(std::move(token)); + + // Standard tree reduction using mbarrier + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + s_data[tid] += s_data[tid + stride]; + } + + // 3. BARRIER SYNCHRONIZATION == __syncthreads + token = bar.arrive(); + bar.wait(std::move(token)); + } + + if (tid == 0) *out = s_data[0]; +} + +int main() { + const int N = BLOCK_SIZE; + const int BYTES = N * sizeof(int); + + // Host data + int* h_in = (int*)malloc(BYTES); + int h_ref_out = 0; + int h_mbar_out = 0; + + // Initialize input (Array of 1s, so Sum should be 256) + for (int i = 0; i < N; i++) h_in[i] = 1; + + // Device data + int *d_in, *d_ref_out, *d_mbar_out; + cudaMalloc(&d_in, BYTES); + cudaMalloc(&d_ref_out, sizeof(int)); + cudaMalloc(&d_mbar_out, sizeof(int)); + cudaMemcpy(d_in, h_in, BYTES, cudaMemcpyHostToDevice); + + // ---------------------------------------------------- + // Run Reference Kernel + // ---------------------------------------------------- + reduction_reference<<<1, BLOCK_SIZE>>>(d_in, d_ref_out); + cudaMemcpy(&h_ref_out, d_ref_out, sizeof(int), cudaMemcpyDeviceToHost); + + // ---------------------------------------------------- + // Run mbarrier Kernel + // ---------------------------------------------------- + reduction_mbarrier<<<1, BLOCK_SIZE>>>(d_in, d_mbar_out); + cudaMemcpy(&h_mbar_out, d_mbar_out, sizeof(int), cudaMemcpyDeviceToHost); + + // ---------------------------------------------------- + // Verify + // ---------------------------------------------------- + printf("Reference Sum: %d\n", h_ref_out); + printf("mbarrier Sum: %d\n", h_mbar_out); + + if (h_ref_out == h_mbar_out && h_ref_out == N) { + printf("SUCCESS: mbarrier matches reference output!\n"); + } else { + printf("FAILURE: Results mismatch.\n"); + printf("Likely Cause: Threads raced ahead without waiting.\n"); + } + + // Cleanup + free(h_in); + cudaFree(d_in); + cudaFree(d_ref_out); + cudaFree(d_mbar_out); + + return 0; +} \ No newline at end of file diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu new file mode 100644 index 000000000..7c9b30d5d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu @@ -0,0 +1,41 @@ +#include +#include // Standard C++ header +#include + +__global__ void mbarrier_kernel(int* out) { + // 1. Declare barrier in Shared Memory + __shared__ cuda::barrier bar; + + // 2. Initialize (Thread 0 only) + if (threadIdx.x == 0) { + init(&bar, blockDim.x); + } + __syncthreads(); + + // 3. Arrive and Wait + // This generates the 'mbarrier.arrive' and 'mbarrier.try_wait' + // instructions automatically. + cuda::barrier::arrival_token token = bar.arrive(); + bar.wait(std::move(token)); + + // Success check + if (threadIdx.x == 0) { + *out = 1; + } +} + +int main() { + int* d_out; + int h_out = 0; + cudaMalloc(&d_out, sizeof(int)); + cudaMemset(d_out, 0, sizeof(int)); + + mbarrier_kernel<<<1, 32>>>(d_out); + + cudaMemcpy(&h_out, d_out, sizeof(int), cudaMemcpyDeviceToHost); + + if (h_out == 1) printf("mbarrier test passed (Standard C++).\n"); + else printf("mbarrier test failed.\n"); + + return 0; +} \ No newline at end of file From 0880076f1a8ce299b5d5ee6722afa996a649f523 Mon Sep 17 00:00:00 2001 From: jyasuumi <144069912+jyasuumi@users.noreply.github.com> Date: Tue, 28 Apr 2026 15:20:55 -0400 Subject: [PATCH 03/35] Refactor Makefiles for TMA benchmarks to include new build targets (multicast, tma bulk variants, etc) and improve organization. --- .../ubench/system/deviceQuery/Makefile | 2 +- .../ubench/tma/mbarrier/Makefile | 30 +- .../ubench/tma/tma_bulk/Makefile | 38 +- .../ubench/tma/{ => tma_bulk}/ublkcp_g_s.cu | 0 .../ubench/tma/{ => tma_bulk}/ublkcp_s_g.cu | 0 .../ubench/tma/tma_bulk/ublkpf.cu | 245 +++++++++++ .../ubench/tma/tma_bulk/ublkred_g_s.cu | 362 ++++++++++++++++ .../ubench/tma/tma_tensor_multicast/Makefile | 28 ++ .../tma_tensor_multicast.cu | 403 ++++++++++++++++++ .../tma_tensor_multicast_4cta.cu | 341 +++++++++++++++ .../tma_tensor_multicast_mask.cu | 344 +++++++++++++++ 11 files changed, 1777 insertions(+), 16 deletions(-) rename src/cuda/GPU_Microbenchmark/ubench/tma/{ => tma_bulk}/ublkcp_g_s.cu (100%) rename src/cuda/GPU_Microbenchmark/ubench/tma/{ => tma_bulk}/ublkcp_s_g.cu (100%) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_4cta.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_mask.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/system/deviceQuery/Makefile b/src/cuda/GPU_Microbenchmark/ubench/system/deviceQuery/Makefile index 167a71704..ef6ca28da 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/system/deviceQuery/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/system/deviceQuery/Makefile @@ -2,7 +2,7 @@ SRC = deviceQuery.cpp EXE = deviceQuery -INCLUDE := -I../../../../cuda-samples/Common/ +INCLUDE := -I../../../../NVIDIA_CUDA-11.0_Samples/common/inc/ NVCC_FLGAS = include ../../../common/common.mk diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/Makefile index 3b7aa310c..2aeee0f5f 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/Makefile @@ -1,12 +1,30 @@ -SRC = mbarrier.cu +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ -EXE = mbarrier +# Build all mbarrier microbenchmarks in this directory. +ALL_BENCHES := mbarrier mbarrier_reduction mbarrier_reduction2 mbarrier_test -ARCH?=sm_80 sm_90a sm_100a sm_101 sm_120 -# Unset the CUDA_CPPFLAGS which is set based on CUDA version +ARCH ?= sm_80 sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. CUDA_CPPFLAGS= -# Generate code for both sm_XXX and compute_XXX (SASS and PTX) +# Generate both SASS and PTX. NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 -include ../../../common/common.mk +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/Makefile index f3c8bcd00..7c8b9909f 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/Makefile @@ -1,13 +1,33 @@ -SRC = tma_bulk.cu +# TMA Bulk microbenchmarks +# Build all benchmarks in this folder when invoked from the top-level +# GPU_Microbenchmark make recursion. -EXE = tma_bulk +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ -# TMA is supported on SM_90a and above -ARCH?=sm_90a sm_100a sm_101 sm_120 -# Unset the CUDA_CPPFLAGS which is set based on CUDA version -# but TMA is only supported on SM_90a and above -CUDA_CPPFLAGS= -# Generate code for both sm_XXX and compute_XXX (SASS and PTX) +# All bulk microbenchmarks in this directory. +ALL_BENCHES := tma_bulk ublkcp_g_s ublkcp_s_g ublkpf ublkred_g_s + +# TMA is supported on SM_90a and above. +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Keep same codegen style used across GPU_Microbenchmark. NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 -include ../../../common/common.mk +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +# Compile .cu -> .a so PTX is preserved, then link with shared cudart. +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_g_s.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_g_s.cu similarity index 100% rename from src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_g_s.cu rename to src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_g_s.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_s_g.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_s_g.cu similarity index 100% rename from src/cuda/GPU_Microbenchmark/ubench/tma/ublkcp_s_g.cu rename to src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_s_g.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu new file mode 100644 index 000000000..acd5ce4aa --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu @@ -0,0 +1,245 @@ +/** + * UBLKPF Unit Test: TMA Bulk Prefetch (cp.async.bulk.prefetch.L2.global) + * ====================================================================== + * Tests that cp.async.bulk.prefetch.L2.global correctly issues a prefetch + * without corrupting memory contents. + * + * The PTX ISA only exposes one variant of 1D bulk prefetch: + * cp.async.bulk.prefetch.L2.global [srcMem], size; + * + * Verification Strategy: + * 1. Global memory is initialized with a known pattern (0, 1, 2, ...) + * 2. Kernel issues cp.async.bulk.prefetch.L2.global for each block's region + * 3. After the prefetch, kernel reads the data back via standard loads + * and writes to a destination buffer + * 4. Host verifies destination matches original source pattern + * + * PASS: Prefetch executed without error, data integrity maintained + * FAIL: Data is corrupted after prefetch (should never happen on correct HW) + * + * Usage: ./ublkpf [-n ] [-i ] [-s ] + * + * -n : number of int32 elements (default: 1024) + * -i : number of prefetch iterations in-kernel (default: 1) + * -s : bytes to prefetch per block (default: buf_len * 4) + * clamped to a multiple of 16, max buf_len * 4 + */ + +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, \ + "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90 or newer for TMA bulk operations."); +#endif + +static constexpr size_t buf_len = 1024; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel: prefetch + readback verification +// +// Thread 0 issues the bulk prefetch, then all threads cooperatively +// read the (now hopefully L2-resident) data via standard loads and +// write it to a separate destination buffer. If the prefetch corrupted +// memory or caused any fault, the readback will expose it. +// ============================================================================ +__global__ void test_UBLKPF(const int32_t *d_src, int32_t *d_dst, + int run_iters, uint32_t prefetch_bytes) +{ + size_t offset = blockIdx.x * blockDim.x; + + // Prefetch phase + if (threadIdx.x == 0) { + uint64_t addr = (uint64_t)(d_src + offset); + for (int iter = 0; iter < run_iters; iter++) { + asm volatile( + "cp.async.bulk.prefetch.L2.global" + " [%0], %1;" + : + : "l"(addr), + "r"(prefetch_bytes) + : "memory"); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } + + __syncthreads(); + + // Readback phase: standard loads to verify data integrity + uint32_t elems = prefetch_bytes / sizeof(int32_t); + for (uint32_t i = threadIdx.x; i < elems; i += blockDim.x) { + d_dst[offset + i] = d_src[offset + i]; + } +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_src, const int32_t *h_dst, int n) +{ + int poison_count = 0; + int mismatch_count = 0; + + for (int i = 0; i < n; i++) { + if (h_dst[i] == POISON) { + if (poison_count == 0) + printf(" Poison still present at index %d: got 0x%08X, expected %d\n", + i, (unsigned)h_dst[i], h_src[i]); + poison_count++; + } else if (h_dst[i] != h_src[i]) { + if (mismatch_count == 0) + printf(" Mismatch at index %d: got %d, expected %d\n", + i, h_dst[i], h_src[i]); + mismatch_count++; + } + } + + if (poison_count > 0) { + printf(" FAIL: %d/%d elements still contain poison (readback did not execute)\n", + poison_count, n); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%d elements have wrong values (data corrupted after prefetch)\n", + mismatch_count, n); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + int n = 1024; + int run_iters = 1; + int user_prefetch_bytes = -1; + + int opt; + while ((opt = getopt(argc, argv, "n:i:s:")) != -1) { + switch (opt) { + case 'n': + n = atoi(optarg); + break; + case 'i': + run_iters = atoi(optarg); + break; + case 's': + user_prefetch_bytes = atoi(optarg); + break; + default: + fprintf(stderr, + "Usage: %s [-n ] [-i ] [-s ]\n", + argv[0]); + fprintf(stderr, " -n : number of int32 elements (default: 1024)\n"); + fprintf(stderr, " -i : number of prefetch iterations (default: 1)\n"); + fprintf(stderr, " -s : bytes to prefetch per block (default: %zu)\n", + buf_len * sizeof(int32_t)); + fprintf(stderr, " must be a multiple of 16, max %zu\n", + buf_len * sizeof(int32_t)); + return 1; + } + } + + uint32_t max_prefetch = (uint32_t)(buf_len * sizeof(int32_t)); + uint32_t prefetch_bytes; + if (user_prefetch_bytes < 0) { + prefetch_bytes = max_prefetch; + } else { + prefetch_bytes = (uint32_t)user_prefetch_bytes; + prefetch_bytes = (prefetch_bytes + 15) & ~15u; + if (prefetch_bytes > max_prefetch) + prefetch_bytes = max_prefetch; + if (prefetch_bytes == 0) + prefetch_bytes = 16; + } + + size_t bytes = (size_t)n * sizeof(int32_t); + uint32_t blockSize = 1024; + uint32_t gridSize = (uint32_t)ceil((float)n / blockSize); + + // Host buffers + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + for (int i = 0; i < n; i++) { + h_src[i] = i; + h_dst[i] = POISON; + } + + // Device buffers + int32_t *d_src, *d_dst; + cudaMalloc(&d_src, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_src, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Run + CUDA_SAFECALL((test_UBLKPF<<>>(d_src, d_dst, run_iters, prefetch_bytes))); + cudaDeviceSynchronize(); + + // Copy back + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + // Verify + uint32_t verified_elems = prefetch_bytes / sizeof(int32_t); + if ((int)verified_elems > n) verified_elems = n; + + printf("=== UBLKPF (TMA Bulk Prefetch: cp.async.bulk.prefetch.L2.global) ===\n"); + printf(" Prefetch size: %u bytes (%u int32 elements)\n", + prefetch_bytes, prefetch_bytes / 4); + printf(" Iterations: %d\n", run_iters); + + bool pass = verify(h_src, h_dst, (int)verified_elems); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + // Dump results to file for easy expected-vs-actual comparison + char filename[128]; + sprintf(filename, "tma_bulk_test_UBLKPF_%d.txt", n); + FILE *f = fopen(filename, "w"); + if (f) { + fprintf(f, "# UBLKPF Test Results (TMA Bulk Prefetch + Readback)\n"); + fprintf(f, "# Prefetch size: %u bytes (%u int32 elements)\n", + prefetch_bytes, prefetch_bytes / 4); + fprintf(f, "# Iterations: %d\n", run_iters); + fprintf(f, "# Format: index, got (hex), expected (hex)\n"); + for (int i = 0; i < (int)verified_elems; i++) { + const char *status = (h_dst[i] == h_src[i]) ? "OK" : + (h_dst[i] == POISON) ? "POISON" : "MISMATCH"; + fprintf(f, "%4d: 0x%08X (expected 0x%08X) [%s]\n", + i, (unsigned)h_dst[i], (unsigned)h_src[i], status); + if ((i + 1) % 512 == 0) fprintf(f, "\n"); + } + fclose(f); + printf("Values dumped to %s\n", filename); + } else { + fprintf(stderr, "Warning: could not open %s for writing\n", filename); + } + + free(h_src); + free(h_dst); + cudaFree(d_src); + cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu new file mode 100644 index 000000000..4ba1c2bbf --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu @@ -0,0 +1,362 @@ +/** + * UBLKRED_G_S Unit Test: TMA Bulk Reduction Global <- Shared + * =========================================================== + * Tests that cp.reduce.async.bulk correctly reduces data from shared + * into global memory for several reduction operators. + * + * Strategy (per element i): + * - Global memory is initialized to an operator-specific BASE value. + * - Shared memory is initialized with the pattern (0, 1, 2, ...). + * - TMA performs a bulk reduction: global[i] = reduce(global[i], shared[i]). + * - Host computes the expected result and compares element-wise. + * + * Supported reduction kinds: + * - add : global += shared + * - min : global = min(global, shared) + * - max : global = max(global, shared) + * - and : global &= shared + * - or : global |= shared + * - xor : global ^= shared + * + * Usage: + * ./ublkred_g_s [-n ] [-i ] [-r ] + * + * -n : number of elements (default: 1024) + * -i : number of reduction iterations in-kernel (default: 1) + * -r : one of {add,min,max,and,or,xor,all} (default: add) + */ + +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, \ + "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90 or newer for TMA bulk operations."); +#endif + +static constexpr size_t buf_len = 1024; + +// Base values used for each reduction kind. +static constexpr int32_t BASE_ADD = 100; +static constexpr int32_t BASE_MIN = 1000000; +static constexpr int32_t BASE_MAX = -1000000; +static constexpr int32_t BASE_AND = ~0; +static constexpr int32_t BASE_OR = 0; +static constexpr int32_t BASE_XOR = 0xAAAAAAAA; + +enum class ReductionKind : int { + ADD = 0, + MIN, + MAX, + AND, + OR, + XOR +}; + +static const char* reduction_kind_name(ReductionKind kind) +{ + switch (kind) { + case ReductionKind::ADD: return "ADD"; + case ReductionKind::MIN: return "MIN"; + case ReductionKind::MAX: return "MAX"; + case ReductionKind::AND: return "AND"; + case ReductionKind::OR: return "OR"; + case ReductionKind::XOR: return "XOR"; + default: return "UNKNOWN"; + } +} + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UBLKRED_G_S(int32_t *data, int run_iters, ReductionKind kind) +{ + // Shared memory buffer. The destination shared memory buffer of + // a bulk operation should be 16-byte aligned. + __shared__ alignas(16) int32_t smem_data[buf_len]; + + size_t offset = blockIdx.x * blockDim.x; + + // Generate the test pattern (0, 1, 2, ...) in shared memory + for (int i = threadIdx.x; i < (int)buf_len; i += blockDim.x) { + smem_data[i] = offset + i; + } + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + for (int iter = 0; iter < run_iters; iter++) { + if (threadIdx.x == 0) { + switch (kind) { + case ReductionKind::ADD: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_add, + data + offset, smem_data, sizeof(smem_data)); + break; + case ReductionKind::MIN: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_min, + data + offset, smem_data, sizeof(smem_data)); + break; + case ReductionKind::MAX: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_max, + data + offset, smem_data, sizeof(smem_data)); + break; + case ReductionKind::AND: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_and_op, + data + offset, smem_data, sizeof(smem_data)); + break; + case ReductionKind::OR: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_or_op, + data + offset, smem_data, sizeof(smem_data)); + break; + case ReductionKind::XOR: + ptx::cp_reduce_async_bulk( + ptx::space_global, + ptx::space_shared, + ptx::op_xor_op, + data + offset, smem_data, sizeof(smem_data)); + break; + } + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Host-side helpers +// ============================================================================ +static int32_t base_for_kind(ReductionKind kind) +{ + switch (kind) { + case ReductionKind::ADD: return BASE_ADD; + case ReductionKind::MIN: return BASE_MIN; + case ReductionKind::MAX: return BASE_MAX; + case ReductionKind::AND: return BASE_AND; + case ReductionKind::OR: return BASE_OR; + case ReductionKind::XOR: return BASE_XOR; + default: return 0; + } +} + +static void init_global(int32_t *h_data, int n, ReductionKind kind) +{ + int32_t base = base_for_kind(kind); + for (int i = 0; i < n; i++) { + h_data[i] = base; + } +} + +static int32_t expected_value_for_index(int idx, ReductionKind kind, int run_iters) +{ + const int32_t i = (int32_t)idx; + + switch (kind) { + case ReductionKind::ADD: + // After run_iters iterations: base + run_iters * i + return BASE_ADD + (int32_t)(run_iters * i); + + case ReductionKind::MIN: + // min(BASE_MIN, i) + return (i < BASE_MIN) ? i : BASE_MIN; + + case ReductionKind::MAX: + // max(BASE_MAX, i) + return (i > BASE_MAX) ? i : BASE_MAX; + + case ReductionKind::AND: + // BASE_AND & i (idempotent across iterations) + return (BASE_AND & i); + + case ReductionKind::OR: + // BASE_OR | i (idempotent across iterations) + return (BASE_OR | i); + + case ReductionKind::XOR: + // XOR toggles each iteration + if ((run_iters & 1) == 0) { + return BASE_XOR; + } else { + return BASE_XOR ^ i; + } + } + return 0; +} + +static bool verify_reduction(int32_t *h_result, int n, ReductionKind kind, int run_iters) +{ + int mismatch_count = 0; + int first_mismatch_idx = -1; + int32_t first_got = 0; + int32_t first_expected = 0; + + for (int idx = 0; idx < n; idx++) { + int32_t expected = expected_value_for_index(idx, kind, run_iters); + if (h_result[idx] != expected) { + mismatch_count++; + if (first_mismatch_idx < 0) { + first_mismatch_idx = idx; + first_got = h_result[idx]; + first_expected = expected; + } + } + } + + if (mismatch_count > 0) { + printf(" First mismatch at index %d: got %d, expected %d\n", + first_mismatch_idx, first_got, first_expected); + printf(" FAIL: %d/%d elements have wrong values\n", mismatch_count, n); + return false; + } + + return true; +} + +static bool run_single_reduction(ReductionKind kind, int n, int run_iters) +{ + size_t bytes = (size_t)n * sizeof(int32_t); + uint32_t blockSize = 1024; + uint32_t gridSize = (uint32_t)ceil((float)n / blockSize); + + int32_t *h_data = (int32_t *)malloc(bytes); + if (!h_data) { + fprintf(stderr, "Host allocation failed\n"); + return false; + } + + int32_t *d_a = nullptr; + cudaMalloc(&d_a, bytes); + + // Initialize global memory on host and copy to device + init_global(h_data, n, kind); + cudaMemcpy(d_a, h_data, bytes, cudaMemcpyHostToDevice); + + // Run kernel + CUDA_SAFECALL((test_UBLKRED_G_S<<>>(d_a, run_iters, kind))); + cudaDeviceSynchronize(); + + // Copy result back + cudaMemcpy(h_data, d_a, bytes, cudaMemcpyDeviceToHost); + + // Verify + printf("=== UBLKRED_G_S (TMA Bulk Reduction: %s) ===\n", reduction_kind_name(kind)); + bool pass = verify_reduction(h_data, n, kind, run_iters); + printf("RESULT: %s\n\n", pass ? "PASS" : "FAIL"); + + free(h_data); + cudaFree(d_a); + + return pass; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + int n = 1024; + int run_iters = 1; + const char *reduction_str = "add"; + bool test_all = false; + + int opt; + while ((opt = getopt(argc, argv, "n:i:r:")) != -1) { + switch (opt) { + case 'n': + n = atoi(optarg); + break; + case 'i': + run_iters = atoi(optarg); + break; + case 'r': + reduction_str = optarg; + break; + default: + fprintf(stderr, + "Usage: %s [-n ] [-i ] [-r ]\n", + argv[0]); + fprintf(stderr, + " -r where is one of: add, min, max, and, or, xor, all\n"); + return 1; + } + } + + ReductionKind selected = ReductionKind::ADD; + + if (strcmp(reduction_str, "add") == 0) { + selected = ReductionKind::ADD; + } else if (strcmp(reduction_str, "min") == 0) { + selected = ReductionKind::MIN; + } else if (strcmp(reduction_str, "max") == 0) { + selected = ReductionKind::MAX; + } else if (strcmp(reduction_str, "and") == 0) { + selected = ReductionKind::AND; + } else if (strcmp(reduction_str, "or") == 0) { + selected = ReductionKind::OR; + } else if (strcmp(reduction_str, "xor") == 0) { + selected = ReductionKind::XOR; + } else if (strcmp(reduction_str, "all") == 0) { + test_all = true; + } else { + fprintf(stderr, + "Invalid reduction type '%s'. Expected one of: add, min, max, and, or, xor, all\n", + reduction_str); + return 1; + } + + bool overall_pass = true; + + if (test_all) { + ReductionKind kinds[] = { + ReductionKind::ADD, + ReductionKind::MIN, + ReductionKind::MAX, + ReductionKind::AND, + ReductionKind::OR, + ReductionKind::XOR + }; + const int num_kinds = (int)(sizeof(kinds) / sizeof(kinds[0])); + + for (int i = 0; i < num_kinds; i++) { + bool pass = run_single_reduction(kinds[i], n, run_iters); + overall_pass = overall_pass && pass; + } + } else { + overall_pass = run_single_reduction(selected, n, run_iters); + } + + return overall_pass ? 0 : 1; +} + diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/Makefile new file mode 100644 index 000000000..f0e572d31 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/Makefile @@ -0,0 +1,28 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# TMA tensor multicast microbenchmarks. +ALL_BENCHES := tma_tensor_multicast tma_tensor_multicast_mask tma_tensor_multicast_4cta + +# TMA multicast requires Hopper-class architectures. +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Generate both SASS and PTX. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast.cu new file mode 100644 index 000000000..6a476e605 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast.cu @@ -0,0 +1,403 @@ +/* + * TMA Tensor Load with Multicast Microbenchmark + * ============================================== + * Tests: cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster + * + * This benchmark verifies TMA tensor load with multicast to multiple CTAs + * within a cluster. Each cluster of CLUSTER_DIM_X CTAs loads a single 2D tile + * from global memory. CTA 0 in each cluster issues the multicast TMA + * instruction, which delivers the tile data to ALL CTAs' shared memories + * simultaneously. Each CTA then copies its shared memory to a unique region + * of a destination buffer for host-side verification. + * + * Usage: ./tma_tensor_multicast [-w ] [-h ] [-i ] + * + * Verification Strategy: + * ---------------------- + * - d_mat (global tensor): initialized with unique values (1, 2, 3, ...) + * - d_dst (verification buffer): poisoned with POISON_VALUE (0xDEADBEEF) + * - Each cluster of 2 CTAs loads the SAME tile via multicast TMA + * - Each CTA copies its smem to a unique d_dst region (standard stores) + * - Verification checks: + * 1. Correctness: every block's d_dst region matches the expected tile + * 2. Multicast: CTAs within the same cluster have identical data + * - PASS: all blocks correct AND cluster-mates identical + * - FAIL_NOP: d_dst still contains poison (TMA had no effect) + * - FAIL_MISMATCH: values differ from expected + * - FAIL_MULTICAST: CTAs in same cluster received different data + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +using barrier = cuda::barrier; +namespace cg = cooperative_groups; +namespace ptx = cuda::ptx; + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) \ + { \ + fprintf( \ + stderr, \ + "Cuda error in function '%s' file '%s' in line %i : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +#define GMEM_WIDTH 128 +#define GMEM_HEIGHT 64 +#define SMEM_WIDTH 32 +#define SMEM_HEIGHT 32 +#define CLUSTER_DIM_X 2 +#define DEFAULT_RUN_ITERS 1 +#define POISON_VALUE 0xDEADBEEF + +// ============================================================================ +// Kernel: TMA tensor load with multicast::cluster +// +// Grid layout (example with defaults): +// grid_dim = (num_tiles_x * CLUSTER_DIM_X, num_tiles_y) +// = (4*2, 2) = (8, 2) = 16 blocks total +// Cluster (i,j) contains blocks (2*i, j) and (2*i+1, j) +// Both blocks in a cluster load the SAME tile from global memory +// +// Each cluster loads tile at: +// tile_x = cluster_idx_x * SMEM_WIDTH +// tile_y = cluster_idx_y * SMEM_HEIGHT +// +// CTA with cluster_rank==0 issues the multicast TMA instruction. +// All CTAs in the cluster receive the data in their shared memory. +// Each CTA writes its smem to d_dst at a unique offset for verification. +// ============================================================================ + +__global__ void __cluster_dims__(CLUSTER_DIM_X, 1, 1) +test_tma_multicast_kernel( + const __grid_constant__ CUtensorMap tensor_map, + int *d_dst, + int run_iters) +{ + __shared__ alignas(128) int smem_buffer[SMEM_HEIGHT][SMEM_WIDTH]; + +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + cg::cluster_group cluster = cg::this_cluster(); + unsigned int cluster_rank = cluster.block_rank(); + + unsigned int cluster_idx_x = blockIdx.x / CLUSTER_DIM_X; + int tile_x = cluster_idx_x * SMEM_WIDTH; + int tile_y = blockIdx.y * SMEM_HEIGHT; + + if (threadIdx.x == 0 && threadIdx.y == 0) { + init(&bar, blockDim.x * blockDim.y); + ptx::fence_proxy_async(ptx::space_shared); + } + __syncthreads(); + + if (blockIdx.x == 0 && blockIdx.y == 0 && + threadIdx.x == 0 && threadIdx.y == 0) { + printf("TensorMap address: %p\n", &tensor_map); + printf("Cluster rank: %u, tile: (%d, %d)\n", cluster_rank, tile_x, tile_y); + } + + for (int iter = 0; iter < run_iters; iter++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + // Thread 0: arrive with expected transaction bytes + auto token = cuda::device::barrier_arrive_tx(bar, 1, sizeof(smem_buffer)); + + if (cluster_rank == 0) { + uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem_buffer)); + uint32_t mbar_addr = static_cast(__cvta_generic_to_shared(&bar)); + uint16_t mcast_mask = static_cast((1u << CLUSTER_DIM_X) - 1u); + + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster" + " [%0], [%1, {%2, %3}], [%4], %5;" + : + : "r"(smem_addr), "l"(&tensor_map), "r"(tile_x), "r"(tile_y), + "r"(mbar_addr), "h"(mcast_mask) + : "memory" + ); + } + + bar.wait(std::move(token)); + } else { + bar.arrive_and_wait(); + } + } + + // Each block writes its smem to a unique region of d_dst + unsigned int block_flat_idx = blockIdx.x + blockIdx.y * gridDim.x; + int dst_offset = block_flat_idx * SMEM_HEIGHT * SMEM_WIDTH; + d_dst[dst_offset + threadIdx.y * SMEM_WIDTH + threadIdx.x] = + smem_buffer[threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Driver API helper +// ============================================================================ + +PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult driver_status; + void *cuTensorMapEncodeTiled_ptr = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion( + "cuTensorMapEncodeTiled", &cuTensorMapEncodeTiled_ptr, + 12000, cudaEnableDefault, &driver_status)); + assert(driver_status == cudaDriverEntryPointSuccess); + return reinterpret_cast(cuTensorMapEncodeTiled_ptr); +} + +// ============================================================================ +// Verification +// ============================================================================ + +enum verify_result_t { + VERIFY_PASS, + VERIFY_FAIL_NOP, + VERIFY_FAIL_MISMATCH, + VERIFY_FAIL_MULTICAST +}; + +static const char* verify_result_str(verify_result_t r) { + switch (r) { + case VERIFY_PASS: return "PASS"; + case VERIFY_FAIL_NOP: return "FAIL (NOP - TMA had no effect)"; + case VERIFY_FAIL_MISMATCH: return "FAIL (MISMATCH - wrong values)"; + case VERIFY_FAIL_MULTICAST: return "FAIL (MULTICAST - CTAs in cluster got different data)"; + default: return "UNKNOWN"; + } +} + +int main(int argc, char *argv[]) +{ + uint64_t width = GMEM_WIDTH; + uint64_t height = GMEM_HEIGHT; + int run_iters = DEFAULT_RUN_ITERS; + + int opt; + while ((opt = getopt(argc, argv, "w:h:i:")) != -1) { + switch (opt) { + case 'w': width = uint64_t(atoi(optarg)); break; + case 'h': height = uint64_t(atoi(optarg)); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-w ] [-h ] [-i ]\n", argv[0]); + fprintf(stderr, " Tile: %dx%d, Cluster: %d CTAs\n", + SMEM_WIDTH, SMEM_HEIGHT, CLUSTER_DIM_X); + return 1; + } + } + + uint64_t height_stride = ((height + (SMEM_HEIGHT - 1)) / SMEM_HEIGHT) * SMEM_HEIGHT; + uint64_t width_stride = ((width + (SMEM_WIDTH - 1)) / SMEM_WIDTH) * SMEM_WIDTH; + + printf("=== TMA Tensor Multicast Benchmark ===\n"); + printf("Global tensor: %lu x %lu (stride: %lu x %lu)\n", + width, height, width_stride, height_stride); + printf("Tile size: %d x %d\n", SMEM_WIDTH, SMEM_HEIGHT); + printf("Cluster: %d CTAs in x\n", CLUSTER_DIM_X); + printf("Iterations: %d\n", run_iters); + + // ---- Allocate and initialize source tensor ---- + size_t mat_bytes = height_stride * width_stride * sizeof(int); + int *mat = (int *)malloc(mat_bytes); + int *d_mat; + cudaMalloc(&d_mat, mat_bytes); + + uint64_t val = 1; + for (uint64_t r = 0; r < height_stride; r++) { + for (uint64_t c = 0; c < width_stride; c++) { + mat[r * width_stride + c] = (r < height && c < width) ? (int)(val++) : 0; + } + } + cudaMemcpy(d_mat, mat, mat_bytes, cudaMemcpyHostToDevice); + + // ---- Create tensor map ---- + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 2; + uint64_t size[rank] = {width, height}; + uint64_t stride[rank - 1] = {width_stride * sizeof(int)}; + uint32_t box_size[rank] = {SMEM_WIDTH, SMEM_HEIGHT}; + uint32_t elem_stride[rank] = {1, 1}; + + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + CUresult res = cuTensorMapEncodeTiled( + &tensor_map, + CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, + d_mat, + size, + stride, + box_size, + elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + if (res != CUDA_SUCCESS) { + fprintf(stderr, "cuTensorMapEncodeTiled failed: %d\n", res); + return 1; + } + + // ---- Grid / block configuration ---- + // Each cluster handles one tile; cluster has CLUSTER_DIM_X blocks + uint32_t num_tiles_x = width_stride / SMEM_WIDTH; + uint32_t num_tiles_y = height_stride / SMEM_HEIGHT; + dim3 grid_dim(num_tiles_x * CLUSTER_DIM_X, num_tiles_y); + dim3 block_dim(SMEM_WIDTH, SMEM_HEIGHT); + uint32_t total_blocks = grid_dim.x * grid_dim.y; + + printf("Grid: (%u, %u), Block: (%u, %u)\n", + grid_dim.x, grid_dim.y, block_dim.x, block_dim.y); + printf("Clusters: %u x %u = %u total (%u blocks/cluster)\n", + num_tiles_x, num_tiles_y, num_tiles_x * num_tiles_y, CLUSTER_DIM_X); + printf("Total blocks: %u\n\n", total_blocks); + + // ---- Allocate and poison destination buffer ---- + size_t dst_elements = (size_t)total_blocks * SMEM_HEIGHT * SMEM_WIDTH; + size_t dst_bytes = dst_elements * sizeof(int); + int *h_dst = (int *)malloc(dst_bytes); + int *d_dst; + cudaMalloc(&d_dst, dst_bytes); + + for (size_t idx = 0; idx < dst_elements; idx++) + h_dst[idx] = (int)POISON_VALUE; + cudaMemcpy(d_dst, h_dst, dst_bytes, cudaMemcpyHostToDevice); + + // ---- Launch kernel ---- + CUDA_SAFECALL((test_tma_multicast_kernel<<>>( + tensor_map, d_dst, run_iters))); + CUDA_SAFECALL(cudaDeviceSynchronize()); + CUDA_SAFECALL(cudaMemcpy(h_dst, d_dst, dst_bytes, cudaMemcpyDeviceToHost)); + + // ---- Verify correctness ---- + int poison_count = 0; + int mismatch_count = 0; + + for (uint32_t bx = 0; bx < grid_dim.x; bx++) { + for (uint32_t by = 0; by < grid_dim.y; by++) { + uint32_t cluster_idx_x = bx / CLUSTER_DIM_X; + int tile_x = cluster_idx_x * SMEM_WIDTH; + int tile_y = by * SMEM_HEIGHT; + + size_t dst_offset = (size_t)(bx + by * grid_dim.x) * SMEM_HEIGHT * SMEM_WIDTH; + + for (int r = 0; r < SMEM_HEIGHT; r++) { + for (int c = 0; c < SMEM_WIDTH; c++) { + int got = h_dst[dst_offset + r * SMEM_WIDTH + c]; + int expected = mat[(tile_y + r) * width_stride + (tile_x + c)]; + + if (got == (int)POISON_VALUE && expected != (int)POISON_VALUE) { + if (poison_count == 0) + printf(" Poison at block(%u,%u)[%d][%d]: " + "got 0x%08X, expected 0x%08X\n", + bx, by, r, c, (unsigned)got, (unsigned)expected); + poison_count++; + } else if (got != expected) { + if (mismatch_count == 0) + printf(" Mismatch at block(%u,%u)[%d][%d]: " + "got 0x%08X, expected 0x%08X\n", + bx, by, r, c, (unsigned)got, (unsigned)expected); + mismatch_count++; + } + } + } + } + } + + // ---- Verify multicast: CTAs in same cluster must have identical data ---- + int multicast_fail = 0; + + for (uint32_t by = 0; by < grid_dim.y; by++) { + for (uint32_t ci = 0; ci < num_tiles_x; ci++) { + uint32_t bx0 = ci * CLUSTER_DIM_X; + size_t off0 = (size_t)(bx0 + by * grid_dim.x) * SMEM_HEIGHT * SMEM_WIDTH; + + for (uint32_t rank = 1; rank < CLUSTER_DIM_X; rank++) { + size_t off1 = (size_t)(bx0 + rank + by * grid_dim.x) + * SMEM_HEIGHT * SMEM_WIDTH; + for (int i = 0; i < SMEM_HEIGHT * SMEM_WIDTH; i++) { + if (h_dst[off0 + i] != h_dst[off1 + i]) { + if (multicast_fail == 0) + printf(" Multicast mismatch: cluster(%u,%u) " + "rank0=0x%08X rank%u=0x%08X at elem %d\n", + ci, by, + (unsigned)h_dst[off0 + i], rank, + (unsigned)h_dst[off1 + i], i); + multicast_fail++; + } + } + } + } + } + + // ---- Report results ---- + verify_result_t result = VERIFY_PASS; + if (poison_count > 0) result = VERIFY_FAIL_NOP; + else if (mismatch_count > 0) result = VERIFY_FAIL_MISMATCH; + else if (multicast_fail > 0) result = VERIFY_FAIL_MULTICAST; + + printf("\n"); + printf("================================================================================\n"); + printf("TEST: TMA Tensor Load with Multicast (Global -> Shared::Cluster)\n"); + printf(" Instruction: cp.async.bulk.tensor.2d.shared::cluster.global\n"); + printf(" .mbarrier::complete_tx::bytes.multicast::cluster\n"); + printf("================================================================================\n"); + printf("RESULT: %s\n", verify_result_str(result)); + if (result == VERIFY_PASS) { + printf(" All %u blocks received correct tile data\n", total_blocks); + printf(" Multicast verified: CTAs in each cluster have identical data\n"); + } + if (poison_count > 0) + printf(" %d elements still contain poison\n", poison_count); + if (mismatch_count > 0) + printf(" %d elements have wrong values\n", mismatch_count); + if (multicast_fail > 0) + printf(" %d elements differ between CTAs in same cluster\n", multicast_fail); + printf("================================================================================\n\n"); + + // ---- Dump raw output for debugging ---- + char filename[128]; + sprintf(filename, "tma_multicast_test_%lu_%lu.txt", height, width); + FILE *f = fopen(filename, "w"); + fprintf(f, "# TMA Multicast Test Output\n"); + fprintf(f, "# Format: block(bx,by) [row][col] = value (hex)\n\n"); + for (uint32_t bx = 0; bx < grid_dim.x; bx++) { + for (uint32_t by = 0; by < grid_dim.y; by++) { + size_t dst_offset = (size_t)(bx + by * grid_dim.x) * SMEM_HEIGHT * SMEM_WIDTH; + fprintf(f, "--- block(%u,%u) cluster_rank=%u tile=(%d,%d) ---\n", + bx, by, bx % CLUSTER_DIM_X, + (int)(bx / CLUSTER_DIM_X) * SMEM_WIDTH, + (int)(by * SMEM_HEIGHT)); + for (int r = 0; r < SMEM_HEIGHT; r++) { + for (int c = 0; c < SMEM_WIDTH; c++) { + fprintf(f, "0x%08x ", (unsigned)h_dst[dst_offset + r * SMEM_WIDTH + c]); + } + fprintf(f, "\n"); + } + fprintf(f, "\n"); + } + } + fclose(f); + printf("Values dumped to %s\n", filename); + + // ---- Cleanup ---- + free(mat); + free(h_dst); + cudaFree(d_mat); + cudaFree(d_dst); + + return (result == VERIFY_PASS) ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_4cta.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_4cta.cu new file mode 100644 index 000000000..32fd4db84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_4cta.cu @@ -0,0 +1,341 @@ +/* + * TMA Tensor Multicast 4-CTA Cluster Test + * ======================================== + * Tests: cp.async.bulk.tensor.2d.shared::cluster.global + * .mbarrier::complete_tx::bytes.multicast::cluster + * + * Minimal configuration: 1 cluster, 4 CTAs, 1 tile (32x32 ints). + * The multicast mask is a runtime parameter (4-bit), enabling targeted + * testing of wider fan-out multicast patterns. + * + * Usage: ./tma_tensor_multicast_4cta [-m ] + * -m 1 (0b0001): only rank 0 + * -m 2 (0b0010): only rank 1 + * -m 4 (0b0100): only rank 2 + * -m 8 (0b1000): only rank 3 (farthest from issuer) + * -m 5 (0b0101): ranks 0 and 2 (alternating) + * -m 10 (0b1010): ranks 1 and 3 (non-issuing CTAs only) + * -m 14 (0b1110): all except issuer + * -m 15 (0b1111): broadcast to all 4 + * + * Verification Strategy (same as 2-CTA variant): + * ----------------------------------------------- + * - Shared memory poisoned with 0xDEADBEEF before TMA + * - CTAs IN mask: mbarrier expects TMA bytes, waits for delivery + * - CTAs NOT in mask: mbarrier expects 0 bytes, completes immediately + * - In-mask CTAs should have tile data; not-in-mask should have POISON + * + * Notable 4-CTA test patterns: + * mask=0b1000 (only rank 3): Data must traverse 3 hops from issuer. + * mask=0b1110 (all except issuer): Issuer sends but keeps nothing. + * mask=0b0101 (alternating): Non-contiguous subset receives. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +using barrier = cuda::barrier; +namespace cg = cooperative_groups; +namespace ptx = cuda::ptx; + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) \ + { \ + fprintf( \ + stderr, \ + "Cuda error in function '%s' file '%s' in line %i : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +#define TILE_DIM 32 +#define CLUSTER_DIM_X 4 +#define POISON_VALUE 0xDEADBEEF + +// ============================================================================ +// Kernel +// +// Layout: grid=(4,1), block=(32,32), cluster_dims=(4,1,1) +// -> 1 cluster containing CTA ranks 0, 1, 2, 3 +// -> All CTAs address the same tile at origin (0,0) +// ============================================================================ + +__global__ void __cluster_dims__(CLUSTER_DIM_X, 1, 1) +test_tma_multicast_4cta_kernel( + const __grid_constant__ CUtensorMap tensor_map, + int *d_dst, + uint16_t mcast_mask) +{ + __shared__ alignas(128) int smem[TILE_DIM][TILE_DIM]; + +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + cg::cluster_group cluster = cg::this_cluster(); + unsigned int rank = cluster.block_rank(); + bool in_mask = (mcast_mask >> rank) & 1; + + smem[threadIdx.y][threadIdx.x] = (int)POISON_VALUE; + + if (threadIdx.x == 0 && threadIdx.y == 0) { + init(&bar, blockDim.x * blockDim.y); + ptx::fence_proxy_async(ptx::space_shared); + } + __syncthreads(); + + if (threadIdx.x == 0 && threadIdx.y == 0) { + printf("[rank %u] in_mask=%d, blockIdx=(%u,%u)\n", + rank, (int)in_mask, blockIdx.x, blockIdx.y); + } + + if (threadIdx.x == 0 && threadIdx.y == 0) { + barrier::arrival_token token; + if (in_mask) { + token = cuda::device::barrier_arrive_tx(bar, 1, sizeof(smem)); + } else { + token = bar.arrive(); + } + + if (rank == 0) { + uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem)); + uint32_t mbar_addr = static_cast(__cvta_generic_to_shared(&bar)); + + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global" + ".mbarrier::complete_tx::bytes.multicast::cluster" + " [%0], [%1, {%2, %3}], [%4], %5;" + : + : "r"(smem_addr), "l"(&tensor_map), + "r"(0), "r"(0), + "r"(mbar_addr), "h"(mcast_mask) + : "memory" + ); + } + + bar.wait(std::move(token)); + } else { + bar.arrive_and_wait(); + } + +__syncthreads(); + + int dst_base = rank * TILE_DIM * TILE_DIM; + d_dst[dst_base + threadIdx.y * TILE_DIM + threadIdx.x] = + smem[threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Driver API helper +// ============================================================================ + +PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult driver_status; + void *cuTensorMapEncodeTiled_ptr = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion( + "cuTensorMapEncodeTiled", &cuTensorMapEncodeTiled_ptr, + 12000, cudaEnableDefault, &driver_status)); + assert(driver_status == cudaDriverEntryPointSuccess); + return reinterpret_cast(cuTensorMapEncodeTiled_ptr); +} + +// ============================================================================ +// Main +// ============================================================================ + +int main(int argc, char *argv[]) +{ + uint16_t mcast_mask = 0xF; + + int opt; + while ((opt = getopt(argc, argv, "m:")) != -1) { + switch (opt) { + case 'm': mcast_mask = (uint16_t)atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-m ]\n", argv[0]); + fprintf(stderr, " 4-bit mask for 4 CTAs in cluster:\n"); + fprintf(stderr, " -m 1 (0b0001): only rank 0\n"); + fprintf(stderr, " -m 2 (0b0010): only rank 1\n"); + fprintf(stderr, " -m 4 (0b0100): only rank 2\n"); + fprintf(stderr, " -m 8 (0b1000): only rank 3\n"); + fprintf(stderr, " -m 5 (0b0101): ranks 0,2\n"); + fprintf(stderr, " -m 10 (0b1010): ranks 1,3\n"); + fprintf(stderr, " -m 14 (0b1110): all except issuer\n"); + fprintf(stderr, " -m 15 (0b1111): broadcast all\n"); + return 1; + } + } + + mcast_mask &= (1u << CLUSTER_DIM_X) - 1; + if (mcast_mask == 0) { + fprintf(stderr, "Error: mask=0 is invalid (no destination CTA)\n"); + return 1; + } + + printf("=== TMA Multicast 4-CTA Cluster Test ===\n"); + printf("Cluster: %d CTAs, Tile: %dx%d ints\n", CLUSTER_DIM_X, TILE_DIM, TILE_DIM); + printf("Multicast mask: 0x%X (0b", mcast_mask); + for (int i = CLUSTER_DIM_X - 1; i >= 0; i--) + printf("%d", (mcast_mask >> i) & 1); + printf(")\n"); + for (int i = 0; i < CLUSTER_DIM_X; i++) + printf(" Rank %d: %s\n", i, + (mcast_mask >> i) & 1 ? "RECEIVES data" : "does NOT receive (expects poison)"); + printf("\n"); + + // ---- Source tensor: 1 tile ---- + const uint64_t width = TILE_DIM; + const uint64_t height = TILE_DIM; + size_t mat_bytes = width * height * sizeof(int); + int *mat = (int *)malloc(mat_bytes); + int *d_mat; + cudaMalloc(&d_mat, mat_bytes); + + for (uint64_t i = 0; i < width * height; i++) + mat[i] = (int)(i + 1); + cudaMemcpy(d_mat, mat, mat_bytes, cudaMemcpyHostToDevice); + + // ---- Tensor map ---- + CUtensorMap tensor_map{}; + constexpr uint32_t tma_rank = 2; + uint64_t size[tma_rank] = {width, height}; + uint64_t stride[tma_rank - 1] = {width * sizeof(int)}; + uint32_t box_size[tma_rank] = {TILE_DIM, TILE_DIM}; + uint32_t elem_stride[tma_rank] = {1, 1}; + + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + CUresult res = cuTensorMapEncodeTiled( + &tensor_map, + CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + tma_rank, + d_mat, + size, + stride, + box_size, + elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + if (res != CUDA_SUCCESS) { + fprintf(stderr, "cuTensorMapEncodeTiled failed: %d\n", res); + return 1; + } + + // ---- Destination buffer: 4 regions, each TILE_DIM*TILE_DIM ---- + size_t dst_elements = (size_t)CLUSTER_DIM_X * TILE_DIM * TILE_DIM; + size_t dst_bytes = dst_elements * sizeof(int); + int *h_dst = (int *)malloc(dst_bytes); + int *d_dst; + cudaMalloc(&d_dst, dst_bytes); + + for (size_t i = 0; i < dst_elements; i++) + h_dst[i] = (int)POISON_VALUE; + cudaMemcpy(d_dst, h_dst, dst_bytes, cudaMemcpyHostToDevice); + + // ---- Launch: 1 cluster = 4 blocks ---- + dim3 grid(CLUSTER_DIM_X, 1); + dim3 block(TILE_DIM, TILE_DIM); + + printf("Grid: (%u, %u), Block: (%u, %u)\n", grid.x, grid.y, block.x, block.y); + printf("Launching 1 cluster of %d CTAs...\n\n", CLUSTER_DIM_X); + + CUDA_SAFECALL((test_tma_multicast_4cta_kernel<<>>( + tensor_map, d_dst, mcast_mask))); + CUDA_SAFECALL(cudaDeviceSynchronize()); + CUDA_SAFECALL(cudaMemcpy(h_dst, d_dst, dst_bytes, cudaMemcpyDeviceToHost)); + + // ---- Verify each rank ---- + int all_pass = 1; + + for (int r = 0; r < CLUSTER_DIM_X; r++) { + bool in_mask = (mcast_mask >> r) & 1; + int base = r * TILE_DIM * TILE_DIM; + int poison_count = 0, mismatch_count = 0, correct_count = 0; + + for (int i = 0; i < TILE_DIM * TILE_DIM; i++) { + int got = h_dst[base + i]; + int expected = in_mask ? mat[i] : (int)POISON_VALUE; + + if (got == expected) { + correct_count++; + } else if (got == (int)POISON_VALUE && in_mask) { + if (poison_count == 0) + printf(" Rank %d: elem %d still POISON (got 0x%08X, expected 0x%08X)\n", + r, i, (unsigned)got, (unsigned)expected); + poison_count++; + } else { + if (mismatch_count == 0) + printf(" Rank %d: elem %d MISMATCH (got 0x%08X, expected 0x%08X)\n", + r, i, (unsigned)got, (unsigned)expected); + mismatch_count++; + } + } + + int total = TILE_DIM * TILE_DIM; + if (correct_count == total) { + if (in_mask) + printf(" Rank %d: PASS - received correct tile data (%d/%d)\n", + r, correct_count, total); + else + printf(" Rank %d: PASS - correctly still has poison (%d/%d)\n", + r, correct_count, total); + } else { + all_pass = 0; + if (in_mask && poison_count > 0) + printf(" Rank %d: FAIL - NOP detected, %d/%d elements still poison\n", + r, poison_count, total); + else + printf(" Rank %d: FAIL - %d/%d elements wrong (%d poison, %d mismatch)\n", + r, poison_count + mismatch_count, total, poison_count, mismatch_count); + } + } + + printf("\n"); + printf("================================================================================\n"); + printf("TEST: TMA Tensor Multicast 4-CTA Cluster (mask=0x%X)\n", mcast_mask); + printf(" Instruction: cp.async.bulk.tensor.2d.shared::cluster.global\n"); + printf(" .mbarrier::complete_tx::bytes.multicast::cluster\n"); + printf("================================================================================\n"); + printf("RESULT: %s\n", all_pass ? "**PASS**" : "**FAIL**"); + printf("================================================================================\n\n"); + + // ---- Dump for debugging ---- + char filename[128]; + sprintf(filename, "tma_multicast_4cta_mask_0x%X.txt", mcast_mask); + FILE *f = fopen(filename, "w"); + fprintf(f, "# Mask=0x%X Cluster=%d CTAs Tile=%dx%d\n\n", + mcast_mask, CLUSTER_DIM_X, TILE_DIM, TILE_DIM); + for (int r = 0; r < CLUSTER_DIM_X; r++) { + bool in_mask = (mcast_mask >> r) & 1; + fprintf(f, "--- Rank %d (%s) ---\n", r, + in_mask ? "in mask" : "NOT in mask"); + int base = r * TILE_DIM * TILE_DIM; + for (int row = 0; row < TILE_DIM; row++) { + for (int col = 0; col < TILE_DIM; col++) + fprintf(f, "0x%08x ", (unsigned)h_dst[base + row * TILE_DIM + col]); + fprintf(f, "\n"); + } + fprintf(f, "\n"); + } + fclose(f); + printf("Values dumped to %s\n", filename); + + free(mat); + free(h_dst); + cudaFree(d_mat); + cudaFree(d_dst); + + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_mask.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_mask.cu new file mode 100644 index 000000000..a1fd9fa65 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor_multicast/tma_tensor_multicast_mask.cu @@ -0,0 +1,344 @@ +/* + * TMA Tensor Multicast Mask Test + * ============================== + * Tests: cp.async.bulk.tensor.2d.shared::cluster.global + * .mbarrier::complete_tx::bytes.multicast::cluster + * + * Minimal configuration: 1 cluster, 2 CTAs, 1 tile (32x32 ints). + * The multicast mask is a runtime parameter, enabling targeted testing + * of which CTAs receive data. + * + * Usage: ./tma_tensor_multicast_mask [-m ] + * -m 1 (0b01): only rank 0 receives data + * -m 2 (0b10): only rank 1 receives data + * -m 3 (0b11): both ranks receive (broadcast) + * + * Verification Strategy: + * ---------------------- + * - Shared memory is initialized to POISON (0xDEADBEEF) before TMA + * - CTA 0 issues multicast TMA with the given mask + * - CTAs IN the mask: mbarrier expects TMA bytes, waits for delivery + * - CTAs NOT in the mask: mbarrier expects 0 bytes, completes immediately + * - After TMA: + * - In-mask CTAs: smem should contain tile data -> PASS + * - Not-in-mask CTAs: smem should still be POISON -> PASS + * - Each CTA copies smem to its own d_dst region for host-side checking + * + * Key mask=0b10 case: + * Rank 0 ISSUES the instruction but does NOT receive data. + * Rank 1 does NOT issue the instruction but DOES receive data. + * This tests that the simulator correctly routes data to a remote CTA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +using barrier = cuda::barrier; +namespace cg = cooperative_groups; +namespace ptx = cuda::ptx; + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) \ + { \ + fprintf( \ + stderr, \ + "Cuda error in function '%s' file '%s' in line %i : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +#define TILE_DIM 32 +#define CLUSTER_DIM_X 2 +#define POISON_VALUE 0xDEADBEEF + +// ============================================================================ +// Kernel +// +// Layout: grid=(2,1), block=(32,32), cluster_dims=(2,1,1) +// -> 1 cluster containing CTA rank 0 and CTA rank 1 +// -> Both CTAs address the same tile at origin (0,0) +// +// Flow per CTA: +// 1. Poison smem with POISON_VALUE +// 2. If in_mask: barrier_arrive_tx with sizeof(smem) expected bytes +// If not in_mask: barrier arrive with 0 expected bytes +// 3. Rank 0 issues multicast TMA (regardless of whether rank 0 is in mask) +// 4. All threads wait on barrier +// 5. Copy smem -> d_dst for host verification +// ============================================================================ + +__global__ void __cluster_dims__(CLUSTER_DIM_X, 1, 1) +test_tma_multicast_mask_kernel( + const __grid_constant__ CUtensorMap tensor_map, + int *d_dst, + uint16_t mcast_mask) +{ + __shared__ alignas(128) int smem[TILE_DIM][TILE_DIM]; + +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + cg::cluster_group cluster = cg::this_cluster(); + unsigned int rank = cluster.block_rank(); + bool in_mask = (mcast_mask >> rank) & 1; + + // Step 1: Poison smem so we can detect whether TMA wrote anything + smem[threadIdx.y][threadIdx.x] = (int)POISON_VALUE; + + if (threadIdx.x == 0 && threadIdx.y == 0) { + init(&bar, blockDim.x * blockDim.y); + ptx::fence_proxy_async(ptx::space_shared); + } + __syncthreads(); + + if (threadIdx.x == 0 && threadIdx.y == 0) { + printf("[rank %u] in_mask=%d, blockIdx=(%u,%u)\n", + rank, (int)in_mask, blockIdx.x, blockIdx.y); + } + + // Step 2 & 3: Arrive on barrier and optionally issue TMA + if (threadIdx.x == 0 && threadIdx.y == 0) { + barrier::arrival_token token; + if (in_mask) { + token = cuda::device::barrier_arrive_tx(bar, 1, sizeof(smem)); + } else { + token = bar.arrive(); + } + + if (rank == 0) { + uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem)); + uint32_t mbar_addr = static_cast(__cvta_generic_to_shared(&bar)); + + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global" + ".mbarrier::complete_tx::bytes.multicast::cluster" + " [%0], [%1, {%2, %3}], [%4], %5;" + : + : "r"(smem_addr), "l"(&tensor_map), + "r"(0), "r"(0), + "r"(mbar_addr), "h"(mcast_mask) + : "memory" + ); + } + + bar.wait(std::move(token)); + } else { + bar.arrive_and_wait(); + } + + __syncthreads(); + + // Step 4: Copy smem -> d_dst for host verification + int dst_base = rank * TILE_DIM * TILE_DIM; + d_dst[dst_base + threadIdx.y * TILE_DIM + threadIdx.x] = + smem[threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Driver API helper +// ============================================================================ + +PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult driver_status; + void *cuTensorMapEncodeTiled_ptr = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion( + "cuTensorMapEncodeTiled", &cuTensorMapEncodeTiled_ptr, + 12000, cudaEnableDefault, &driver_status)); + assert(driver_status == cudaDriverEntryPointSuccess); + return reinterpret_cast(cuTensorMapEncodeTiled_ptr); +} + +// ============================================================================ +// Main +// ============================================================================ + +int main(int argc, char *argv[]) +{ + uint16_t mcast_mask = 0x3; + + int opt; + while ((opt = getopt(argc, argv, "m:")) != -1) { + switch (opt) { + case 'm': mcast_mask = (uint16_t)atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-m ]\n", argv[0]); + fprintf(stderr, " -m 1 (0b01): only rank 0 receives\n"); + fprintf(stderr, " -m 2 (0b10): only rank 1 receives\n"); + fprintf(stderr, " -m 3 (0b11): both ranks receive\n"); + return 1; + } + } + + mcast_mask &= (1u << CLUSTER_DIM_X) - 1; + if (mcast_mask == 0) { + fprintf(stderr, "Error: mask=0 is invalid (no destination CTA)\n"); + return 1; + } + + printf("=== TMA Multicast Mask Test ===\n"); + printf("Cluster: %d CTAs, Tile: %dx%d ints\n", CLUSTER_DIM_X, TILE_DIM, TILE_DIM); + printf("Multicast mask: 0x%X (0b", mcast_mask); + for (int i = CLUSTER_DIM_X - 1; i >= 0; i--) + printf("%d", (mcast_mask >> i) & 1); + printf(")\n"); + for (int i = 0; i < CLUSTER_DIM_X; i++) + printf(" Rank %d: %s\n", i, + (mcast_mask >> i) & 1 ? "RECEIVES data" : "does NOT receive (expects poison)"); + printf("\n"); + + // ---- Source tensor: 1 tile, values 1..TILE_DIM*TILE_DIM ---- + const uint64_t width = TILE_DIM; + const uint64_t height = TILE_DIM; + size_t mat_bytes = width * height * sizeof(int); + int *mat = (int *)malloc(mat_bytes); + int *d_mat; + cudaMalloc(&d_mat, mat_bytes); + + for (uint64_t i = 0; i < width * height; i++) + mat[i] = (int)(i + 1); + cudaMemcpy(d_mat, mat, mat_bytes, cudaMemcpyHostToDevice); + + // ---- Tensor map ---- + CUtensorMap tensor_map{}; + constexpr uint32_t tma_rank = 2; + uint64_t size[tma_rank] = {width, height}; + uint64_t stride[tma_rank - 1] = {width * sizeof(int)}; + uint32_t box_size[tma_rank] = {TILE_DIM, TILE_DIM}; + uint32_t elem_stride[tma_rank] = {1, 1}; + + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + CUresult res = cuTensorMapEncodeTiled( + &tensor_map, + CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + tma_rank, + d_mat, + size, + stride, + box_size, + elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + if (res != CUDA_SUCCESS) { + fprintf(stderr, "cuTensorMapEncodeTiled failed: %d\n", res); + return 1; + } + + // ---- Destination buffer: CLUSTER_DIM_X regions, each TILE_DIM*TILE_DIM ---- + size_t dst_elements = (size_t)CLUSTER_DIM_X * TILE_DIM * TILE_DIM; + size_t dst_bytes = dst_elements * sizeof(int); + int *h_dst = (int *)malloc(dst_bytes); + int *d_dst; + cudaMalloc(&d_dst, dst_bytes); + + for (size_t i = 0; i < dst_elements; i++) + h_dst[i] = (int)POISON_VALUE; + cudaMemcpy(d_dst, h_dst, dst_bytes, cudaMemcpyHostToDevice); + + // ---- Launch: 1 cluster = CLUSTER_DIM_X blocks ---- + dim3 grid(CLUSTER_DIM_X, 1); + dim3 block(TILE_DIM, TILE_DIM); + + printf("Grid: (%u, %u), Block: (%u, %u)\n", grid.x, grid.y, block.x, block.y); + printf("Launching 1 cluster of %d CTAs...\n\n", CLUSTER_DIM_X); + + CUDA_SAFECALL((test_tma_multicast_mask_kernel<<>>( + tensor_map, d_dst, mcast_mask))); + CUDA_SAFECALL(cudaDeviceSynchronize()); + CUDA_SAFECALL(cudaMemcpy(h_dst, d_dst, dst_bytes, cudaMemcpyDeviceToHost)); + + // ---- Verify each rank ---- + int all_pass = 1; + + for (int r = 0; r < CLUSTER_DIM_X; r++) { + bool in_mask = (mcast_mask >> r) & 1; + int base = r * TILE_DIM * TILE_DIM; + int poison_count = 0, mismatch_count = 0, correct_count = 0; + + for (int i = 0; i < TILE_DIM * TILE_DIM; i++) { + int got = h_dst[base + i]; + int expected = in_mask ? mat[i] : (int)POISON_VALUE; + + if (got == expected) { + correct_count++; + } else if (got == (int)POISON_VALUE && in_mask) { + if (poison_count == 0) + printf(" Rank %d: elem %d still POISON (got 0x%08X, expected 0x%08X)\n", + r, i, (unsigned)got, (unsigned)expected); + poison_count++; + } else { + if (mismatch_count == 0) + printf(" Rank %d: elem %d MISMATCH (got 0x%08X, expected 0x%08X)\n", + r, i, (unsigned)got, (unsigned)expected); + mismatch_count++; + } + } + + int total = TILE_DIM * TILE_DIM; + if (correct_count == total) { + if (in_mask) + printf(" Rank %d: PASS - received correct tile data (%d/%d)\n", + r, correct_count, total); + else + printf(" Rank %d: PASS - correctly still has poison (%d/%d)\n", + r, correct_count, total); + } else { + all_pass = 0; + if (in_mask && poison_count > 0) + printf(" Rank %d: FAIL - NOP detected, %d/%d elements still poison\n", + r, poison_count, total); + else + printf(" Rank %d: FAIL - %d/%d elements wrong (%d poison, %d mismatch)\n", + r, poison_count + mismatch_count, total, poison_count, mismatch_count); + } + } + + printf("\n"); + printf("================================================================================\n"); + printf("TEST: TMA Tensor Multicast Mask (mask=0x%X)\n", mcast_mask); + printf(" Instruction: cp.async.bulk.tensor.2d.shared::cluster.global\n"); + printf(" .mbarrier::complete_tx::bytes.multicast::cluster\n"); + printf("================================================================================\n"); + printf("RESULT: %s\n", all_pass ? "**PASS**" : "**FAIL**"); + printf("================================================================================\n\n"); + + // ---- Dump for debugging ---- + char filename[128]; + sprintf(filename, "tma_multicast_mask_0x%X.txt", mcast_mask); + FILE *f = fopen(filename, "w"); + fprintf(f, "# Mask=0x%X Cluster=%d CTAs Tile=%dx%d\n\n", mcast_mask, CLUSTER_DIM_X, TILE_DIM, TILE_DIM); + for (int r = 0; r < CLUSTER_DIM_X; r++) { + bool in_mask = (mcast_mask >> r) & 1; + fprintf(f, "--- Rank %d (%s) ---\n", r, + in_mask ? "in mask" : "NOT in mask"); + int base = r * TILE_DIM * TILE_DIM; + for (int row = 0; row < TILE_DIM; row++) { + for (int col = 0; col < TILE_DIM; col++) + fprintf(f, "0x%08x ", (unsigned)h_dst[base + row * TILE_DIM + col]); + fprintf(f, "\n"); + } + fprintf(f, "\n"); + } + fclose(f); + printf("Values dumped to %s\n", filename); + + free(mat); + free(h_dst); + cudaFree(d_mat); + cudaFree(d_dst); + + return all_pass ? 0 : 1; +} From 8a86b8ca3c11e67e1e80c8bb4fb089d09325b379 Mon Sep 17 00:00:00 2001 From: jyasuumi <144069912+jyasuumi@users.noreply.github.com> Date: Tue, 28 Apr 2026 15:31:29 -0400 Subject: [PATCH 04/35] changed error status to return 1 on fail for mbarrier ubenches --- .../ubench/tma/mbarrier/mbarrier_reduction.cu | 5 +++-- .../ubench/tma/mbarrier/mbarrier_reduction2.cu | 5 +++-- .../GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu index dc7e14af0..4c40a89f0 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu @@ -105,7 +105,8 @@ int main() { printf("Reference Sum: %d\n", h_ref_out); printf("mbarrier Sum: %d\n", h_mbar_out); - if (h_ref_out == h_mbar_out && h_ref_out == N) { + bool pass = (h_ref_out == h_mbar_out && h_ref_out == N); + if (pass) { printf("SUCCESS: mbarrier matches reference output!\n"); } else { printf("FAILURE: Results mismatch.\n"); @@ -118,5 +119,5 @@ int main() { cudaFree(d_ref_out); cudaFree(d_mbar_out); - return 0; + return pass ? 0 : 1; } \ No newline at end of file diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu index f258b5633..9357de736 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu @@ -127,7 +127,8 @@ int main() { printf("Reference Sum: %d\n", h_ref_out); printf("mbarrier Sum: %d\n", h_mbar_out); - if (h_ref_out == h_mbar_out && h_ref_out == N) { + bool pass = (h_ref_out == h_mbar_out && h_ref_out == N); + if (pass) { printf("SUCCESS: mbarrier matches reference output!\n"); } else { printf("FAILURE: Results mismatch.\n"); @@ -140,5 +141,5 @@ int main() { cudaFree(d_ref_out); cudaFree(d_mbar_out); - return 0; + return pass ? 0 : 1; } \ No newline at end of file diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu index 7c9b30d5d..0bbf9ade0 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_test.cu @@ -34,8 +34,9 @@ int main() { cudaMemcpy(&h_out, d_out, sizeof(int), cudaMemcpyDeviceToHost); - if (h_out == 1) printf("mbarrier test passed (Standard C++).\n"); + bool pass = (h_out == 1); + if (pass) printf("mbarrier test passed (Standard C++).\n"); else printf("mbarrier test failed.\n"); - return 0; + return pass ? 0 : 1; } \ No newline at end of file From 924fc2410effd3c4202ac18044560870bf652d47 Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 29 Apr 2026 13:53:16 -0700 Subject: [PATCH 05/35] Add standalone GPGPU-Sim ubench tests --- .../ubench/gpgpusim_tests/README.md | 17 + .../gpgpusim_tests/fp_lowp_cvt/Makefile | 46 + .../gpgpusim_tests/fp_lowp_cvt/README.md | 26 + .../lowp_cvt_diff/kernels_sm120.ptx | 295 ++ .../lowp_cvt_runtime_hw_sim_diff.cu | 2364 +++++++++++++++++ .../lowp_cvt_runtime_hw_sim_diff_rs.cu | 748 ++++++ .../fp_lowp_cvt/ptx_datatype_cases/README.md | 14 + .../declaration_param_bf16_not_allowed.expect | 2 + .../declaration_param_bf16_not_allowed.ptx | 7 + ...eclaration_param_bf16x2_not_allowed.expect | 2 + .../declaration_param_bf16x2_not_allowed.ptx | 7 + .../declaration_param_e2m1_not_allowed.expect | 2 + .../declaration_param_e2m1_not_allowed.ptx | 7 + ...eclaration_param_e2m1x2_not_allowed.expect | 2 + .../declaration_param_e2m1x2_not_allowed.ptx | 7 + ...eclaration_param_e2m1x4_not_allowed.expect | 2 + .../declaration_param_e2m1x4_not_allowed.ptx | 7 + .../declaration_param_e2m3_not_allowed.expect | 2 + .../declaration_param_e2m3_not_allowed.ptx | 7 + ...eclaration_param_e2m3x2_not_allowed.expect | 2 + .../declaration_param_e2m3x2_not_allowed.ptx | 7 + ...eclaration_param_e2m3x4_not_allowed.expect | 2 + .../declaration_param_e2m3x4_not_allowed.ptx | 7 + .../declaration_param_e3m2_not_allowed.expect | 2 + .../declaration_param_e3m2_not_allowed.ptx | 7 + ...eclaration_param_e3m2x2_not_allowed.expect | 2 + .../declaration_param_e3m2x2_not_allowed.ptx | 7 + ...eclaration_param_e3m2x4_not_allowed.expect | 2 + .../declaration_param_e3m2x4_not_allowed.ptx | 7 + .../declaration_param_e4m3_not_allowed.expect | 2 + .../declaration_param_e4m3_not_allowed.ptx | 7 + ...eclaration_param_e4m3x2_not_allowed.expect | 2 + .../declaration_param_e4m3x2_not_allowed.ptx | 7 + ...eclaration_param_e4m3x4_not_allowed.expect | 2 + .../declaration_param_e4m3x4_not_allowed.ptx | 7 + .../declaration_param_e5m2_not_allowed.expect | 2 + .../declaration_param_e5m2_not_allowed.ptx | 7 + ...eclaration_param_e5m2x2_not_allowed.expect | 2 + .../declaration_param_e5m2x2_not_allowed.ptx | 7 + ...eclaration_param_e5m2x4_not_allowed.expect | 2 + .../declaration_param_e5m2x4_not_allowed.ptx | 7 + ...declaration_param_f16x2_not_allowed.expect | 1 + .../declaration_param_f16x2_not_allowed.ptx | 7 + ...declaration_param_f32x2_not_allowed.expect | 2 + .../declaration_param_f32x2_not_allowed.ptx | 7 + ...declaration_param_s16x2_not_allowed.expect | 2 + .../declaration_param_s16x2_not_allowed.ptx | 7 + .../declaration_param_s2f6_not_allowed.expect | 2 + .../declaration_param_s2f6_not_allowed.ptx | 7 + ...eclaration_param_s2f6x2_not_allowed.expect | 2 + .../declaration_param_s2f6x2_not_allowed.ptx | 7 + .../declaration_param_tf32_not_allowed.expect | 2 + .../declaration_param_tf32_not_allowed.ptx | 7 + ...declaration_param_u16x2_not_allowed.expect | 2 + .../declaration_param_u16x2_not_allowed.ptx | 7 + ...declaration_param_ue4m3_not_allowed.expect | 2 + .../declaration_param_ue4m3_not_allowed.ptx | 7 + ...declaration_param_ue8m0_not_allowed.expect | 2 + .../declaration_param_ue8m0_not_allowed.ptx | 7 + ...claration_param_ue8m0x2_not_allowed.expect | 2 + .../declaration_param_ue8m0x2_not_allowed.ptx | 7 + .../declaration_reg_bf16_not_allowed.expect | 2 + .../fail/declaration_reg_bf16_not_allowed.ptx | 8 + .../declaration_reg_bf16x2_not_allowed.expect | 2 + .../declaration_reg_bf16x2_not_allowed.ptx | 8 + .../declaration_reg_e2m1_not_allowed.expect | 2 + .../fail/declaration_reg_e2m1_not_allowed.ptx | 8 + .../declaration_reg_e2m1x2_not_allowed.expect | 2 + .../declaration_reg_e2m1x2_not_allowed.ptx | 8 + .../declaration_reg_e2m1x4_not_allowed.expect | 2 + .../declaration_reg_e2m1x4_not_allowed.ptx | 8 + .../declaration_reg_e2m3_not_allowed.expect | 2 + .../fail/declaration_reg_e2m3_not_allowed.ptx | 8 + .../declaration_reg_e2m3x2_not_allowed.expect | 2 + .../declaration_reg_e2m3x2_not_allowed.ptx | 8 + .../declaration_reg_e2m3x4_not_allowed.expect | 2 + .../declaration_reg_e2m3x4_not_allowed.ptx | 8 + .../declaration_reg_e3m2_not_allowed.expect | 2 + .../fail/declaration_reg_e3m2_not_allowed.ptx | 8 + .../declaration_reg_e3m2x2_not_allowed.expect | 2 + .../declaration_reg_e3m2x2_not_allowed.ptx | 8 + .../declaration_reg_e3m2x4_not_allowed.expect | 2 + .../declaration_reg_e3m2x4_not_allowed.ptx | 8 + .../declaration_reg_e4m3_not_allowed.expect | 2 + .../fail/declaration_reg_e4m3_not_allowed.ptx | 8 + .../declaration_reg_e4m3x2_not_allowed.expect | 2 + .../declaration_reg_e4m3x2_not_allowed.ptx | 8 + .../declaration_reg_e4m3x4_not_allowed.expect | 2 + .../declaration_reg_e4m3x4_not_allowed.ptx | 8 + .../declaration_reg_e5m2_not_allowed.expect | 2 + .../fail/declaration_reg_e5m2_not_allowed.ptx | 8 + .../declaration_reg_e5m2x2_not_allowed.expect | 2 + .../declaration_reg_e5m2x2_not_allowed.ptx | 8 + .../declaration_reg_e5m2x4_not_allowed.expect | 2 + .../declaration_reg_e5m2x4_not_allowed.ptx | 8 + .../declaration_reg_f32x2_not_allowed.expect | 2 + .../declaration_reg_f32x2_not_allowed.ptx | 8 + .../declaration_reg_s16x2_not_allowed.expect | 2 + .../declaration_reg_s16x2_not_allowed.ptx | 8 + .../declaration_reg_s2f6_not_allowed.expect | 2 + .../fail/declaration_reg_s2f6_not_allowed.ptx | 8 + .../declaration_reg_s2f6x2_not_allowed.expect | 2 + .../declaration_reg_s2f6x2_not_allowed.ptx | 8 + .../declaration_reg_tf32_not_allowed.expect | 2 + .../fail/declaration_reg_tf32_not_allowed.ptx | 8 + .../declaration_reg_u16x2_not_allowed.expect | 2 + .../declaration_reg_u16x2_not_allowed.ptx | 8 + .../declaration_reg_ue4m3_not_allowed.expect | 2 + .../declaration_reg_ue4m3_not_allowed.ptx | 8 + .../declaration_reg_ue8m0_not_allowed.expect | 2 + .../declaration_reg_ue8m0_not_allowed.ptx | 8 + ...declaration_reg_ue8m0x2_not_allowed.expect | 2 + .../declaration_reg_ue8m0x2_not_allowed.ptx | 8 + .../initializer_f16_brace_not_allowed.expect | 1 + .../initializer_f16_brace_not_allowed.ptx | 8 + .../fail/initializer_f16_not_allowed.expect | 1 + .../fail/initializer_f16_not_allowed.ptx | 8 + ...initializer_f16x2_brace_not_allowed.expect | 1 + .../initializer_f16x2_brace_not_allowed.ptx | 8 + .../fail/initializer_f16x2_not_allowed.expect | 1 + .../fail/initializer_f16x2_not_allowed.ptx | 8 + .../initializer_pred_brace_not_allowed.expect | 1 + .../initializer_pred_brace_not_allowed.ptx | 8 + .../fail/initializer_pred_not_allowed.expect | 1 + .../fail/initializer_pred_not_allowed.ptx | 8 + ...uction_cvt_bf16_f16_ftz_not_allowed.expect | 1 + ...struction_cvt_bf16_f16_ftz_not_allowed.ptx | 11 + ...ction_cvt_bf16_f16_sm80_not_allowed.expect | 1 + ...truction_cvt_bf16_f16_sm80_not_allowed.ptx | 11 + ...16_f32_missing_rounding_not_allowed.expect | 1 + ..._bf16_f32_missing_rounding_not_allowed.ptx | 11 + ...uction_cvt_bf16_f32_sat_not_allowed.expect | 1 + ...struction_cvt_bf16_f32_sat_not_allowed.ptx | 11 + ...t_bf16_f32_satfinite_rm_not_allowed.expect | 1 + ..._cvt_bf16_f32_satfinite_rm_not_allowed.ptx | 11 + ...ction_cvt_bf16_f32_sm75_not_allowed.expect | 1 + ...truction_cvt_bf16_f32_sm75_not_allowed.ptx | 11 + ...ction_cvt_bf16_s32_sm80_not_allowed.expect | 1 + ...truction_cvt_bf16_s32_sm80_not_allowed.ptx | 11 + ...t_bf16x2_f32_single_src_not_allowed.expect | 2 + ..._cvt_bf16x2_f32_single_src_not_allowed.ptx | 11 + ...f16x2_ue8m0x2_satfinite_not_allowed.expect | 1 + ...t_bf16x2_ue8m0x2_satfinite_not_allowed.ptx | 11 + ...on_cvt_e2m1x2_dst_sm100_not_allowed.expect | 1 + ...ction_cvt_e2m1x2_dst_sm100_not_allowed.ptx | 11 + ...on_cvt_e2m1x2_f32_sm120_not_allowed.expect | 1 + ...ction_cvt_e2m1x2_f32_sm120_not_allowed.ptx | 12 + ...on_cvt_e2m1x2_src_sm100_not_allowed.expect | 1 + ...ction_cvt_e2m1x2_src_sm100_not_allowed.ptx | 11 + ...uction_cvt_e2m3x2_sm100_not_allowed.expect | 1 + ...struction_cvt_e2m3x2_sm100_not_allowed.ptx | 11 + ...ruction_cvt_e4m3x2_sm80_not_allowed.expect | 1 + ...nstruction_cvt_e4m3x2_sm80_not_allowed.ptx | 12 + ...ction_cvt_f16_bf16_sm80_not_allowed.expect | 1 + ...truction_cvt_f16_bf16_sm80_not_allowed.ptx | 11 + ...16_f32_missing_rounding_not_allowed.expect | 1 + ...t_f16_f32_missing_rounding_not_allowed.ptx | 11 + ...vt_f16x2_f32_single_src_not_allowed.expect | 2 + ...n_cvt_f16x2_f32_single_src_not_allowed.ptx | 11 + ...tion_cvt_f16x2_f32_sm75_not_allowed.expect | 1 + ...ruction_cvt_f16x2_f32_sm75_not_allowed.ptx | 11 + ...ction_cvt_f16x2_ue8m0x2_not_allowed.expect | 2 + ...truction_cvt_f16x2_ue8m0x2_not_allowed.ptx | 10 + ...n_cvt_f32_bf16_ftz_sm80_not_allowed.expect | 1 + ...tion_cvt_f32_bf16_ftz_sm80_not_allowed.ptx | 11 + ...32_s32_missing_rounding_not_allowed.expect | 1 + ...t_f32_s32_missing_rounding_not_allowed.ptx | 11 + ...nstruction_cvt_f32_tf32_not_allowed.expect | 2 + .../instruction_cvt_f32_tf32_not_allowed.ptx | 11 + ...struction_cvt_f32x2_dst_not_allowed.expect | 1 + .../instruction_cvt_f32x2_dst_not_allowed.ptx | 10 + ...struction_cvt_f32x2_src_not_allowed.expect | 1 + .../instruction_cvt_f32x2_src_not_allowed.ptx | 10 + .../instruction_cvt_pred_not_allowed.expect | 2 + .../fail/instruction_cvt_pred_not_allowed.ptx | 10 + .../instruction_cvt_requires_two_types.expect | 1 + .../instruction_cvt_requires_two_types.ptx | 10 + ...nstruction_cvt_rs_sm100_not_allowed.expect | 1 + .../instruction_cvt_rs_sm100_not_allowed.ptx | 11 + ...struction_cvt_rs_sm100f_not_allowed.expect | 1 + .../instruction_cvt_rs_sm100f_not_allowed.ptx | 11 + ...struction_cvt_rs_sm110a_not_allowed.expect | 1 + .../instruction_cvt_rs_sm110a_not_allowed.ptx | 13 + ...struction_cvt_rs_sm120a_not_allowed.expect | 1 + .../instruction_cvt_rs_sm120a_not_allowed.ptx | 11 + ...struction_cvt_rs_sm120f_not_allowed.expect | 1 + .../instruction_cvt_rs_sm120f_not_allowed.ptx | 11 + ...f16x2_missing_satfinite_not_allowed.expect | 2 + ...2_bf16x2_missing_satfinite_not_allowed.ptx | 11 + ..._operand_without_scaled_not_allowed.expect | 2 + ...6x2_operand_without_scaled_not_allowed.ptx | 11 + ..._scaled_without_operand_not_allowed.expect | 2 + ...6x2_scaled_without_operand_not_allowed.ptx | 11 + ...vt_s2f6x2_bf16x2_sm100f_not_allowed.expect | 1 + ...n_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx | 11 + ...vt_s2f6x2_bf16x2_sm120f_not_allowed.expect | 1 + ...n_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx | 11 + ...uction_cvt_s2f6x2_sm100_not_allowed.expect | 1 + ...struction_cvt_s2f6x2_sm100_not_allowed.ptx | 11 + ...uction_cvt_s32_bf16_sat_not_allowed.expect | 1 + ...struction_cvt_s32_bf16_sat_not_allowed.ptx | 11 + ...ction_cvt_s32_bf16_sm80_not_allowed.expect | 1 + ...truction_cvt_s32_bf16_sm80_not_allowed.ptx | 11 + ...ruction_cvt_s32_s32_sat_not_allowed.expect | 1 + ...nstruction_cvt_s32_s32_sat_not_allowed.ptx | 11 + ...ruction_cvt_scalar_relu_not_allowed.expect | 1 + ...nstruction_cvt_scalar_relu_not_allowed.ptx | 11 + ...on_cvt_scalar_satfinite_not_allowed.expect | 1 + ...ction_cvt_scalar_satfinite_not_allowed.ptx | 11 + ...32_f32_missing_rounding_not_allowed.expect | 2 + ..._tf32_f32_missing_rounding_not_allowed.ptx | 11 + ...on_cvt_tf32_f32_rn_sm89_not_allowed.expect | 1 + ...ction_cvt_tf32_f32_rn_sm89_not_allowed.ptx | 11 + ...n_cvt_tf32_f32_rna_relu_not_allowed.expect | 1 + ...tion_cvt_tf32_f32_rna_relu_not_allowed.ptx | 11 + ...2_f32_satfinite_rn_sm90_not_allowed.expect | 1 + ...tf32_f32_satfinite_rn_sm90_not_allowed.ptx | 11 + ...nstruction_cvt_tf32_u32_not_allowed.expect | 2 + .../instruction_cvt_tf32_u32_not_allowed.ptx | 10 + .../instruction_cvt_u16x2_not_allowed.expect | 2 + .../instruction_cvt_u16x2_not_allowed.ptx | 10 + ...struction_cvt_ue4m3_f32_not_allowed.expect | 1 + .../instruction_cvt_ue4m3_f32_not_allowed.ptx | 11 + ...bf16x2_missing_rounding_not_allowed.expect | 2 + ...x2_bf16x2_missing_rounding_not_allowed.ptx | 11 + ...ction_cvt_ue8m0x2_f16x2_not_allowed.expect | 2 + ...truction_cvt_ue8m0x2_f16x2_not_allowed.ptx | 10 + ...on_cvt_ue8m0x2_f32_relu_not_allowed.expect | 1 + ...ction_cvt_ue8m0x2_f32_relu_not_allowed.ptx | 11 + ...n_cvt_ue8m0x2_f32_sm120_not_allowed.expect | 1 + ...tion_cvt_ue8m0x2_f32_sm120_not_allowed.ptx | 12 + ...tion_cvt_ue8m0x2_scaled_not_allowed.expect | 1 + ...ruction_cvt_ue8m0x2_scaled_not_allowed.ptx | 12 + ...128_pack_v2_mixed_width_not_allowed.expect | 1 + ...v_b128_pack_v2_mixed_width_not_allowed.ptx | 12 + ...ruction_mov_b128_scalar_not_allowed.expect | 1 + ...nstruction_mov_b128_scalar_not_allowed.ptx | 11 + ...uction_mov_bb128_scalar_not_allowed.expect | 1 + ...struction_mov_bb128_scalar_not_allowed.ptx | 11 + .../instruction_mov_bf16_not_allowed.expect | 2 + .../fail/instruction_mov_bf16_not_allowed.ptx | 9 + .../instruction_mov_f16_not_allowed.expect | 2 + .../fail/instruction_mov_f16_not_allowed.ptx | 9 + .../fail/unsupported_token_s32x1.expect | 1 + .../fail/unsupported_token_s32x1.ptx | 9 + .../fail/unsupported_token_u32x1.expect | 1 + .../fail/unsupported_token_u32x1.ptx | 9 + .../pass/all_new_instruction_types.ptx | 32 + ...on_param_b32_for_f16x2_storage_allowed.ptx | 7 + .../pass/declaration_reg_b128_allowed.ptx | 12 + .../pass/declaration_reg_f16x2_allowed.ptx | 8 + .../pass/declaration_underlying_bit_types.ptx | 16 + .../instruction_cvt_bf16_f16_sm90_allowed.ptx | 11 + ...on_cvt_bf16_f32_satfinite_relu_allowed.ptx | 11 + .../instruction_cvt_bf16_s32_sm90_allowed.ptx | 11 + ...nstruction_cvt_bf16x2_f32_pair_allowed.ptx | 11 + .../instruction_cvt_bf16x2_s2f6x2_allowed.ptx | 11 + ...ction_cvt_bf16x2_s2f6x2_sm103a_allowed.ptx | 11 + ...ction_cvt_bf16x2_s2f6x2_sm121a_allowed.ptx | 11 + ...instruction_cvt_bf16x2_ue8m0x2_allowed.ptx | 11 + ...truction_cvt_e2m1x2_f32_sm100a_allowed.ptx | 12 + ...truction_cvt_e2m1x2_f32_sm100f_allowed.ptx | 12 + ...truction_cvt_e2m1x2_f32_sm103a_allowed.ptx | 12 + ...truction_cvt_e2m1x2_f32_sm120a_allowed.ptx | 12 + ...truction_cvt_e2m1x2_f32_sm120f_allowed.ptx | 12 + ...truction_cvt_e2m1x2_f32_sm121a_allowed.ptx | 12 + ...truction_cvt_e2m1x2_f32_sm121f_allowed.ptx | 12 + .../instruction_cvt_e2m1x4_f32_rs_allowed.ptx | 12 + .../instruction_cvt_e2m3x4_f32_rs_allowed.ptx | 11 + .../instruction_cvt_e3m2x4_f32_rs_allowed.ptx | 11 + .../instruction_cvt_e4m3x4_f32_rs_allowed.ptx | 11 + .../instruction_cvt_e5m2x4_f32_rs_allowed.ptx | 11 + .../instruction_cvt_f16_bf16_sm90_allowed.ptx | 11 + ...ion_cvt_f16_f32_satfinite_relu_allowed.ptx | 11 + ...uction_cvt_f16x2_e2m1x2_sm100a_allowed.ptx | 11 + ...uction_cvt_f16x2_e2m3x2_sm100a_allowed.ptx | 11 + ...instruction_cvt_f16x2_f32_pair_allowed.ptx | 11 + ...truction_cvt_f32_bf16_ftz_sm90_allowed.ptx | 11 + .../instruction_cvt_rs_sm103a_allowed.ptx | 11 + .../instruction_cvt_s2f6x2_bf16x2_allowed.ptx | 11 + ...ction_cvt_s2f6x2_bf16x2_scaled_allowed.ptx | 11 + ...ction_cvt_s2f6x2_bf16x2_sm103a_allowed.ptx | 11 + ...ction_cvt_s2f6x2_bf16x2_sm120a_allowed.ptx | 11 + ...ction_cvt_s2f6x2_bf16x2_sm121a_allowed.ptx | 11 + .../instruction_cvt_s32_bf16_sm90_allowed.ptx | 11 + .../instruction_cvt_s32_f32_sat_allowed.ptx | 11 + ...on_cvt_tf32_f32_satfinite_relu_allowed.ptx | 11 + ...instruction_cvt_tf32_roundtrip_allowed.ptx | 11 + .../instruction_cvt_u32_f64_sat_allowed.ptx | 11 + ...instruction_cvt_ue8m0x2_bf16x2_allowed.ptx | 11 + ...uction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx | 11 + ...8m0x2_bf16x2_without_satfinite_allowed.ptx | 11 + ...ruction_cvt_ue8m0x2_f32_sm100a_allowed.ptx | 12 + ...ruction_cvt_ue8m0x2_f32_sm100f_allowed.ptx | 12 + ...ruction_cvt_ue8m0x2_f32_sm103f_allowed.ptx | 12 + ...ruction_cvt_ue8m0x2_f32_sm110f_allowed.ptx | 12 + ...ruction_cvt_ue8m0x2_f32_sm120a_allowed.ptx | 12 + ...ruction_cvt_ue8m0x2_f32_sm120f_allowed.ptx | 12 + ...ruction_cvt_ue8m0x2_f32_sm121a_allowed.ptx | 12 + .../instruction_mov_b128_pack_allowed.ptx | 11 + .../instruction_mov_b128_pack_v2_allowed.ptx | 11 + ...struction_mov_b128_pack_v2_u64_allowed.ptx | 11 + .../instruction_mov_b128_scalar_allowed.ptx | 10 + .../instruction_mov_b128_unpack_allowed.ptx | 11 + ...instruction_mov_b128_unpack_v2_allowed.ptx | 11 + ...uction_mov_b128_unpack_v4_sink_allowed.ptx | 11 + ...ruction_mov_b128_unpack_v4_u32_allowed.ptx | 11 + .../run_lowp_cvt_ptxas_crosscheck.sh | 174 ++ .../gpgpusim_tests/tma_tensormap/Makefile | 50 + .../gpgpusim_tests/tma_tensormap/README.md | 22 + .../tma_tensor_test_REGULAR_LOAD_64_64.txt | 8 + .../tma_tensor_test_UTMALDG_64_64.txt | 8 + .../expected/tma_tensor_test_UTMAPF_64_64.txt | 8 + .../tma_tensor_test_UTMAREDG_64_64.txt | 8 + .../tma_tensor_test_UTMASTG_50_50.txt | 8 + .../tma_tensor_test_UTMASTG_64_64.txt | 8 + .../tma_tensormap/tma_tensor_benchmark.cu | 415 +++ 317 files changed, 6207 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/README.md create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/README.md create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_diff/kernels_sm120.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/README.md create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.expect create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/all_new_instruction_types.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_param_b32_for_f16x2_storage_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_b128_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_f16x2_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_underlying_bit_types.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f16_sm90_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f32_satfinite_relu_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_s32_sm90_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_f32_pair_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm103a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm121a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_ue8m0x2_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100f_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm103a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120f_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121f_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x4_f32_rs_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m3x4_f32_rs_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e3m2x4_f32_rs_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e4m3x4_f32_rs_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e5m2x4_f32_rs_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_bf16_sm90_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_f32_satfinite_relu_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m1x2_sm100a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m3x2_sm100a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_f32_pair_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f32_bf16_ftz_sm90_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_rs_sm103a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_scaled_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm103a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm120a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm121a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_bf16_sm90_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_f32_sat_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_f32_satfinite_relu_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_roundtrip_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_u32_f64_sat_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_without_satfinite_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100f_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm103f_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm110f_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120f_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm121a_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_u64_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_scalar_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v2_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_sink_allowed.ptx create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_u32_allowed.ptx create mode 100755 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/run_lowp_cvt_ptxas_crosscheck.sh create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/README.md create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_REGULAR_LOAD_64_64.txt create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMALDG_64_64.txt create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAPF_64_64.txt create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAREDG_64_64.txt create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_50_50.txt create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_64_64.txt create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/tma_tensor_benchmark.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/README.md b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/README.md new file mode 100644 index 000000000..8609fa268 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/README.md @@ -0,0 +1,17 @@ +# GPGPU-Sim Standalone Tests + +This directory collects standalone CUDA/PTX tests that were originally kept +with GPGPU-Sim feature branches. They are placed under `GPU_Microbenchmark` so +the app-collection build can compile them with the same flow as the other +microbenchmarks. + +The tests here intentionally avoid linking against GPGPU-Sim internals. C++ +unit tests that need simulator headers or libraries should stay in the +GPGPU-Sim source tree. + +## Test Groups + +- `fp_lowp_cvt`: runtime low-precision conversion kernels and PTX cases used + for `ptxas` cross-checking. +- `tma_tensormap`: standalone TMA tensor-map benchmark plus expected output + fixtures from the simulator branch. diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile new file mode 100644 index 000000000..6800d6860 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile @@ -0,0 +1,46 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin + +CC := nvcc + +LOWP_ARCH ?= sm_120a +LOWP_RS_ARCH ?= sm_100a +CUDA_CPPFLAGS = + +LOWP_EXE := lowp_cvt_runtime_hw_sim_diff +LOWP_RS_EXE := lowp_cvt_runtime_hw_sim_diff_rs + +LOWP_NVCC_FLAGS := -gencode=arch=compute_$(subst sm_,,$(LOWP_ARCH)),code=$(LOWP_ARCH) \ + -gencode=arch=compute_$(subst sm_,,$(LOWP_ARCH)),code=compute_$(subst sm_,,$(LOWP_ARCH)) \ + -std=c++17 -O2 +LOWP_RS_NVCC_FLAGS := -gencode=arch=compute_$(subst sm_,,$(LOWP_RS_ARCH)),code=$(LOWP_RS_ARCH) \ + -gencode=arch=compute_$(subst sm_,,$(LOWP_RS_ARCH)),code=compute_$(subst sm_,,$(LOWP_RS_ARCH)) \ + -std=c++17 -O2 + +.DEFAULT_GOAL := all + +all: $(BIN_DIR)/$(LOWP_EXE) $(BIN_DIR)/$(LOWP_RS_EXE) + +$(BIN_DIR): + mkdir -p $@ + +$(LOWP_EXE).a: $(LOWP_EXE).cu + $(CC) $(LOWP_NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(LOWP_RS_EXE).a: $(LOWP_RS_EXE).cu + $(CC) $(LOWP_RS_NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(BIN_DIR)/$(LOWP_EXE): $(LOWP_EXE).a | $(BIN_DIR) + $(CC) $(LOWP_NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + +$(BIN_DIR)/$(LOWP_RS_EXE): $(LOWP_RS_EXE).a | $(BIN_DIR) + $(CC) $(LOWP_RS_NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + +ptxas-crosscheck: + ./run_lowp_cvt_ptxas_crosscheck.sh ptx_datatype_cases + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(LOWP_EXE) $(LOWP_RS_EXE) + rm -f $(BIN_DIR)/$(LOWP_EXE) $(BIN_DIR)/$(LOWP_RS_EXE) + +.PHONY: all clean ptxas-crosscheck diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/README.md b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/README.md new file mode 100644 index 000000000..e9c1becff --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/README.md @@ -0,0 +1,26 @@ +# FP Low-Precision Conversion Tests + +This folder contains standalone CUDA/PTX low-precision conversion tests from +the GPGPU-Sim `fp-work` branch. + +## Build + +```sh +make +``` + +The default build creates these binaries under `GPU_Microbenchmark/bin`: + +- `lowp_cvt_runtime_hw_sim_diff` +- `lowp_cvt_runtime_hw_sim_diff_rs` + +Default architectures are `LOWP_ARCH=sm_120a` and +`LOWP_RS_ARCH=sm_100a`. Override them on the make command line if needed. + +## PTX Cross-Check + +```sh +make ptxas-crosscheck +``` + +This target runs the imported PTX accept/reject cases through `ptxas`. diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_diff/kernels_sm120.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_diff/kernels_sm120.ptx new file mode 100644 index 000000000..376d5dd92 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_diff/kernels_sm120.ptx @@ -0,0 +1,295 @@ +.version 9.1 +.target sm_120a +.address_size 64 + +.visible .entry k_f16x2_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<4>; + .reg .f32 %f<3>; + .reg .b32 %r<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + add.u64 %rd2, %rd1, 4; + ld.global.f32 %f1, [%rd2]; + cvt.rn.f16x2.f32 %r0, %f0, %f1; + st.global.u32 [%rd0], %r0; + ret; +} + +.visible .entry k_f16x2_rz( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<4>; + .reg .f32 %f<3>; + .reg .b32 %r<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + add.u64 %rd2, %rd1, 4; + ld.global.f32 %f1, [%rd2]; + cvt.rz.f16x2.f32 %r0, %f0, %f1; + st.global.u32 [%rd0], %r0; + ret; +} + +.visible .entry k_bf16x2_rz( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<4>; + .reg .f32 %f<3>; + .reg .b32 %r<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + add.u64 %rd2, %rd1, 4; + ld.global.f32 %f1, [%rd2]; + cvt.rz.bf16x2.f32 %r0, %f0, %f1; + st.global.u32 [%rd0], %r0; + ret; +} + +.visible .entry k_bf16x2_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<4>; + .reg .f32 %f<3>; + .reg .b32 %r<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + add.u64 %rd2, %rd1, 4; + ld.global.f32 %f1, [%rd2]; + cvt.rn.bf16x2.f32 %r0, %f0, %f1; + st.global.u32 [%rd0], %r0; + ret; +} + +.visible .entry k_e4m3x2_satfinite_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<4>; + .reg .f32 %f<3>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + add.u64 %rd2, %rd1, 4; + ld.global.f32 %f1, [%rd2]; + cvt.rn.satfinite.e4m3x2.f32 %h0, %f0, %f1; + st.global.u16 [%rd0], %h0; + ret; +} + +.visible .entry k_e5m2x2_satfinite_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<4>; + .reg .f32 %f<3>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + add.u64 %rd2, %rd1, 4; + ld.global.f32 %f1, [%rd2]; + cvt.rn.satfinite.e5m2x2.f32 %h0, %f0, %f1; + st.global.u16 [%rd0], %h0; + ret; +} + +.visible .entry k_e2m3x2_satfinite_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<4>; + .reg .f32 %f<3>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + add.u64 %rd2, %rd1, 4; + ld.global.f32 %f1, [%rd2]; + cvt.rn.satfinite.e2m3x2.f32 %h0, %f0, %f1; + st.global.u16 [%rd0], %h0; + ret; +} + +.visible .entry k_e3m2x2_satfinite_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<4>; + .reg .f32 %f<3>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + add.u64 %rd2, %rd1, 4; + ld.global.f32 %f1, [%rd2]; + cvt.rn.satfinite.e3m2x2.f32 %h0, %f0, %f1; + st.global.u16 [%rd0], %h0; + ret; +} + +.visible .entry k_ue8m0x2_satfinite_rz( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<4>; + .reg .f32 %f<3>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + add.u64 %rd2, %rd1, 4; + ld.global.f32 %f1, [%rd2]; + cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; + st.global.u16 [%rd0], %h0; + ret; +} + +.visible .entry k_s2f6x2_satfinite_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<4>; + .reg .f32 %f<3>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + add.u64 %rd2, %rd1, 4; + ld.global.f32 %f1, [%rd2]; + cvt.rn.satfinite.s2f6x2.f32 %h0, %f0, %f1; + st.global.u16 [%rd0], %h0; + ret; +} + +.visible .entry k_tf32_satfinite_relu_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<3>; + .reg .f32 %f<2>; + .reg .b32 %r<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + cvt.rn.satfinite.relu.tf32.f32 %r0, %f0; + st.global.u32 [%rd0], %r0; + ret; +} + +.visible .entry k_tf32_rz( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<3>; + .reg .f32 %f<2>; + .reg .b32 %r<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + cvt.rz.tf32.f32 %r0, %f0; + st.global.u32 [%rd0], %r0; + ret; +} + +.visible .entry k_bf16_satfinite_relu_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<3>; + .reg .f32 %f<2>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + cvt.rn.satfinite.relu.bf16.f32 %h0, %f0; + st.global.b16 [%rd0], %h0; + ret; +} + +.visible .entry k_bf16_rz( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<3>; + .reg .f32 %f<2>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + cvt.rz.bf16.f32 %h0, %f0; + st.global.b16 [%rd0], %h0; + ret; +} + +.visible .entry k_f16_satfinite_relu_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<3>; + .reg .f32 %f<2>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + cvt.rn.satfinite.relu.f16.f32 %h0, %f0; + st.global.b16 [%rd0], %h0; + ret; +} + +.visible .entry k_f16_rz( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<3>; + .reg .f32 %f<2>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + cvt.rz.f16.f32 %h0, %f0; + st.global.b16 [%rd0], %h0; + ret; +} + +.visible .entry k_f16_sat_rn( + .param .u64 out_ptr, + .param .u64 in_ptr +) { + .reg .u64 %rd<3>; + .reg .f32 %f<2>; + .reg .b16 %h<2>; + + ld.param.u64 %rd0, [out_ptr]; + ld.param.u64 %rd1, [in_ptr]; + ld.global.f32 %f0, [%rd1]; + cvt.rn.sat.f16.f32 %h0, %f0; + st.global.b16 [%rd0], %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu new file mode 100644 index 000000000..2bad3c250 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu @@ -0,0 +1,2364 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef LOWP_CVT_HAS_UE8M0X2_BF16X2_RELU +#define LOWP_CVT_HAS_UE8M0X2_BF16X2_RELU 0 +#endif + +namespace { + +#define CUDACHK(call) \ + do { \ + cudaError_t _err = (call); \ + if (_err != cudaSuccess) { \ + std::cerr << "CUDA error: " << cudaGetErrorString(_err) << " @ " \ + << #call << "\n"; \ + std::exit(2); \ + } \ + } while (0) + +float bits_to_f32(uint32_t bits) { + float out = 0.0f; + std::memcpy(&out, &bits, sizeof(out)); + return out; +} + +uint32_t f32_to_bits(float v) { + uint32_t bits = 0u; + std::memcpy(&bits, &v, sizeof(bits)); + return bits; +} + +uint16_t bf16_bits_from_f32(float v) { + return static_cast(f32_to_bits(v) >> 16); +} + +double bits_to_f64(uint64_t bits) { + double out = 0.0; + std::memcpy(&out, &bits, sizeof(out)); + return out; +} + +enum class minifloat_family_t { + kE4M3, + kE5M2, + kE2M1, + kE2M3, + kE3M2, + kUE8M0, +}; + +struct minifloat_entry_t { + uint16_t code; + float value; +}; + +unsigned minifloat_code_limit(minifloat_family_t fmt) { + switch (fmt) { + case minifloat_family_t::kE4M3: + case minifloat_family_t::kE5M2: + case minifloat_family_t::kUE8M0: + return 256u; + case minifloat_family_t::kE2M1: + return 16u; + case minifloat_family_t::kE2M3: + case minifloat_family_t::kE3M2: + return 64u; + } + return 0u; +} + +float decode_minifloat_code(minifloat_family_t fmt, uint16_t code) { + if (fmt == minifloat_family_t::kUE8M0) { + if (code == 0xffu) return std::numeric_limits::quiet_NaN(); + return std::ldexp(1.0f, static_cast(code) - 127); + } + + int exp_bits = 0; + int mant_bits = 0; + int bias = 0; + bool has_inf = false; + switch (fmt) { + case minifloat_family_t::kE4M3: + exp_bits = 4; + mant_bits = 3; + bias = 7; + has_inf = false; + break; + case minifloat_family_t::kE5M2: + exp_bits = 5; + mant_bits = 2; + bias = 15; + has_inf = true; + break; + case minifloat_family_t::kE2M1: + exp_bits = 2; + mant_bits = 1; + bias = 1; + has_inf = false; + break; + case minifloat_family_t::kE2M3: + exp_bits = 2; + mant_bits = 3; + bias = 1; + has_inf = false; + break; + case minifloat_family_t::kE3M2: + exp_bits = 3; + mant_bits = 2; + bias = 3; + has_inf = false; + break; + case minifloat_family_t::kUE8M0: + break; + } + + const int sign_shift = exp_bits + mant_bits; + const bool neg = ((code >> sign_shift) & 0x1u) != 0u; + const int exp_mask = (1 << exp_bits) - 1; + const int mant_mask = (1 << mant_bits) - 1; + const int exp = (code >> mant_bits) & exp_mask; + const int mant = code & mant_mask; + + if (fmt == minifloat_family_t::kE4M3 && (code == 0x7fu || code == 0xffu)) { + return std::numeric_limits::quiet_NaN(); + } + if (has_inf && exp == exp_mask) { + if (mant == 0) { + return neg ? -std::numeric_limits::infinity() + : std::numeric_limits::infinity(); + } + return std::numeric_limits::quiet_NaN(); + } + + float value = 0.0f; + if (exp == 0) { + if (mant != 0) { + value = std::ldexp(static_cast(mant), 1 - bias - mant_bits); + } + } else { + const int sig = (1 << mant_bits) | mant; + value = std::ldexp(static_cast(sig), exp - bias - mant_bits); + } + return neg ? -value : value; +} + +std::vector collect_minifloat_midpoint_ties(minifloat_family_t fmt, + size_t max_count) { + std::vector entries; + entries.reserve(minifloat_code_limit(fmt)); + for (unsigned code = 0; code < minifloat_code_limit(fmt); ++code) { + const float value = decode_minifloat_code(fmt, static_cast(code)); + if (!std::isfinite(value)) continue; + entries.push_back({static_cast(code), value}); + } + + std::sort(entries.begin(), entries.end(), + [](const minifloat_entry_t &a, const minifloat_entry_t &b) { + if (a.value < b.value) return true; + if (a.value > b.value) return false; + return a.code < b.code; + }); + + std::vector out; + out.reserve(max_count); + for (size_t i = 0; i + 1 < entries.size() && out.size() < max_count; ++i) { + const float lo = entries[i].value; + const float hi = entries[i + 1].value; + if (!(lo < hi)) continue; + const long double midpoint = + (static_cast(lo) + static_cast(hi)) * 0.5L; + if (!std::isfinite(static_cast(midpoint))) continue; + const float mid_f32 = static_cast(midpoint); + if (!std::isfinite(mid_f32)) continue; + if (static_cast(mid_f32) != midpoint) continue; + out.push_back(mid_f32); + } + return out; +} + +std::vector build_minifloat_midpoint_probe_values() { + const minifloat_family_t families[] = { + minifloat_family_t::kE4M3, minifloat_family_t::kE5M2, + minifloat_family_t::kE2M1, minifloat_family_t::kE2M3, + minifloat_family_t::kE3M2, minifloat_family_t::kUE8M0, + }; + + std::vector out; + for (minifloat_family_t fmt : families) { + const std::vector ties = collect_minifloat_midpoint_ties(fmt, 12); + for (float tie : ties) { + out.push_back(tie); + const float down = + std::nextafter(tie, -std::numeric_limits::infinity()); + const float up = + std::nextafter(tie, std::numeric_limits::infinity()); + if (std::isfinite(down)) out.push_back(down); + if (std::isfinite(up)) out.push_back(up); + } + } + + std::sort(out.begin(), out.end(), + [](float a, float b) { return f32_to_bits(a) < f32_to_bits(b); }); + out.erase(std::unique(out.begin(), out.end(), + [](float a, float b) { + return f32_to_bits(a) == f32_to_bits(b); + }), + out.end()); + return out; +} + +std::vector edge_f32_patterns() { + return { + 0x00000000u, 0x80000000u, 0x00000001u, 0x80000001u, 0x007fffffu, + 0x807fffffu, 0x00800000u, 0x80800000u, 0x00800001u, 0x80800001u, + 0x3f7fffffu, 0x3f800000u, 0x3f800001u, 0xbf7fffffu, 0xbf800000u, + 0xbf800001u, 0x7f7fffffu, 0xff7fffffu, 0x7f800000u, 0xff800000u, + 0x7fc00000u, 0xffc00000u, 0x7fa00001u, 0xffa00001u, + }; +} + +std::vector build_scalar_inputs() { + std::vector out; + out.reserve(384); + const std::vector edges = edge_f32_patterns(); + for (size_t i = 0; i < edges.size(); ++i) { + out.push_back(bits_to_f32(edges[i])); + } + std::mt19937 rng(0x40f8b37du); + while (out.size() < 384) out.push_back(bits_to_f32(rng())); + return out; +} + +void build_pair_inputs(std::vector &a, std::vector &b) { + a.clear(); + b.clear(); + a.reserve(448); + b.reserve(448); + + const std::vector edges = edge_f32_patterns(); + for (size_t i = 0; i + 1 < edges.size(); i += 2) { + a.push_back(bits_to_f32(edges[i])); + b.push_back(bits_to_f32(edges[i + 1])); + } + + const std::vector midpoint_probes = build_minifloat_midpoint_probe_values(); + for (size_t i = 0; i < midpoint_probes.size(); i += 2) { + const float lhs = midpoint_probes[i]; + const float rhs = + (i + 1 < midpoint_probes.size()) ? midpoint_probes[i + 1] : lhs; + a.push_back(lhs); + b.push_back(rhs); + } + + std::mt19937 rng(0x7b91d00du); + while (a.size() < 384) { + a.push_back(bits_to_f32(rng())); + b.push_back(bits_to_f32(rng())); + } +} + +std::vector build_bf16_inputs() { + std::vector out = { + 0x0000u, 0x8000u, 0x0001u, 0x8001u, 0x007fu, 0x807fu, 0x0080u, 0x8080u, + 0x3f80u, 0xbf80u, 0x7f80u, 0xff80u, 0x7fffu, + }; + + const float int_edges[] = { + -65536.0f, -32768.0f, -32767.0f, -256.0f, -255.0f, -128.0f, -127.5f, + -127.0f, -1.5f, -1.0f, -0.5f, 0.0f, 0.5f, 1.0f, + 1.5f, 127.0f, 127.5f, 128.0f, 255.0f, 256.0f, 32767.0f, + 32768.0f, 65535.0f, 65536.0f, + }; + for (float v : int_edges) out.push_back(bf16_bits_from_f32(v)); + + out.reserve(384); + std::mt19937 rng(0x13572468u); + while (out.size() < 384) out.push_back(static_cast(rng())); + return out; +} + +std::vector build_f16_inputs() { + std::vector out = { + 0x0000u, 0x8000u, 0x0001u, 0x8001u, 0x03ffu, 0x83ffu, 0x0400u, + 0x8400u, 0x3c00u, 0xbc00u, 0x7bffu, 0xfbffu, 0x7c00u, 0xfc00u, + 0x7e00u, 0xfe00u, + }; + out.reserve(384); + std::mt19937 rng(0x97531bdcu); + while (out.size() < 384) out.push_back(static_cast(rng())); + return out; +} + +std::vector build_s32_inputs() { + std::vector out = { + 0, 1, -1, 2, -2, 127, -127, 255, -255, 1024, -1024, + std::numeric_limits::max(), + std::numeric_limits::min(), + 0x00ffffff, static_cast(0xff000001u), + }; + out.reserve(384); + std::mt19937 rng(0x2468ace0u); + while (out.size() < 384) out.push_back(static_cast(rng())); + return out; +} + +std::vector build_u32_inputs() { + std::vector out = { + 0u, 1u, 2u, 255u, 65535u, 65536u, 0x00ffffffu, 0x7fffffffu, 0xffffffffu, + }; + out.reserve(384); + std::mt19937 rng(0x10293847u); + while (out.size() < 384) out.push_back(rng()); + return out; +} + +std::vector build_f64_inputs() { + std::vector out; + const std::vector edges = { + 0x0000000000000000ull, 0x8000000000000000ull, 0x0000000000000001ull, + 0x8000000000000001ull, 0x000fffffffffffffull, 0x800fffffffffffffull, + 0x0010000000000000ull, 0x8010000000000000ull, 0x3ff0000000000000ull, + 0xbff0000000000000ull, 0x3fefffffffffffffull, 0xbfefffffffffffffull, + 0x7fefffffffffffffull, 0xffefffffffffffffull, 0x7ff0000000000000ull, + 0xfff0000000000000ull, 0x7ff8000000000000ull, 0xfff8000000000000ull, + 0x7ff4000000000001ull, 0xfff4000000000001ull, + }; + out.reserve(384); + for (size_t i = 0; i < edges.size(); ++i) out.push_back(bits_to_f64(edges[i])); + std::mt19937_64 rng(0x1122334455667788ull); + while (out.size() < 384) out.push_back(bits_to_f64(rng())); + return out; +} + +std::vector build_e2m1x2_inputs_exhaustive() { + std::vector out; + out.reserve(256); + for (unsigned packed = 0; packed <= 0xff; ++packed) { + out.push_back(static_cast(packed)); + } + return out; +} + +std::vector build_e2m3x2_inputs_exhaustive() { + std::vector out; + out.reserve(64u * 64u); + for (unsigned hi = 0; hi < 64; ++hi) { + for (unsigned lo = 0; lo < 64; ++lo) { + out.push_back(static_cast((hi << 8) | lo)); + } + } + return out; +} + +std::vector build_e3m2x2_inputs_exhaustive() { + std::vector out; + out.reserve(64u * 64u); + for (unsigned hi = 0; hi < 64; ++hi) { + for (unsigned lo = 0; lo < 64; ++lo) { + out.push_back(static_cast((hi << 8) | lo)); + } + } + return out; +} + +std::vector build_u16_inputs_exhaustive() { + std::vector out; + out.reserve(1u << 16); + for (unsigned packed = 0; packed <= 0xffff; ++packed) { + out.push_back(static_cast(packed)); + } + return out; +} + +std::vector build_scale_inputs(size_t count, uint32_t seed) { + std::vector out; + out.reserve(count); + + // Packed ue8m0x2 scale-factors; include notable edge patterns and random. + const uint16_t edge_scales[] = { + 0x0000u, 0x0101u, 0x7f7fu, 0xfefeu, 0x0080u, + 0x8000u, 0x7f00u, 0x007fu, 0xff00u, 0x00ffu, + }; + for (size_t i = 0; i < sizeof(edge_scales) / sizeof(edge_scales[0]) && + out.size() < count; + ++i) { + out.push_back(edge_scales[i]); + } + + std::mt19937 rng(seed); + while (out.size() < count) out.push_back(static_cast(rng())); + return out; +} + +void build_s2f6_relu_focus_inputs(std::vector &a, std::vector &b, + std::vector &scale) { + a.clear(); + b.clear(); + scale.clear(); + + const uint32_t a_bits[] = { + 0xff800000u, 0x7fc00000u, 0x80000000u, 0xbf800000u, 0x00000000u, + 0x7f800000u, 0x7f7fffffu, 0xff7fffffu, + }; + const uint32_t b_bits[] = { + 0x7f800000u, 0xffc00000u, 0x00000000u, 0x3f800000u, 0x80000000u, + 0xbf800000u, 0xff800000u, 0x7f800000u, + }; + const uint16_t scale_words[] = { + 0x0000u, 0x0000u, 0x7f7fu, 0x0101u, 0x8000u, 0x0080u, 0x00ffu, 0xff00u, + }; + + const size_t n = sizeof(a_bits) / sizeof(a_bits[0]); + for (size_t i = 0; i < n; ++i) { + a.push_back(bits_to_f32(a_bits[i])); + b.push_back(bits_to_f32(b_bits[i])); + scale.push_back(scale_words[i]); + } +} + +__global__ void k_f16_rz(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rz.f16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_f16_rm(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rm.f16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_f16_rp(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rp.f16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_f16_sat_rn(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.sat.f16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_f16_satfinite_relu_rn(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.relu.f16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_bf16_rz(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rz.bf16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_bf16_rm(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rm.bf16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_bf16_rp(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rp.bf16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_bf16_satfinite_relu_rn(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.relu.bf16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_tf32_rz(uint32_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rz.tf32.f32 %0, %1;" : "=r"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_tf32_rna(uint32_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rna.tf32.f32 %0, %1;" : "=r"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_tf32_satfinite_relu_rn(uint32_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rn.satfinite.relu.tf32.f32 %0, %1;" : "=r"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_f32_bf16_ftz_rn(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + float d = 0.0f; + asm volatile("cvt.rn.ftz.f32.bf16 %0, %1;" : "=f"(d) : "h"(in[i])); + out[i] = __float_as_uint(d); +} + +__global__ void k_f16_bf16_rn(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.f16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_f64_bf16(uint64_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + double d = 0.0; + asm volatile("cvt.f64.bf16 %0, %1;" : "=d"(d) : "h"(in[i])); + out[i] = __double_as_longlong(d); +} + +__global__ void k_s32_bf16_rzi(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rzi.s32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_u32_bf16_rzi(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rzi.u32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_s32_bf16_rni(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rni.s32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_s32_bf16_rmi(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rmi.s32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_s32_bf16_rpi(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rpi.s32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_u32_bf16_rni(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rni.u32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_u32_bf16_rmi(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rmi.u32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_u32_bf16_rpi(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rpi.u32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_s16_bf16_rzi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + short d = 0; + asm volatile("cvt.rzi.s16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_s16_bf16_rni(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + short d = 0; + asm volatile("cvt.rni.s16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_s16_bf16_rmi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + short d = 0; + asm volatile("cvt.rmi.s16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_s16_bf16_rpi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + short d = 0; + asm volatile("cvt.rpi.s16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_u16_bf16_rzi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rzi.u16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_u16_bf16_rni(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rni.u16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_u16_bf16_rmi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rmi.u16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_u16_bf16_rpi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rpi.u16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_s8_bf16_rzi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + short d = 0; + asm volatile("{ .reg .s8 t; cvt.rzi.s8.bf16 t, %1; cvt.s16.s8 %0, t; }" + : "=h"(d) + : "h"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_s8_bf16_rni(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + short d = 0; + asm volatile("{ .reg .s8 t; cvt.rni.s8.bf16 t, %1; cvt.s16.s8 %0, t; }" + : "=h"(d) + : "h"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_s8_bf16_rmi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + short d = 0; + asm volatile("{ .reg .s8 t; cvt.rmi.s8.bf16 t, %1; cvt.s16.s8 %0, t; }" + : "=h"(d) + : "h"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_s8_bf16_rpi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + short d = 0; + asm volatile("{ .reg .s8 t; cvt.rpi.s8.bf16 t, %1; cvt.s16.s8 %0, t; }" + : "=h"(d) + : "h"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_u8_bf16_rzi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("{ .reg .u8 t; cvt.rzi.u8.bf16 t, %1; cvt.u16.u8 %0, t; }" + : "=h"(d) + : "h"(in[i])); + out[i] = d; +} + +__global__ void k_u8_bf16_rni(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("{ .reg .u8 t; cvt.rni.u8.bf16 t, %1; cvt.u16.u8 %0, t; }" + : "=h"(d) + : "h"(in[i])); + out[i] = d; +} + +__global__ void k_u8_bf16_rmi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("{ .reg .u8 t; cvt.rmi.u8.bf16 t, %1; cvt.u16.u8 %0, t; }" + : "=h"(d) + : "h"(in[i])); + out[i] = d; +} + +__global__ void k_u8_bf16_rpi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("{ .reg .u8 t; cvt.rpi.u8.bf16 t, %1; cvt.u16.u8 %0, t; }" + : "=h"(d) + : "h"(in[i])); + out[i] = d; +} + +__global__ void k_s64_bf16_rzi(uint64_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + long long d = 0; + asm volatile("cvt.rzi.s64.bf16 %0, %1;" : "=l"(d) : "h"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_u64_bf16_rzi(uint64_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned long long d = 0; + asm volatile("cvt.rzi.u64.bf16 %0, %1;" : "=l"(d) : "h"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_s64_f32_rzi(uint64_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + long long d = 0; + asm volatile("cvt.rzi.s64.f32 %0, %1;" : "=l"(d) : "f"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_u64_f32_rzi(uint64_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned long long d = 0; + asm volatile("cvt.rzi.u64.f32 %0, %1;" : "=l"(d) : "f"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_s32_f32_rzi_sat(uint32_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rzi.sat.s32.f32 %0, %1;" : "=r"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_u32_f32_rzi_sat(uint32_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rzi.sat.u32.f32 %0, %1;" : "=r"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_s32_f64_rzi_sat(uint32_t *out, const double *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rzi.sat.s32.f64 %0, %1;" : "=r"(d) : "d"(in[i])); + out[i] = d; +} + +__global__ void k_u32_f64_rzi_sat(uint32_t *out, const double *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rzi.sat.u32.f64 %0, %1;" : "=r"(d) : "d"(in[i])); + out[i] = d; +} + +__global__ void k_s16_f32_rzi_sat(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + short d = 0; + asm volatile("cvt.rzi.sat.s16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_u16_f32_rzi_sat(uint16_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rzi.sat.u16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); + out[i] = d; +} + +__global__ void k_s64_f32_rzi_sat(uint64_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + long long d = 0; + asm volatile("cvt.rzi.sat.s64.f32 %0, %1;" : "=l"(d) : "f"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_u64_f32_rzi_sat(uint64_t *out, const float *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned long long d = 0; + asm volatile("cvt.rzi.sat.u64.f32 %0, %1;" : "=l"(d) : "f"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_s16_f64_rzi_sat(uint16_t *out, const double *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + short d = 0; + asm volatile("cvt.rzi.sat.s16.f64 %0, %1;" : "=h"(d) : "d"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_u16_f64_rzi_sat(uint16_t *out, const double *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rzi.sat.u16.f64 %0, %1;" : "=h"(d) : "d"(in[i])); + out[i] = d; +} + +__global__ void k_s64_f64_rzi_sat(uint64_t *out, const double *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + long long d = 0; + asm volatile("cvt.rzi.sat.s64.f64 %0, %1;" : "=l"(d) : "d"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_u64_f64_rzi_sat(uint64_t *out, const double *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned long long d = 0; + asm volatile("cvt.rzi.sat.u64.f64 %0, %1;" : "=l"(d) : "d"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_s64_f64_rzi(uint64_t *out, const double *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + long long d = 0; + asm volatile("cvt.rzi.s64.f64 %0, %1;" : "=l"(d) : "d"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_u64_f64_rzi(uint64_t *out, const double *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned long long d = 0; + asm volatile("cvt.rzi.u64.f64 %0, %1;" : "=l"(d) : "d"(in[i])); + out[i] = static_cast(d); +} + +__global__ void k_bf16_f16_rn(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.bf16.f16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_bf16_s32_rn(uint16_t *out, const int32_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.bf16.s32 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_bf16_u32_rz(uint16_t *out, const uint32_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rz.bf16.u32 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_bf16_f64_rn(uint16_t *out, const double *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.bf16.f64 %0, %1;" : "=h"(d) : "d"(in[i])); + out[i] = d; +} + +__global__ void k_bf16_bf16(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.bf16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_bf16_bf16_rzi(uint16_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rzi.bf16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_f16x2_rn(uint32_t *out, const float *a, const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;" : "=r"(d) : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_bf16x2_rz(uint32_t *out, const float *a, const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rz.bf16x2.f32 %0, %1, %2;" : "=r"(d) : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_e2m1x2_satfinite_rn(uint16_t *out, const float *a, + const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("{ .reg .b8 t; " + "cvt.rn.satfinite.e2m1x2.f32 t, %1, %2; " + "cvt.u16.u8 %0, t; }" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_e4m3x2_satfinite_rn(uint16_t *out, const float *a, + const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_e5m2x2_satfinite_rn(uint16_t *out, const float *a, + const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e5m2x2.f32 %0, %1, %2;" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_e2m3x2_satfinite_rn(uint16_t *out, const float *a, + const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e2m3x2.f32 %0, %1, %2;" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_e3m2x2_satfinite_rn(uint16_t *out, const float *a, + const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e3m2x2.f32 %0, %1, %2;" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_f16x2_e4m3x2_rn(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rn.f16x2.e4m3x2 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_f16x2_e5m2x2_rn(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rn.f16x2.e5m2x2 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_f16x2_e2m1x2_rn(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("{ .reg .b8 t; " + "cvt.u8.u16 t, %1; " + "cvt.rn.f16x2.e2m1x2 %0, t; }" + : "=r"(d) + : "h"(in[i])); + out[i] = d; +} + +__global__ void k_f16x2_e2m3x2_rn(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rn.f16x2.e2m3x2 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_f16x2_e3m2x2_rn(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rn.f16x2.e3m2x2 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_ue8m0x2_satfinite_rz(uint16_t *out, const float *a, + const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rz.satfinite.ue8m0x2.f32 %0, %1, %2;" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_ue8m0x2_rz(uint16_t *out, const float *a, const float *b, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rz.ue8m0x2.f32 %0, %1, %2;" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_ue8m0x2_satfinite_rp(uint16_t *out, const float *a, + const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rp.satfinite.ue8m0x2.f32 %0, %1, %2;" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_ue8m0x2_rp(uint16_t *out, const float *a, const float *b, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rp.ue8m0x2.f32 %0, %1, %2;" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_s2f6x2_satfinite_rn(uint16_t *out, const float *a, + const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.s2f6x2.f32 %0, %1, %2;" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_s2f6x2_satfinite_relu_rn(uint16_t *out, const float *a, + const float *b, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.relu.s2f6x2.f32 %0, %1, %2;" + : "=h"(d) + : "f"(a[i]), "f"(b[i])); + out[i] = d; +} + +__global__ void k_e4m3x2_f16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e4m3x2.f16x2 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_e5m2x2_f16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e5m2x2.f16x2 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_e2m1x2_f16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("{ .reg .b8 t; " + "cvt.rn.satfinite.e2m1x2.f16x2 t, %1; " + "cvt.u16.u8 %0, t; }" + : "=h"(d) + : "r"(in[i])); + out[i] = d; +} + +__global__ void k_e2m3x2_f16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e2m3x2.f16x2 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_e3m2x2_f16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e3m2x2.f16x2 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_e4m3x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_e5m2x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e5m2x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_e2m1x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("{ .reg .b8 t; " + "cvt.rn.satfinite.e2m1x2.bf16x2 t, %1; " + "cvt.u16.u8 %0, t; }" + : "=h"(d) + : "r"(in[i])); + out[i] = d; +} + +__global__ void k_e2m3x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e2m3x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_e3m2x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.e3m2x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_ue8m0x2_bf16x2_satfinite_rz(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rz.satfinite.ue8m0x2.bf16x2 %0, %1;" + : "=h"(d) + : "r"(in[i])); + out[i] = d; +} + +__global__ void k_ue8m0x2_bf16x2_rz(uint16_t *out, const uint32_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rz.ue8m0x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +__global__ void k_ue8m0x2_bf16x2_satfinite_rp(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rp.satfinite.ue8m0x2.bf16x2 %0, %1;" + : "=h"(d) + : "r"(in[i])); + out[i] = d; +} + +__global__ void k_ue8m0x2_bf16x2_rp(uint16_t *out, const uint32_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rp.ue8m0x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); + out[i] = d; +} + +#if LOWP_CVT_HAS_UE8M0X2_BF16X2_RELU +__global__ void k_ue8m0x2_bf16x2_satfinite_relu_rz(uint16_t *out, + const uint32_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rz.satfinite.relu.ue8m0x2.bf16x2 %0, %1;" + : "=h"(d) + : "r"(in[i])); + out[i] = d; +} +#endif + +__global__ void k_bf16x2_ue8m0x2_rn(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rn.bf16x2.ue8m0x2 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_s2f6x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.s2f6x2.bf16x2 %0, %1;" + : "=h"(d) + : "r"(in[i])); + out[i] = d; +} + +__global__ void k_s2f6x2_bf16x2_satfinite_relu_rn(uint16_t *out, + const uint32_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.relu.s2f6x2.bf16x2 %0, %1;" + : "=h"(d) + : "r"(in[i])); + out[i] = d; +} + +__global__ void k_bf16x2_s2f6x2_rn(uint32_t *out, const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rn.bf16x2.s2f6x2 %0, %1;" : "=r"(d) : "h"(in[i])); + out[i] = d; +} + +__global__ void k_bf16x2_s2f6x2_satfinite_relu_rn(uint32_t *out, + const uint16_t *in, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rn.satfinite.relu.bf16x2.s2f6x2 %0, %1;" + : "=r"(d) + : "h"(in[i])); + out[i] = d; +} + +__global__ void k_s2f6x2_f32_sc(uint16_t *out, const float *a, const float *b, + const uint16_t *scale, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rn.satfinite.scaled::n2::ue8m0.s2f6x2.f32 %0, %1, %2, %3;" + : "=h"(d) + : "f"(a[i]), "f"(b[i]), "h"(scale[i])); + out[i] = d; +} + +__global__ void k_s2f6x2_f32_sc_relu(uint16_t *out, const float *a, + const float *b, const uint16_t *scale, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile( + "cvt.rn.satfinite.relu.scaled::n2::ue8m0.s2f6x2.f32 %0, %1, %2, %3;" + : "=h"(d) + : "f"(a[i]), "f"(b[i]), "h"(scale[i])); + out[i] = d; +} + +__global__ void k_s2f6x2_bf16x2_sc(uint16_t *out, const uint32_t *in, + const uint16_t *scale, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile( + "cvt.rn.satfinite.scaled::n2::ue8m0.s2f6x2.bf16x2 %0, %1, %2;" + : "=h"(d) + : "r"(in[i]), "h"(scale[i])); + out[i] = d; +} + +__global__ void k_s2f6x2_bf16x2_sc_relu(uint16_t *out, const uint32_t *in, + const uint16_t *scale, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile( + "cvt.rn.satfinite.relu.scaled::n2::ue8m0.s2f6x2.bf16x2 %0, %1, %2;" + : "=h"(d) + : "r"(in[i]), "h"(scale[i])); + out[i] = d; +} + +__global__ void k_bf16x2_s2f6x2_sc(uint32_t *out, const uint16_t *in, + const uint16_t *scale, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile( + "cvt.rn.satfinite.scaled::n2::ue8m0.bf16x2.s2f6x2 %0, %1, %2;" + : "=r"(d) + : "h"(in[i]), "h"(scale[i])); + out[i] = d; +} + +__global__ void k_bf16x2_s2f6x2_sc_relu(uint32_t *out, const uint16_t *in, + const uint16_t *scale, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile( + "cvt.rn.satfinite.relu.scaled::n2::ue8m0.bf16x2.s2f6x2 %0, %1, %2;" + : "=r"(d) + : "h"(in[i]), "h"(scale[i])); + out[i] = d; +} + +template +void run_unary_kernel(const std::vector &in, std::vector &out, + LaunchFn launch_kernel) { + TIn *d_in = nullptr; + TOut *d_out = nullptr; + const int n = static_cast(in.size()); + CUDACHK(cudaMalloc(&d_in, sizeof(TIn) * in.size())); + CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * in.size())); + CUDACHK(cudaMemcpy(d_in, in.data(), sizeof(TIn) * in.size(), + cudaMemcpyHostToDevice)); + const int block = 128; + const int grid = (n + block - 1) / block; + launch_kernel(grid, block, d_out, d_in, n); + CUDACHK(cudaDeviceSynchronize()); + out.resize(in.size()); + CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), + cudaMemcpyDeviceToHost)); + CUDACHK(cudaFree(d_out)); + CUDACHK(cudaFree(d_in)); +} + +template +void run_pair_kernel(const std::vector &a, const std::vector &b, + std::vector &out, LaunchFn launch_kernel) { + float *d_a = nullptr; + float *d_b = nullptr; + TOut *d_out = nullptr; + const int n = static_cast(a.size()); + CUDACHK(cudaMalloc(&d_a, sizeof(float) * a.size())); + CUDACHK(cudaMalloc(&d_b, sizeof(float) * b.size())); + CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * a.size())); + CUDACHK(cudaMemcpy(d_a, a.data(), sizeof(float) * a.size(), + cudaMemcpyHostToDevice)); + CUDACHK(cudaMemcpy(d_b, b.data(), sizeof(float) * b.size(), + cudaMemcpyHostToDevice)); + const int block = 128; + const int grid = (n + block - 1) / block; + launch_kernel(grid, block, d_out, d_a, d_b, n); + CUDACHK(cudaDeviceSynchronize()); + out.resize(a.size()); + CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), + cudaMemcpyDeviceToHost)); + CUDACHK(cudaFree(d_out)); + CUDACHK(cudaFree(d_b)); + CUDACHK(cudaFree(d_a)); +} + +template +void run_unary_kernel_with_scale(const std::vector &in, + const std::vector &scale, + std::vector &out, + LaunchFn launch_kernel) { + if (in.size() != scale.size()) { + std::cerr << "input/scale size mismatch\n"; + std::exit(2); + } + TIn *d_in = nullptr; + TScale *d_scale = nullptr; + TOut *d_out = nullptr; + const int n = static_cast(in.size()); + CUDACHK(cudaMalloc(&d_in, sizeof(TIn) * in.size())); + CUDACHK(cudaMalloc(&d_scale, sizeof(TScale) * scale.size())); + CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * in.size())); + CUDACHK( + cudaMemcpy(d_in, in.data(), sizeof(TIn) * in.size(), cudaMemcpyHostToDevice)); + CUDACHK(cudaMemcpy(d_scale, scale.data(), sizeof(TScale) * scale.size(), + cudaMemcpyHostToDevice)); + const int block = 128; + const int grid = (n + block - 1) / block; + launch_kernel(grid, block, d_out, d_in, d_scale, n); + CUDACHK(cudaDeviceSynchronize()); + out.resize(in.size()); + CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), + cudaMemcpyDeviceToHost)); + CUDACHK(cudaFree(d_out)); + CUDACHK(cudaFree(d_scale)); + CUDACHK(cudaFree(d_in)); +} + +template +void run_pair_kernel_with_scale(const std::vector &a, + const std::vector &b, + const std::vector &scale, + std::vector &out, + LaunchFn launch_kernel) { + if (a.size() != b.size() || a.size() != scale.size()) { + std::cerr << "pair/scale size mismatch\n"; + std::exit(2); + } + float *d_a = nullptr; + float *d_b = nullptr; + uint16_t *d_scale = nullptr; + TOut *d_out = nullptr; + const int n = static_cast(a.size()); + CUDACHK(cudaMalloc(&d_a, sizeof(float) * a.size())); + CUDACHK(cudaMalloc(&d_b, sizeof(float) * b.size())); + CUDACHK(cudaMalloc(&d_scale, sizeof(uint16_t) * scale.size())); + CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * a.size())); + CUDACHK(cudaMemcpy(d_a, a.data(), sizeof(float) * a.size(), + cudaMemcpyHostToDevice)); + CUDACHK(cudaMemcpy(d_b, b.data(), sizeof(float) * b.size(), + cudaMemcpyHostToDevice)); + CUDACHK(cudaMemcpy(d_scale, scale.data(), sizeof(uint16_t) * scale.size(), + cudaMemcpyHostToDevice)); + const int block = 128; + const int grid = (n + block - 1) / block; + launch_kernel(grid, block, d_out, d_a, d_b, d_scale, n); + CUDACHK(cudaDeviceSynchronize()); + out.resize(a.size()); + CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), + cudaMemcpyDeviceToHost)); + CUDACHK(cudaFree(d_out)); + CUDACHK(cudaFree(d_scale)); + CUDACHK(cudaFree(d_b)); + CUDACHK(cudaFree(d_a)); +} + +template +void dump_hex(std::ofstream &ofs, const std::string &label, + const std::vector &vals, unsigned width) { + for (size_t i = 0; i < vals.size(); ++i) { + ofs << label << " " << i << " 0x" << std::hex << std::setw(width) + << std::setfill('0') + << static_cast(vals[i]) << std::dec << "\n"; + } +} + +int run_and_dump(const char *out_path) { + const std::vector scalars = build_scalar_inputs(); + std::vector pair_a, pair_b; + build_pair_inputs(pair_a, pair_b); + const std::vector bf16_inputs = build_bf16_inputs(); + const std::vector f16_inputs = build_f16_inputs(); + const std::vector s32_inputs = build_s32_inputs(); + const std::vector u32_inputs = build_u32_inputs(); + const std::vector f64_inputs = build_f64_inputs(); + const std::vector e2m1x2_inputs_exhaustive = + build_e2m1x2_inputs_exhaustive(); + const std::vector e2m3x2_inputs_exhaustive = + build_e2m3x2_inputs_exhaustive(); + const std::vector e3m2x2_inputs_exhaustive = + build_e3m2x2_inputs_exhaustive(); + const std::vector u16_inputs_exhaustive = + build_u16_inputs_exhaustive(); + const std::vector pair_scales = + build_scale_inputs(pair_a.size(), 0x4f2a9c13u); + std::vector s2f6_relu_focus_a, s2f6_relu_focus_b; + std::vector s2f6_relu_focus_scales; + build_s2f6_relu_focus_inputs(s2f6_relu_focus_a, s2f6_relu_focus_b, + s2f6_relu_focus_scales); + const std::vector exhaustive_scales = + build_scale_inputs(u16_inputs_exhaustive.size(), 0x6be03491u); + + std::vector out_u16; + std::vector out_u32; + std::vector out_u64; + std::vector tmp_u16; + std::vector tmp_u32; + + std::ofstream ofs(out_path, std::ios::out | std::ios::trunc); + if (!ofs) { + std::cerr << "failed to open output file: " << out_path << "\n"; + return 2; + } + + run_unary_kernel( + scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, + int n) { k_f16_rz<<>>(d_out, d_in, n); }); + dump_hex(ofs, "k_f16_rz", out_u16, 4); + run_unary_kernel( + scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, + int n) { k_f16_rm<<>>(d_out, d_in, n); }); + dump_hex(ofs, "k_f16_rm", out_u16, 4); + run_unary_kernel( + scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, + int n) { k_f16_rp<<>>(d_out, d_in, n); }); + dump_hex(ofs, "k_f16_rp", out_u16, 4); + run_unary_kernel( + scalars, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_in, int n) { + k_f16_sat_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16_sat_rn", out_u16, 4); + run_unary_kernel( + scalars, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_in, int n) { + k_f16_satfinite_relu_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16_satfinite_relu_rn", out_u16, 4); + + run_unary_kernel( + scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, + int n) { k_bf16_rz<<>>(d_out, d_in, n); }); + dump_hex(ofs, "k_bf16_rz", out_u16, 4); + run_unary_kernel( + scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, + int n) { k_bf16_rm<<>>(d_out, d_in, n); }); + dump_hex(ofs, "k_bf16_rm", out_u16, 4); + run_unary_kernel( + scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, + int n) { k_bf16_rp<<>>(d_out, d_in, n); }); + dump_hex(ofs, "k_bf16_rp", out_u16, 4); + run_unary_kernel( + scalars, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_in, int n) { + k_bf16_satfinite_relu_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16_satfinite_relu_rn", out_u16, 4); + + run_unary_kernel( + scalars, out_u32, [](int grid, int block, uint32_t *d_out, const float *d_in, + int n) { k_tf32_rz<<>>(d_out, d_in, n); }); + dump_hex(ofs, "k_tf32_rz", out_u32, 8); + run_unary_kernel( + scalars, out_u32, [](int grid, int block, uint32_t *d_out, const float *d_in, + int n) { k_tf32_rna<<>>(d_out, d_in, n); }); + dump_hex(ofs, "k_tf32_rna", out_u32, 8); + run_unary_kernel( + scalars, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_in, int n) { + k_tf32_satfinite_relu_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_tf32_satfinite_relu_rn", out_u32, 8); + + run_unary_kernel( + bf16_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f32_bf16_ftz_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f32_bf16_ftz_rn", out_u32, 8); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_f16_bf16_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16_bf16_rn", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u64, + [](int grid, int block, uint64_t *d_out, const uint16_t *d_in, int n) { + k_f64_bf16<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f64_bf16", out_u64, 16); + run_unary_kernel( + bf16_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_s32_bf16_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s32_bf16_rzi", out_u32, 8); + run_unary_kernel( + bf16_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_u32_bf16_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u32_bf16_rzi", out_u32, 8); + run_unary_kernel( + bf16_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_s32_bf16_rni<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s32_bf16_rni", out_u32, 8); + run_unary_kernel( + bf16_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_s32_bf16_rmi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s32_bf16_rmi", out_u32, 8); + run_unary_kernel( + bf16_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_s32_bf16_rpi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s32_bf16_rpi", out_u32, 8); + run_unary_kernel( + bf16_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_u32_bf16_rni<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u32_bf16_rni", out_u32, 8); + run_unary_kernel( + bf16_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_u32_bf16_rmi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u32_bf16_rmi", out_u32, 8); + run_unary_kernel( + bf16_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_u32_bf16_rpi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u32_bf16_rpi", out_u32, 8); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_s16_bf16_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s16_bf16_rzi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_s16_bf16_rni<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s16_bf16_rni", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_s16_bf16_rmi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s16_bf16_rmi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_s16_bf16_rpi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s16_bf16_rpi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_u16_bf16_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u16_bf16_rzi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_u16_bf16_rni<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u16_bf16_rni", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_u16_bf16_rmi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u16_bf16_rmi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_u16_bf16_rpi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u16_bf16_rpi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_s8_bf16_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s8_bf16_rzi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_s8_bf16_rni<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s8_bf16_rni", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_s8_bf16_rmi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s8_bf16_rmi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_s8_bf16_rpi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s8_bf16_rpi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_u8_bf16_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u8_bf16_rzi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_u8_bf16_rni<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u8_bf16_rni", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_u8_bf16_rmi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u8_bf16_rmi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_u8_bf16_rpi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u8_bf16_rpi", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u64, + [](int grid, int block, uint64_t *d_out, const uint16_t *d_in, int n) { + k_s64_bf16_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s64_bf16_rzi", out_u64, 16); + run_unary_kernel( + bf16_inputs, out_u64, + [](int grid, int block, uint64_t *d_out, const uint16_t *d_in, int n) { + k_u64_bf16_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u64_bf16_rzi", out_u64, 16); + run_unary_kernel( + scalars, out_u64, [](int grid, int block, uint64_t *d_out, const float *d_in, + int n) { + k_s64_f32_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s64_f32_rzi", out_u64, 16); + run_unary_kernel( + scalars, out_u64, [](int grid, int block, uint64_t *d_out, const float *d_in, + int n) { + k_u64_f32_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u64_f32_rzi", out_u64, 16); + run_unary_kernel( + scalars, out_u32, [](int grid, int block, uint32_t *d_out, const float *d_in, + int n) { + k_s32_f32_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s32_f32_rzi_sat", out_u32, 8); + run_unary_kernel( + scalars, out_u32, [](int grid, int block, uint32_t *d_out, const float *d_in, + int n) { + k_u32_f32_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u32_f32_rzi_sat", out_u32, 8); + run_unary_kernel( + f64_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const double *d_in, int n) { + k_s32_f64_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s32_f64_rzi_sat", out_u32, 8); + run_unary_kernel( + f64_inputs, out_u32, + [](int grid, int block, uint32_t *d_out, const double *d_in, int n) { + k_u32_f64_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u32_f64_rzi_sat", out_u32, 8); + run_unary_kernel( + scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, + int n) { + k_s16_f32_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s16_f32_rzi_sat", out_u16, 4); + run_unary_kernel( + scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, + int n) { + k_u16_f32_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u16_f32_rzi_sat", out_u16, 4); + run_unary_kernel( + scalars, out_u64, [](int grid, int block, uint64_t *d_out, const float *d_in, + int n) { + k_s64_f32_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s64_f32_rzi_sat", out_u64, 16); + run_unary_kernel( + scalars, out_u64, [](int grid, int block, uint64_t *d_out, const float *d_in, + int n) { + k_u64_f32_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u64_f32_rzi_sat", out_u64, 16); + run_unary_kernel( + f64_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const double *d_in, int n) { + k_s16_f64_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s16_f64_rzi_sat", out_u16, 4); + run_unary_kernel( + f64_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const double *d_in, int n) { + k_u16_f64_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u16_f64_rzi_sat", out_u16, 4); + run_unary_kernel( + f64_inputs, out_u64, + [](int grid, int block, uint64_t *d_out, const double *d_in, int n) { + k_s64_f64_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s64_f64_rzi", out_u64, 16); + run_unary_kernel( + f64_inputs, out_u64, + [](int grid, int block, uint64_t *d_out, const double *d_in, int n) { + k_u64_f64_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u64_f64_rzi", out_u64, 16); + run_unary_kernel( + f64_inputs, out_u64, + [](int grid, int block, uint64_t *d_out, const double *d_in, int n) { + k_s64_f64_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s64_f64_rzi_sat", out_u64, 16); + run_unary_kernel( + f64_inputs, out_u64, + [](int grid, int block, uint64_t *d_out, const double *d_in, int n) { + k_u64_f64_rzi_sat<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_u64_f64_rzi_sat", out_u64, 16); + + run_unary_kernel( + f16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_bf16_f16_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16_f16_rn", out_u16, 4); + run_unary_kernel( + s32_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const int32_t *d_in, int n) { + k_bf16_s32_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16_s32_rn", out_u16, 4); + run_unary_kernel( + u32_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_bf16_u32_rz<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16_u32_rz", out_u16, 4); + run_unary_kernel( + f64_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const double *d_in, int n) { + k_bf16_f64_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16_f64_rn", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_bf16_bf16<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16_bf16", out_u16, 4); + run_unary_kernel( + bf16_inputs, out_u16, + [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { + k_bf16_bf16_rzi<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16_bf16_rzi", out_u16, 4); + + run_pair_kernel( + pair_a, pair_b, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, const float *d_b, + int n) { k_f16x2_rn<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_f16x2_rn", out_u32, 8); + const std::vector f16x2_pairs = out_u32; + run_pair_kernel( + pair_a, pair_b, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, const float *d_b, + int n) { k_bf16x2_rz<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_bf16x2_rz", out_u32, 8); + const std::vector bf16x2_pairs = out_u32; + + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_e4m3x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_e4m3x2_satfinite_rn", out_u16, 4); + const std::vector e4m3x2_pairs = out_u16; + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_e5m2x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_e5m2x2_satfinite_rn", out_u16, 4); + const std::vector e5m2x2_pairs = out_u16; + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_e2m1x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_e2m1x2_satfinite_rn", out_u16, 4); + const std::vector e2m1x2_pairs = out_u16; + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_e2m3x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_e2m3x2_satfinite_rn", out_u16, 4); + const std::vector e2m3x2_pairs = out_u16; + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_e3m2x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_e3m2x2_satfinite_rn", out_u16, 4); + const std::vector e3m2x2_pairs = out_u16; + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_ue8m0x2_satfinite_rz<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_ue8m0x2_satfinite_rz", out_u16, 4); + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_ue8m0x2_rz<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_ue8m0x2_rz", out_u16, 4); + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_ue8m0x2_satfinite_rp<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_ue8m0x2_satfinite_rp", out_u16, 4); + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_ue8m0x2_rp<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_ue8m0x2_rp", out_u16, 4); + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_s2f6x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_s2f6x2_satfinite_rn", out_u16, 4); + run_pair_kernel( + pair_a, pair_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_s2f6x2_satfinite_relu_rn<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_s2f6x2_satfinite_relu_rn", out_u16, 4); + run_pair_kernel_with_scale( + pair_a, pair_b, pair_scales, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, + const float *d_b, const uint16_t *d_scale, int n) { + k_s2f6x2_f32_sc<<>>(d_out, d_a, d_b, d_scale, n); + }); + dump_hex(ofs, "k_s2f6x2_f32_satfinite_scaled_rn", out_u16, 4); + run_pair_kernel_with_scale( + pair_a, pair_b, pair_scales, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, + const float *d_b, const uint16_t *d_scale, int n) { + k_s2f6x2_f32_sc_relu<<>>(d_out, d_a, d_b, d_scale, n); + }); + dump_hex(ofs, "k_s2f6x2_f32_satfinite_relu_scaled_rn", out_u16, 4); + run_pair_kernel( + s2f6_relu_focus_a, s2f6_relu_focus_b, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, + int n) { k_s2f6x2_satfinite_relu_rn<<>>(d_out, d_a, d_b, n); }); + dump_hex(ofs, "k_s2f6x2_satfinite_relu_rn_focus", out_u16, 4); + run_pair_kernel_with_scale( + s2f6_relu_focus_a, s2f6_relu_focus_b, s2f6_relu_focus_scales, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, + const float *d_b, const uint16_t *d_scale, int n) { + k_s2f6x2_f32_sc_relu<<>>(d_out, d_a, d_b, d_scale, n); + }); + dump_hex(ofs, "k_s2f6x2_f32_satfinite_relu_scaled_rn_focus", out_u16, 4); + run_pair_kernel( + s2f6_relu_focus_a, s2f6_relu_focus_b, tmp_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, const float *d_b, + int n) { k_bf16x2_rz<<>>(d_out, d_a, d_b, n); }); + const std::vector s2f6_relu_focus_bf16x2 = tmp_u32; + run_unary_kernel( + s2f6_relu_focus_bf16x2, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_s2f6x2_bf16x2_satfinite_relu_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_relu_rn_focus", tmp_u16, 4); + run_unary_kernel_with_scale( + s2f6_relu_focus_bf16x2, s2f6_relu_focus_scales, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, + const uint16_t *d_scale, int n) { + k_s2f6x2_bf16x2_sc_relu<<>>(d_out, d_in, d_scale, n); + }); + dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_relu_scaled_rn_focus", tmp_u16, 4); + + run_unary_kernel( + e4m3x2_pairs, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f16x2_e4m3x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16x2_e4m3x2_rn", tmp_u32, 8); + run_unary_kernel( + e5m2x2_pairs, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f16x2_e5m2x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16x2_e5m2x2_rn", tmp_u32, 8); + run_unary_kernel( + e2m1x2_pairs, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f16x2_e2m1x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16x2_e2m1x2_rn", tmp_u32, 8); + run_unary_kernel( + e2m3x2_pairs, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f16x2_e2m3x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16x2_e2m3x2_rn", tmp_u32, 8); + run_unary_kernel( + e3m2x2_pairs, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f16x2_e3m2x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16x2_e3m2x2_rn", tmp_u32, 8); + + run_unary_kernel( + f16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e4m3x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e4m3x2_f16x2_satfinite_rn", tmp_u16, 4); + run_unary_kernel( + f16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e5m2x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e5m2x2_f16x2_satfinite_rn", tmp_u16, 4); + run_unary_kernel( + f16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e2m1x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e2m1x2_f16x2_satfinite_rn", tmp_u16, 4); + run_unary_kernel( + f16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e2m3x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e2m3x2_f16x2_satfinite_rn", tmp_u16, 4); + run_unary_kernel( + f16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e3m2x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e3m2x2_f16x2_satfinite_rn", tmp_u16, 4); + + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e4m3x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e4m3x2_bf16x2_satfinite_rn", tmp_u16, 4); + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e5m2x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e5m2x2_bf16x2_satfinite_rn", tmp_u16, 4); + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e2m1x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e2m1x2_bf16x2_satfinite_rn", tmp_u16, 4); + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e2m3x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e2m3x2_bf16x2_satfinite_rn", tmp_u16, 4); + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e3m2x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e3m2x2_bf16x2_satfinite_rn", tmp_u16, 4); + + run_unary_kernel( + u16_inputs_exhaustive, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f16x2_e4m3x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16x2_e4m3x2_rn_exhaustive", tmp_u32, 8); + run_unary_kernel( + tmp_u32, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e4m3x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e4m3x2_f16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, + 4); + + run_unary_kernel( + u16_inputs_exhaustive, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f16x2_e5m2x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16x2_e5m2x2_rn_exhaustive", tmp_u32, 8); + run_unary_kernel( + tmp_u32, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e5m2x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e5m2x2_f16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, + 4); + + run_unary_kernel( + e2m1x2_inputs_exhaustive, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f16x2_e2m1x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16x2_e2m1x2_rn_exhaustive", tmp_u32, 8); + run_unary_kernel( + tmp_u32, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e2m1x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e2m1x2_f16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, + 4); + + run_unary_kernel( + e2m3x2_inputs_exhaustive, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f16x2_e2m3x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16x2_e2m3x2_rn_exhaustive", tmp_u32, 8); + run_unary_kernel( + tmp_u32, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e2m3x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e2m3x2_f16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, + 4); + + run_unary_kernel( + e3m2x2_inputs_exhaustive, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_f16x2_e3m2x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_f16x2_e3m2x2_rn_exhaustive", tmp_u32, 8); + run_unary_kernel( + tmp_u32, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_e3m2x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_e3m2x2_f16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, + 4); + + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_ue8m0x2_bf16x2_satfinite_rz<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_ue8m0x2_bf16x2_satfinite_rz", tmp_u16, 4); + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_ue8m0x2_bf16x2_rz<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_ue8m0x2_bf16x2_rz", tmp_u16, 4); + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_ue8m0x2_bf16x2_satfinite_rp<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_ue8m0x2_bf16x2_satfinite_rp", tmp_u16, 4); + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_ue8m0x2_bf16x2_rp<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_ue8m0x2_bf16x2_rp", tmp_u16, 4); +#if LOWP_CVT_HAS_UE8M0X2_BF16X2_RELU + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_ue8m0x2_bf16x2_satfinite_relu_rz<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_ue8m0x2_bf16x2_satfinite_relu_rz", tmp_u16, 4); +#endif + run_unary_kernel( + tmp_u16, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_bf16x2_ue8m0x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16x2_ue8m0x2_rn", tmp_u32, 8); + + run_unary_kernel( + u16_inputs_exhaustive, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_bf16x2_ue8m0x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16x2_ue8m0x2_rn_exhaustive", tmp_u32, 8); + run_unary_kernel( + tmp_u32, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_ue8m0x2_bf16x2_satfinite_rz<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_ue8m0x2_bf16x2_satfinite_rz_roundtrip_exhaustive", tmp_u16, + 4); + + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_s2f6x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_rn", tmp_u16, 4); + const std::vector s2f6_pairs = tmp_u16; + run_unary_kernel( + bf16x2_pairs, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_s2f6x2_bf16x2_satfinite_relu_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_relu_rn", tmp_u16, 4); + run_unary_kernel( + s2f6_pairs, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_bf16x2_s2f6x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16x2_s2f6x2_rn", tmp_u32, 8); + run_unary_kernel( + s2f6_pairs, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_bf16x2_s2f6x2_satfinite_relu_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16x2_s2f6x2_satfinite_relu_rn", tmp_u32, 8); + run_unary_kernel_with_scale( + bf16x2_pairs, pair_scales, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, + const uint16_t *d_scale, int n) { + k_s2f6x2_bf16x2_sc<<>>(d_out, d_in, d_scale, n); + }); + dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_scaled_rn", tmp_u16, 4); + run_unary_kernel_with_scale( + bf16x2_pairs, pair_scales, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, + const uint16_t *d_scale, int n) { + k_s2f6x2_bf16x2_sc_relu<<>>(d_out, d_in, d_scale, n); + }); + dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_relu_scaled_rn", tmp_u16, 4); + run_unary_kernel_with_scale( + s2f6_pairs, pair_scales, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, + const uint16_t *d_scale, int n) { + k_bf16x2_s2f6x2_sc<<>>(d_out, d_in, d_scale, n); + }); + dump_hex(ofs, "k_bf16x2_s2f6x2_satfinite_scaled_rn", tmp_u32, 8); + run_unary_kernel_with_scale( + s2f6_pairs, pair_scales, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, + const uint16_t *d_scale, int n) { + k_bf16x2_s2f6x2_sc_relu<<>>(d_out, d_in, d_scale, n); + }); + dump_hex(ofs, "k_bf16x2_s2f6x2_satfinite_relu_scaled_rn", tmp_u32, 8); + + run_unary_kernel( + u16_inputs_exhaustive, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { + k_bf16x2_s2f6x2_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_bf16x2_s2f6x2_rn_exhaustive", tmp_u32, 8); + run_unary_kernel( + tmp_u32, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { + k_s2f6x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); + }); + dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, + 4); + run_unary_kernel_with_scale( + u16_inputs_exhaustive, exhaustive_scales, tmp_u32, + [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, + const uint16_t *d_scale, int n) { + k_bf16x2_s2f6x2_sc<<>>(d_out, d_in, d_scale, n); + }); + dump_hex(ofs, "k_bf16x2_s2f6x2_satfinite_scaled_rn_exhaustive", tmp_u32, 8); + run_unary_kernel_with_scale( + tmp_u32, exhaustive_scales, tmp_u16, + [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, + const uint16_t *d_scale, int n) { + k_s2f6x2_bf16x2_sc<<>>(d_out, d_in, d_scale, n); + }); + dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_scaled_rn_roundtrip_exhaustive", + tmp_u16, 4); + + return 0; +} + +} // namespace + +int main(int argc, char **argv) { + if (argc != 3 || std::string(argv[1]) != "--output") { + std::cerr << "usage: " << argv[0] << " --output \n"; + return 2; + } + return run_and_dump(argv[2]); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu new file mode 100644 index 000000000..977dcb1a4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu @@ -0,0 +1,748 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +#define CUDACHK(call) \ + do { \ + cudaError_t _err = (call); \ + if (_err != cudaSuccess) { \ + std::cerr << "CUDA error: " << cudaGetErrorString(_err) << " @ " \ + << #call << "\n"; \ + std::exit(2); \ + } \ + } while (0) + +float bits_to_f32(uint32_t bits) { + float out = 0.0f; + std::memcpy(&out, &bits, sizeof(out)); + return out; +} + +enum class rs_minifloat_family_t { + kE4M3, + kE5M2, + kE2M1, + kE2M3, + kE3M2, +}; + +struct rs_minifloat_entry_t { + uint16_t code; + float value; +}; + +unsigned rs_minifloat_code_limit(rs_minifloat_family_t fmt) { + switch (fmt) { + case rs_minifloat_family_t::kE4M3: + case rs_minifloat_family_t::kE5M2: + return 256u; + case rs_minifloat_family_t::kE2M1: + return 16u; + case rs_minifloat_family_t::kE2M3: + case rs_minifloat_family_t::kE3M2: + return 64u; + } + return 0u; +} + +float decode_rs_minifloat_code(rs_minifloat_family_t fmt, uint16_t code) { + int exp_bits = 0; + int mant_bits = 0; + int bias = 0; + bool has_inf = false; + switch (fmt) { + case rs_minifloat_family_t::kE4M3: + exp_bits = 4; + mant_bits = 3; + bias = 7; + has_inf = false; + break; + case rs_minifloat_family_t::kE5M2: + exp_bits = 5; + mant_bits = 2; + bias = 15; + has_inf = true; + break; + case rs_minifloat_family_t::kE2M1: + exp_bits = 2; + mant_bits = 1; + bias = 1; + has_inf = false; + break; + case rs_minifloat_family_t::kE2M3: + exp_bits = 2; + mant_bits = 3; + bias = 1; + has_inf = false; + break; + case rs_minifloat_family_t::kE3M2: + exp_bits = 3; + mant_bits = 2; + bias = 3; + has_inf = false; + break; + } + + const int sign_shift = exp_bits + mant_bits; + const bool neg = ((code >> sign_shift) & 0x1u) != 0u; + const int exp_mask = (1 << exp_bits) - 1; + const int mant_mask = (1 << mant_bits) - 1; + const int exp = (code >> mant_bits) & exp_mask; + const int mant = code & mant_mask; + + if (fmt == rs_minifloat_family_t::kE4M3 && + (code == 0x7fu || code == 0xffu)) { + return std::numeric_limits::quiet_NaN(); + } + if (has_inf && exp == exp_mask) { + if (mant == 0) { + return neg ? -std::numeric_limits::infinity() + : std::numeric_limits::infinity(); + } + return std::numeric_limits::quiet_NaN(); + } + + float value = 0.0f; + if (exp == 0) { + if (mant != 0) { + value = std::ldexp(static_cast(mant), 1 - bias - mant_bits); + } + } else { + const int sig = (1 << mant_bits) | mant; + value = std::ldexp(static_cast(sig), exp - bias - mant_bits); + } + return neg ? -value : value; +} + +std::vector collect_rs_midpoint_ties(rs_minifloat_family_t fmt, + size_t max_count) { + std::vector entries; + entries.reserve(rs_minifloat_code_limit(fmt)); + for (unsigned code = 0; code < rs_minifloat_code_limit(fmt); ++code) { + const float value = decode_rs_minifloat_code(fmt, static_cast(code)); + if (!std::isfinite(value)) continue; + entries.push_back({static_cast(code), value}); + } + + std::sort(entries.begin(), entries.end(), + [](const rs_minifloat_entry_t &a, const rs_minifloat_entry_t &b) { + if (a.value < b.value) return true; + if (a.value > b.value) return false; + return a.code < b.code; + }); + + std::vector out; + out.reserve(max_count); + for (size_t i = 0; i + 1 < entries.size() && out.size() < max_count; ++i) { + const float lo = entries[i].value; + const float hi = entries[i + 1].value; + if (!(lo < hi)) continue; + const long double midpoint = + (static_cast(lo) + static_cast(hi)) * 0.5L; + if (!std::isfinite(static_cast(midpoint))) continue; + const float mid_f32 = static_cast(midpoint); + if (!std::isfinite(mid_f32)) continue; + if (static_cast(mid_f32) != midpoint) continue; + out.push_back(mid_f32); + } + return out; +} + +std::vector edge_f32_patterns() { + return { + 0x00000000u, 0x80000000u, 0x00000001u, 0x80000001u, 0x007fffffu, + 0x807fffffu, 0x00800000u, 0x80800000u, 0x00800001u, 0x80800001u, + 0x3f7fffffu, 0x3f800000u, 0x3f800001u, 0xbf7fffffu, 0xbf800000u, + 0xbf800001u, 0x7f7fffffu, 0xff7fffffu, 0x7f800000u, 0xff800000u, + 0x7fc00000u, 0xffc00000u, 0x7fa00001u, 0xffa00001u, + }; +} + +void build_pair_rs_inputs(std::vector &a, std::vector &b, + std::vector &rbits) { + a.clear(); + b.clear(); + rbits.clear(); + + std::mt19937 rng(0x84f0a313u); + + const std::vector edges = edge_f32_patterns(); + for (size_t i = 0; i + 1 < edges.size(); i += 2) { + a.push_back(bits_to_f32(edges[i])); + b.push_back(bits_to_f32(edges[i + 1])); + + const uint32_t ra = rng() & 0x1fffu; + const uint32_t rb = rng() & 0x1fffu; + rbits.push_back((ra << 16) | rb); + } + + const uint32_t manual_rbits[] = { + 0x00000000u, 0x00010001u, 0x1fff1fffu, 0x1fff0000u, + 0x00001fffu, 0x12340789u, 0x15550aaau, + }; + for (uint32_t r : manual_rbits) { + a.push_back(bits_to_f32(rng())); + b.push_back(bits_to_f32(rng())); + rbits.push_back(static_cast(((r >> 16) & 0x1fffu) << 16) | + (r & 0x1fffu)); + } + + // Deterministic tie-focused probes for .rs pair conversions. + const uint32_t midpoint_bits[] = { + 0x3f801000u, 0xbf801000u, 0x3f001000u, 0xbf001000u, + }; + const uint16_t tie_rbits[] = {0x0fffu, 0x1000u}; + for (uint32_t bits : midpoint_bits) { + const float x = bits_to_f32(bits); + for (uint16_t r : tie_rbits) { + a.push_back(x); + b.push_back(x); + rbits.push_back((static_cast(r) << 16) | r); + } + } + + while (a.size() < 1024) { + a.push_back(bits_to_f32(rng())); + b.push_back(bits_to_f32(rng())); + + const uint32_t ra = rng() & 0x1fffu; + const uint32_t rb = rng() & 0x1fffu; + rbits.push_back((ra << 16) | rb); + } +} + +void build_quad_rs_inputs(std::vector &a, std::vector &b, + std::vector &e, std::vector &f, + std::vector &rbits) { + a.clear(); + b.clear(); + e.clear(); + f.clear(); + rbits.clear(); + + std::mt19937 rng(0x2f08bc41u); + + const std::vector edges = edge_f32_patterns(); + for (size_t i = 0; i + 3 < edges.size(); i += 4) { + a.push_back(bits_to_f32(edges[i])); + b.push_back(bits_to_f32(edges[i + 1])); + e.push_back(bits_to_f32(edges[i + 2])); + f.push_back(bits_to_f32(edges[i + 3])); + rbits.push_back(rng()); + } + + const uint32_t manual_rbits[] = { + 0x00000000u, 0xffffffffu, 0x01020304u, 0x11223344u, + 0x89abcdefu, 0xfedcba98u, + }; + for (uint32_t r : manual_rbits) { + a.push_back(bits_to_f32(rng())); + b.push_back(bits_to_f32(rng())); + e.push_back(bits_to_f32(rng())); + f.push_back(bits_to_f32(rng())); + rbits.push_back(r); + } + + // Deterministic midpoint/tie probes for each minifloat x4 .rs family. + const rs_minifloat_family_t families[] = { + rs_minifloat_family_t::kE4M3, rs_minifloat_family_t::kE5M2, + rs_minifloat_family_t::kE2M1, rs_minifloat_family_t::kE2M3, + rs_minifloat_family_t::kE3M2, + }; + for (rs_minifloat_family_t fmt : families) { + const std::vector ties = collect_rs_midpoint_ties(fmt, 8); + for (float tie : ties) { + a.push_back(tie); + b.push_back(tie); + e.push_back(tie); + f.push_back(tie); + rbits.push_back(0x7f7f7f7fu); + + a.push_back(tie); + b.push_back(tie); + e.push_back(tie); + f.push_back(tie); + rbits.push_back(0x80808080u); + + const float down = + std::nextafter(tie, -std::numeric_limits::infinity()); + const float up = + std::nextafter(tie, std::numeric_limits::infinity()); + if (std::isfinite(down) && std::isfinite(up)) { + a.push_back(down); + b.push_back(tie); + e.push_back(up); + f.push_back(tie); + rbits.push_back(0x7f80807fu); + } + } + } + + while (a.size() < 768) { + a.push_back(bits_to_f32(rng())); + b.push_back(bits_to_f32(rng())); + e.push_back(bits_to_f32(rng())); + f.push_back(bits_to_f32(rng())); + rbits.push_back(rng()); + } +} + +__global__ void k_f16x2_rs(uint32_t *out, const float *a, const float *b, + const uint32_t *rbits, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.f16x2.f32 %0, %1, %2, %3;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_f16x2_satfinite_relu_rs(uint32_t *out, const float *a, + const float *b, + const uint32_t *rbits, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.satfinite.relu.f16x2.f32 %0, %1, %2, %3;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_bf16x2_rs(uint32_t *out, const float *a, const float *b, + const uint32_t *rbits, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.bf16x2.f32 %0, %1, %2, %3;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_bf16x2_satfinite_relu_rs(uint32_t *out, const float *a, + const float *b, + const uint32_t *rbits, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.satfinite.relu.bf16x2.f32 %0, %1, %2, %3;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_e4m3x4_satfinite_rs(uint32_t *out, const float *a, + const float *b, const float *e, + const float *f, const uint32_t *rbits, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.satfinite.e4m3x4.f32 %0, {%1, %2, %3, %4}, %5;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_e4m3x4_satfinite_relu_rs(uint32_t *out, const float *a, + const float *b, const float *e, + const float *f, + const uint32_t *rbits, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.satfinite.relu.e4m3x4.f32 %0, {%1, %2, %3, %4}, %5;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_e5m2x4_satfinite_rs(uint32_t *out, const float *a, + const float *b, const float *e, + const float *f, const uint32_t *rbits, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.satfinite.e5m2x4.f32 %0, {%1, %2, %3, %4}, %5;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_e5m2x4_satfinite_relu_rs(uint32_t *out, const float *a, + const float *b, const float *e, + const float *f, + const uint32_t *rbits, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.satfinite.relu.e5m2x4.f32 %0, {%1, %2, %3, %4}, %5;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_e2m1x4_satfinite_rs(uint16_t *out, const float *a, + const float *b, const float *e, + const float *f, const uint32_t *rbits, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rs.satfinite.e2m1x4.f32 %0, {%1, %2, %3, %4}, %5;" + : "=h"(d) + : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_e2m1x4_satfinite_relu_rs(uint16_t *out, const float *a, + const float *b, const float *e, + const float *f, + const uint32_t *rbits, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned short d = 0; + asm volatile("cvt.rs.satfinite.relu.e2m1x4.f32 %0, {%1, %2, %3, %4}, %5;" + : "=h"(d) + : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_e2m3x4_satfinite_rs(uint32_t *out, const float *a, + const float *b, const float *e, + const float *f, const uint32_t *rbits, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.satfinite.e2m3x4.f32 %0, {%1, %2, %3, %4}, %5;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_e2m3x4_satfinite_relu_rs(uint32_t *out, const float *a, + const float *b, const float *e, + const float *f, + const uint32_t *rbits, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.satfinite.relu.e2m3x4.f32 %0, {%1, %2, %3, %4}, %5;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_e3m2x4_satfinite_rs(uint32_t *out, const float *a, + const float *b, const float *e, + const float *f, const uint32_t *rbits, + int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.satfinite.e3m2x4.f32 %0, {%1, %2, %3, %4}, %5;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); + out[i] = d; +} + +__global__ void k_e3m2x4_satfinite_relu_rs(uint32_t *out, const float *a, + const float *b, const float *e, + const float *f, + const uint32_t *rbits, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + unsigned d = 0; + asm volatile("cvt.rs.satfinite.relu.e3m2x4.f32 %0, {%1, %2, %3, %4}, %5;" + : "=r"(d) + : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); + out[i] = d; +} + +template +void run_pair_kernel_with_rbits(const std::vector &a, + const std::vector &b, + const std::vector &rbits, + std::vector &out, + LaunchFn launch_kernel) { + float *d_a = nullptr; + float *d_b = nullptr; + uint32_t *d_rbits = nullptr; + TOut *d_out = nullptr; + const int n = static_cast(a.size()); + + CUDACHK(cudaMalloc(&d_a, sizeof(float) * a.size())); + CUDACHK(cudaMalloc(&d_b, sizeof(float) * b.size())); + CUDACHK(cudaMalloc(&d_rbits, sizeof(uint32_t) * rbits.size())); + CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * a.size())); + + CUDACHK(cudaMemcpy(d_a, a.data(), sizeof(float) * a.size(), + cudaMemcpyHostToDevice)); + CUDACHK(cudaMemcpy(d_b, b.data(), sizeof(float) * b.size(), + cudaMemcpyHostToDevice)); + CUDACHK(cudaMemcpy(d_rbits, rbits.data(), sizeof(uint32_t) * rbits.size(), + cudaMemcpyHostToDevice)); + + const int block = 128; + const int grid = (n + block - 1) / block; + launch_kernel(grid, block, d_out, d_a, d_b, d_rbits, n); + CUDACHK(cudaDeviceSynchronize()); + + out.resize(a.size()); + CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), + cudaMemcpyDeviceToHost)); + + CUDACHK(cudaFree(d_out)); + CUDACHK(cudaFree(d_rbits)); + CUDACHK(cudaFree(d_b)); + CUDACHK(cudaFree(d_a)); +} + +template +void run_quad_kernel_with_rbits(const std::vector &a, + const std::vector &b, + const std::vector &e, + const std::vector &f, + const std::vector &rbits, + std::vector &out, + LaunchFn launch_kernel) { + float *d_a = nullptr; + float *d_b = nullptr; + float *d_e = nullptr; + float *d_f = nullptr; + uint32_t *d_rbits = nullptr; + TOut *d_out = nullptr; + const int n = static_cast(a.size()); + + CUDACHK(cudaMalloc(&d_a, sizeof(float) * a.size())); + CUDACHK(cudaMalloc(&d_b, sizeof(float) * b.size())); + CUDACHK(cudaMalloc(&d_e, sizeof(float) * e.size())); + CUDACHK(cudaMalloc(&d_f, sizeof(float) * f.size())); + CUDACHK(cudaMalloc(&d_rbits, sizeof(uint32_t) * rbits.size())); + CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * a.size())); + + CUDACHK(cudaMemcpy(d_a, a.data(), sizeof(float) * a.size(), + cudaMemcpyHostToDevice)); + CUDACHK(cudaMemcpy(d_b, b.data(), sizeof(float) * b.size(), + cudaMemcpyHostToDevice)); + CUDACHK(cudaMemcpy(d_e, e.data(), sizeof(float) * e.size(), + cudaMemcpyHostToDevice)); + CUDACHK(cudaMemcpy(d_f, f.data(), sizeof(float) * f.size(), + cudaMemcpyHostToDevice)); + CUDACHK(cudaMemcpy(d_rbits, rbits.data(), sizeof(uint32_t) * rbits.size(), + cudaMemcpyHostToDevice)); + + const int block = 128; + const int grid = (n + block - 1) / block; + launch_kernel(grid, block, d_out, d_a, d_b, d_e, d_f, d_rbits, n); + CUDACHK(cudaDeviceSynchronize()); + + out.resize(a.size()); + CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), + cudaMemcpyDeviceToHost)); + + CUDACHK(cudaFree(d_out)); + CUDACHK(cudaFree(d_rbits)); + CUDACHK(cudaFree(d_f)); + CUDACHK(cudaFree(d_e)); + CUDACHK(cudaFree(d_b)); + CUDACHK(cudaFree(d_a)); +} + +template +void dump_hex(std::ofstream &ofs, const std::string &label, + const std::vector &vals, unsigned width) { + for (size_t i = 0; i < vals.size(); ++i) { + ofs << label << " " << i << " 0x" << std::hex << std::setw(width) + << std::setfill('0') + << static_cast(vals[i]) << std::dec << "\n"; + } +} + +int run_and_dump(const char *out_path) { + std::vector pair_a, pair_b; + std::vector pair_rbits; + build_pair_rs_inputs(pair_a, pair_b, pair_rbits); + + std::vector quad_a, quad_b, quad_e, quad_f; + std::vector quad_rbits; + build_quad_rs_inputs(quad_a, quad_b, quad_e, quad_f, quad_rbits); + + std::vector out_u16; + std::vector out_u32; + + std::ofstream ofs(out_path, std::ios::out | std::ios::trunc); + if (!ofs) { + std::cerr << "failed to open output file: " << out_path << "\n"; + return 2; + } + + run_pair_kernel_with_rbits( + pair_a, pair_b, pair_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const uint32_t *d_rbits, int n) { + k_f16x2_rs<<>>(d_out, d_a, d_b, d_rbits, n); + }); + bool all_zero = true; + for (size_t i = 0; i < out_u32.size(); ++i) { + if (out_u32[i] != 0u) { + all_zero = false; + break; + } + } + if (all_zero) { + std::cerr << "RS_FEATURE_UNAVAILABLE: k_f16x2_rs produced all-zero output" + << "\n"; + return 3; + } + dump_hex(ofs, "k_f16x2_rs", out_u32, 8); + + run_pair_kernel_with_rbits( + pair_a, pair_b, pair_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const uint32_t *d_rbits, int n) { + k_f16x2_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_rbits, + n); + }); + dump_hex(ofs, "k_f16x2_satfinite_relu_rs", out_u32, 8); + + run_pair_kernel_with_rbits( + pair_a, pair_b, pair_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const uint32_t *d_rbits, int n) { + k_bf16x2_rs<<>>(d_out, d_a, d_b, d_rbits, n); + }); + dump_hex(ofs, "k_bf16x2_rs", out_u32, 8); + + run_pair_kernel_with_rbits( + pair_a, pair_b, pair_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const uint32_t *d_rbits, int n) { + k_bf16x2_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_rbits, + n); + }); + dump_hex(ofs, "k_bf16x2_satfinite_relu_rs", out_u32, 8); + + run_quad_kernel_with_rbits( + quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const float *d_e, const float *d_f, + const uint32_t *d_rbits, int n) { + k_e4m3x4_satfinite_rs<<>>(d_out, d_a, d_b, d_e, d_f, + d_rbits, n); + }); + dump_hex(ofs, "k_e4m3x4_satfinite_rs", out_u32, 8); + run_quad_kernel_with_rbits( + quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const float *d_e, const float *d_f, + const uint32_t *d_rbits, int n) { + k_e4m3x4_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_e, d_f, + d_rbits, n); + }); + dump_hex(ofs, "k_e4m3x4_satfinite_relu_rs", out_u32, 8); + + run_quad_kernel_with_rbits( + quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const float *d_e, const float *d_f, + const uint32_t *d_rbits, int n) { + k_e5m2x4_satfinite_rs<<>>(d_out, d_a, d_b, d_e, d_f, + d_rbits, n); + }); + dump_hex(ofs, "k_e5m2x4_satfinite_rs", out_u32, 8); + run_quad_kernel_with_rbits( + quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const float *d_e, const float *d_f, + const uint32_t *d_rbits, int n) { + k_e5m2x4_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_e, d_f, + d_rbits, n); + }); + dump_hex(ofs, "k_e5m2x4_satfinite_relu_rs", out_u32, 8); + + run_quad_kernel_with_rbits( + quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, + const float *d_b, const float *d_e, const float *d_f, + const uint32_t *d_rbits, int n) { + k_e2m1x4_satfinite_rs<<>>(d_out, d_a, d_b, d_e, d_f, + d_rbits, n); + }); + dump_hex(ofs, "k_e2m1x4_satfinite_rs", out_u16, 4); + run_quad_kernel_with_rbits( + quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u16, + [](int grid, int block, uint16_t *d_out, const float *d_a, + const float *d_b, const float *d_e, const float *d_f, + const uint32_t *d_rbits, int n) { + k_e2m1x4_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_e, d_f, + d_rbits, n); + }); + dump_hex(ofs, "k_e2m1x4_satfinite_relu_rs", out_u16, 4); + + run_quad_kernel_with_rbits( + quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const float *d_e, const float *d_f, + const uint32_t *d_rbits, int n) { + k_e2m3x4_satfinite_rs<<>>(d_out, d_a, d_b, d_e, d_f, + d_rbits, n); + }); + dump_hex(ofs, "k_e2m3x4_satfinite_rs", out_u32, 8); + run_quad_kernel_with_rbits( + quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const float *d_e, const float *d_f, + const uint32_t *d_rbits, int n) { + k_e2m3x4_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_e, d_f, + d_rbits, n); + }); + dump_hex(ofs, "k_e2m3x4_satfinite_relu_rs", out_u32, 8); + + run_quad_kernel_with_rbits( + quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const float *d_e, const float *d_f, + const uint32_t *d_rbits, int n) { + k_e3m2x4_satfinite_rs<<>>(d_out, d_a, d_b, d_e, d_f, + d_rbits, n); + }); + dump_hex(ofs, "k_e3m2x4_satfinite_rs", out_u32, 8); + run_quad_kernel_with_rbits( + quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, + [](int grid, int block, uint32_t *d_out, const float *d_a, + const float *d_b, const float *d_e, const float *d_f, + const uint32_t *d_rbits, int n) { + k_e3m2x4_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_e, d_f, + d_rbits, n); + }); + dump_hex(ofs, "k_e3m2x4_satfinite_relu_rs", out_u32, 8); + + return 0; +} + +} // namespace + +int main(int argc, char **argv) { + if (argc != 3 || std::string(argv[1]) != "--output") { + std::cerr << "usage: " << argv[0] << " --output \n"; + return 2; + } + return run_and_dump(argv[2]); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/README.md b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/README.md new file mode 100644 index 000000000..bc8e104d7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/README.md @@ -0,0 +1,14 @@ +# PTX Datatype Parser Smoke Cases + +`run_ptx_datatype_parser_smoke.sh` validates lexer/parser + IR construction only. +Does not execute PTX instructions, so `instructions.cc:cvt_impl()` runtime +bounds checking for `g_cvt_fn[src_fmt][dst_fmt]` cannot be exercised here. + +The cvt bounds guard remains in runtime code to protect execution paths that +use datatype format IDs beyond the legacy conversion table dimensions. + +`run_lowp_cvt_ptxas_crosscheck.sh` includes a spec-tracking gate for +`cvt.*.ue8m0x2.bf16x2` with `.relu`. By default +`LOWP_CVT_SPEC_TRACK_UE8M0X2_BF16X2_RELU=auto` (track current toolchain +behavior); set `LOWP_CVT_SPEC_TRACK_UE8M0X2_BF16X2_RELU=ptxas_accepts` when +toolchains begin accepting the form. diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.ptx new file mode 100644 index 000000000..964b028a6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_bf16_not_allowed(.param .bf16 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.ptx new file mode 100644 index 000000000..a61ca6199 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_bf16x2_not_allowed(.param .bf16x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.ptx new file mode 100644 index 000000000..59548fd27 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e2m1_not_allowed(.param .e2m1 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.ptx new file mode 100644 index 000000000..03917b7d3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e2m1x2_not_allowed(.param .e2m1x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.ptx new file mode 100644 index 000000000..03fe0839c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e2m1x4_not_allowed(.param .e2m1x4 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.ptx new file mode 100644 index 000000000..88412e00b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e2m3_not_allowed(.param .e2m3 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.ptx new file mode 100644 index 000000000..683b918c7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e2m3x2_not_allowed(.param .e2m3x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.ptx new file mode 100644 index 000000000..3c006fe78 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e2m3x4_not_allowed(.param .e2m3x4 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.ptx new file mode 100644 index 000000000..de7bf36f1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e3m2_not_allowed(.param .e3m2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.ptx new file mode 100644 index 000000000..9db04a5f2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e3m2x2_not_allowed(.param .e3m2x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.ptx new file mode 100644 index 000000000..aa28891ca --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e3m2x4_not_allowed(.param .e3m2x4 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.ptx new file mode 100644 index 000000000..a6c89a5c4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e4m3_not_allowed(.param .e4m3 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.ptx new file mode 100644 index 000000000..d5f16ad39 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e4m3x2_not_allowed(.param .e4m3x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.ptx new file mode 100644 index 000000000..7772ce13b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e4m3x4_not_allowed(.param .e4m3x4 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.ptx new file mode 100644 index 000000000..547d5c8b4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e5m2_not_allowed(.param .e5m2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.ptx new file mode 100644 index 000000000..dade9ba4f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e5m2x2_not_allowed(.param .e5m2x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.ptx new file mode 100644 index 000000000..bf76505b2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_e5m2x4_not_allowed(.param .e5m2x4 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.expect new file mode 100644 index 000000000..aee0192e4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.expect @@ -0,0 +1 @@ +datatype ".f16x2" cannot be used in .param declarations; use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.ptx new file mode 100644 index 000000000..afb973b77 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_f16x2_not_allowed(.param .f16x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.expect new file mode 100644 index 000000000..1e942bed4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b64" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.ptx new file mode 100644 index 000000000..61908015f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_f32x2_not_allowed(.param .f32x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.ptx new file mode 100644 index 000000000..c6d8ae3dd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_s16x2_not_allowed(.param .s16x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.ptx new file mode 100644 index 000000000..17b67ba5b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_s2f6_not_allowed(.param .s2f6 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.ptx new file mode 100644 index 000000000..07622f91d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_s2f6x2_not_allowed(.param .s2f6x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.ptx new file mode 100644 index 000000000..d352545dc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_tf32_not_allowed(.param .tf32 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.ptx new file mode 100644 index 000000000..80b83a672 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_u16x2_not_allowed(.param .u16x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.ptx new file mode 100644 index 000000000..dce1d62c7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_ue4m3_not_allowed(.param .ue4m3 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.ptx new file mode 100644 index 000000000..001d8f540 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_ue8m0_not_allowed(.param .ue8m0 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.ptx new file mode 100644 index 000000000..cbc81ee0b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_ue8m0x2_not_allowed(.param .ue8m0x2 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.ptx new file mode 100644 index 000000000..57866e700 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_bf16_not_allowed() { + .reg .bf16 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.ptx new file mode 100644 index 000000000..d2530e948 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_bf16x2_not_allowed() { + .reg .bf16x2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.ptx new file mode 100644 index 000000000..5f7d93375 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e2m1_not_allowed() { + .reg .e2m1 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.ptx new file mode 100644 index 000000000..e53289fc3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e2m1x2_not_allowed() { + .reg .e2m1x2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.ptx new file mode 100644 index 000000000..496cd925c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e2m1x4_not_allowed() { + .reg .e2m1x4 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.ptx new file mode 100644 index 000000000..7483aa0db --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e2m3_not_allowed() { + .reg .e2m3 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.ptx new file mode 100644 index 000000000..9856e3d53 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e2m3x2_not_allowed() { + .reg .e2m3x2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.ptx new file mode 100644 index 000000000..e71d70b02 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e2m3x4_not_allowed() { + .reg .e2m3x4 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.ptx new file mode 100644 index 000000000..8736a7840 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e3m2_not_allowed() { + .reg .e3m2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.ptx new file mode 100644 index 000000000..e38893d83 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e3m2x2_not_allowed() { + .reg .e3m2x2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.ptx new file mode 100644 index 000000000..7e3a1172b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e3m2x4_not_allowed() { + .reg .e3m2x4 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.ptx new file mode 100644 index 000000000..48eae06d3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e4m3_not_allowed() { + .reg .e4m3 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.ptx new file mode 100644 index 000000000..716827cbe --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e4m3x2_not_allowed() { + .reg .e4m3x2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.ptx new file mode 100644 index 000000000..d8dbfb07e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e4m3x4_not_allowed() { + .reg .e4m3x4 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.ptx new file mode 100644 index 000000000..8bc5a699a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e5m2_not_allowed() { + .reg .e5m2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.ptx new file mode 100644 index 000000000..31d73358e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e5m2x2_not_allowed() { + .reg .e5m2x2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.ptx new file mode 100644 index 000000000..7c83ac280 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_e5m2x4_not_allowed() { + .reg .e5m2x4 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.expect new file mode 100644 index 000000000..1e942bed4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b64" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.ptx new file mode 100644 index 000000000..623974f41 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_f32x2_not_allowed() { + .reg .f32x2 %d0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.ptx new file mode 100644 index 000000000..68401ca6c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_s16x2_not_allowed() { + .reg .s16x2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.ptx new file mode 100644 index 000000000..cd8974de8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_s2f6_not_allowed() { + .reg .s2f6 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.ptx new file mode 100644 index 000000000..39a20c848 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_s2f6x2_not_allowed() { + .reg .s2f6x2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.ptx new file mode 100644 index 000000000..340ea8208 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_tf32_not_allowed() { + .reg .tf32 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.expect new file mode 100644 index 000000000..bf0ed7010 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.ptx new file mode 100644 index 000000000..9f61e6a23 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_u16x2_not_allowed() { + .reg .u16x2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.ptx new file mode 100644 index 000000000..8204785be --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_ue4m3_not_allowed() { + .reg .ue4m3 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.expect new file mode 100644 index 000000000..ed2cbdf84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.ptx new file mode 100644 index 000000000..8f1a807b3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_ue8m0_not_allowed() { + .reg .ue8m0 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.expect new file mode 100644 index 000000000..8a09ac699 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.expect @@ -0,0 +1,2 @@ +non-fundamental instruction-only datatype +use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.ptx new file mode 100644 index 000000000..50674d5ba --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_ue8m0x2_not_allowed() { + .reg .ue8m0x2 %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.expect new file mode 100644 index 000000000..97b500494 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.expect @@ -0,0 +1 @@ +declaration initializers are not supported for datatype ".f16" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.ptx new file mode 100644 index 000000000..13fae918a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry initializer_f16_brace_not_allowed() { + .reg .f16 %h0 = {0}; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.expect new file mode 100644 index 000000000..97b500494 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.expect @@ -0,0 +1 @@ +declaration initializers are not supported for datatype ".f16" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.ptx new file mode 100644 index 000000000..cf52c08dc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry initializer_f16_not_allowed() { + .reg .f16 %h0 = 0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.expect new file mode 100644 index 000000000..a45fde33c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.expect @@ -0,0 +1 @@ +declaration initializers are not supported for datatype ".f16x2" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.ptx new file mode 100644 index 000000000..516b6ace6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry initializer_f16x2_brace_not_allowed() { + .reg .f16x2 %r0 = {0}; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.expect new file mode 100644 index 000000000..a45fde33c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.expect @@ -0,0 +1 @@ +declaration initializers are not supported for datatype ".f16x2" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.ptx new file mode 100644 index 000000000..d19272d2e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry initializer_f16x2_not_allowed() { + .reg .f16x2 %r0 = 0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.expect new file mode 100644 index 000000000..0e6e35910 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.expect @@ -0,0 +1 @@ +declaration initializers are not supported for datatype ".pred" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.ptx new file mode 100644 index 000000000..791f17631 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry initializer_pred_brace_not_allowed() { + .reg .pred %p0 = {0}; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.expect new file mode 100644 index 000000000..0e6e35910 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.expect @@ -0,0 +1 @@ +declaration initializers are not supported for datatype ".pred" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.ptx new file mode 100644 index 000000000..3797a84e6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry initializer_pred_not_allowed() { + .reg .pred %p0 = 0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.expect new file mode 100644 index 000000000..979531235 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.expect @@ -0,0 +1 @@ +does not allow modifier ".ftz" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.ptx new file mode 100644 index 000000000..1fc32d9d1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_bf16_f16_ftz_not_allowed() { + .reg .b16 %h0; + .reg .b16 %h1; + + cvt.rn.ftz.bf16.f16 %h0, %h1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.expect new file mode 100644 index 000000000..a2fbb57f2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.ptx new file mode 100644 index 000000000..b7530c178 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_80 +.address_size 64 + +.visible .entry instruction_cvt_bf16_f16_sm80_not_allowed() { + .reg .b16 %h0; + .reg .b16 %h1; + + cvt.rn.bf16.f16 %h0, %h1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.expect new file mode 100644 index 000000000..a9a8eae8d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.expect @@ -0,0 +1 @@ +requires a floating-point rounding modifier in {.rn, .rz, .rm, .rp} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.ptx new file mode 100644 index 000000000..b6aad08df --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_bf16_f32_missing_rounding_not_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + + cvt.bf16.f32 %h0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.expect new file mode 100644 index 000000000..c383bd768 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.expect @@ -0,0 +1 @@ +allows modifier ".sat" only for floating-point destination types {.f16, .f32, .f64} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.ptx new file mode 100644 index 000000000..0fa02bac7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_120 +.address_size 64 + +.visible .entry instruction_cvt_bf16_f32_sat_not_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + + cvt.rn.sat.bf16.f32 %h0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.expect new file mode 100644 index 000000000..34a42399c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.expect @@ -0,0 +1 @@ +with modifiers ".satfinite" or ".relu" requires rounding modifier ".rn" or ".rz" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.ptx new file mode 100644 index 000000000..9befe5494 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_120 +.address_size 64 + +.visible .entry instruction_cvt_bf16_f32_satfinite_rm_not_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + + cvt.rm.satfinite.bf16.f32 %h0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.expect new file mode 100644 index 000000000..50a346065 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_80" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.ptx new file mode 100644 index 000000000..60beec746 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_75 +.address_size 64 + +.visible .entry instruction_cvt_bf16_f32_sm75_not_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + + cvt.rn.bf16.f32 %h0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.expect new file mode 100644 index 000000000..a2fbb57f2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.ptx new file mode 100644 index 000000000..34db9bcf0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_80 +.address_size 64 + +.visible .entry instruction_cvt_bf16_s32_sm80_not_allowed() { + .reg .b16 %h0; + .reg .s32 %r0; + + cvt.rn.bf16.s32 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.expect new file mode 100644 index 000000000..759efe784 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" datatype combination "BF16X2_TYPE.F32_TYPE" +requires operand shape "d, a, b" or "d, a, b, rbits" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.ptx new file mode 100644 index 000000000..ce44c747b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_bf16x2_f32_single_src_not_allowed() { + .reg .b32 %r0; + .reg .f32 %f0; + + cvt.rz.bf16x2.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.expect new file mode 100644 index 000000000..0d36b3f06 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.expect @@ -0,0 +1 @@ +does not allow modifier ".satfinite" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.ptx new file mode 100644 index 000000000..db4aaef74 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.satfinite.bf16x2.ue8m0x2 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.expect new file mode 100644 index 000000000..6c2280c3d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.expect @@ -0,0 +1 @@ +requires an ".target sm_100a" (or newer arch-a) target or ".target sm_100f" (or newer arch-f) target for datatype destination "E2M1X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.ptx new file mode 100644 index 000000000..4b730968c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_e2m1x2_dst_sm100_not_allowed() { + .reg .b8 %b0; + .reg .b32 %r0; + + cvt.rn.satfinite.e2m1x2.f16x2 %b0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.expect new file mode 100644 index 000000000..f085c3721 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.expect @@ -0,0 +1 @@ +requires an ".target sm_100a" (or newer arch-a) target or ".target sm_100f" (or newer arch-f) target diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.ptx new file mode 100644 index 000000000..ffe0525e3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_120 +.address_size 64 + +.visible .entry instruction_cvt_e2m1x2_f32_sm120_not_allowed() { + .reg .b8 %b0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.expect new file mode 100644 index 000000000..ee57e96b5 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.expect @@ -0,0 +1 @@ +requires an ".target sm_100a" (or newer arch-a) target or ".target sm_100f" (or newer arch-f) target for datatype source "E2M1X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.ptx new file mode 100644 index 000000000..72f4c1cea --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_e2m1x2_src_sm100_not_allowed() { + .reg .b8 %b0; + .reg .b32 %r0; + + cvt.rn.f16x2.e2m1x2 %r0, %b0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.expect new file mode 100644 index 000000000..52709d518 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.expect @@ -0,0 +1 @@ +requires an ".target sm_100a" (or newer arch-a) target or ".target sm_100f" (or newer arch-f) target for datatype source "E2M3X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.ptx new file mode 100644 index 000000000..7613e344b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_e2m3x2_sm100_not_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.f16x2.e2m3x2 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.expect new file mode 100644 index 000000000..fb2759887 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_89" or higher for datatype destination "E4M3X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.ptx new file mode 100644 index 000000000..06e51d33a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_80 +.address_size 64 + +.visible .entry instruction_cvt_e4m3x2_sm80_not_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rn.satfinite.e4m3x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.expect new file mode 100644 index 000000000..a2fbb57f2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.ptx new file mode 100644 index 000000000..96e615a82 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_80 +.address_size 64 + +.visible .entry instruction_cvt_f16_bf16_sm80_not_allowed() { + .reg .b16 %h0; + .reg .b16 %h1; + + cvt.rn.f16.bf16 %h0, %h1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.expect new file mode 100644 index 000000000..a9a8eae8d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.expect @@ -0,0 +1 @@ +requires a floating-point rounding modifier in {.rn, .rz, .rm, .rp} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.ptx new file mode 100644 index 000000000..4bc06f445 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_f16_f32_missing_rounding_not_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + + cvt.f16.f32 %h0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.expect new file mode 100644 index 000000000..1c86804d8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" datatype combination "F16X2_TYPE.F32_TYPE" +requires operand shape "d, a, b" or "d, a, b, rbits" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.ptx new file mode 100644 index 000000000..34ef10edd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_f16x2_f32_single_src_not_allowed() { + .reg .b32 %r0; + .reg .f32 %f0; + + cvt.rn.f16x2.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.expect new file mode 100644 index 000000000..50a346065 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_80" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.ptx new file mode 100644 index 000000000..4ebe2e2fa --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_75 +.address_size 64 + +.visible .entry instruction_cvt_f16x2_f32_sm75_not_allowed() { + .reg .b32 %r0; + .reg .f32 %f0, %f1; + + cvt.rn.f16x2.f32 %r0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.expect new file mode 100644 index 000000000..5009bf073 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" does not support datatype combination +F16X2_TYPE.UE8M0X2_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.ptx new file mode 100644 index 000000000..22ed7b173 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.ptx @@ -0,0 +1,10 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_f16x2_ue8m0x2_not_allowed() { + .reg .b32 %r0; + .reg .b16 %h0; + cvt.rn.f16x2.ue8m0x2 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.expect new file mode 100644 index 000000000..a2fbb57f2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.ptx new file mode 100644 index 000000000..91e1b917b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_80 +.address_size 64 + +.visible .entry instruction_cvt_f32_bf16_ftz_sm80_not_allowed() { + .reg .f32 %f0; + .reg .b16 %h0; + + cvt.rn.ftz.f32.bf16 %f0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.expect new file mode 100644 index 000000000..a9a8eae8d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.expect @@ -0,0 +1 @@ +requires a floating-point rounding modifier in {.rn, .rz, .rm, .rp} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.ptx new file mode 100644 index 000000000..90f098dc7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_f32_s32_missing_rounding_not_allowed() { + .reg .f32 %f0; + .reg .s32 %s0; + + cvt.f32.s32 %f0, %s0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.expect new file mode 100644 index 000000000..8e5841f5d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" does not support datatype combination +F32_TYPE.TF32_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.ptx new file mode 100644 index 000000000..9ac99951d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_f32_tf32_not_allowed() { + .reg .f32 %f0; + .reg .b32 %r0; + + cvt.rn.f32.tf32 %f0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.expect new file mode 100644 index 000000000..524a955e7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.expect @@ -0,0 +1 @@ +instruction "cvt" does not support datatype combination "F32X2_TYPE.F32_TYPE" per PTX ISA diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.ptx new file mode 100644 index 000000000..a4504b9d5 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.ptx @@ -0,0 +1,10 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_f32x2_dst_not_allowed() { + .reg .b64 %d0; + .reg .f32 %f0; + cvt.f32x2.f32 %d0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.expect new file mode 100644 index 000000000..c6e48c4c0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.expect @@ -0,0 +1 @@ +instruction "cvt" does not support datatype combination "F32_TYPE.F32X2_TYPE" per PTX ISA diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.ptx new file mode 100644 index 000000000..e49a901a8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.ptx @@ -0,0 +1,10 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_f32x2_src_not_allowed() { + .reg .f32 %f0; + .reg .b64 %d0; + cvt.f32.f32x2 %f0, %d0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.expect new file mode 100644 index 000000000..bbcf66a01 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" does not support datatype combination +PRED_TYPE.U32_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.ptx new file mode 100644 index 000000000..237934210 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.ptx @@ -0,0 +1,10 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_pred_not_allowed() { + .reg .pred %p0; + .reg .u32 %u0; + cvt.pred.u32 %p0, %u0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.expect new file mode 100644 index 000000000..ad9846e6d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.expect @@ -0,0 +1 @@ +instruction "cvt" requires exactly two datatypes ., found 1 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.ptx new file mode 100644 index 000000000..c3c1a0c4a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.ptx @@ -0,0 +1,10 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_requires_two_types() { + .reg .u32 %u0; + .reg .s32 %s0; + cvt.u32 %u0, %s0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.expect new file mode 100644 index 000000000..c1e03a0aa --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_100a" or ".target sm_103a" for rounding modifier ".rs" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.ptx new file mode 100644 index 000000000..58f8c4aa7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_rs_sm100_not_allowed() { + .reg .b32 %r0, %r1; + .reg .f32 %f0, %f1; + + cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.expect new file mode 100644 index 000000000..c1e03a0aa --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_100a" or ".target sm_103a" for rounding modifier ".rs" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.ptx new file mode 100644 index 000000000..c81f294b4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100f +.address_size 64 + +.visible .entry instruction_cvt_rs_sm100f_not_allowed() { + .reg .b32 %r0, %r1; + .reg .f32 %f0, %f1; + + cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.expect new file mode 100644 index 000000000..c1e03a0aa --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_100a" or ".target sm_103a" for rounding modifier ".rs" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.ptx new file mode 100644 index 000000000..b587d5a37 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.ptx @@ -0,0 +1,13 @@ +.version 9.1 +.target sm_110a +.address_size 64 + +.visible .entry instruction_cvt_rs_sm110a_not_allowed() { + .reg .b32 %r0; + .reg .f32 %f0; + .reg .f32 %f1; + .reg .b32 %r1; + + cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.expect new file mode 100644 index 000000000..c1e03a0aa --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_100a" or ".target sm_103a" for rounding modifier ".rs" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.ptx new file mode 100644 index 000000000..2e9d27501 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_120a +.address_size 64 + +.visible .entry instruction_cvt_rs_sm120a_not_allowed() { + .reg .b32 %r0, %r1; + .reg .f32 %f0, %f1; + + cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.expect new file mode 100644 index 000000000..c1e03a0aa --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_100a" or ".target sm_103a" for rounding modifier ".rs" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.ptx new file mode 100644 index 000000000..42cfad069 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_120f +.address_size 64 + +.visible .entry instruction_cvt_rs_sm120f_not_allowed() { + .reg .b32 %r0, %r1; + .reg .f32 %f0, %f1; + + cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.expect new file mode 100644 index 000000000..fbf1f0128 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" datatype combination "S2F6X2_TYPE.BF16X2_TYPE" +requires ".satfinite" modifier diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.ptx new file mode 100644 index 000000000..b1a72724d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.s2f6x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.expect new file mode 100644 index 000000000..899e8cf13 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" datatype combination "S2F6X2_TYPE.BF16X2_TYPE" +requires ".scaled::n2::ue8m0" qualifier and scale-factor operand to appear together diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.ptx new file mode 100644 index 000000000..299ab9908 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed() { + .reg .b16 %h0, %h1; + .reg .b32 %r0; + + cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0, %h1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.expect new file mode 100644 index 000000000..899e8cf13 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" datatype combination "S2F6X2_TYPE.BF16X2_TYPE" +requires ".scaled::n2::ue8m0" qualifier and scale-factor operand to appear together diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.ptx new file mode 100644 index 000000000..25fa29256 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.satfinite.scaled::n2::ue8m0.s2f6x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.expect new file mode 100644 index 000000000..924b7f25e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.expect @@ -0,0 +1 @@ +requires an ".target sm_100a/sm_101a/sm_103a/sm_110a/sm_120a/sm_121a" target for datatype destination "S2F6X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx new file mode 100644 index 000000000..e58ace0bc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100f +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.expect new file mode 100644 index 000000000..924b7f25e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.expect @@ -0,0 +1 @@ +requires an ".target sm_100a/sm_101a/sm_103a/sm_110a/sm_120a/sm_121a" target for datatype destination "S2F6X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx new file mode 100644 index 000000000..a17478691 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_120f +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.expect new file mode 100644 index 000000000..924b7f25e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.expect @@ -0,0 +1 @@ +requires an ".target sm_100a/sm_101a/sm_103a/sm_110a/sm_120a/sm_121a" target for datatype destination "S2F6X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.ptx new file mode 100644 index 000000000..21366a9cb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_sm100_not_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.expect new file mode 100644 index 000000000..3f36dd30a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.expect @@ -0,0 +1 @@ +allows modifier ".sat" only for floating-point destination types {.f16, .f32, .f64} or integer destination types with source {.f16, .f32, .f64} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.ptx new file mode 100644 index 000000000..6a6b6dca1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_90 +.address_size 64 + +.visible .entry instruction_cvt_s32_bf16_sat_not_allowed() { + .reg .s32 %r0; + .reg .b16 %h0; + + cvt.rzi.sat.s32.bf16 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.expect new file mode 100644 index 000000000..a2fbb57f2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.ptx new file mode 100644 index 000000000..b9e71675a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_80 +.address_size 64 + +.visible .entry instruction_cvt_s32_bf16_sm80_not_allowed() { + .reg .s32 %r0; + .reg .b16 %h0; + + cvt.rzi.s32.bf16 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.expect new file mode 100644 index 000000000..3f36dd30a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.expect @@ -0,0 +1 @@ +allows modifier ".sat" only for floating-point destination types {.f16, .f32, .f64} or integer destination types with source {.f16, .f32, .f64} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.ptx new file mode 100644 index 000000000..1629c9ef7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_90 +.address_size 64 + +.visible .entry instruction_cvt_s32_s32_sat_not_allowed() { + .reg .s32 %r0; + .reg .s32 %r1; + + cvt.sat.s32.s32 %r0, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.expect new file mode 100644 index 000000000..68ae1fd84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.expect @@ -0,0 +1 @@ +allows modifiers ".satfinite" and ".relu" only for destination {.f16, .bf16} with source .f32 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.ptx new file mode 100644 index 000000000..06e34235f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_scalar_relu_not_allowed() { + .reg .s32 %r0; + .reg .f32 %f0; + + cvt.rni.relu.s32.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.expect new file mode 100644 index 000000000..68ae1fd84 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.expect @@ -0,0 +1 @@ +allows modifiers ".satfinite" and ".relu" only for destination {.f16, .bf16} with source .f32 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.ptx new file mode 100644 index 000000000..787134c31 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_scalar_satfinite_not_allowed() { + .reg .s32 %r0; + .reg .f32 %f0; + + cvt.rni.satfinite.s32.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.expect new file mode 100644 index 000000000..2dc5a8fb4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" datatype combination "TF32_TYPE.F32_TYPE" +requires a rounding modifier diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.ptx new file mode 100644 index 000000000..54a937815 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_tf32_f32_missing_rounding_not_allowed() { + .reg .b32 %r0; + .reg .f32 %f0; + + cvt.tf32.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.expect new file mode 100644 index 000000000..a2fbb57f2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.ptx new file mode 100644 index 000000000..9cbe1eb37 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_89 +.address_size 64 + +.visible .entry instruction_cvt_tf32_f32_rn_sm89_not_allowed() { + .reg .b32 %r0; + .reg .f32 %f0; + + cvt.rn.tf32.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.expect new file mode 100644 index 000000000..91af51d18 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.expect @@ -0,0 +1 @@ +does not allow modifier combination ".relu" with rounding ".rna" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.ptx new file mode 100644 index 000000000..3fecb3c7f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_tf32_f32_rna_relu_not_allowed() { + .reg .b32 %r0; + .reg .f32 %f0; + + cvt.rna.relu.tf32.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.expect new file mode 100644 index 000000000..5ecd319f3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.expect @@ -0,0 +1 @@ +requires ".target sm_100" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.ptx new file mode 100644 index 000000000..f48243604 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_90 +.address_size 64 + +.visible .entry instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed() { + .reg .b32 %r0; + .reg .f32 %f0; + + cvt.rn.satfinite.tf32.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.expect new file mode 100644 index 000000000..845d16fea --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" does not support datatype combination +TF32_TYPE.U32_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.ptx new file mode 100644 index 000000000..3b2d3b6cf --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.ptx @@ -0,0 +1,10 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_tf32_u32_not_allowed() { + .reg .b32 %r0; + .reg .u32 %u0; + cvt.tf32.u32 %r0, %u0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.expect new file mode 100644 index 000000000..191081fed --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" does not support datatype combination +U16X2_TYPE.S32_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.ptx new file mode 100644 index 000000000..228273ccf --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.ptx @@ -0,0 +1,10 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_u16x2_not_allowed() { + .reg .b32 %r0; + .reg .s32 %s0; + cvt.u16x2.s32 %r0, %s0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.expect new file mode 100644 index 000000000..2339eef9c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.expect @@ -0,0 +1 @@ +instruction "cvt" does not support datatype combination diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.ptx new file mode 100644 index 000000000..331235b1b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_120a +.address_size 64 + +.visible .entry instruction_cvt_ue4m3_f32_not_allowed() { + .reg .b8 %b0; + .reg .f32 %f0; + + cvt.rn.satfinite.ue4m3.f32 %b0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.expect new file mode 100644 index 000000000..c0b0d5bfa --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" datatype combination "UE8M0X2_TYPE.BF16X2_TYPE" +requires a rounding modifier diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.ptx new file mode 100644 index 000000000..18d20da69 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.satfinite.ue8m0x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.expect new file mode 100644 index 000000000..1bdfb0b11 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "cvt" does not support datatype combination +UE8M0X2_TYPE.F16X2_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.ptx new file mode 100644 index 000000000..915d1fa3c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.ptx @@ -0,0 +1,10 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_f16x2_not_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + cvt.rz.ue8m0x2.f16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.expect new file mode 100644 index 000000000..a4e52a0a2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.expect @@ -0,0 +1 @@ +does not allow modifier ".relu" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.ptx new file mode 100644 index 000000000..4176a68b2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_110f +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_f32_relu_not_allowed() { + .reg .f32 %f<3>; + .reg .b16 %h0; + + cvt.rz.satfinite.relu.ue8m0x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.expect new file mode 100644 index 000000000..f085c3721 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.expect @@ -0,0 +1 @@ +requires an ".target sm_100a" (or newer arch-a) target or ".target sm_100f" (or newer arch-f) target diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.ptx new file mode 100644 index 000000000..722336440 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_120 +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_f32_sm120_not_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.expect new file mode 100644 index 000000000..fb009a1c6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.expect @@ -0,0 +1 @@ +does not allow modifier ".scaled::n2::ue8m0" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.ptx new file mode 100644 index 000000000..79f0b4f2f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_scaled_not_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + .reg .f32 %f0, %f1; + + cvt.rz.scaled::n2::ue8m0.ue8m0x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.expect new file mode 100644 index 000000000..a707d2e85 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.expect @@ -0,0 +1 @@ +instruction "mov" datatype ".b128/.bb128" requires operand shapes diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.ptx new file mode 100644 index 000000000..49df0b963 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_b128_pack_v2_mixed_width_not_allowed() { + .reg .b128 %q0; + .reg .b64 %d0; + .reg .b32 %r0; + + mov.b128 %q0, {%d0, %r0}; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.expect new file mode 100644 index 000000000..a707d2e85 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.expect @@ -0,0 +1 @@ +instruction "mov" datatype ".b128/.bb128" requires operand shapes diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.ptx new file mode 100644 index 000000000..3b7dd5c23 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_b128_scalar_not_allowed() { + .reg .b128 %q0; + .reg .b32 %r0, %r1, %r2; + + mov.b128 %q0, {%r0, %r1, %r2}; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.expect new file mode 100644 index 000000000..a707d2e85 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.expect @@ -0,0 +1 @@ +instruction "mov" datatype ".b128/.bb128" requires operand shapes diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.ptx new file mode 100644 index 000000000..60a63d26f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_bb128_scalar_not_allowed() { + .reg .bb128 %q0; + .reg .b32 %r0, %r1, %r2; + + mov.bb128 %q0, {%r0, %r1, %r2}; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.expect new file mode 100644 index 000000000..395433de9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "mov" does not support datatype +BF16_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.ptx new file mode 100644 index 000000000..eb43d3310 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.ptx @@ -0,0 +1,9 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_bf16_not_allowed() { + .reg .b16 %h0, %h1; + mov.bf16 %h0, %h1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.expect new file mode 100644 index 000000000..b1bc1f01c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.expect @@ -0,0 +1,2 @@ +instruction "mov" does not support datatype +F16_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.ptx new file mode 100644 index 000000000..6b1af2fd4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.ptx @@ -0,0 +1,9 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_f16_not_allowed() { + .reg .b16 %h0, %h1; + mov.f16 %h0, %h1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.expect new file mode 100644 index 000000000..fa930aec0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.expect @@ -0,0 +1 @@ +unsupported PTX datatype token ".s32x1" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.ptx new file mode 100644 index 000000000..29590683d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.ptx @@ -0,0 +1,9 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry unsupported_token_s32x1() { + .reg .b32 %r0, %r1; + mov.s32x1 %r0, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.expect new file mode 100644 index 000000000..c92cc8b79 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.expect @@ -0,0 +1 @@ +unsupported PTX datatype token ".u32x1" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.ptx new file mode 100644 index 000000000..d1370ce53 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.ptx @@ -0,0 +1,9 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry unsupported_token_u32x1() { + .reg .b32 %r0, %r1; + mov.u32x1 %r0, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/all_new_instruction_types.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/all_new_instruction_types.ptx new file mode 100644 index 000000000..9fd5e3d22 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/all_new_instruction_types.ptx @@ -0,0 +1,32 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry all_new_instruction_types() { + .reg .pred %p0, %p1; + .reg .b16 %h0, %h1; + .reg .b32 %r0, %r1; + .reg .b64 %d0, %d1; + .reg .s32 %s0, %s1; + .reg .u32 %u0, %u1; + .reg .f32 %f0, %f1; + .reg .f64 %fd0, %fd1; + + mov.pred %p0, %p1; + mov.b16 %h0, %h1; + mov.b32 %r0, %r1; + mov.b64 %d0, %d1; + mov.s32 %s0, %s1; + mov.u32 %u0, %u1; + mov.f32 %f0, %f1; + mov.f64 %fd0, %fd1; + + cvt.u32.s32 %u0, %s0; + cvt.s32.u32 %s0, %u0; + cvt.rn.f32.s32 %f0, %s0; + cvt.f64.f32 %fd0, %f0; + cvt.rn.f16.f32 %h0, %f0; + cvt.rn.bf16.f32 %h0, %f0; + + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_param_b32_for_f16x2_storage_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_param_b32_for_f16x2_storage_allowed.ptx new file mode 100644 index 000000000..2ae08b936 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_param_b32_for_f16x2_storage_allowed.ptx @@ -0,0 +1,7 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_param_b32_for_f16x2_storage_allowed(.param .b32 p0) { + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_b128_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_b128_allowed.ptx new file mode 100644 index 000000000..4e9b2fccb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_b128_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_b128_allowed() { + .reg .b128 %q0; + .reg .b32 %r0, %r1, %r2, %r3; + + mov.b128 %q0, {%r0, %r1, %r2, %r3}; + mov.b128 {%r0, %r1, %r2, %r3}, %q0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_f16x2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_f16x2_allowed.ptx new file mode 100644 index 000000000..c8c5f17db --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_f16x2_allowed.ptx @@ -0,0 +1,8 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_reg_f16x2_allowed() { + .reg .f16x2 %r0, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_underlying_bit_types.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_underlying_bit_types.ptx new file mode 100644 index 000000000..47cc7a9dc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_underlying_bit_types.ptx @@ -0,0 +1,16 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry declaration_underlying_bit_types() { + .reg .b8 %b0, %b1; + .reg .b16 %h0, %h1; + .reg .b32 %r0, %r1; + .reg .b64 %d0, %d1; + .reg .b128 %q0; + + mov.b16 %h0, %h1; + mov.b32 %r0, %r1; + mov.b64 %d0, %d1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f16_sm90_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f16_sm90_allowed.ptx new file mode 100644 index 000000000..e65c07e09 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f16_sm90_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_90 +.address_size 64 + +.visible .entry instruction_cvt_bf16_f16_sm90_allowed() { + .reg .b16 %h0; + .reg .b16 %h1; + + cvt.rn.bf16.f16 %h0, %h1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f32_satfinite_relu_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f32_satfinite_relu_allowed.ptx new file mode 100644 index 000000000..dffaf4164 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f32_satfinite_relu_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_120 +.address_size 64 + +.visible .entry instruction_cvt_bf16_f32_satfinite_relu_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + + cvt.rn.satfinite.relu.bf16.f32 %h0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_s32_sm90_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_s32_sm90_allowed.ptx new file mode 100644 index 000000000..ca33c74af --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_s32_sm90_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_90 +.address_size 64 + +.visible .entry instruction_cvt_bf16_s32_sm90_allowed() { + .reg .b16 %h0; + .reg .s32 %r0; + + cvt.rn.bf16.s32 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_f32_pair_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_f32_pair_allowed.ptx new file mode 100644 index 000000000..1cad380a8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_f32_pair_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_bf16x2_f32_pair_allowed() { + .reg .b32 %r0; + .reg .f32 %f0, %f1; + + cvt.rz.bf16x2.f32 %r0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_allowed.ptx new file mode 100644 index 000000000..fdd8e24a5 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_bf16x2_s2f6x2_allowed() { + .reg .b32 %r0; + .reg .b16 %h0; + + cvt.rn.bf16x2.s2f6x2 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm103a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm103a_allowed.ptx new file mode 100644 index 000000000..10ee8e316 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm103a_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_103a +.address_size 64 + +.visible .entry instruction_cvt_bf16x2_s2f6x2_sm103a_allowed() { + .reg .b32 %r0; + .reg .b16 %h0; + + cvt.rn.bf16x2.s2f6x2 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm121a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm121a_allowed.ptx new file mode 100644 index 000000000..eeac3f0b8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm121a_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_121a +.address_size 64 + +.visible .entry instruction_cvt_bf16x2_s2f6x2_sm121a_allowed() { + .reg .b32 %r0; + .reg .b16 %h0; + + cvt.rn.bf16x2.s2f6x2 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_ue8m0x2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_ue8m0x2_allowed.ptx new file mode 100644 index 000000000..945fe46da --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_ue8m0x2_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_bf16x2_ue8m0x2_allowed() { + .reg .b32 %r0; + .reg .b16 %h0; + + cvt.rn.bf16x2.ue8m0x2 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100a_allowed.ptx new file mode 100644 index 000000000..5817036ea --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100a_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_e2m1x2_f32_sm100a_allowed() { + .reg .b8 %b0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100f_allowed.ptx new file mode 100644 index 000000000..2776a2a9a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100f_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_100f +.address_size 64 + +.visible .entry instruction_cvt_e2m1x2_f32_sm100f_allowed() { + .reg .b8 %b0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm103a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm103a_allowed.ptx new file mode 100644 index 000000000..36a539b10 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm103a_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_103a +.address_size 64 + +.visible .entry instruction_cvt_e2m1x2_f32_sm103a_allowed() { + .reg .b8 %b0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120a_allowed.ptx new file mode 100644 index 000000000..5395515eb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120a_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_120a +.address_size 64 + +.visible .entry instruction_cvt_e2m1x2_f32_sm120a_allowed() { + .reg .b8 %b0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120f_allowed.ptx new file mode 100644 index 000000000..0c1e71d6b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120f_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_120f +.address_size 64 + +.visible .entry instruction_cvt_e2m1x2_f32_sm120f_allowed() { + .reg .b8 %b0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121a_allowed.ptx new file mode 100644 index 000000000..001de537b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121a_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_121a +.address_size 64 + +.visible .entry instruction_cvt_e2m1x2_f32_sm121a_allowed() { + .reg .b8 %b0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121f_allowed.ptx new file mode 100644 index 000000000..b9a36f978 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121f_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_121f +.address_size 64 + +.visible .entry instruction_cvt_e2m1x2_f32_sm121f_allowed() { + .reg .b8 %b0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x4_f32_rs_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x4_f32_rs_allowed.ptx new file mode 100644 index 000000000..290232e8e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x4_f32_rs_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_e2m1x4_f32_rs_allowed() { + .reg .b16 %h0; + .reg .f32 %f0, %f1, %f2, %f3; + .reg .b32 %r0; + + cvt.rs.satfinite.e2m1x4.f32 %h0, {%f0, %f1, %f2, %f3}, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m3x4_f32_rs_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m3x4_f32_rs_allowed.ptx new file mode 100644 index 000000000..909131eb7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m3x4_f32_rs_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_e2m3x4_f32_rs_allowed() { + .reg .b32 %r0, %r1; + .reg .f32 %f0, %f1, %f2, %f3; + + cvt.rs.satfinite.e2m3x4.f32 %r0, {%f0, %f1, %f2, %f3}, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e3m2x4_f32_rs_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e3m2x4_f32_rs_allowed.ptx new file mode 100644 index 000000000..e16bcfcfb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e3m2x4_f32_rs_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_e3m2x4_f32_rs_allowed() { + .reg .b32 %r0, %r1; + .reg .f32 %f0, %f1, %f2, %f3; + + cvt.rs.satfinite.e3m2x4.f32 %r0, {%f0, %f1, %f2, %f3}, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e4m3x4_f32_rs_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e4m3x4_f32_rs_allowed.ptx new file mode 100644 index 000000000..7b074bf48 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e4m3x4_f32_rs_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_e4m3x4_f32_rs_allowed() { + .reg .b32 %r0, %r1; + .reg .f32 %f0, %f1, %f2, %f3; + + cvt.rs.satfinite.e4m3x4.f32 %r0, {%f0, %f1, %f2, %f3}, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e5m2x4_f32_rs_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e5m2x4_f32_rs_allowed.ptx new file mode 100644 index 000000000..58b38a4ce --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e5m2x4_f32_rs_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_e5m2x4_f32_rs_allowed() { + .reg .b32 %r0, %r1; + .reg .f32 %f0, %f1, %f2, %f3; + + cvt.rs.satfinite.e5m2x4.f32 %r0, {%f0, %f1, %f2, %f3}, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_bf16_sm90_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_bf16_sm90_allowed.ptx new file mode 100644 index 000000000..b2629cbee --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_bf16_sm90_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_90 +.address_size 64 + +.visible .entry instruction_cvt_f16_bf16_sm90_allowed() { + .reg .b16 %h0; + .reg .b16 %h1; + + cvt.rn.f16.bf16 %h0, %h1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_f32_satfinite_relu_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_f32_satfinite_relu_allowed.ptx new file mode 100644 index 000000000..2fcc9451f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_f32_satfinite_relu_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_120 +.address_size 64 + +.visible .entry instruction_cvt_f16_f32_satfinite_relu_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + + cvt.rn.satfinite.relu.f16.f32 %h0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m1x2_sm100a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m1x2_sm100a_allowed.ptx new file mode 100644 index 000000000..7b358719c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m1x2_sm100a_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_f16x2_e2m1x2_sm100a_allowed() { + .reg .b8 %b0; + .reg .b32 %r0; + + cvt.rn.f16x2.e2m1x2 %r0, %b0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m3x2_sm100a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m3x2_sm100a_allowed.ptx new file mode 100644 index 000000000..6c3acd631 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m3x2_sm100a_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_f16x2_e2m3x2_sm100a_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.f16x2.e2m3x2 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_f32_pair_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_f32_pair_allowed.ptx new file mode 100644 index 000000000..998e2d44d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_f32_pair_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_f16x2_f32_pair_allowed() { + .reg .b32 %r0; + .reg .f32 %f0, %f1; + + cvt.rn.f16x2.f32 %r0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f32_bf16_ftz_sm90_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f32_bf16_ftz_sm90_allowed.ptx new file mode 100644 index 000000000..c7dad4186 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f32_bf16_ftz_sm90_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_90 +.address_size 64 + +.visible .entry instruction_cvt_f32_bf16_ftz_sm90_allowed() { + .reg .f32 %f0; + .reg .b16 %h0; + + cvt.rn.ftz.f32.bf16 %f0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_rs_sm103a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_rs_sm103a_allowed.ptx new file mode 100644 index 000000000..e8fec2072 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_rs_sm103a_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_103a +.address_size 64 + +.visible .entry instruction_cvt_rs_sm103a_allowed() { + .reg .b32 %r0, %r1; + .reg .f32 %f0, %f1; + + cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_allowed.ptx new file mode 100644 index 000000000..258ad0662 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_bf16x2_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_scaled_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_scaled_allowed.ptx new file mode 100644 index 000000000..c8755a6d1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_scaled_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_bf16x2_scaled_allowed() { + .reg .b16 %h0, %h1; + .reg .b32 %r0; + + cvt.rn.satfinite.scaled::n2::ue8m0.s2f6x2.bf16x2 %h0, %r0, %h1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm103a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm103a_allowed.ptx new file mode 100644 index 000000000..722918a82 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm103a_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_103a +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_bf16x2_sm103a_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm120a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm120a_allowed.ptx new file mode 100644 index 000000000..849fda4ff --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm120a_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_120a +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_bf16x2_sm120a_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm121a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm121a_allowed.ptx new file mode 100644 index 000000000..50ec72018 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm121a_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_121a +.address_size 64 + +.visible .entry instruction_cvt_s2f6x2_bf16x2_sm121a_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_bf16_sm90_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_bf16_sm90_allowed.ptx new file mode 100644 index 000000000..a0113c74d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_bf16_sm90_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_90 +.address_size 64 + +.visible .entry instruction_cvt_s32_bf16_sm90_allowed() { + .reg .s32 %r0; + .reg .b16 %h0; + + cvt.rzi.s32.bf16 %r0, %h0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_f32_sat_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_f32_sat_allowed.ptx new file mode 100644 index 000000000..7559b90b8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_f32_sat_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_90 +.address_size 64 + +.visible .entry instruction_cvt_s32_f32_sat_allowed() { + .reg .s32 %r0; + .reg .f32 %f0; + + cvt.rzi.sat.s32.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_f32_satfinite_relu_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_f32_satfinite_relu_allowed.ptx new file mode 100644 index 000000000..be2fde912 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_f32_satfinite_relu_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_tf32_f32_satfinite_relu_allowed() { + .reg .b32 %r0; + .reg .f32 %f0; + + cvt.rn.satfinite.relu.tf32.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_roundtrip_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_roundtrip_allowed.ptx new file mode 100644 index 000000000..b7664f656 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_roundtrip_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_cvt_tf32_roundtrip_allowed() { + .reg .f32 %f0; + .reg .b32 %r0; + + cvt.rna.tf32.f32 %r0, %f0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_u32_f64_sat_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_u32_f64_sat_allowed.ptx new file mode 100644 index 000000000..0b9eaacc7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_u32_f64_sat_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_90 +.address_size 64 + +.visible .entry instruction_cvt_u32_f64_sat_allowed() { + .reg .u32 %r0; + .reg .f64 %fd0; + + cvt.rzi.sat.u32.f64 %r0, %fd0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_allowed.ptx new file mode 100644 index 000000000..8b4046cc5 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_bf16x2_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rz.satfinite.ue8m0x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx new file mode 100644 index 000000000..654a8de5b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_bf16x2_relu_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rz.satfinite.relu.ue8m0x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_without_satfinite_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_without_satfinite_allowed.ptx new file mode 100644 index 000000000..dc0c0cece --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_without_satfinite_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_bf16x2_without_satfinite_allowed() { + .reg .b16 %h0; + .reg .b32 %r0; + + cvt.rz.ue8m0x2.bf16x2 %h0, %r0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100a_allowed.ptx new file mode 100644 index 000000000..939e4a856 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100a_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_100a +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_f32_sm100a_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100f_allowed.ptx new file mode 100644 index 000000000..b5c82a27d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100f_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_100f +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_f32_sm100f_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm103f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm103f_allowed.ptx new file mode 100644 index 000000000..ec99ca1b4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm103f_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_103f +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_f32_sm103f_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm110f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm110f_allowed.ptx new file mode 100644 index 000000000..02d6a0d07 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm110f_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_110f +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_f32_sm110f_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120a_allowed.ptx new file mode 100644 index 000000000..46999edc6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120a_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_120a +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_f32_sm120a_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120f_allowed.ptx new file mode 100644 index 000000000..431ec0f2c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120f_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_120f +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_f32_sm120f_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm121a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm121a_allowed.ptx new file mode 100644 index 000000000..d157676b9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm121a_allowed.ptx @@ -0,0 +1,12 @@ +.version 9.1 +.target sm_121a +.address_size 64 + +.visible .entry instruction_cvt_ue8m0x2_f32_sm121a_allowed() { + .reg .b16 %h0; + .reg .f32 %f0; + .reg .f32 %f1; + + cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_allowed.ptx new file mode 100644 index 000000000..f2fd64a41 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_b128_pack_allowed() { + .reg .b128 %q0; + .reg .b32 %r0, %r1, %r2, %r3; + + mov.b128 %q0, {%r0, %r1, %r2, %r3}; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_allowed.ptx new file mode 100644 index 000000000..652df6487 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_b128_pack_v2_allowed() { + .reg .b128 %q0; + .reg .b64 %d0, %d1; + + mov.b128 %q0, {%d0, %d1}; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_u64_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_u64_allowed.ptx new file mode 100644 index 000000000..c29306eb9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_u64_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_b128_pack_v2_u64_allowed() { + .reg .b128 %q0; + .reg .u64 %u0, %u1; + + mov.b128 %q0, {%u0, %u1}; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_scalar_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_scalar_allowed.ptx new file mode 100644 index 000000000..838da831f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_scalar_allowed.ptx @@ -0,0 +1,10 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_b128_scalar_allowed() { + .reg .b128 %q0, %q1; + + mov.b128 %q0, %q1; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_allowed.ptx new file mode 100644 index 000000000..c192e3dbf --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_b128_unpack_allowed() { + .reg .b128 %q0; + .reg .b32 %r0, %r1, %r2, %r3; + + mov.b128 {%r0, %r1, %r2, %r3}, %q0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v2_allowed.ptx new file mode 100644 index 000000000..ac73dfb6c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v2_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_b128_unpack_v2_allowed() { + .reg .b128 %q0; + .reg .b64 %d0, %d1; + + mov.b128 {%d0, %d1}, %q0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_sink_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_sink_allowed.ptx new file mode 100644 index 000000000..1f72a20c5 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_sink_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_b128_unpack_v4_sink_allowed() { + .reg .b128 %q0; + .reg .b32 %r0, %r2; + + mov.b128 {%r0, _, %r2, _}, %q0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_u32_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_u32_allowed.ptx new file mode 100644 index 000000000..9e249ac68 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_u32_allowed.ptx @@ -0,0 +1,11 @@ +.version 9.1 +.target sm_100 +.address_size 64 + +.visible .entry instruction_mov_b128_unpack_v4_u32_allowed() { + .reg .b128 %q0; + .reg .u32 %u0, %u1, %u2, %u3; + + mov.b128 {%u0, %u1, %u2, %u3}, %q0; + ret; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/run_lowp_cvt_ptxas_crosscheck.sh b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/run_lowp_cvt_ptxas_crosscheck.sh new file mode 100755 index 000000000..240c93d4d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/run_lowp_cvt_ptxas_crosscheck.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +cases_root="$1" + +if ! command -v ptxas >/dev/null 2>&1; then + echo "ptxas not found; lowp cvt ptxas cross-check cannot run" >&2 + exit 1 +fi + +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT + +failures=0 +ran=0 + +ptx_target() { + awk '$1 == ".target" {print $2; exit}' "$1" +} + +run_expect_pass() { + local rel="$1" + local file="$cases_root/$rel" + local arch + local out="$work_dir/$(basename "$file").out" + local cubin="$work_dir/$(basename "$file").cubin" + + ((ran += 1)) + arch="$(ptx_target "$file")" + if [[ -z "$arch" ]]; then + echo "[FAIL] missing .target in $file" + ((failures += 1)) + return + fi + + if ptxas -arch="$arch" "$file" -o "$cubin" >"$out" 2>&1; then + echo "[PASS] ptxas accepted: $rel" + else + echo "[FAIL] expected ptxas success: $rel" + cat "$out" + ((failures += 1)) + fi +} + +run_expect_fail() { + local rel="$1" + local pattern="$2" + local file="$cases_root/$rel" + local arch + local out="$work_dir/$(basename "$file").out" + local cubin="$work_dir/$(basename "$file").cubin" + + ((ran += 1)) + arch="$(ptx_target "$file")" + if [[ -z "$arch" ]]; then + echo "[FAIL] missing .target in $file" + ((failures += 1)) + return + fi + + if ptxas -arch="$arch" "$file" -o "$cubin" >"$out" 2>&1; then + echo "[FAIL] expected ptxas failure: $rel" + ((failures += 1)) + return + fi + + if [[ -n "$pattern" ]] && ! grep -Fq -- "$pattern" "$out"; then + echo "[FAIL] expected ptxas error text not found: $rel" + echo "[FAIL] missing pattern: $pattern" + cat "$out" + ((failures += 1)) + return + fi + + echo "[PASS] ptxas rejected as expected: $rel" +} + +ptxas_accepts_case() { + local rel="$1" + local file="$cases_root/$rel" + local arch + local out="$work_dir/probe_$(basename "$file").out" + local cubin="$work_dir/probe_$(basename "$file").cubin" + arch="$(ptx_target "$file")" + if [[ -z "$arch" ]]; then + return 1 + fi + if ptxas -arch="$arch" "$file" -o "$cubin" >"$out" 2>&1; then + return 0 + fi + return 1 +} + +spec_tracking_ue8m0x2_bf16x2_relu="${LOWP_CVT_SPEC_TRACK_UE8M0X2_BF16X2_RELU:-auto}" +spec_tracking_case_rel="pass/instruction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx" +pass_case_count=0 +while IFS= read -r case_file; do + rel="${case_file#$cases_root/}" + if [[ "$rel" == "$spec_tracking_case_rel" ]]; then + continue + fi + run_expect_pass "$rel" + ((pass_case_count += 1)) +done < <( + find "$cases_root/pass" -maxdepth 1 -type f \ + -name "instruction_cvt_*_allowed.ptx" | sort +) +echo "[INFO] auto-covered parser-pass cvt forms: $pass_case_count" + +case "$spec_tracking_ue8m0x2_bf16x2_relu" in + auto) + if ptxas_accepts_case "$spec_tracking_case_rel"; then + echo "[SPEC-TRACKING:ue8m0x2_bf16x2_relu] ptxas currently accepts this PTX-legal form" + run_expect_pass "$spec_tracking_case_rel" + else + echo "[SPEC-TRACKING:ue8m0x2_bf16x2_relu] ptxas currently rejects this PTX-legal form (tracked divergence)" + run_expect_fail "$spec_tracking_case_rel" "" + fi + ;; + ptxas_rejects) + echo "[SPEC-TRACKING:ue8m0x2_bf16x2_relu] expecting ptxas rejection" + run_expect_fail "$spec_tracking_case_rel" "Illegal modifier '.relu'" + ;; + ptxas_accepts) + echo "[SPEC-TRACKING:ue8m0x2_bf16x2_relu] expecting ptxas acceptance" + run_expect_pass "$spec_tracking_case_rel" + ;; + spec_accepts) + echo "[SPEC-TRACKING:ue8m0x2_bf16x2_relu] enforcing spec expectation (accept)" + run_expect_pass "$spec_tracking_case_rel" + ;; + *) + echo "[FAIL] invalid LOWP_CVT_SPEC_TRACK_UE8M0X2_BF16X2_RELU value: $spec_tracking_ue8m0x2_bf16x2_relu (expected auto|ptxas_rejects|ptxas_accepts|spec_accepts)" >&2 + exit 2 + ;; +esac + +run_expect_fail "fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.ptx" "Illegal modifier '.relu'" +run_expect_fail "fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.ptx" "not supported on .target 'sm_120'" +run_expect_fail "fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.ptx" "not supported on .target 'sm_120'" +run_expect_fail "fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.ptx" "requires .target sm_90 or higher" +run_expect_fail "fail/instruction_cvt_bf16_f16_sm80_not_allowed.ptx" "requires .target sm_90 or higher" +run_expect_fail "fail/instruction_cvt_f16_bf16_sm80_not_allowed.ptx" "requires .target sm_90 or higher" +run_expect_fail "fail/instruction_cvt_bf16_s32_sm80_not_allowed.ptx" "requires .target sm_90 or higher" +run_expect_fail "fail/instruction_cvt_s32_bf16_sm80_not_allowed.ptx" "requires .target sm_90 or higher" +run_expect_fail "fail/instruction_cvt_rs_sm110a_not_allowed.ptx" "Feature '.rs' not supported" +run_expect_fail "fail/instruction_cvt_rs_sm100_not_allowed.ptx" "Feature '.rs' not supported" +run_expect_fail "fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.ptx" "Rounding modifier required for instruction 'cvt'" +run_expect_fail "fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.ptx" "'.satfinite' modifier required" +run_expect_fail "fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.ptx" "Illegal modifier '.scaled::n2::ue8m0'" +run_expect_fail "fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.ptx" "Arguments mismatch for instruction 'cvt'" +run_expect_fail "fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx" "not supported on .target 'sm_100f'" +run_expect_fail "fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx" "not supported on .target 'sm_120f'" +run_expect_fail "fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.ptx" "requires .target sm_100 or higher" +run_expect_fail "fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.ptx" "cannot be combined with modifier '.rna'" +run_expect_fail "fail/instruction_cvt_ue4m3_f32_not_allowed.ptx" "Unexpected instruction types specified for 'cvt'" +run_expect_fail "fail/instruction_cvt_ue8m0x2_scaled_not_allowed.ptx" "Illegal modifier '.scaled::n2::ue8m0'" +run_expect_fail "fail/instruction_cvt_rs_sm100f_not_allowed.ptx" "Feature '.rs' not supported" +run_expect_fail "fail/instruction_cvt_rs_sm120a_not_allowed.ptx" "Feature '.rs' not supported" +run_expect_fail "fail/instruction_cvt_rs_sm120f_not_allowed.ptx" "Feature '.rs' not supported" +run_expect_fail "fail/instruction_cvt_s32_bf16_sat_not_allowed.ptx" "Illegal modifier '.sat'" +run_expect_fail "fail/instruction_cvt_s32_s32_sat_not_allowed.ptx" "Illegal modifier '.sat'" + +if [[ $failures -ne 0 ]]; then + echo "lowp cvt ptxas cross-check failed: $failures / $ran cases failed" >&2 + exit 1 +fi + +echo "lowp cvt ptxas cross-check passed: $ran cases" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/Makefile b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/Makefile new file mode 100644 index 000000000..0cdf5fee4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/Makefile @@ -0,0 +1,50 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin + +CC := nvcc +EXE := gpgpusim_tma_tensor_benchmark +SRC := tma_tensor_benchmark.cu + +ifneq ($(filter 13 14 15 16,$(CUDA_VERSION_MAJOR)),) +ARCH ?= sm_90a sm_100a sm_110 sm_120 +else +ARCH ?= sm_90a sm_100a sm_101 sm_120 +endif + +CUDA_CPPFLAGS = +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(BIN_DIR)/$(EXE) + +$(BIN_DIR): + mkdir -p $@ + +$(EXE).a: $(SRC) + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(BIN_DIR)/$(EXE): $(EXE).a | $(BIN_DIR) + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + +run-fixtures: $(BIN_DIR)/$(EXE) + set -e; \ + tmp_dir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmp_dir"' EXIT; \ + for expected in expected/tma_tensor_test_*.txt; do \ + base="$$(basename "$$expected" .txt)"; \ + spec="$${base#tma_tensor_test_}"; \ + width="$${spec##*_}"; \ + spec="$${spec%_*}"; \ + height="$${spec##*_}"; \ + opcode="$${spec%_*}"; \ + ( cd "$$tmp_dir" && "$(BIN_DIR)/$(EXE)" -w "$$width" -h "$$height" -o "$$opcode" ); \ + actual="$$tmp_dir/$$(basename "$$expected")"; \ + cmp -s "$$expected" "$$actual" || diff -u "$$expected" "$$actual"; \ + done + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(EXE) + rm -f $(BIN_DIR)/$(EXE) + +.PHONY: all clean run-fixtures diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/README.md b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/README.md new file mode 100644 index 000000000..ea41ab04f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/README.md @@ -0,0 +1,22 @@ +# TMA Tensor-Map Test + +This folder contains the standalone TMA tensor-map benchmark from the +GPGPU-Sim `tma` branch. Expected fixture outputs are kept under `expected/`. + +## Build + +```sh +make +``` + +The default build creates `gpgpusim_tma_tensor_benchmark` under +`GPU_Microbenchmark/bin`. + +## Fixture Check + +```sh +make run-fixtures +``` + +This target runs the imported fixture cases and compares generated output +against `expected/`. diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_REGULAR_LOAD_64_64.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_REGULAR_LOAD_64_64.txt new file mode 100644 index 000000000..e3f4714b7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_REGULAR_LOAD_64_64.txt @@ -0,0 +1,8 @@ +0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff 0x100 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x135 0x136 0x137 0x138 0x139 0x13a 0x13b 0x13c 0x13d 0x13e 0x13f 0x140 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x175 0x176 0x177 0x178 0x179 0x17a 0x17b 0x17c 0x17d 0x17e 0x17f 0x180 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x1b5 0x1b6 0x1b7 0x1b8 0x1b9 0x1ba 0x1bb 0x1bc 0x1bd 0x1be 0x1bf 0x1c0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x1f5 0x1f6 0x1f7 0x1f8 0x1f9 0x1fa 0x1fb 0x1fc 0x1fd 0x1fe 0x1ff 0x200 0x201 +0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x235 0x236 0x237 0x238 0x239 0x23a 0x23b 0x23c 0x23d 0x23e 0x23f 0x240 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x275 0x276 0x277 0x278 0x279 0x27a 0x27b 0x27c 0x27d 0x27e 0x27f 0x280 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x2b5 0x2b6 0x2b7 0x2b8 0x2b9 0x2ba 0x2bb 0x2bc 0x2bd 0x2be 0x2bf 0x2c0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x2f5 0x2f6 0x2f7 0x2f8 0x2f9 0x2fa 0x2fb 0x2fc 0x2fd 0x2fe 0x2ff 0x300 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x335 0x336 0x337 0x338 0x339 0x33a 0x33b 0x33c 0x33d 0x33e 0x33f 0x340 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x375 0x376 0x377 0x378 0x379 0x37a 0x37b 0x37c 0x37d 0x37e 0x37f 0x380 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x3b5 0x3b6 0x3b7 0x3b8 0x3b9 0x3ba 0x3bb 0x3bc 0x3bd 0x3be 0x3bf 0x3c0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x3f5 0x3f6 0x3f7 0x3f8 0x3f9 0x3fa 0x3fb 0x3fc 0x3fd 0x3fe 0x3ff 0x400 0x401 +0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x435 0x436 0x437 0x438 0x439 0x43a 0x43b 0x43c 0x43d 0x43e 0x43f 0x440 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x475 0x476 0x477 0x478 0x479 0x47a 0x47b 0x47c 0x47d 0x47e 0x47f 0x480 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x4b5 0x4b6 0x4b7 0x4b8 0x4b9 0x4ba 0x4bb 0x4bc 0x4bd 0x4be 0x4bf 0x4c0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x4f5 0x4f6 0x4f7 0x4f8 0x4f9 0x4fa 0x4fb 0x4fc 0x4fd 0x4fe 0x4ff 0x500 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x535 0x536 0x537 0x538 0x539 0x53a 0x53b 0x53c 0x53d 0x53e 0x53f 0x540 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x575 0x576 0x577 0x578 0x579 0x57a 0x57b 0x57c 0x57d 0x57e 0x57f 0x580 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x5b5 0x5b6 0x5b7 0x5b8 0x5b9 0x5ba 0x5bb 0x5bc 0x5bd 0x5be 0x5bf 0x5c0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x5f5 0x5f6 0x5f7 0x5f8 0x5f9 0x5fa 0x5fb 0x5fc 0x5fd 0x5fe 0x5ff 0x600 0x601 +0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x635 0x636 0x637 0x638 0x639 0x63a 0x63b 0x63c 0x63d 0x63e 0x63f 0x640 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x675 0x676 0x677 0x678 0x679 0x67a 0x67b 0x67c 0x67d 0x67e 0x67f 0x680 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x6b5 0x6b6 0x6b7 0x6b8 0x6b9 0x6ba 0x6bb 0x6bc 0x6bd 0x6be 0x6bf 0x6c0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x6f5 0x6f6 0x6f7 0x6f8 0x6f9 0x6fa 0x6fb 0x6fc 0x6fd 0x6fe 0x6ff 0x700 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x735 0x736 0x737 0x738 0x739 0x73a 0x73b 0x73c 0x73d 0x73e 0x73f 0x740 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x775 0x776 0x777 0x778 0x779 0x77a 0x77b 0x77c 0x77d 0x77e 0x77f 0x780 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x7b5 0x7b6 0x7b7 0x7b8 0x7b9 0x7ba 0x7bb 0x7bc 0x7bd 0x7be 0x7bf 0x7c0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x7f5 0x7f6 0x7f7 0x7f8 0x7f9 0x7fa 0x7fb 0x7fc 0x7fd 0x7fe 0x7ff 0x800 0x801 +0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x835 0x836 0x837 0x838 0x839 0x83a 0x83b 0x83c 0x83d 0x83e 0x83f 0x840 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x875 0x876 0x877 0x878 0x879 0x87a 0x87b 0x87c 0x87d 0x87e 0x87f 0x880 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x8b5 0x8b6 0x8b7 0x8b8 0x8b9 0x8ba 0x8bb 0x8bc 0x8bd 0x8be 0x8bf 0x8c0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x8f5 0x8f6 0x8f7 0x8f8 0x8f9 0x8fa 0x8fb 0x8fc 0x8fd 0x8fe 0x8ff 0x900 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x935 0x936 0x937 0x938 0x939 0x93a 0x93b 0x93c 0x93d 0x93e 0x93f 0x940 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x975 0x976 0x977 0x978 0x979 0x97a 0x97b 0x97c 0x97d 0x97e 0x97f 0x980 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x9b5 0x9b6 0x9b7 0x9b8 0x9b9 0x9ba 0x9bb 0x9bc 0x9bd 0x9be 0x9bf 0x9c0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x9f5 0x9f6 0x9f7 0x9f8 0x9f9 0x9fa 0x9fb 0x9fc 0x9fd 0x9fe 0x9ff 0xa00 0xa01 +0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0xa35 0xa36 0xa37 0xa38 0xa39 0xa3a 0xa3b 0xa3c 0xa3d 0xa3e 0xa3f 0xa40 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0xa75 0xa76 0xa77 0xa78 0xa79 0xa7a 0xa7b 0xa7c 0xa7d 0xa7e 0xa7f 0xa80 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0xab5 0xab6 0xab7 0xab8 0xab9 0xaba 0xabb 0xabc 0xabd 0xabe 0xabf 0xac0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0xaf5 0xaf6 0xaf7 0xaf8 0xaf9 0xafa 0xafb 0xafc 0xafd 0xafe 0xaff 0xb00 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0xb35 0xb36 0xb37 0xb38 0xb39 0xb3a 0xb3b 0xb3c 0xb3d 0xb3e 0xb3f 0xb40 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0xb75 0xb76 0xb77 0xb78 0xb79 0xb7a 0xb7b 0xb7c 0xb7d 0xb7e 0xb7f 0xb80 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0xbb5 0xbb6 0xbb7 0xbb8 0xbb9 0xbba 0xbbb 0xbbc 0xbbd 0xbbe 0xbbf 0xbc0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0xbf5 0xbf6 0xbf7 0xbf8 0xbf9 0xbfa 0xbfb 0xbfc 0xbfd 0xbfe 0xbff 0xc00 0xc01 +0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0xc35 0xc36 0xc37 0xc38 0xc39 0xc3a 0xc3b 0xc3c 0xc3d 0xc3e 0xc3f 0xc40 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0xc75 0xc76 0xc77 0xc78 0xc79 0xc7a 0xc7b 0xc7c 0xc7d 0xc7e 0xc7f 0xc80 0xc81 0xc82 0xc83 0xc84 0xc85 0xc86 0xc87 0xc88 0xc89 0xc8a 0xc8b 0xc8c 0xc8d 0xc8e 0xc8f 0xc90 0xc91 0xc92 0xc93 0xc94 0xc95 0xc96 0xc97 0xc98 0xc99 0xc9a 0xc9b 0xc9c 0xc9d 0xc9e 0xc9f 0xca0 0xca1 0xca2 0xca3 0xca4 0xca5 0xca6 0xca7 0xca8 0xca9 0xcaa 0xcab 0xcac 0xcad 0xcae 0xcaf 0xcb0 0xcb1 0xcb2 0xcb3 0xcb4 0xcb5 0xcb6 0xcb7 0xcb8 0xcb9 0xcba 0xcbb 0xcbc 0xcbd 0xcbe 0xcbf 0xcc0 0xcc1 0xcc2 0xcc3 0xcc4 0xcc5 0xcc6 0xcc7 0xcc8 0xcc9 0xcca 0xccb 0xccc 0xccd 0xcce 0xccf 0xcd0 0xcd1 0xcd2 0xcd3 0xcd4 0xcd5 0xcd6 0xcd7 0xcd8 0xcd9 0xcda 0xcdb 0xcdc 0xcdd 0xcde 0xcdf 0xce0 0xce1 0xce2 0xce3 0xce4 0xce5 0xce6 0xce7 0xce8 0xce9 0xcea 0xceb 0xcec 0xced 0xcee 0xcef 0xcf0 0xcf1 0xcf2 0xcf3 0xcf4 0xcf5 0xcf6 0xcf7 0xcf8 0xcf9 0xcfa 0xcfb 0xcfc 0xcfd 0xcfe 0xcff 0xd00 0xd01 0xd02 0xd03 0xd04 0xd05 0xd06 0xd07 0xd08 0xd09 0xd0a 0xd0b 0xd0c 0xd0d 0xd0e 0xd0f 0xd10 0xd11 0xd12 0xd13 0xd14 0xd15 0xd16 0xd17 0xd18 0xd19 0xd1a 0xd1b 0xd1c 0xd1d 0xd1e 0xd1f 0xd20 0xd21 0xd22 0xd23 0xd24 0xd25 0xd26 0xd27 0xd28 0xd29 0xd2a 0xd2b 0xd2c 0xd2d 0xd2e 0xd2f 0xd30 0xd31 0xd32 0xd33 0xd34 0xd35 0xd36 0xd37 0xd38 0xd39 0xd3a 0xd3b 0xd3c 0xd3d 0xd3e 0xd3f 0xd40 0xd41 0xd42 0xd43 0xd44 0xd45 0xd46 0xd47 0xd48 0xd49 0xd4a 0xd4b 0xd4c 0xd4d 0xd4e 0xd4f 0xd50 0xd51 0xd52 0xd53 0xd54 0xd55 0xd56 0xd57 0xd58 0xd59 0xd5a 0xd5b 0xd5c 0xd5d 0xd5e 0xd5f 0xd60 0xd61 0xd62 0xd63 0xd64 0xd65 0xd66 0xd67 0xd68 0xd69 0xd6a 0xd6b 0xd6c 0xd6d 0xd6e 0xd6f 0xd70 0xd71 0xd72 0xd73 0xd74 0xd75 0xd76 0xd77 0xd78 0xd79 0xd7a 0xd7b 0xd7c 0xd7d 0xd7e 0xd7f 0xd80 0xd81 0xd82 0xd83 0xd84 0xd85 0xd86 0xd87 0xd88 0xd89 0xd8a 0xd8b 0xd8c 0xd8d 0xd8e 0xd8f 0xd90 0xd91 0xd92 0xd93 0xd94 0xd95 0xd96 0xd97 0xd98 0xd99 0xd9a 0xd9b 0xd9c 0xd9d 0xd9e 0xd9f 0xda0 0xda1 0xda2 0xda3 0xda4 0xda5 0xda6 0xda7 0xda8 0xda9 0xdaa 0xdab 0xdac 0xdad 0xdae 0xdaf 0xdb0 0xdb1 0xdb2 0xdb3 0xdb4 0xdb5 0xdb6 0xdb7 0xdb8 0xdb9 0xdba 0xdbb 0xdbc 0xdbd 0xdbe 0xdbf 0xdc0 0xdc1 0xdc2 0xdc3 0xdc4 0xdc5 0xdc6 0xdc7 0xdc8 0xdc9 0xdca 0xdcb 0xdcc 0xdcd 0xdce 0xdcf 0xdd0 0xdd1 0xdd2 0xdd3 0xdd4 0xdd5 0xdd6 0xdd7 0xdd8 0xdd9 0xdda 0xddb 0xddc 0xddd 0xdde 0xddf 0xde0 0xde1 0xde2 0xde3 0xde4 0xde5 0xde6 0xde7 0xde8 0xde9 0xdea 0xdeb 0xdec 0xded 0xdee 0xdef 0xdf0 0xdf1 0xdf2 0xdf3 0xdf4 0xdf5 0xdf6 0xdf7 0xdf8 0xdf9 0xdfa 0xdfb 0xdfc 0xdfd 0xdfe 0xdff 0xe00 0xe01 +0xe02 0xe03 0xe04 0xe05 0xe06 0xe07 0xe08 0xe09 0xe0a 0xe0b 0xe0c 0xe0d 0xe0e 0xe0f 0xe10 0xe11 0xe12 0xe13 0xe14 0xe15 0xe16 0xe17 0xe18 0xe19 0xe1a 0xe1b 0xe1c 0xe1d 0xe1e 0xe1f 0xe20 0xe21 0xe22 0xe23 0xe24 0xe25 0xe26 0xe27 0xe28 0xe29 0xe2a 0xe2b 0xe2c 0xe2d 0xe2e 0xe2f 0xe30 0xe31 0xe32 0xe33 0xe34 0xe35 0xe36 0xe37 0xe38 0xe39 0xe3a 0xe3b 0xe3c 0xe3d 0xe3e 0xe3f 0xe40 0xe41 0xe42 0xe43 0xe44 0xe45 0xe46 0xe47 0xe48 0xe49 0xe4a 0xe4b 0xe4c 0xe4d 0xe4e 0xe4f 0xe50 0xe51 0xe52 0xe53 0xe54 0xe55 0xe56 0xe57 0xe58 0xe59 0xe5a 0xe5b 0xe5c 0xe5d 0xe5e 0xe5f 0xe60 0xe61 0xe62 0xe63 0xe64 0xe65 0xe66 0xe67 0xe68 0xe69 0xe6a 0xe6b 0xe6c 0xe6d 0xe6e 0xe6f 0xe70 0xe71 0xe72 0xe73 0xe74 0xe75 0xe76 0xe77 0xe78 0xe79 0xe7a 0xe7b 0xe7c 0xe7d 0xe7e 0xe7f 0xe80 0xe81 0xe82 0xe83 0xe84 0xe85 0xe86 0xe87 0xe88 0xe89 0xe8a 0xe8b 0xe8c 0xe8d 0xe8e 0xe8f 0xe90 0xe91 0xe92 0xe93 0xe94 0xe95 0xe96 0xe97 0xe98 0xe99 0xe9a 0xe9b 0xe9c 0xe9d 0xe9e 0xe9f 0xea0 0xea1 0xea2 0xea3 0xea4 0xea5 0xea6 0xea7 0xea8 0xea9 0xeaa 0xeab 0xeac 0xead 0xeae 0xeaf 0xeb0 0xeb1 0xeb2 0xeb3 0xeb4 0xeb5 0xeb6 0xeb7 0xeb8 0xeb9 0xeba 0xebb 0xebc 0xebd 0xebe 0xebf 0xec0 0xec1 0xec2 0xec3 0xec4 0xec5 0xec6 0xec7 0xec8 0xec9 0xeca 0xecb 0xecc 0xecd 0xece 0xecf 0xed0 0xed1 0xed2 0xed3 0xed4 0xed5 0xed6 0xed7 0xed8 0xed9 0xeda 0xedb 0xedc 0xedd 0xede 0xedf 0xee0 0xee1 0xee2 0xee3 0xee4 0xee5 0xee6 0xee7 0xee8 0xee9 0xeea 0xeeb 0xeec 0xeed 0xeee 0xeef 0xef0 0xef1 0xef2 0xef3 0xef4 0xef5 0xef6 0xef7 0xef8 0xef9 0xefa 0xefb 0xefc 0xefd 0xefe 0xeff 0xf00 0xf01 0xf02 0xf03 0xf04 0xf05 0xf06 0xf07 0xf08 0xf09 0xf0a 0xf0b 0xf0c 0xf0d 0xf0e 0xf0f 0xf10 0xf11 0xf12 0xf13 0xf14 0xf15 0xf16 0xf17 0xf18 0xf19 0xf1a 0xf1b 0xf1c 0xf1d 0xf1e 0xf1f 0xf20 0xf21 0xf22 0xf23 0xf24 0xf25 0xf26 0xf27 0xf28 0xf29 0xf2a 0xf2b 0xf2c 0xf2d 0xf2e 0xf2f 0xf30 0xf31 0xf32 0xf33 0xf34 0xf35 0xf36 0xf37 0xf38 0xf39 0xf3a 0xf3b 0xf3c 0xf3d 0xf3e 0xf3f 0xf40 0xf41 0xf42 0xf43 0xf44 0xf45 0xf46 0xf47 0xf48 0xf49 0xf4a 0xf4b 0xf4c 0xf4d 0xf4e 0xf4f 0xf50 0xf51 0xf52 0xf53 0xf54 0xf55 0xf56 0xf57 0xf58 0xf59 0xf5a 0xf5b 0xf5c 0xf5d 0xf5e 0xf5f 0xf60 0xf61 0xf62 0xf63 0xf64 0xf65 0xf66 0xf67 0xf68 0xf69 0xf6a 0xf6b 0xf6c 0xf6d 0xf6e 0xf6f 0xf70 0xf71 0xf72 0xf73 0xf74 0xf75 0xf76 0xf77 0xf78 0xf79 0xf7a 0xf7b 0xf7c 0xf7d 0xf7e 0xf7f 0xf80 0xf81 0xf82 0xf83 0xf84 0xf85 0xf86 0xf87 0xf88 0xf89 0xf8a 0xf8b 0xf8c 0xf8d 0xf8e 0xf8f 0xf90 0xf91 0xf92 0xf93 0xf94 0xf95 0xf96 0xf97 0xf98 0xf99 0xf9a 0xf9b 0xf9c 0xf9d 0xf9e 0xf9f 0xfa0 0xfa1 0xfa2 0xfa3 0xfa4 0xfa5 0xfa6 0xfa7 0xfa8 0xfa9 0xfaa 0xfab 0xfac 0xfad 0xfae 0xfaf 0xfb0 0xfb1 0xfb2 0xfb3 0xfb4 0xfb5 0xfb6 0xfb7 0xfb8 0xfb9 0xfba 0xfbb 0xfbc 0xfbd 0xfbe 0xfbf 0xfc0 0xfc1 0xfc2 0xfc3 0xfc4 0xfc5 0xfc6 0xfc7 0xfc8 0xfc9 0xfca 0xfcb 0xfcc 0xfcd 0xfce 0xfcf 0xfd0 0xfd1 0xfd2 0xfd3 0xfd4 0xfd5 0xfd6 0xfd7 0xfd8 0xfd9 0xfda 0xfdb 0xfdc 0xfdd 0xfde 0xfdf 0xfe0 0xfe1 0xfe2 0xfe3 0xfe4 0xfe5 0xfe6 0xfe7 0xfe8 0xfe9 0xfea 0xfeb 0xfec 0xfed 0xfee 0xfef 0xff0 0xff1 0xff2 0xff3 0xff4 0xff5 0xff6 0xff7 0xff8 0xff9 0xffa 0xffb 0xffc 0xffd 0xffe 0xfff 0x1000 0x1001 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMALDG_64_64.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMALDG_64_64.txt new file mode 100644 index 000000000..166808918 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMALDG_64_64.txt @@ -0,0 +1,8 @@ +0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff 0x100 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x135 0x136 0x137 0x138 0x139 0x13a 0x13b 0x13c 0x13d 0x13e 0x13f 0x140 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x175 0x176 0x177 0x178 0x179 0x17a 0x17b 0x17c 0x17d 0x17e 0x17f 0x180 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x1b5 0x1b6 0x1b7 0x1b8 0x1b9 0x1ba 0x1bb 0x1bc 0x1bd 0x1be 0x1bf 0x1c0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x1f5 0x1f6 0x1f7 0x1f8 0x1f9 0x1fa 0x1fb 0x1fc 0x1fd 0x1fe 0x1ff 0x200 +0x201 0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x235 0x236 0x237 0x238 0x239 0x23a 0x23b 0x23c 0x23d 0x23e 0x23f 0x240 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x275 0x276 0x277 0x278 0x279 0x27a 0x27b 0x27c 0x27d 0x27e 0x27f 0x280 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x2b5 0x2b6 0x2b7 0x2b8 0x2b9 0x2ba 0x2bb 0x2bc 0x2bd 0x2be 0x2bf 0x2c0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x2f5 0x2f6 0x2f7 0x2f8 0x2f9 0x2fa 0x2fb 0x2fc 0x2fd 0x2fe 0x2ff 0x300 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x335 0x336 0x337 0x338 0x339 0x33a 0x33b 0x33c 0x33d 0x33e 0x33f 0x340 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x375 0x376 0x377 0x378 0x379 0x37a 0x37b 0x37c 0x37d 0x37e 0x37f 0x380 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x3b5 0x3b6 0x3b7 0x3b8 0x3b9 0x3ba 0x3bb 0x3bc 0x3bd 0x3be 0x3bf 0x3c0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x3f5 0x3f6 0x3f7 0x3f8 0x3f9 0x3fa 0x3fb 0x3fc 0x3fd 0x3fe 0x3ff 0x400 +0x401 0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x435 0x436 0x437 0x438 0x439 0x43a 0x43b 0x43c 0x43d 0x43e 0x43f 0x440 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x475 0x476 0x477 0x478 0x479 0x47a 0x47b 0x47c 0x47d 0x47e 0x47f 0x480 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x4b5 0x4b6 0x4b7 0x4b8 0x4b9 0x4ba 0x4bb 0x4bc 0x4bd 0x4be 0x4bf 0x4c0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x4f5 0x4f6 0x4f7 0x4f8 0x4f9 0x4fa 0x4fb 0x4fc 0x4fd 0x4fe 0x4ff 0x500 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x535 0x536 0x537 0x538 0x539 0x53a 0x53b 0x53c 0x53d 0x53e 0x53f 0x540 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x575 0x576 0x577 0x578 0x579 0x57a 0x57b 0x57c 0x57d 0x57e 0x57f 0x580 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x5b5 0x5b6 0x5b7 0x5b8 0x5b9 0x5ba 0x5bb 0x5bc 0x5bd 0x5be 0x5bf 0x5c0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x5f5 0x5f6 0x5f7 0x5f8 0x5f9 0x5fa 0x5fb 0x5fc 0x5fd 0x5fe 0x5ff 0x600 +0x601 0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x635 0x636 0x637 0x638 0x639 0x63a 0x63b 0x63c 0x63d 0x63e 0x63f 0x640 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x675 0x676 0x677 0x678 0x679 0x67a 0x67b 0x67c 0x67d 0x67e 0x67f 0x680 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x6b5 0x6b6 0x6b7 0x6b8 0x6b9 0x6ba 0x6bb 0x6bc 0x6bd 0x6be 0x6bf 0x6c0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x6f5 0x6f6 0x6f7 0x6f8 0x6f9 0x6fa 0x6fb 0x6fc 0x6fd 0x6fe 0x6ff 0x700 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x735 0x736 0x737 0x738 0x739 0x73a 0x73b 0x73c 0x73d 0x73e 0x73f 0x740 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x775 0x776 0x777 0x778 0x779 0x77a 0x77b 0x77c 0x77d 0x77e 0x77f 0x780 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x7b5 0x7b6 0x7b7 0x7b8 0x7b9 0x7ba 0x7bb 0x7bc 0x7bd 0x7be 0x7bf 0x7c0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x7f5 0x7f6 0x7f7 0x7f8 0x7f9 0x7fa 0x7fb 0x7fc 0x7fd 0x7fe 0x7ff 0x800 +0x801 0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x835 0x836 0x837 0x838 0x839 0x83a 0x83b 0x83c 0x83d 0x83e 0x83f 0x840 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x875 0x876 0x877 0x878 0x879 0x87a 0x87b 0x87c 0x87d 0x87e 0x87f 0x880 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x8b5 0x8b6 0x8b7 0x8b8 0x8b9 0x8ba 0x8bb 0x8bc 0x8bd 0x8be 0x8bf 0x8c0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x8f5 0x8f6 0x8f7 0x8f8 0x8f9 0x8fa 0x8fb 0x8fc 0x8fd 0x8fe 0x8ff 0x900 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x935 0x936 0x937 0x938 0x939 0x93a 0x93b 0x93c 0x93d 0x93e 0x93f 0x940 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x975 0x976 0x977 0x978 0x979 0x97a 0x97b 0x97c 0x97d 0x97e 0x97f 0x980 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x9b5 0x9b6 0x9b7 0x9b8 0x9b9 0x9ba 0x9bb 0x9bc 0x9bd 0x9be 0x9bf 0x9c0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x9f5 0x9f6 0x9f7 0x9f8 0x9f9 0x9fa 0x9fb 0x9fc 0x9fd 0x9fe 0x9ff 0xa00 +0xa01 0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0xa35 0xa36 0xa37 0xa38 0xa39 0xa3a 0xa3b 0xa3c 0xa3d 0xa3e 0xa3f 0xa40 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0xa75 0xa76 0xa77 0xa78 0xa79 0xa7a 0xa7b 0xa7c 0xa7d 0xa7e 0xa7f 0xa80 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0xab5 0xab6 0xab7 0xab8 0xab9 0xaba 0xabb 0xabc 0xabd 0xabe 0xabf 0xac0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0xaf5 0xaf6 0xaf7 0xaf8 0xaf9 0xafa 0xafb 0xafc 0xafd 0xafe 0xaff 0xb00 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0xb35 0xb36 0xb37 0xb38 0xb39 0xb3a 0xb3b 0xb3c 0xb3d 0xb3e 0xb3f 0xb40 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0xb75 0xb76 0xb77 0xb78 0xb79 0xb7a 0xb7b 0xb7c 0xb7d 0xb7e 0xb7f 0xb80 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0xbb5 0xbb6 0xbb7 0xbb8 0xbb9 0xbba 0xbbb 0xbbc 0xbbd 0xbbe 0xbbf 0xbc0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0xbf5 0xbf6 0xbf7 0xbf8 0xbf9 0xbfa 0xbfb 0xbfc 0xbfd 0xbfe 0xbff 0xc00 +0xc01 0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0xc35 0xc36 0xc37 0xc38 0xc39 0xc3a 0xc3b 0xc3c 0xc3d 0xc3e 0xc3f 0xc40 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0xc75 0xc76 0xc77 0xc78 0xc79 0xc7a 0xc7b 0xc7c 0xc7d 0xc7e 0xc7f 0xc80 0xc81 0xc82 0xc83 0xc84 0xc85 0xc86 0xc87 0xc88 0xc89 0xc8a 0xc8b 0xc8c 0xc8d 0xc8e 0xc8f 0xc90 0xc91 0xc92 0xc93 0xc94 0xc95 0xc96 0xc97 0xc98 0xc99 0xc9a 0xc9b 0xc9c 0xc9d 0xc9e 0xc9f 0xca0 0xca1 0xca2 0xca3 0xca4 0xca5 0xca6 0xca7 0xca8 0xca9 0xcaa 0xcab 0xcac 0xcad 0xcae 0xcaf 0xcb0 0xcb1 0xcb2 0xcb3 0xcb4 0xcb5 0xcb6 0xcb7 0xcb8 0xcb9 0xcba 0xcbb 0xcbc 0xcbd 0xcbe 0xcbf 0xcc0 0xcc1 0xcc2 0xcc3 0xcc4 0xcc5 0xcc6 0xcc7 0xcc8 0xcc9 0xcca 0xccb 0xccc 0xccd 0xcce 0xccf 0xcd0 0xcd1 0xcd2 0xcd3 0xcd4 0xcd5 0xcd6 0xcd7 0xcd8 0xcd9 0xcda 0xcdb 0xcdc 0xcdd 0xcde 0xcdf 0xce0 0xce1 0xce2 0xce3 0xce4 0xce5 0xce6 0xce7 0xce8 0xce9 0xcea 0xceb 0xcec 0xced 0xcee 0xcef 0xcf0 0xcf1 0xcf2 0xcf3 0xcf4 0xcf5 0xcf6 0xcf7 0xcf8 0xcf9 0xcfa 0xcfb 0xcfc 0xcfd 0xcfe 0xcff 0xd00 0xd01 0xd02 0xd03 0xd04 0xd05 0xd06 0xd07 0xd08 0xd09 0xd0a 0xd0b 0xd0c 0xd0d 0xd0e 0xd0f 0xd10 0xd11 0xd12 0xd13 0xd14 0xd15 0xd16 0xd17 0xd18 0xd19 0xd1a 0xd1b 0xd1c 0xd1d 0xd1e 0xd1f 0xd20 0xd21 0xd22 0xd23 0xd24 0xd25 0xd26 0xd27 0xd28 0xd29 0xd2a 0xd2b 0xd2c 0xd2d 0xd2e 0xd2f 0xd30 0xd31 0xd32 0xd33 0xd34 0xd35 0xd36 0xd37 0xd38 0xd39 0xd3a 0xd3b 0xd3c 0xd3d 0xd3e 0xd3f 0xd40 0xd41 0xd42 0xd43 0xd44 0xd45 0xd46 0xd47 0xd48 0xd49 0xd4a 0xd4b 0xd4c 0xd4d 0xd4e 0xd4f 0xd50 0xd51 0xd52 0xd53 0xd54 0xd55 0xd56 0xd57 0xd58 0xd59 0xd5a 0xd5b 0xd5c 0xd5d 0xd5e 0xd5f 0xd60 0xd61 0xd62 0xd63 0xd64 0xd65 0xd66 0xd67 0xd68 0xd69 0xd6a 0xd6b 0xd6c 0xd6d 0xd6e 0xd6f 0xd70 0xd71 0xd72 0xd73 0xd74 0xd75 0xd76 0xd77 0xd78 0xd79 0xd7a 0xd7b 0xd7c 0xd7d 0xd7e 0xd7f 0xd80 0xd81 0xd82 0xd83 0xd84 0xd85 0xd86 0xd87 0xd88 0xd89 0xd8a 0xd8b 0xd8c 0xd8d 0xd8e 0xd8f 0xd90 0xd91 0xd92 0xd93 0xd94 0xd95 0xd96 0xd97 0xd98 0xd99 0xd9a 0xd9b 0xd9c 0xd9d 0xd9e 0xd9f 0xda0 0xda1 0xda2 0xda3 0xda4 0xda5 0xda6 0xda7 0xda8 0xda9 0xdaa 0xdab 0xdac 0xdad 0xdae 0xdaf 0xdb0 0xdb1 0xdb2 0xdb3 0xdb4 0xdb5 0xdb6 0xdb7 0xdb8 0xdb9 0xdba 0xdbb 0xdbc 0xdbd 0xdbe 0xdbf 0xdc0 0xdc1 0xdc2 0xdc3 0xdc4 0xdc5 0xdc6 0xdc7 0xdc8 0xdc9 0xdca 0xdcb 0xdcc 0xdcd 0xdce 0xdcf 0xdd0 0xdd1 0xdd2 0xdd3 0xdd4 0xdd5 0xdd6 0xdd7 0xdd8 0xdd9 0xdda 0xddb 0xddc 0xddd 0xdde 0xddf 0xde0 0xde1 0xde2 0xde3 0xde4 0xde5 0xde6 0xde7 0xde8 0xde9 0xdea 0xdeb 0xdec 0xded 0xdee 0xdef 0xdf0 0xdf1 0xdf2 0xdf3 0xdf4 0xdf5 0xdf6 0xdf7 0xdf8 0xdf9 0xdfa 0xdfb 0xdfc 0xdfd 0xdfe 0xdff 0xe00 +0xe01 0xe02 0xe03 0xe04 0xe05 0xe06 0xe07 0xe08 0xe09 0xe0a 0xe0b 0xe0c 0xe0d 0xe0e 0xe0f 0xe10 0xe11 0xe12 0xe13 0xe14 0xe15 0xe16 0xe17 0xe18 0xe19 0xe1a 0xe1b 0xe1c 0xe1d 0xe1e 0xe1f 0xe20 0xe21 0xe22 0xe23 0xe24 0xe25 0xe26 0xe27 0xe28 0xe29 0xe2a 0xe2b 0xe2c 0xe2d 0xe2e 0xe2f 0xe30 0xe31 0xe32 0xe33 0xe34 0xe35 0xe36 0xe37 0xe38 0xe39 0xe3a 0xe3b 0xe3c 0xe3d 0xe3e 0xe3f 0xe40 0xe41 0xe42 0xe43 0xe44 0xe45 0xe46 0xe47 0xe48 0xe49 0xe4a 0xe4b 0xe4c 0xe4d 0xe4e 0xe4f 0xe50 0xe51 0xe52 0xe53 0xe54 0xe55 0xe56 0xe57 0xe58 0xe59 0xe5a 0xe5b 0xe5c 0xe5d 0xe5e 0xe5f 0xe60 0xe61 0xe62 0xe63 0xe64 0xe65 0xe66 0xe67 0xe68 0xe69 0xe6a 0xe6b 0xe6c 0xe6d 0xe6e 0xe6f 0xe70 0xe71 0xe72 0xe73 0xe74 0xe75 0xe76 0xe77 0xe78 0xe79 0xe7a 0xe7b 0xe7c 0xe7d 0xe7e 0xe7f 0xe80 0xe81 0xe82 0xe83 0xe84 0xe85 0xe86 0xe87 0xe88 0xe89 0xe8a 0xe8b 0xe8c 0xe8d 0xe8e 0xe8f 0xe90 0xe91 0xe92 0xe93 0xe94 0xe95 0xe96 0xe97 0xe98 0xe99 0xe9a 0xe9b 0xe9c 0xe9d 0xe9e 0xe9f 0xea0 0xea1 0xea2 0xea3 0xea4 0xea5 0xea6 0xea7 0xea8 0xea9 0xeaa 0xeab 0xeac 0xead 0xeae 0xeaf 0xeb0 0xeb1 0xeb2 0xeb3 0xeb4 0xeb5 0xeb6 0xeb7 0xeb8 0xeb9 0xeba 0xebb 0xebc 0xebd 0xebe 0xebf 0xec0 0xec1 0xec2 0xec3 0xec4 0xec5 0xec6 0xec7 0xec8 0xec9 0xeca 0xecb 0xecc 0xecd 0xece 0xecf 0xed0 0xed1 0xed2 0xed3 0xed4 0xed5 0xed6 0xed7 0xed8 0xed9 0xeda 0xedb 0xedc 0xedd 0xede 0xedf 0xee0 0xee1 0xee2 0xee3 0xee4 0xee5 0xee6 0xee7 0xee8 0xee9 0xeea 0xeeb 0xeec 0xeed 0xeee 0xeef 0xef0 0xef1 0xef2 0xef3 0xef4 0xef5 0xef6 0xef7 0xef8 0xef9 0xefa 0xefb 0xefc 0xefd 0xefe 0xeff 0xf00 0xf01 0xf02 0xf03 0xf04 0xf05 0xf06 0xf07 0xf08 0xf09 0xf0a 0xf0b 0xf0c 0xf0d 0xf0e 0xf0f 0xf10 0xf11 0xf12 0xf13 0xf14 0xf15 0xf16 0xf17 0xf18 0xf19 0xf1a 0xf1b 0xf1c 0xf1d 0xf1e 0xf1f 0xf20 0xf21 0xf22 0xf23 0xf24 0xf25 0xf26 0xf27 0xf28 0xf29 0xf2a 0xf2b 0xf2c 0xf2d 0xf2e 0xf2f 0xf30 0xf31 0xf32 0xf33 0xf34 0xf35 0xf36 0xf37 0xf38 0xf39 0xf3a 0xf3b 0xf3c 0xf3d 0xf3e 0xf3f 0xf40 0xf41 0xf42 0xf43 0xf44 0xf45 0xf46 0xf47 0xf48 0xf49 0xf4a 0xf4b 0xf4c 0xf4d 0xf4e 0xf4f 0xf50 0xf51 0xf52 0xf53 0xf54 0xf55 0xf56 0xf57 0xf58 0xf59 0xf5a 0xf5b 0xf5c 0xf5d 0xf5e 0xf5f 0xf60 0xf61 0xf62 0xf63 0xf64 0xf65 0xf66 0xf67 0xf68 0xf69 0xf6a 0xf6b 0xf6c 0xf6d 0xf6e 0xf6f 0xf70 0xf71 0xf72 0xf73 0xf74 0xf75 0xf76 0xf77 0xf78 0xf79 0xf7a 0xf7b 0xf7c 0xf7d 0xf7e 0xf7f 0xf80 0xf81 0xf82 0xf83 0xf84 0xf85 0xf86 0xf87 0xf88 0xf89 0xf8a 0xf8b 0xf8c 0xf8d 0xf8e 0xf8f 0xf90 0xf91 0xf92 0xf93 0xf94 0xf95 0xf96 0xf97 0xf98 0xf99 0xf9a 0xf9b 0xf9c 0xf9d 0xf9e 0xf9f 0xfa0 0xfa1 0xfa2 0xfa3 0xfa4 0xfa5 0xfa6 0xfa7 0xfa8 0xfa9 0xfaa 0xfab 0xfac 0xfad 0xfae 0xfaf 0xfb0 0xfb1 0xfb2 0xfb3 0xfb4 0xfb5 0xfb6 0xfb7 0xfb8 0xfb9 0xfba 0xfbb 0xfbc 0xfbd 0xfbe 0xfbf 0xfc0 0xfc1 0xfc2 0xfc3 0xfc4 0xfc5 0xfc6 0xfc7 0xfc8 0xfc9 0xfca 0xfcb 0xfcc 0xfcd 0xfce 0xfcf 0xfd0 0xfd1 0xfd2 0xfd3 0xfd4 0xfd5 0xfd6 0xfd7 0xfd8 0xfd9 0xfda 0xfdb 0xfdc 0xfdd 0xfde 0xfdf 0xfe0 0xfe1 0xfe2 0xfe3 0xfe4 0xfe5 0xfe6 0xfe7 0xfe8 0xfe9 0xfea 0xfeb 0xfec 0xfed 0xfee 0xfef 0xff0 0xff1 0xff2 0xff3 0xff4 0xff5 0xff6 0xff7 0xff8 0xff9 0xffa 0xffb 0xffc 0xffd 0xffe 0xfff 0x1000 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAPF_64_64.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAPF_64_64.txt new file mode 100644 index 000000000..166808918 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAPF_64_64.txt @@ -0,0 +1,8 @@ +0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff 0x100 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x135 0x136 0x137 0x138 0x139 0x13a 0x13b 0x13c 0x13d 0x13e 0x13f 0x140 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x175 0x176 0x177 0x178 0x179 0x17a 0x17b 0x17c 0x17d 0x17e 0x17f 0x180 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x1b5 0x1b6 0x1b7 0x1b8 0x1b9 0x1ba 0x1bb 0x1bc 0x1bd 0x1be 0x1bf 0x1c0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x1f5 0x1f6 0x1f7 0x1f8 0x1f9 0x1fa 0x1fb 0x1fc 0x1fd 0x1fe 0x1ff 0x200 +0x201 0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x235 0x236 0x237 0x238 0x239 0x23a 0x23b 0x23c 0x23d 0x23e 0x23f 0x240 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x275 0x276 0x277 0x278 0x279 0x27a 0x27b 0x27c 0x27d 0x27e 0x27f 0x280 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x2b5 0x2b6 0x2b7 0x2b8 0x2b9 0x2ba 0x2bb 0x2bc 0x2bd 0x2be 0x2bf 0x2c0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x2f5 0x2f6 0x2f7 0x2f8 0x2f9 0x2fa 0x2fb 0x2fc 0x2fd 0x2fe 0x2ff 0x300 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x335 0x336 0x337 0x338 0x339 0x33a 0x33b 0x33c 0x33d 0x33e 0x33f 0x340 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x375 0x376 0x377 0x378 0x379 0x37a 0x37b 0x37c 0x37d 0x37e 0x37f 0x380 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x3b5 0x3b6 0x3b7 0x3b8 0x3b9 0x3ba 0x3bb 0x3bc 0x3bd 0x3be 0x3bf 0x3c0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x3f5 0x3f6 0x3f7 0x3f8 0x3f9 0x3fa 0x3fb 0x3fc 0x3fd 0x3fe 0x3ff 0x400 +0x401 0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x435 0x436 0x437 0x438 0x439 0x43a 0x43b 0x43c 0x43d 0x43e 0x43f 0x440 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x475 0x476 0x477 0x478 0x479 0x47a 0x47b 0x47c 0x47d 0x47e 0x47f 0x480 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x4b5 0x4b6 0x4b7 0x4b8 0x4b9 0x4ba 0x4bb 0x4bc 0x4bd 0x4be 0x4bf 0x4c0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x4f5 0x4f6 0x4f7 0x4f8 0x4f9 0x4fa 0x4fb 0x4fc 0x4fd 0x4fe 0x4ff 0x500 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x535 0x536 0x537 0x538 0x539 0x53a 0x53b 0x53c 0x53d 0x53e 0x53f 0x540 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x575 0x576 0x577 0x578 0x579 0x57a 0x57b 0x57c 0x57d 0x57e 0x57f 0x580 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x5b5 0x5b6 0x5b7 0x5b8 0x5b9 0x5ba 0x5bb 0x5bc 0x5bd 0x5be 0x5bf 0x5c0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x5f5 0x5f6 0x5f7 0x5f8 0x5f9 0x5fa 0x5fb 0x5fc 0x5fd 0x5fe 0x5ff 0x600 +0x601 0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x635 0x636 0x637 0x638 0x639 0x63a 0x63b 0x63c 0x63d 0x63e 0x63f 0x640 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x675 0x676 0x677 0x678 0x679 0x67a 0x67b 0x67c 0x67d 0x67e 0x67f 0x680 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x6b5 0x6b6 0x6b7 0x6b8 0x6b9 0x6ba 0x6bb 0x6bc 0x6bd 0x6be 0x6bf 0x6c0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x6f5 0x6f6 0x6f7 0x6f8 0x6f9 0x6fa 0x6fb 0x6fc 0x6fd 0x6fe 0x6ff 0x700 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x735 0x736 0x737 0x738 0x739 0x73a 0x73b 0x73c 0x73d 0x73e 0x73f 0x740 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x775 0x776 0x777 0x778 0x779 0x77a 0x77b 0x77c 0x77d 0x77e 0x77f 0x780 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x7b5 0x7b6 0x7b7 0x7b8 0x7b9 0x7ba 0x7bb 0x7bc 0x7bd 0x7be 0x7bf 0x7c0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x7f5 0x7f6 0x7f7 0x7f8 0x7f9 0x7fa 0x7fb 0x7fc 0x7fd 0x7fe 0x7ff 0x800 +0x801 0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x835 0x836 0x837 0x838 0x839 0x83a 0x83b 0x83c 0x83d 0x83e 0x83f 0x840 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x875 0x876 0x877 0x878 0x879 0x87a 0x87b 0x87c 0x87d 0x87e 0x87f 0x880 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x8b5 0x8b6 0x8b7 0x8b8 0x8b9 0x8ba 0x8bb 0x8bc 0x8bd 0x8be 0x8bf 0x8c0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x8f5 0x8f6 0x8f7 0x8f8 0x8f9 0x8fa 0x8fb 0x8fc 0x8fd 0x8fe 0x8ff 0x900 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x935 0x936 0x937 0x938 0x939 0x93a 0x93b 0x93c 0x93d 0x93e 0x93f 0x940 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x975 0x976 0x977 0x978 0x979 0x97a 0x97b 0x97c 0x97d 0x97e 0x97f 0x980 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x9b5 0x9b6 0x9b7 0x9b8 0x9b9 0x9ba 0x9bb 0x9bc 0x9bd 0x9be 0x9bf 0x9c0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x9f5 0x9f6 0x9f7 0x9f8 0x9f9 0x9fa 0x9fb 0x9fc 0x9fd 0x9fe 0x9ff 0xa00 +0xa01 0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0xa35 0xa36 0xa37 0xa38 0xa39 0xa3a 0xa3b 0xa3c 0xa3d 0xa3e 0xa3f 0xa40 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0xa75 0xa76 0xa77 0xa78 0xa79 0xa7a 0xa7b 0xa7c 0xa7d 0xa7e 0xa7f 0xa80 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0xab5 0xab6 0xab7 0xab8 0xab9 0xaba 0xabb 0xabc 0xabd 0xabe 0xabf 0xac0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0xaf5 0xaf6 0xaf7 0xaf8 0xaf9 0xafa 0xafb 0xafc 0xafd 0xafe 0xaff 0xb00 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0xb35 0xb36 0xb37 0xb38 0xb39 0xb3a 0xb3b 0xb3c 0xb3d 0xb3e 0xb3f 0xb40 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0xb75 0xb76 0xb77 0xb78 0xb79 0xb7a 0xb7b 0xb7c 0xb7d 0xb7e 0xb7f 0xb80 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0xbb5 0xbb6 0xbb7 0xbb8 0xbb9 0xbba 0xbbb 0xbbc 0xbbd 0xbbe 0xbbf 0xbc0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0xbf5 0xbf6 0xbf7 0xbf8 0xbf9 0xbfa 0xbfb 0xbfc 0xbfd 0xbfe 0xbff 0xc00 +0xc01 0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0xc35 0xc36 0xc37 0xc38 0xc39 0xc3a 0xc3b 0xc3c 0xc3d 0xc3e 0xc3f 0xc40 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0xc75 0xc76 0xc77 0xc78 0xc79 0xc7a 0xc7b 0xc7c 0xc7d 0xc7e 0xc7f 0xc80 0xc81 0xc82 0xc83 0xc84 0xc85 0xc86 0xc87 0xc88 0xc89 0xc8a 0xc8b 0xc8c 0xc8d 0xc8e 0xc8f 0xc90 0xc91 0xc92 0xc93 0xc94 0xc95 0xc96 0xc97 0xc98 0xc99 0xc9a 0xc9b 0xc9c 0xc9d 0xc9e 0xc9f 0xca0 0xca1 0xca2 0xca3 0xca4 0xca5 0xca6 0xca7 0xca8 0xca9 0xcaa 0xcab 0xcac 0xcad 0xcae 0xcaf 0xcb0 0xcb1 0xcb2 0xcb3 0xcb4 0xcb5 0xcb6 0xcb7 0xcb8 0xcb9 0xcba 0xcbb 0xcbc 0xcbd 0xcbe 0xcbf 0xcc0 0xcc1 0xcc2 0xcc3 0xcc4 0xcc5 0xcc6 0xcc7 0xcc8 0xcc9 0xcca 0xccb 0xccc 0xccd 0xcce 0xccf 0xcd0 0xcd1 0xcd2 0xcd3 0xcd4 0xcd5 0xcd6 0xcd7 0xcd8 0xcd9 0xcda 0xcdb 0xcdc 0xcdd 0xcde 0xcdf 0xce0 0xce1 0xce2 0xce3 0xce4 0xce5 0xce6 0xce7 0xce8 0xce9 0xcea 0xceb 0xcec 0xced 0xcee 0xcef 0xcf0 0xcf1 0xcf2 0xcf3 0xcf4 0xcf5 0xcf6 0xcf7 0xcf8 0xcf9 0xcfa 0xcfb 0xcfc 0xcfd 0xcfe 0xcff 0xd00 0xd01 0xd02 0xd03 0xd04 0xd05 0xd06 0xd07 0xd08 0xd09 0xd0a 0xd0b 0xd0c 0xd0d 0xd0e 0xd0f 0xd10 0xd11 0xd12 0xd13 0xd14 0xd15 0xd16 0xd17 0xd18 0xd19 0xd1a 0xd1b 0xd1c 0xd1d 0xd1e 0xd1f 0xd20 0xd21 0xd22 0xd23 0xd24 0xd25 0xd26 0xd27 0xd28 0xd29 0xd2a 0xd2b 0xd2c 0xd2d 0xd2e 0xd2f 0xd30 0xd31 0xd32 0xd33 0xd34 0xd35 0xd36 0xd37 0xd38 0xd39 0xd3a 0xd3b 0xd3c 0xd3d 0xd3e 0xd3f 0xd40 0xd41 0xd42 0xd43 0xd44 0xd45 0xd46 0xd47 0xd48 0xd49 0xd4a 0xd4b 0xd4c 0xd4d 0xd4e 0xd4f 0xd50 0xd51 0xd52 0xd53 0xd54 0xd55 0xd56 0xd57 0xd58 0xd59 0xd5a 0xd5b 0xd5c 0xd5d 0xd5e 0xd5f 0xd60 0xd61 0xd62 0xd63 0xd64 0xd65 0xd66 0xd67 0xd68 0xd69 0xd6a 0xd6b 0xd6c 0xd6d 0xd6e 0xd6f 0xd70 0xd71 0xd72 0xd73 0xd74 0xd75 0xd76 0xd77 0xd78 0xd79 0xd7a 0xd7b 0xd7c 0xd7d 0xd7e 0xd7f 0xd80 0xd81 0xd82 0xd83 0xd84 0xd85 0xd86 0xd87 0xd88 0xd89 0xd8a 0xd8b 0xd8c 0xd8d 0xd8e 0xd8f 0xd90 0xd91 0xd92 0xd93 0xd94 0xd95 0xd96 0xd97 0xd98 0xd99 0xd9a 0xd9b 0xd9c 0xd9d 0xd9e 0xd9f 0xda0 0xda1 0xda2 0xda3 0xda4 0xda5 0xda6 0xda7 0xda8 0xda9 0xdaa 0xdab 0xdac 0xdad 0xdae 0xdaf 0xdb0 0xdb1 0xdb2 0xdb3 0xdb4 0xdb5 0xdb6 0xdb7 0xdb8 0xdb9 0xdba 0xdbb 0xdbc 0xdbd 0xdbe 0xdbf 0xdc0 0xdc1 0xdc2 0xdc3 0xdc4 0xdc5 0xdc6 0xdc7 0xdc8 0xdc9 0xdca 0xdcb 0xdcc 0xdcd 0xdce 0xdcf 0xdd0 0xdd1 0xdd2 0xdd3 0xdd4 0xdd5 0xdd6 0xdd7 0xdd8 0xdd9 0xdda 0xddb 0xddc 0xddd 0xdde 0xddf 0xde0 0xde1 0xde2 0xde3 0xde4 0xde5 0xde6 0xde7 0xde8 0xde9 0xdea 0xdeb 0xdec 0xded 0xdee 0xdef 0xdf0 0xdf1 0xdf2 0xdf3 0xdf4 0xdf5 0xdf6 0xdf7 0xdf8 0xdf9 0xdfa 0xdfb 0xdfc 0xdfd 0xdfe 0xdff 0xe00 +0xe01 0xe02 0xe03 0xe04 0xe05 0xe06 0xe07 0xe08 0xe09 0xe0a 0xe0b 0xe0c 0xe0d 0xe0e 0xe0f 0xe10 0xe11 0xe12 0xe13 0xe14 0xe15 0xe16 0xe17 0xe18 0xe19 0xe1a 0xe1b 0xe1c 0xe1d 0xe1e 0xe1f 0xe20 0xe21 0xe22 0xe23 0xe24 0xe25 0xe26 0xe27 0xe28 0xe29 0xe2a 0xe2b 0xe2c 0xe2d 0xe2e 0xe2f 0xe30 0xe31 0xe32 0xe33 0xe34 0xe35 0xe36 0xe37 0xe38 0xe39 0xe3a 0xe3b 0xe3c 0xe3d 0xe3e 0xe3f 0xe40 0xe41 0xe42 0xe43 0xe44 0xe45 0xe46 0xe47 0xe48 0xe49 0xe4a 0xe4b 0xe4c 0xe4d 0xe4e 0xe4f 0xe50 0xe51 0xe52 0xe53 0xe54 0xe55 0xe56 0xe57 0xe58 0xe59 0xe5a 0xe5b 0xe5c 0xe5d 0xe5e 0xe5f 0xe60 0xe61 0xe62 0xe63 0xe64 0xe65 0xe66 0xe67 0xe68 0xe69 0xe6a 0xe6b 0xe6c 0xe6d 0xe6e 0xe6f 0xe70 0xe71 0xe72 0xe73 0xe74 0xe75 0xe76 0xe77 0xe78 0xe79 0xe7a 0xe7b 0xe7c 0xe7d 0xe7e 0xe7f 0xe80 0xe81 0xe82 0xe83 0xe84 0xe85 0xe86 0xe87 0xe88 0xe89 0xe8a 0xe8b 0xe8c 0xe8d 0xe8e 0xe8f 0xe90 0xe91 0xe92 0xe93 0xe94 0xe95 0xe96 0xe97 0xe98 0xe99 0xe9a 0xe9b 0xe9c 0xe9d 0xe9e 0xe9f 0xea0 0xea1 0xea2 0xea3 0xea4 0xea5 0xea6 0xea7 0xea8 0xea9 0xeaa 0xeab 0xeac 0xead 0xeae 0xeaf 0xeb0 0xeb1 0xeb2 0xeb3 0xeb4 0xeb5 0xeb6 0xeb7 0xeb8 0xeb9 0xeba 0xebb 0xebc 0xebd 0xebe 0xebf 0xec0 0xec1 0xec2 0xec3 0xec4 0xec5 0xec6 0xec7 0xec8 0xec9 0xeca 0xecb 0xecc 0xecd 0xece 0xecf 0xed0 0xed1 0xed2 0xed3 0xed4 0xed5 0xed6 0xed7 0xed8 0xed9 0xeda 0xedb 0xedc 0xedd 0xede 0xedf 0xee0 0xee1 0xee2 0xee3 0xee4 0xee5 0xee6 0xee7 0xee8 0xee9 0xeea 0xeeb 0xeec 0xeed 0xeee 0xeef 0xef0 0xef1 0xef2 0xef3 0xef4 0xef5 0xef6 0xef7 0xef8 0xef9 0xefa 0xefb 0xefc 0xefd 0xefe 0xeff 0xf00 0xf01 0xf02 0xf03 0xf04 0xf05 0xf06 0xf07 0xf08 0xf09 0xf0a 0xf0b 0xf0c 0xf0d 0xf0e 0xf0f 0xf10 0xf11 0xf12 0xf13 0xf14 0xf15 0xf16 0xf17 0xf18 0xf19 0xf1a 0xf1b 0xf1c 0xf1d 0xf1e 0xf1f 0xf20 0xf21 0xf22 0xf23 0xf24 0xf25 0xf26 0xf27 0xf28 0xf29 0xf2a 0xf2b 0xf2c 0xf2d 0xf2e 0xf2f 0xf30 0xf31 0xf32 0xf33 0xf34 0xf35 0xf36 0xf37 0xf38 0xf39 0xf3a 0xf3b 0xf3c 0xf3d 0xf3e 0xf3f 0xf40 0xf41 0xf42 0xf43 0xf44 0xf45 0xf46 0xf47 0xf48 0xf49 0xf4a 0xf4b 0xf4c 0xf4d 0xf4e 0xf4f 0xf50 0xf51 0xf52 0xf53 0xf54 0xf55 0xf56 0xf57 0xf58 0xf59 0xf5a 0xf5b 0xf5c 0xf5d 0xf5e 0xf5f 0xf60 0xf61 0xf62 0xf63 0xf64 0xf65 0xf66 0xf67 0xf68 0xf69 0xf6a 0xf6b 0xf6c 0xf6d 0xf6e 0xf6f 0xf70 0xf71 0xf72 0xf73 0xf74 0xf75 0xf76 0xf77 0xf78 0xf79 0xf7a 0xf7b 0xf7c 0xf7d 0xf7e 0xf7f 0xf80 0xf81 0xf82 0xf83 0xf84 0xf85 0xf86 0xf87 0xf88 0xf89 0xf8a 0xf8b 0xf8c 0xf8d 0xf8e 0xf8f 0xf90 0xf91 0xf92 0xf93 0xf94 0xf95 0xf96 0xf97 0xf98 0xf99 0xf9a 0xf9b 0xf9c 0xf9d 0xf9e 0xf9f 0xfa0 0xfa1 0xfa2 0xfa3 0xfa4 0xfa5 0xfa6 0xfa7 0xfa8 0xfa9 0xfaa 0xfab 0xfac 0xfad 0xfae 0xfaf 0xfb0 0xfb1 0xfb2 0xfb3 0xfb4 0xfb5 0xfb6 0xfb7 0xfb8 0xfb9 0xfba 0xfbb 0xfbc 0xfbd 0xfbe 0xfbf 0xfc0 0xfc1 0xfc2 0xfc3 0xfc4 0xfc5 0xfc6 0xfc7 0xfc8 0xfc9 0xfca 0xfcb 0xfcc 0xfcd 0xfce 0xfcf 0xfd0 0xfd1 0xfd2 0xfd3 0xfd4 0xfd5 0xfd6 0xfd7 0xfd8 0xfd9 0xfda 0xfdb 0xfdc 0xfdd 0xfde 0xfdf 0xfe0 0xfe1 0xfe2 0xfe3 0xfe4 0xfe5 0xfe6 0xfe7 0xfe8 0xfe9 0xfea 0xfeb 0xfec 0xfed 0xfee 0xfef 0xff0 0xff1 0xff2 0xff3 0xff4 0xff5 0xff6 0xff7 0xff8 0xff9 0xffa 0xffb 0xffc 0xffd 0xffe 0xfff 0x1000 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAREDG_64_64.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAREDG_64_64.txt new file mode 100644 index 000000000..166808918 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAREDG_64_64.txt @@ -0,0 +1,8 @@ +0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff 0x100 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x135 0x136 0x137 0x138 0x139 0x13a 0x13b 0x13c 0x13d 0x13e 0x13f 0x140 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x175 0x176 0x177 0x178 0x179 0x17a 0x17b 0x17c 0x17d 0x17e 0x17f 0x180 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x1b5 0x1b6 0x1b7 0x1b8 0x1b9 0x1ba 0x1bb 0x1bc 0x1bd 0x1be 0x1bf 0x1c0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x1f5 0x1f6 0x1f7 0x1f8 0x1f9 0x1fa 0x1fb 0x1fc 0x1fd 0x1fe 0x1ff 0x200 +0x201 0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x235 0x236 0x237 0x238 0x239 0x23a 0x23b 0x23c 0x23d 0x23e 0x23f 0x240 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x275 0x276 0x277 0x278 0x279 0x27a 0x27b 0x27c 0x27d 0x27e 0x27f 0x280 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x2b5 0x2b6 0x2b7 0x2b8 0x2b9 0x2ba 0x2bb 0x2bc 0x2bd 0x2be 0x2bf 0x2c0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x2f5 0x2f6 0x2f7 0x2f8 0x2f9 0x2fa 0x2fb 0x2fc 0x2fd 0x2fe 0x2ff 0x300 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x335 0x336 0x337 0x338 0x339 0x33a 0x33b 0x33c 0x33d 0x33e 0x33f 0x340 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x375 0x376 0x377 0x378 0x379 0x37a 0x37b 0x37c 0x37d 0x37e 0x37f 0x380 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x3b5 0x3b6 0x3b7 0x3b8 0x3b9 0x3ba 0x3bb 0x3bc 0x3bd 0x3be 0x3bf 0x3c0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x3f5 0x3f6 0x3f7 0x3f8 0x3f9 0x3fa 0x3fb 0x3fc 0x3fd 0x3fe 0x3ff 0x400 +0x401 0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x435 0x436 0x437 0x438 0x439 0x43a 0x43b 0x43c 0x43d 0x43e 0x43f 0x440 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x475 0x476 0x477 0x478 0x479 0x47a 0x47b 0x47c 0x47d 0x47e 0x47f 0x480 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x4b5 0x4b6 0x4b7 0x4b8 0x4b9 0x4ba 0x4bb 0x4bc 0x4bd 0x4be 0x4bf 0x4c0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x4f5 0x4f6 0x4f7 0x4f8 0x4f9 0x4fa 0x4fb 0x4fc 0x4fd 0x4fe 0x4ff 0x500 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x535 0x536 0x537 0x538 0x539 0x53a 0x53b 0x53c 0x53d 0x53e 0x53f 0x540 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x575 0x576 0x577 0x578 0x579 0x57a 0x57b 0x57c 0x57d 0x57e 0x57f 0x580 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x5b5 0x5b6 0x5b7 0x5b8 0x5b9 0x5ba 0x5bb 0x5bc 0x5bd 0x5be 0x5bf 0x5c0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x5f5 0x5f6 0x5f7 0x5f8 0x5f9 0x5fa 0x5fb 0x5fc 0x5fd 0x5fe 0x5ff 0x600 +0x601 0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x635 0x636 0x637 0x638 0x639 0x63a 0x63b 0x63c 0x63d 0x63e 0x63f 0x640 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x675 0x676 0x677 0x678 0x679 0x67a 0x67b 0x67c 0x67d 0x67e 0x67f 0x680 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x6b5 0x6b6 0x6b7 0x6b8 0x6b9 0x6ba 0x6bb 0x6bc 0x6bd 0x6be 0x6bf 0x6c0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x6f5 0x6f6 0x6f7 0x6f8 0x6f9 0x6fa 0x6fb 0x6fc 0x6fd 0x6fe 0x6ff 0x700 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x735 0x736 0x737 0x738 0x739 0x73a 0x73b 0x73c 0x73d 0x73e 0x73f 0x740 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x775 0x776 0x777 0x778 0x779 0x77a 0x77b 0x77c 0x77d 0x77e 0x77f 0x780 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x7b5 0x7b6 0x7b7 0x7b8 0x7b9 0x7ba 0x7bb 0x7bc 0x7bd 0x7be 0x7bf 0x7c0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x7f5 0x7f6 0x7f7 0x7f8 0x7f9 0x7fa 0x7fb 0x7fc 0x7fd 0x7fe 0x7ff 0x800 +0x801 0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x835 0x836 0x837 0x838 0x839 0x83a 0x83b 0x83c 0x83d 0x83e 0x83f 0x840 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x875 0x876 0x877 0x878 0x879 0x87a 0x87b 0x87c 0x87d 0x87e 0x87f 0x880 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x8b5 0x8b6 0x8b7 0x8b8 0x8b9 0x8ba 0x8bb 0x8bc 0x8bd 0x8be 0x8bf 0x8c0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x8f5 0x8f6 0x8f7 0x8f8 0x8f9 0x8fa 0x8fb 0x8fc 0x8fd 0x8fe 0x8ff 0x900 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x935 0x936 0x937 0x938 0x939 0x93a 0x93b 0x93c 0x93d 0x93e 0x93f 0x940 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x975 0x976 0x977 0x978 0x979 0x97a 0x97b 0x97c 0x97d 0x97e 0x97f 0x980 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x9b5 0x9b6 0x9b7 0x9b8 0x9b9 0x9ba 0x9bb 0x9bc 0x9bd 0x9be 0x9bf 0x9c0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x9f5 0x9f6 0x9f7 0x9f8 0x9f9 0x9fa 0x9fb 0x9fc 0x9fd 0x9fe 0x9ff 0xa00 +0xa01 0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0xa35 0xa36 0xa37 0xa38 0xa39 0xa3a 0xa3b 0xa3c 0xa3d 0xa3e 0xa3f 0xa40 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0xa75 0xa76 0xa77 0xa78 0xa79 0xa7a 0xa7b 0xa7c 0xa7d 0xa7e 0xa7f 0xa80 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0xab5 0xab6 0xab7 0xab8 0xab9 0xaba 0xabb 0xabc 0xabd 0xabe 0xabf 0xac0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0xaf5 0xaf6 0xaf7 0xaf8 0xaf9 0xafa 0xafb 0xafc 0xafd 0xafe 0xaff 0xb00 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0xb35 0xb36 0xb37 0xb38 0xb39 0xb3a 0xb3b 0xb3c 0xb3d 0xb3e 0xb3f 0xb40 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0xb75 0xb76 0xb77 0xb78 0xb79 0xb7a 0xb7b 0xb7c 0xb7d 0xb7e 0xb7f 0xb80 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0xbb5 0xbb6 0xbb7 0xbb8 0xbb9 0xbba 0xbbb 0xbbc 0xbbd 0xbbe 0xbbf 0xbc0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0xbf5 0xbf6 0xbf7 0xbf8 0xbf9 0xbfa 0xbfb 0xbfc 0xbfd 0xbfe 0xbff 0xc00 +0xc01 0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0xc35 0xc36 0xc37 0xc38 0xc39 0xc3a 0xc3b 0xc3c 0xc3d 0xc3e 0xc3f 0xc40 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0xc75 0xc76 0xc77 0xc78 0xc79 0xc7a 0xc7b 0xc7c 0xc7d 0xc7e 0xc7f 0xc80 0xc81 0xc82 0xc83 0xc84 0xc85 0xc86 0xc87 0xc88 0xc89 0xc8a 0xc8b 0xc8c 0xc8d 0xc8e 0xc8f 0xc90 0xc91 0xc92 0xc93 0xc94 0xc95 0xc96 0xc97 0xc98 0xc99 0xc9a 0xc9b 0xc9c 0xc9d 0xc9e 0xc9f 0xca0 0xca1 0xca2 0xca3 0xca4 0xca5 0xca6 0xca7 0xca8 0xca9 0xcaa 0xcab 0xcac 0xcad 0xcae 0xcaf 0xcb0 0xcb1 0xcb2 0xcb3 0xcb4 0xcb5 0xcb6 0xcb7 0xcb8 0xcb9 0xcba 0xcbb 0xcbc 0xcbd 0xcbe 0xcbf 0xcc0 0xcc1 0xcc2 0xcc3 0xcc4 0xcc5 0xcc6 0xcc7 0xcc8 0xcc9 0xcca 0xccb 0xccc 0xccd 0xcce 0xccf 0xcd0 0xcd1 0xcd2 0xcd3 0xcd4 0xcd5 0xcd6 0xcd7 0xcd8 0xcd9 0xcda 0xcdb 0xcdc 0xcdd 0xcde 0xcdf 0xce0 0xce1 0xce2 0xce3 0xce4 0xce5 0xce6 0xce7 0xce8 0xce9 0xcea 0xceb 0xcec 0xced 0xcee 0xcef 0xcf0 0xcf1 0xcf2 0xcf3 0xcf4 0xcf5 0xcf6 0xcf7 0xcf8 0xcf9 0xcfa 0xcfb 0xcfc 0xcfd 0xcfe 0xcff 0xd00 0xd01 0xd02 0xd03 0xd04 0xd05 0xd06 0xd07 0xd08 0xd09 0xd0a 0xd0b 0xd0c 0xd0d 0xd0e 0xd0f 0xd10 0xd11 0xd12 0xd13 0xd14 0xd15 0xd16 0xd17 0xd18 0xd19 0xd1a 0xd1b 0xd1c 0xd1d 0xd1e 0xd1f 0xd20 0xd21 0xd22 0xd23 0xd24 0xd25 0xd26 0xd27 0xd28 0xd29 0xd2a 0xd2b 0xd2c 0xd2d 0xd2e 0xd2f 0xd30 0xd31 0xd32 0xd33 0xd34 0xd35 0xd36 0xd37 0xd38 0xd39 0xd3a 0xd3b 0xd3c 0xd3d 0xd3e 0xd3f 0xd40 0xd41 0xd42 0xd43 0xd44 0xd45 0xd46 0xd47 0xd48 0xd49 0xd4a 0xd4b 0xd4c 0xd4d 0xd4e 0xd4f 0xd50 0xd51 0xd52 0xd53 0xd54 0xd55 0xd56 0xd57 0xd58 0xd59 0xd5a 0xd5b 0xd5c 0xd5d 0xd5e 0xd5f 0xd60 0xd61 0xd62 0xd63 0xd64 0xd65 0xd66 0xd67 0xd68 0xd69 0xd6a 0xd6b 0xd6c 0xd6d 0xd6e 0xd6f 0xd70 0xd71 0xd72 0xd73 0xd74 0xd75 0xd76 0xd77 0xd78 0xd79 0xd7a 0xd7b 0xd7c 0xd7d 0xd7e 0xd7f 0xd80 0xd81 0xd82 0xd83 0xd84 0xd85 0xd86 0xd87 0xd88 0xd89 0xd8a 0xd8b 0xd8c 0xd8d 0xd8e 0xd8f 0xd90 0xd91 0xd92 0xd93 0xd94 0xd95 0xd96 0xd97 0xd98 0xd99 0xd9a 0xd9b 0xd9c 0xd9d 0xd9e 0xd9f 0xda0 0xda1 0xda2 0xda3 0xda4 0xda5 0xda6 0xda7 0xda8 0xda9 0xdaa 0xdab 0xdac 0xdad 0xdae 0xdaf 0xdb0 0xdb1 0xdb2 0xdb3 0xdb4 0xdb5 0xdb6 0xdb7 0xdb8 0xdb9 0xdba 0xdbb 0xdbc 0xdbd 0xdbe 0xdbf 0xdc0 0xdc1 0xdc2 0xdc3 0xdc4 0xdc5 0xdc6 0xdc7 0xdc8 0xdc9 0xdca 0xdcb 0xdcc 0xdcd 0xdce 0xdcf 0xdd0 0xdd1 0xdd2 0xdd3 0xdd4 0xdd5 0xdd6 0xdd7 0xdd8 0xdd9 0xdda 0xddb 0xddc 0xddd 0xdde 0xddf 0xde0 0xde1 0xde2 0xde3 0xde4 0xde5 0xde6 0xde7 0xde8 0xde9 0xdea 0xdeb 0xdec 0xded 0xdee 0xdef 0xdf0 0xdf1 0xdf2 0xdf3 0xdf4 0xdf5 0xdf6 0xdf7 0xdf8 0xdf9 0xdfa 0xdfb 0xdfc 0xdfd 0xdfe 0xdff 0xe00 +0xe01 0xe02 0xe03 0xe04 0xe05 0xe06 0xe07 0xe08 0xe09 0xe0a 0xe0b 0xe0c 0xe0d 0xe0e 0xe0f 0xe10 0xe11 0xe12 0xe13 0xe14 0xe15 0xe16 0xe17 0xe18 0xe19 0xe1a 0xe1b 0xe1c 0xe1d 0xe1e 0xe1f 0xe20 0xe21 0xe22 0xe23 0xe24 0xe25 0xe26 0xe27 0xe28 0xe29 0xe2a 0xe2b 0xe2c 0xe2d 0xe2e 0xe2f 0xe30 0xe31 0xe32 0xe33 0xe34 0xe35 0xe36 0xe37 0xe38 0xe39 0xe3a 0xe3b 0xe3c 0xe3d 0xe3e 0xe3f 0xe40 0xe41 0xe42 0xe43 0xe44 0xe45 0xe46 0xe47 0xe48 0xe49 0xe4a 0xe4b 0xe4c 0xe4d 0xe4e 0xe4f 0xe50 0xe51 0xe52 0xe53 0xe54 0xe55 0xe56 0xe57 0xe58 0xe59 0xe5a 0xe5b 0xe5c 0xe5d 0xe5e 0xe5f 0xe60 0xe61 0xe62 0xe63 0xe64 0xe65 0xe66 0xe67 0xe68 0xe69 0xe6a 0xe6b 0xe6c 0xe6d 0xe6e 0xe6f 0xe70 0xe71 0xe72 0xe73 0xe74 0xe75 0xe76 0xe77 0xe78 0xe79 0xe7a 0xe7b 0xe7c 0xe7d 0xe7e 0xe7f 0xe80 0xe81 0xe82 0xe83 0xe84 0xe85 0xe86 0xe87 0xe88 0xe89 0xe8a 0xe8b 0xe8c 0xe8d 0xe8e 0xe8f 0xe90 0xe91 0xe92 0xe93 0xe94 0xe95 0xe96 0xe97 0xe98 0xe99 0xe9a 0xe9b 0xe9c 0xe9d 0xe9e 0xe9f 0xea0 0xea1 0xea2 0xea3 0xea4 0xea5 0xea6 0xea7 0xea8 0xea9 0xeaa 0xeab 0xeac 0xead 0xeae 0xeaf 0xeb0 0xeb1 0xeb2 0xeb3 0xeb4 0xeb5 0xeb6 0xeb7 0xeb8 0xeb9 0xeba 0xebb 0xebc 0xebd 0xebe 0xebf 0xec0 0xec1 0xec2 0xec3 0xec4 0xec5 0xec6 0xec7 0xec8 0xec9 0xeca 0xecb 0xecc 0xecd 0xece 0xecf 0xed0 0xed1 0xed2 0xed3 0xed4 0xed5 0xed6 0xed7 0xed8 0xed9 0xeda 0xedb 0xedc 0xedd 0xede 0xedf 0xee0 0xee1 0xee2 0xee3 0xee4 0xee5 0xee6 0xee7 0xee8 0xee9 0xeea 0xeeb 0xeec 0xeed 0xeee 0xeef 0xef0 0xef1 0xef2 0xef3 0xef4 0xef5 0xef6 0xef7 0xef8 0xef9 0xefa 0xefb 0xefc 0xefd 0xefe 0xeff 0xf00 0xf01 0xf02 0xf03 0xf04 0xf05 0xf06 0xf07 0xf08 0xf09 0xf0a 0xf0b 0xf0c 0xf0d 0xf0e 0xf0f 0xf10 0xf11 0xf12 0xf13 0xf14 0xf15 0xf16 0xf17 0xf18 0xf19 0xf1a 0xf1b 0xf1c 0xf1d 0xf1e 0xf1f 0xf20 0xf21 0xf22 0xf23 0xf24 0xf25 0xf26 0xf27 0xf28 0xf29 0xf2a 0xf2b 0xf2c 0xf2d 0xf2e 0xf2f 0xf30 0xf31 0xf32 0xf33 0xf34 0xf35 0xf36 0xf37 0xf38 0xf39 0xf3a 0xf3b 0xf3c 0xf3d 0xf3e 0xf3f 0xf40 0xf41 0xf42 0xf43 0xf44 0xf45 0xf46 0xf47 0xf48 0xf49 0xf4a 0xf4b 0xf4c 0xf4d 0xf4e 0xf4f 0xf50 0xf51 0xf52 0xf53 0xf54 0xf55 0xf56 0xf57 0xf58 0xf59 0xf5a 0xf5b 0xf5c 0xf5d 0xf5e 0xf5f 0xf60 0xf61 0xf62 0xf63 0xf64 0xf65 0xf66 0xf67 0xf68 0xf69 0xf6a 0xf6b 0xf6c 0xf6d 0xf6e 0xf6f 0xf70 0xf71 0xf72 0xf73 0xf74 0xf75 0xf76 0xf77 0xf78 0xf79 0xf7a 0xf7b 0xf7c 0xf7d 0xf7e 0xf7f 0xf80 0xf81 0xf82 0xf83 0xf84 0xf85 0xf86 0xf87 0xf88 0xf89 0xf8a 0xf8b 0xf8c 0xf8d 0xf8e 0xf8f 0xf90 0xf91 0xf92 0xf93 0xf94 0xf95 0xf96 0xf97 0xf98 0xf99 0xf9a 0xf9b 0xf9c 0xf9d 0xf9e 0xf9f 0xfa0 0xfa1 0xfa2 0xfa3 0xfa4 0xfa5 0xfa6 0xfa7 0xfa8 0xfa9 0xfaa 0xfab 0xfac 0xfad 0xfae 0xfaf 0xfb0 0xfb1 0xfb2 0xfb3 0xfb4 0xfb5 0xfb6 0xfb7 0xfb8 0xfb9 0xfba 0xfbb 0xfbc 0xfbd 0xfbe 0xfbf 0xfc0 0xfc1 0xfc2 0xfc3 0xfc4 0xfc5 0xfc6 0xfc7 0xfc8 0xfc9 0xfca 0xfcb 0xfcc 0xfcd 0xfce 0xfcf 0xfd0 0xfd1 0xfd2 0xfd3 0xfd4 0xfd5 0xfd6 0xfd7 0xfd8 0xfd9 0xfda 0xfdb 0xfdc 0xfdd 0xfde 0xfdf 0xfe0 0xfe1 0xfe2 0xfe3 0xfe4 0xfe5 0xfe6 0xfe7 0xfe8 0xfe9 0xfea 0xfeb 0xfec 0xfed 0xfee 0xfef 0xff0 0xff1 0xff2 0xff3 0xff4 0xff5 0xff6 0xff7 0xff8 0xff9 0xffa 0xffb 0xffc 0xffd 0xffe 0xfff 0x1000 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_50_50.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_50_50.txt new file mode 100644 index 000000000..f75acfe46 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_50_50.txt @@ -0,0 +1,8 @@ +0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 +0x201 0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 +0x401 0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 +0x601 0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 +0x801 0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 +0xa01 0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 +0xc01 0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 +0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_64_64.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_64_64.txt new file mode 100644 index 000000000..166808918 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_64_64.txt @@ -0,0 +1,8 @@ +0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff 0x100 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x135 0x136 0x137 0x138 0x139 0x13a 0x13b 0x13c 0x13d 0x13e 0x13f 0x140 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x175 0x176 0x177 0x178 0x179 0x17a 0x17b 0x17c 0x17d 0x17e 0x17f 0x180 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x1b5 0x1b6 0x1b7 0x1b8 0x1b9 0x1ba 0x1bb 0x1bc 0x1bd 0x1be 0x1bf 0x1c0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x1f5 0x1f6 0x1f7 0x1f8 0x1f9 0x1fa 0x1fb 0x1fc 0x1fd 0x1fe 0x1ff 0x200 +0x201 0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x235 0x236 0x237 0x238 0x239 0x23a 0x23b 0x23c 0x23d 0x23e 0x23f 0x240 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x275 0x276 0x277 0x278 0x279 0x27a 0x27b 0x27c 0x27d 0x27e 0x27f 0x280 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x2b5 0x2b6 0x2b7 0x2b8 0x2b9 0x2ba 0x2bb 0x2bc 0x2bd 0x2be 0x2bf 0x2c0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x2f5 0x2f6 0x2f7 0x2f8 0x2f9 0x2fa 0x2fb 0x2fc 0x2fd 0x2fe 0x2ff 0x300 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x335 0x336 0x337 0x338 0x339 0x33a 0x33b 0x33c 0x33d 0x33e 0x33f 0x340 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x375 0x376 0x377 0x378 0x379 0x37a 0x37b 0x37c 0x37d 0x37e 0x37f 0x380 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x3b5 0x3b6 0x3b7 0x3b8 0x3b9 0x3ba 0x3bb 0x3bc 0x3bd 0x3be 0x3bf 0x3c0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x3f5 0x3f6 0x3f7 0x3f8 0x3f9 0x3fa 0x3fb 0x3fc 0x3fd 0x3fe 0x3ff 0x400 +0x401 0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x435 0x436 0x437 0x438 0x439 0x43a 0x43b 0x43c 0x43d 0x43e 0x43f 0x440 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x475 0x476 0x477 0x478 0x479 0x47a 0x47b 0x47c 0x47d 0x47e 0x47f 0x480 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x4b5 0x4b6 0x4b7 0x4b8 0x4b9 0x4ba 0x4bb 0x4bc 0x4bd 0x4be 0x4bf 0x4c0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x4f5 0x4f6 0x4f7 0x4f8 0x4f9 0x4fa 0x4fb 0x4fc 0x4fd 0x4fe 0x4ff 0x500 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x535 0x536 0x537 0x538 0x539 0x53a 0x53b 0x53c 0x53d 0x53e 0x53f 0x540 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x575 0x576 0x577 0x578 0x579 0x57a 0x57b 0x57c 0x57d 0x57e 0x57f 0x580 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x5b5 0x5b6 0x5b7 0x5b8 0x5b9 0x5ba 0x5bb 0x5bc 0x5bd 0x5be 0x5bf 0x5c0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x5f5 0x5f6 0x5f7 0x5f8 0x5f9 0x5fa 0x5fb 0x5fc 0x5fd 0x5fe 0x5ff 0x600 +0x601 0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x635 0x636 0x637 0x638 0x639 0x63a 0x63b 0x63c 0x63d 0x63e 0x63f 0x640 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x675 0x676 0x677 0x678 0x679 0x67a 0x67b 0x67c 0x67d 0x67e 0x67f 0x680 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x6b5 0x6b6 0x6b7 0x6b8 0x6b9 0x6ba 0x6bb 0x6bc 0x6bd 0x6be 0x6bf 0x6c0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x6f5 0x6f6 0x6f7 0x6f8 0x6f9 0x6fa 0x6fb 0x6fc 0x6fd 0x6fe 0x6ff 0x700 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x735 0x736 0x737 0x738 0x739 0x73a 0x73b 0x73c 0x73d 0x73e 0x73f 0x740 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x775 0x776 0x777 0x778 0x779 0x77a 0x77b 0x77c 0x77d 0x77e 0x77f 0x780 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x7b5 0x7b6 0x7b7 0x7b8 0x7b9 0x7ba 0x7bb 0x7bc 0x7bd 0x7be 0x7bf 0x7c0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x7f5 0x7f6 0x7f7 0x7f8 0x7f9 0x7fa 0x7fb 0x7fc 0x7fd 0x7fe 0x7ff 0x800 +0x801 0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x835 0x836 0x837 0x838 0x839 0x83a 0x83b 0x83c 0x83d 0x83e 0x83f 0x840 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x875 0x876 0x877 0x878 0x879 0x87a 0x87b 0x87c 0x87d 0x87e 0x87f 0x880 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x8b5 0x8b6 0x8b7 0x8b8 0x8b9 0x8ba 0x8bb 0x8bc 0x8bd 0x8be 0x8bf 0x8c0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x8f5 0x8f6 0x8f7 0x8f8 0x8f9 0x8fa 0x8fb 0x8fc 0x8fd 0x8fe 0x8ff 0x900 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x935 0x936 0x937 0x938 0x939 0x93a 0x93b 0x93c 0x93d 0x93e 0x93f 0x940 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x975 0x976 0x977 0x978 0x979 0x97a 0x97b 0x97c 0x97d 0x97e 0x97f 0x980 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x9b5 0x9b6 0x9b7 0x9b8 0x9b9 0x9ba 0x9bb 0x9bc 0x9bd 0x9be 0x9bf 0x9c0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x9f5 0x9f6 0x9f7 0x9f8 0x9f9 0x9fa 0x9fb 0x9fc 0x9fd 0x9fe 0x9ff 0xa00 +0xa01 0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0xa35 0xa36 0xa37 0xa38 0xa39 0xa3a 0xa3b 0xa3c 0xa3d 0xa3e 0xa3f 0xa40 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0xa75 0xa76 0xa77 0xa78 0xa79 0xa7a 0xa7b 0xa7c 0xa7d 0xa7e 0xa7f 0xa80 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0xab5 0xab6 0xab7 0xab8 0xab9 0xaba 0xabb 0xabc 0xabd 0xabe 0xabf 0xac0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0xaf5 0xaf6 0xaf7 0xaf8 0xaf9 0xafa 0xafb 0xafc 0xafd 0xafe 0xaff 0xb00 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0xb35 0xb36 0xb37 0xb38 0xb39 0xb3a 0xb3b 0xb3c 0xb3d 0xb3e 0xb3f 0xb40 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0xb75 0xb76 0xb77 0xb78 0xb79 0xb7a 0xb7b 0xb7c 0xb7d 0xb7e 0xb7f 0xb80 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0xbb5 0xbb6 0xbb7 0xbb8 0xbb9 0xbba 0xbbb 0xbbc 0xbbd 0xbbe 0xbbf 0xbc0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0xbf5 0xbf6 0xbf7 0xbf8 0xbf9 0xbfa 0xbfb 0xbfc 0xbfd 0xbfe 0xbff 0xc00 +0xc01 0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0xc35 0xc36 0xc37 0xc38 0xc39 0xc3a 0xc3b 0xc3c 0xc3d 0xc3e 0xc3f 0xc40 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0xc75 0xc76 0xc77 0xc78 0xc79 0xc7a 0xc7b 0xc7c 0xc7d 0xc7e 0xc7f 0xc80 0xc81 0xc82 0xc83 0xc84 0xc85 0xc86 0xc87 0xc88 0xc89 0xc8a 0xc8b 0xc8c 0xc8d 0xc8e 0xc8f 0xc90 0xc91 0xc92 0xc93 0xc94 0xc95 0xc96 0xc97 0xc98 0xc99 0xc9a 0xc9b 0xc9c 0xc9d 0xc9e 0xc9f 0xca0 0xca1 0xca2 0xca3 0xca4 0xca5 0xca6 0xca7 0xca8 0xca9 0xcaa 0xcab 0xcac 0xcad 0xcae 0xcaf 0xcb0 0xcb1 0xcb2 0xcb3 0xcb4 0xcb5 0xcb6 0xcb7 0xcb8 0xcb9 0xcba 0xcbb 0xcbc 0xcbd 0xcbe 0xcbf 0xcc0 0xcc1 0xcc2 0xcc3 0xcc4 0xcc5 0xcc6 0xcc7 0xcc8 0xcc9 0xcca 0xccb 0xccc 0xccd 0xcce 0xccf 0xcd0 0xcd1 0xcd2 0xcd3 0xcd4 0xcd5 0xcd6 0xcd7 0xcd8 0xcd9 0xcda 0xcdb 0xcdc 0xcdd 0xcde 0xcdf 0xce0 0xce1 0xce2 0xce3 0xce4 0xce5 0xce6 0xce7 0xce8 0xce9 0xcea 0xceb 0xcec 0xced 0xcee 0xcef 0xcf0 0xcf1 0xcf2 0xcf3 0xcf4 0xcf5 0xcf6 0xcf7 0xcf8 0xcf9 0xcfa 0xcfb 0xcfc 0xcfd 0xcfe 0xcff 0xd00 0xd01 0xd02 0xd03 0xd04 0xd05 0xd06 0xd07 0xd08 0xd09 0xd0a 0xd0b 0xd0c 0xd0d 0xd0e 0xd0f 0xd10 0xd11 0xd12 0xd13 0xd14 0xd15 0xd16 0xd17 0xd18 0xd19 0xd1a 0xd1b 0xd1c 0xd1d 0xd1e 0xd1f 0xd20 0xd21 0xd22 0xd23 0xd24 0xd25 0xd26 0xd27 0xd28 0xd29 0xd2a 0xd2b 0xd2c 0xd2d 0xd2e 0xd2f 0xd30 0xd31 0xd32 0xd33 0xd34 0xd35 0xd36 0xd37 0xd38 0xd39 0xd3a 0xd3b 0xd3c 0xd3d 0xd3e 0xd3f 0xd40 0xd41 0xd42 0xd43 0xd44 0xd45 0xd46 0xd47 0xd48 0xd49 0xd4a 0xd4b 0xd4c 0xd4d 0xd4e 0xd4f 0xd50 0xd51 0xd52 0xd53 0xd54 0xd55 0xd56 0xd57 0xd58 0xd59 0xd5a 0xd5b 0xd5c 0xd5d 0xd5e 0xd5f 0xd60 0xd61 0xd62 0xd63 0xd64 0xd65 0xd66 0xd67 0xd68 0xd69 0xd6a 0xd6b 0xd6c 0xd6d 0xd6e 0xd6f 0xd70 0xd71 0xd72 0xd73 0xd74 0xd75 0xd76 0xd77 0xd78 0xd79 0xd7a 0xd7b 0xd7c 0xd7d 0xd7e 0xd7f 0xd80 0xd81 0xd82 0xd83 0xd84 0xd85 0xd86 0xd87 0xd88 0xd89 0xd8a 0xd8b 0xd8c 0xd8d 0xd8e 0xd8f 0xd90 0xd91 0xd92 0xd93 0xd94 0xd95 0xd96 0xd97 0xd98 0xd99 0xd9a 0xd9b 0xd9c 0xd9d 0xd9e 0xd9f 0xda0 0xda1 0xda2 0xda3 0xda4 0xda5 0xda6 0xda7 0xda8 0xda9 0xdaa 0xdab 0xdac 0xdad 0xdae 0xdaf 0xdb0 0xdb1 0xdb2 0xdb3 0xdb4 0xdb5 0xdb6 0xdb7 0xdb8 0xdb9 0xdba 0xdbb 0xdbc 0xdbd 0xdbe 0xdbf 0xdc0 0xdc1 0xdc2 0xdc3 0xdc4 0xdc5 0xdc6 0xdc7 0xdc8 0xdc9 0xdca 0xdcb 0xdcc 0xdcd 0xdce 0xdcf 0xdd0 0xdd1 0xdd2 0xdd3 0xdd4 0xdd5 0xdd6 0xdd7 0xdd8 0xdd9 0xdda 0xddb 0xddc 0xddd 0xdde 0xddf 0xde0 0xde1 0xde2 0xde3 0xde4 0xde5 0xde6 0xde7 0xde8 0xde9 0xdea 0xdeb 0xdec 0xded 0xdee 0xdef 0xdf0 0xdf1 0xdf2 0xdf3 0xdf4 0xdf5 0xdf6 0xdf7 0xdf8 0xdf9 0xdfa 0xdfb 0xdfc 0xdfd 0xdfe 0xdff 0xe00 +0xe01 0xe02 0xe03 0xe04 0xe05 0xe06 0xe07 0xe08 0xe09 0xe0a 0xe0b 0xe0c 0xe0d 0xe0e 0xe0f 0xe10 0xe11 0xe12 0xe13 0xe14 0xe15 0xe16 0xe17 0xe18 0xe19 0xe1a 0xe1b 0xe1c 0xe1d 0xe1e 0xe1f 0xe20 0xe21 0xe22 0xe23 0xe24 0xe25 0xe26 0xe27 0xe28 0xe29 0xe2a 0xe2b 0xe2c 0xe2d 0xe2e 0xe2f 0xe30 0xe31 0xe32 0xe33 0xe34 0xe35 0xe36 0xe37 0xe38 0xe39 0xe3a 0xe3b 0xe3c 0xe3d 0xe3e 0xe3f 0xe40 0xe41 0xe42 0xe43 0xe44 0xe45 0xe46 0xe47 0xe48 0xe49 0xe4a 0xe4b 0xe4c 0xe4d 0xe4e 0xe4f 0xe50 0xe51 0xe52 0xe53 0xe54 0xe55 0xe56 0xe57 0xe58 0xe59 0xe5a 0xe5b 0xe5c 0xe5d 0xe5e 0xe5f 0xe60 0xe61 0xe62 0xe63 0xe64 0xe65 0xe66 0xe67 0xe68 0xe69 0xe6a 0xe6b 0xe6c 0xe6d 0xe6e 0xe6f 0xe70 0xe71 0xe72 0xe73 0xe74 0xe75 0xe76 0xe77 0xe78 0xe79 0xe7a 0xe7b 0xe7c 0xe7d 0xe7e 0xe7f 0xe80 0xe81 0xe82 0xe83 0xe84 0xe85 0xe86 0xe87 0xe88 0xe89 0xe8a 0xe8b 0xe8c 0xe8d 0xe8e 0xe8f 0xe90 0xe91 0xe92 0xe93 0xe94 0xe95 0xe96 0xe97 0xe98 0xe99 0xe9a 0xe9b 0xe9c 0xe9d 0xe9e 0xe9f 0xea0 0xea1 0xea2 0xea3 0xea4 0xea5 0xea6 0xea7 0xea8 0xea9 0xeaa 0xeab 0xeac 0xead 0xeae 0xeaf 0xeb0 0xeb1 0xeb2 0xeb3 0xeb4 0xeb5 0xeb6 0xeb7 0xeb8 0xeb9 0xeba 0xebb 0xebc 0xebd 0xebe 0xebf 0xec0 0xec1 0xec2 0xec3 0xec4 0xec5 0xec6 0xec7 0xec8 0xec9 0xeca 0xecb 0xecc 0xecd 0xece 0xecf 0xed0 0xed1 0xed2 0xed3 0xed4 0xed5 0xed6 0xed7 0xed8 0xed9 0xeda 0xedb 0xedc 0xedd 0xede 0xedf 0xee0 0xee1 0xee2 0xee3 0xee4 0xee5 0xee6 0xee7 0xee8 0xee9 0xeea 0xeeb 0xeec 0xeed 0xeee 0xeef 0xef0 0xef1 0xef2 0xef3 0xef4 0xef5 0xef6 0xef7 0xef8 0xef9 0xefa 0xefb 0xefc 0xefd 0xefe 0xeff 0xf00 0xf01 0xf02 0xf03 0xf04 0xf05 0xf06 0xf07 0xf08 0xf09 0xf0a 0xf0b 0xf0c 0xf0d 0xf0e 0xf0f 0xf10 0xf11 0xf12 0xf13 0xf14 0xf15 0xf16 0xf17 0xf18 0xf19 0xf1a 0xf1b 0xf1c 0xf1d 0xf1e 0xf1f 0xf20 0xf21 0xf22 0xf23 0xf24 0xf25 0xf26 0xf27 0xf28 0xf29 0xf2a 0xf2b 0xf2c 0xf2d 0xf2e 0xf2f 0xf30 0xf31 0xf32 0xf33 0xf34 0xf35 0xf36 0xf37 0xf38 0xf39 0xf3a 0xf3b 0xf3c 0xf3d 0xf3e 0xf3f 0xf40 0xf41 0xf42 0xf43 0xf44 0xf45 0xf46 0xf47 0xf48 0xf49 0xf4a 0xf4b 0xf4c 0xf4d 0xf4e 0xf4f 0xf50 0xf51 0xf52 0xf53 0xf54 0xf55 0xf56 0xf57 0xf58 0xf59 0xf5a 0xf5b 0xf5c 0xf5d 0xf5e 0xf5f 0xf60 0xf61 0xf62 0xf63 0xf64 0xf65 0xf66 0xf67 0xf68 0xf69 0xf6a 0xf6b 0xf6c 0xf6d 0xf6e 0xf6f 0xf70 0xf71 0xf72 0xf73 0xf74 0xf75 0xf76 0xf77 0xf78 0xf79 0xf7a 0xf7b 0xf7c 0xf7d 0xf7e 0xf7f 0xf80 0xf81 0xf82 0xf83 0xf84 0xf85 0xf86 0xf87 0xf88 0xf89 0xf8a 0xf8b 0xf8c 0xf8d 0xf8e 0xf8f 0xf90 0xf91 0xf92 0xf93 0xf94 0xf95 0xf96 0xf97 0xf98 0xf99 0xf9a 0xf9b 0xf9c 0xf9d 0xf9e 0xf9f 0xfa0 0xfa1 0xfa2 0xfa3 0xfa4 0xfa5 0xfa6 0xfa7 0xfa8 0xfa9 0xfaa 0xfab 0xfac 0xfad 0xfae 0xfaf 0xfb0 0xfb1 0xfb2 0xfb3 0xfb4 0xfb5 0xfb6 0xfb7 0xfb8 0xfb9 0xfba 0xfbb 0xfbc 0xfbd 0xfbe 0xfbf 0xfc0 0xfc1 0xfc2 0xfc3 0xfc4 0xfc5 0xfc6 0xfc7 0xfc8 0xfc9 0xfca 0xfcb 0xfcc 0xfcd 0xfce 0xfcf 0xfd0 0xfd1 0xfd2 0xfd3 0xfd4 0xfd5 0xfd6 0xfd7 0xfd8 0xfd9 0xfda 0xfdb 0xfdc 0xfdd 0xfde 0xfdf 0xfe0 0xfe1 0xfe2 0xfe3 0xfe4 0xfe5 0xfe6 0xfe7 0xfe8 0xfe9 0xfea 0xfeb 0xfec 0xfed 0xfee 0xfef 0xff0 0xff1 0xff2 0xff3 0xff4 0xff5 0xff6 0xff7 0xff8 0xff9 0xffa 0xffb 0xffc 0xffd 0xffe 0xfff 0x1000 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/tma_tensor_benchmark.cu b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/tma_tensor_benchmark.cu new file mode 100644 index 000000000..a33ac4b13 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/tma_tensor_benchmark.cu @@ -0,0 +1,415 @@ +// Adapt from https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html?highlight=tma#using-tma-to-transfer-multi-dimensional-arrays +#include // PFN_cuTensorMapEncodeTiled, CUtensorMap +#include // CUtensormap +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using barrier = cuda::barrier; +namespace ptx = cuda::ptx; + +/* + * Test application for TMA tensor operations. + * + * Usage: ./tma_tensor -w -h -o + * + */ + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) \ + { \ + fprintf( \ + stderr, \ + "Cuda error in function '%s' file '%s' in line %i : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); \ + exit(EXIT_FAILURE); \ + } \ + } + +#define GMEM_WIDTH 1024 +#define GMEM_HEIGHT 1024 +#define SMEM_WIDTH 32 +#define SMEM_HEIGHT 32 +#define DEFAULT_RUN_ITERS 128 + +enum class TestType { + UTMAPF, + UTMALDG, + UTMALDG_L2Hint, + UTMASTG, + UTMAREDG, + REGULAR_LOAD +}; + +static const std::unordered_map opcode_map = { + {"UTMAPF", TestType::UTMAPF}, + {"UTMALDG", TestType::UTMALDG}, + {"UTMASTG", TestType::UTMASTG}, + {"UTMAREDG", TestType::UTMAREDG}, + {"REGULAR_LOAD", TestType::REGULAR_LOAD} +}; + +__global__ void test_kernel(const CUtensorMap *tensor_map, int *mat, TestType test_type, int width_stride, int run_iters); +__device__ void test_UTMAPF_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters); +__device__ void test_UTMALDG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters); +__device__ void test_UTMASTG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters); +__device__ void test_UTMAREDG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters); +__device__ void test_REGULAR_LOAD_kernel(int *mat, int x, int y, int width_stride, int run_iters); + +__global__ void test_kernel(const CUtensorMap *tensor_map, int *mat, TestType test_type, int width_stride, int run_iters) { + int x = blockDim.x * blockIdx.x; + int y = blockDim.y * blockIdx.y; + if (blockIdx.x == 0 && blockIdx.y == 0 && + threadIdx.x == 0 && threadIdx.y == 0) { + printf("TensorMap address: %p\n", tensor_map); + } + switch (test_type) { + case TestType::UTMAPF: + test_UTMAPF_kernel(tensor_map, x, y, run_iters); + break; + case TestType::UTMALDG: + test_UTMALDG_kernel(tensor_map, x, y, run_iters); + break; + case TestType::UTMASTG: + test_UTMASTG_kernel(tensor_map, x, y, run_iters); + break; + case TestType::UTMAREDG: + test_UTMAREDG_kernel(tensor_map, x, y, run_iters); + break; + case TestType::REGULAR_LOAD: + test_REGULAR_LOAD_kernel(mat, x, y, width_stride, run_iters); + break; + default: + test_UTMAPF_kernel(tensor_map, x, y, run_iters); + break; + } +} + +__device__ void test_UTMAPF_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters) { + // TensorMap prefetch at tensor_map with tensor coord {x, y} + if (threadIdx.x == 0 && threadIdx.y == 0) { + asm volatile ( + "cp.async.bulk.prefetch.tensor.2d.L2.global.tile" + " [%0, {%1, %2}];" + : + : "l"(tensor_map), + "r"(x), + "r"(y) + : "memory"); + } +} + +__device__ void test_UTMALDG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters) { + // The destination shared memory buffer of a bulk tensor operation should be + // 128 byte aligned. + __shared__ alignas(128) int smem_buffer[SMEM_HEIGHT][SMEM_WIDTH]; + +// Initialize shared memory barrier with the number of threads participating in the barrier. +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + if (threadIdx.x == 0 && threadIdx.y == 0) + { + // Initialize barrier. All threads in block participate. + init(&bar, blockDim.x * blockDim.y); + // Make initialized barrier visible in async proxy. + ptx::fence_proxy_async(ptx::space_shared); + } + // Syncthreads so initialized barrier is visible to all threads. + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + barrier::arrival_token token; + if (threadIdx.x == 0 && threadIdx.y == 0) { + // Initiate bulk tensor copy. + ptx::cp_async_bulk_tensor( + ptx::space_cluster, + ptx::space_global, + &smem_buffer, + tensor_map, + {x, y}, + cuda::device::barrier_native_handle(bar) + ); + // Arrive on the barrier and tell how many bytes are expected to come in. + token = cuda::device::barrier_arrive_tx(bar, 1, sizeof(smem_buffer)); + } + else + { + // Other threads just arrive. + token = bar.arrive(); + } + // Wait for the data to have arrived. + bar.wait(std::move(token)); + } +} + +__device__ void test_UTMASTG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters) { + __shared__ alignas(128) int smem_buffer[SMEM_HEIGHT][SMEM_WIDTH]; + + // Compute a unique value for the thread + int thread_x = threadIdx.x + x; + int thread_y = threadIdx.y + y; + smem_buffer[threadIdx.y][threadIdx.x] = 1 + thread_x + thread_y * blockDim.x * gridDim.x; + + // Wait for shared memory writes to be visible to TMA engine. + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + // Initiate TMA transfer to copy shared memory to global memory + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + ptx::cp_async_bulk_tensor( + ptx::space_global, + ptx::space_shared, + tensor_map, + {x, y}, + &smem_buffer + ); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +__device__ void test_UTMAREDG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters) { + __shared__ alignas(128) int smem_buffer[SMEM_HEIGHT][SMEM_WIDTH]; + + // Compute a unique value for the thread + int thread_x = threadIdx.x + x; + int thread_y = threadIdx.y + y; + // Add 1 so the max op will not be effective + smem_buffer[threadIdx.y][threadIdx.x] = 1 + thread_x + thread_y * blockDim.x * gridDim.x; + + // Wait for shared memory writes to be visible to TMA engine. + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + // Initiate TMA transfer to copy shared memory to global memory + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + ptx::cp_reduce_async_bulk_tensor( + ptx::space_global, + ptx::space_shared, + ptx::op_max, + tensor_map, + {x, y}, + &smem_buffer + ); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +__device__ void test_REGULAR_LOAD_kernel(int *mat, int x, int y, int width_stride, int run_iters) { + __shared__ alignas(128) int smem_buffer[SMEM_HEIGHT][SMEM_WIDTH]; + + // Compute a unique value for the thread + int thread_x = threadIdx.x + x; + int thread_y = threadIdx.y + y; + // Mimic a TMA load pattern here + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + for (int row = 0; row < SMEM_HEIGHT; row++) { + for (int col = 0; col < SMEM_WIDTH; col++) { + smem_buffer[row][col] = mat[(y + row) * width_stride + (x + col)] + 1; + } + } + } + __syncthreads(); + // Mimic a TMA store pattern here to make compiler happy + if (threadIdx.x == 0 && threadIdx.y == 0) { + for (int row = 0; row < SMEM_HEIGHT; row++) { + for (int col = 0; col < SMEM_WIDTH; col++) { + mat[(y + row) * width_stride + (x + col)] = smem_buffer[row][col]; + } + } + } + __syncthreads(); + } +} + +PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + // Get pointer to cuTensorMapEncodeTiled + cudaDriverEntryPointQueryResult driver_status; + void *cuTensorMapEncodeTiled_ptr = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", &cuTensorMapEncodeTiled_ptr, 12000, cudaEnableDefault, &driver_status)); + assert(driver_status == cudaDriverEntryPointSuccess); + + return reinterpret_cast(cuTensorMapEncodeTiled_ptr); +} + + +int main(int argc, char *argv[]) { + uint64_t width = GMEM_WIDTH; + uint64_t height = GMEM_HEIGHT; + std::string opcode = "UTMAPF"; + TestType test_type = TestType::UTMAPF; + int run_iters = DEFAULT_RUN_ITERS; + int opt; + while ((opt = getopt(argc, argv, "w:h:o:i:")) != -1) { + switch (opt) { + case 'w': + width = uint64_t(atoi(optarg)); + break; + case 'h': + height = uint64_t(atoi(optarg)); + break; + case 'o': + opcode = std::string(optarg); + break; + case 'i': + run_iters = atoi(optarg); + break; + default: + fprintf(stderr, "Usage: %s -w -h -o \n", argv[0]); + fprintf(stderr, " Block size: %d x %d\n", SMEM_WIDTH, SMEM_HEIGHT); + fprintf(stderr, " -w : width of the matrix\n"); + fprintf(stderr, " -h : height of the matrix\n"); + fprintf(stderr, " -o : opcode\n"); + fprintf(stderr, " -o UTMAPF: tensor prefetch\n"); + fprintf(stderr, " -o UTMALDG: tensor load async\n"); + fprintf(stderr, " -o UTMASTG: tensor store async\n"); + fprintf(stderr, " -o UTMAREDG: tensor reduce async\n"); + fprintf(stderr, " -i : number of iterations\n"); + return 1; + } + } + + // Check if opcode is valid + if (opcode_map.find(opcode) == opcode_map.end()) { + fprintf(stderr, "Invalid opcode\n"); + return 1; + } + + test_type = opcode_map.at(opcode); + + // Initialize data matrix + int *mat, *out_mat, *d_mat; + CUtensorMap *d_tensor_map; + // height|width_stride must be a multiple of 16 and must be greater than height|width + // Here we make it multiple of SMEM_HEIGHT and SMEM_WIDTH to fit our shmem size + uint64_t height_stride = ((height + (SMEM_HEIGHT - 1)) / SMEM_HEIGHT) * SMEM_HEIGHT; + uint64_t width_stride = ((width + (SMEM_WIDTH - 1)) / SMEM_WIDTH) * SMEM_WIDTH; + printf("height: %lu, width: %lu\n", height, width); + printf("height_stride: %lu, width_stride: %lu\n", height_stride, width_stride); + size_t byte_count = height_stride * width_stride * sizeof(int); + mat = (int *)malloc(byte_count); + out_mat = (int *)malloc(byte_count); + cudaMalloc(&d_mat, byte_count); + cudaMalloc(&d_tensor_map, sizeof(CUtensorMap)); + + // Initialize the matrix, set the values to 0 for out of bounds + uint64_t i = 1; + for (uint64_t r = 0; r < height_stride; r++) { + for (uint64_t c = 0; c < width_stride; c++) { + // If within the tensor, set some unique values + if (r < height && c < width) { + mat[r * width_stride + c] = i; + i++; + } + // Otherwise, set the value to 0 + else { + mat[r * width_stride + c] = 0; + } + } + } + + // TMA tensor map object + CUtensorMap tensor_map{}; + // rank is the number of dimensions of the array. + constexpr uint32_t rank = 2; + // The tensor size + uint64_t size[rank] = {width, height}; + // The stride is the number of bytes to traverse from the first element of one row to the next. + // It must be a multiple of 16. + uint64_t stride[rank - 1] = {width_stride * sizeof(int)}; + // The box_size is the size of the shared memory buffer that is used as the + // destination of a TMA transfer. + uint32_t box_size[rank] = {SMEM_WIDTH, SMEM_HEIGHT}; + // The distance between elements in units of sizeof(element). A stride of 2 + // can be used to load only the real component of a complex-valued tensor, for instance. + uint32_t elem_stride[rank] = {1, 1}; + + // Get a function pointer to the cuTensorMapEncodeTiled driver API. + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + + // Create the tensor descriptor. + CUresult res = cuTensorMapEncodeTiled( + &tensor_map, // CUtensorMap *tensorMap, + CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, // cuuint32_t tensorRank, + d_mat, // void *globalAddress, + size, // const cuuint64_t *globalDim, + stride, // const cuuint64_t *globalStrides, + box_size, // const cuuint32_t *boxDim, + elem_stride, // const cuuint32_t *elementStrides, + // Interleave patterns can be used to accelerate loading of values that + // are less than 4 bytes long. + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + // Swizzling can be used to avoid shared memory bank conflicts. + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + // L2 Promotion can be used to widen the effect of a cache-policy to a wider + // set of L2 cache lines. + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + // Any element that is outside of bounds will be set to zero by the TMA transfer. + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + if (res != CUDA_SUCCESS) { + fprintf(stderr, "cuTensorMapEncodeTiled failed with CUresult=%d\n", (int)res); + return 1; + } + cudaMemcpy(d_tensor_map, &tensor_map, sizeof(CUtensorMap), cudaMemcpyHostToDevice); + + // Kernel launch + dim3 grid_dim(width_stride / SMEM_WIDTH, height_stride / SMEM_HEIGHT); + dim3 block_dim(SMEM_WIDTH, SMEM_HEIGHT); + printf("grid_dim: x: %d, y: %d\n", grid_dim.x, grid_dim.y); + printf("block_dim: x: %d, y: %d\n", block_dim.x, block_dim.y); + cudaMemcpy(d_mat, mat, byte_count, cudaMemcpyHostToDevice); + CUDA_SAFECALL((test_kernel<<>>(d_tensor_map, d_mat, test_type, width_stride, run_iters))); + CUDA_SAFECALL(cudaDeviceSynchronize()); + CUDA_SAFECALL(cudaMemcpy(out_mat, d_mat, byte_count, cudaMemcpyDeviceToHost)); + + // Print the matrix to output file + // Dump the values to a file in hex format + // For load operations, use host source array + int *ptr = mat; + + // For store operations, use destination array + if (test_type == TestType::UTMASTG || test_type == TestType::UTMAREDG) { + ptr = out_mat; + } + + char filename[100]; + sprintf(filename, "tma_tensor_test_%s_%lu_%lu.txt", opcode.c_str(), height, width); + FILE *f = fopen(filename, "w"); + for (int i = 0; i < height_stride * width_stride; i++) { + if (i != 0) { + fprintf(f, (i % 512 == 0) ? "\n" : " "); + } + fprintf(f, "0x%x", ptr[i]); + } + fprintf(f, "\n"); + fclose(f); + printf("Values dumped to %s\n", filename); + + // Release device memory + cudaFree(d_tensor_map); + cudaFree(d_mat); + + // Release host memory + free(out_mat); + free(mat); + return 0; +} From ce4ae0256a8889f8e6d8553bb158856ed94ab638 Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 29 Apr 2026 14:13:51 -0700 Subject: [PATCH 06/35] Fix GPU_Microbenchmark ubench build --- .../ubench/gpgpusim_tests/fp_lowp_cvt/Makefile | 14 ++++++++------ .../fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu | 10 +++++----- .../fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu | 6 +++--- .../ubench/tma/tma_bulk/tma_bulk.cu | 1 + .../ubench/tma/tma_tensor/tma_tensor.cu | 2 ++ 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile index 6800d6860..f3376e39d 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile @@ -5,17 +5,19 @@ CC := nvcc LOWP_ARCH ?= sm_120a LOWP_RS_ARCH ?= sm_100a +LOWP_COMPUTE = compute_$(subst sm_,,$(LOWP_ARCH)) +LOWP_RS_COMPUTE = compute_$(subst sm_,,$(LOWP_RS_ARCH)) +LOWP_CODE ?= $(LOWP_COMPUTE) +LOWP_RS_CODE ?= $(LOWP_RS_COMPUTE) CUDA_CPPFLAGS = LOWP_EXE := lowp_cvt_runtime_hw_sim_diff LOWP_RS_EXE := lowp_cvt_runtime_hw_sim_diff_rs -LOWP_NVCC_FLAGS := -gencode=arch=compute_$(subst sm_,,$(LOWP_ARCH)),code=$(LOWP_ARCH) \ - -gencode=arch=compute_$(subst sm_,,$(LOWP_ARCH)),code=compute_$(subst sm_,,$(LOWP_ARCH)) \ - -std=c++17 -O2 -LOWP_RS_NVCC_FLAGS := -gencode=arch=compute_$(subst sm_,,$(LOWP_RS_ARCH)),code=$(LOWP_RS_ARCH) \ - -gencode=arch=compute_$(subst sm_,,$(LOWP_RS_ARCH)),code=compute_$(subst sm_,,$(LOWP_RS_ARCH)) \ - -std=c++17 -O2 +# Keep the default artifact PTX-only: these microbenchmarks intentionally cover +# low-precision PTX forms that can be ahead of CUDA 12.x host ptxas support. +LOWP_NVCC_FLAGS := -gencode=arch=$(LOWP_COMPUTE),code=$(LOWP_CODE) -std=c++17 -O2 +LOWP_RS_NVCC_FLAGS := -gencode=arch=$(LOWP_RS_COMPUTE),code=$(LOWP_RS_CODE) -std=c++17 -O2 .DEFAULT_GOAL := all diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu index 2bad3c250..be4247245 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu @@ -24,7 +24,7 @@ namespace { if (_err != cudaSuccess) { \ std::cerr << "CUDA error: " << cudaGetErrorString(_err) << " @ " \ << #call << "\n"; \ - std::exit(2); \ + std::exit(1); \ } \ } while (0) @@ -1455,7 +1455,7 @@ void run_unary_kernel_with_scale(const std::vector &in, LaunchFn launch_kernel) { if (in.size() != scale.size()) { std::cerr << "input/scale size mismatch\n"; - std::exit(2); + std::exit(1); } TIn *d_in = nullptr; TScale *d_scale = nullptr; @@ -1488,7 +1488,7 @@ void run_pair_kernel_with_scale(const std::vector &a, LaunchFn launch_kernel) { if (a.size() != b.size() || a.size() != scale.size()) { std::cerr << "pair/scale size mismatch\n"; - std::exit(2); + std::exit(1); } float *d_a = nullptr; float *d_b = nullptr; @@ -1563,7 +1563,7 @@ int run_and_dump(const char *out_path) { std::ofstream ofs(out_path, std::ios::out | std::ios::trunc); if (!ofs) { std::cerr << "failed to open output file: " << out_path << "\n"; - return 2; + return 1; } run_unary_kernel( @@ -2358,7 +2358,7 @@ int run_and_dump(const char *out_path) { int main(int argc, char **argv) { if (argc != 3 || std::string(argv[1]) != "--output") { std::cerr << "usage: " << argv[0] << " --output \n"; - return 2; + return 1; } return run_and_dump(argv[2]); } diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu index 977dcb1a4..f8873d205 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu @@ -20,7 +20,7 @@ namespace { if (_err != cudaSuccess) { \ std::cerr << "CUDA error: " << cudaGetErrorString(_err) << " @ " \ << #call << "\n"; \ - std::exit(2); \ + std::exit(1); \ } \ } while (0) @@ -590,7 +590,7 @@ int run_and_dump(const char *out_path) { std::ofstream ofs(out_path, std::ios::out | std::ios::trunc); if (!ofs) { std::cerr << "failed to open output file: " << out_path << "\n"; - return 2; + return 1; } run_pair_kernel_with_rbits( @@ -742,7 +742,7 @@ int run_and_dump(const char *out_path) { int main(int argc, char **argv) { if (argc != 3 || std::string(argv[1]) != "--output") { std::cerr << "usage: " << argv[0] << " --output \n"; - return 2; + return 1; } return run_and_dump(argv[2]); } diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/tma_bulk.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/tma_bulk.cu index fc885b515..7190422c8 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/tma_bulk.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/tma_bulk.cu @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/tma_tensor.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/tma_tensor.cu index d9f7879d7..4e6bbedc6 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/tma_tensor.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/tma_tensor.cu @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include using barrier = cuda::barrier; namespace ptx = cuda::ptx; From 373a72e45e0110bf411c4da225b5f4aecbea4e32 Mon Sep 17 00:00:00 2001 From: Ivorchu <41135347+Ivorchu@users.noreply.github.com> Date: Wed, 29 Apr 2026 22:40:26 -0400 Subject: [PATCH 07/35] wgmma tests --- .../ubench/wgmma/wgmma/Makefile | 35 + .../ubench/wgmma/wgmma/wgmma_no_tma.cu | 306 ++++++ .../ubench/wgmma/wgmma/wgmma_verify.cu | 980 ++++++++++++++++++ .../ubench/wgmma/wgmma_scale/Makefile | 29 + .../ubench/wgmma/wgmma_scale/wgmma_scale.cu | 300 ++++++ .../ubench/wgmma/wgmma_shapes/Makefile | 29 + .../ubench/wgmma/wgmma_shapes/wgmma_shapes.cu | 248 +++++ .../ubench/wgmma/wgmma_sync/Makefile | 30 + .../ubench/wgmma/wgmma_sync/wgmma_fence.cu | 331 ++++++ .../ubench/wgmma/wgmma_sync/wgmma_sync.cu | 290 ++++++ 10 files changed, 2578 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile new file mode 100644 index 000000000..f43d1c4a5 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile @@ -0,0 +1,35 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ +CUTLASS_ROOT ?= $(BASE_DIR)/../../../../cutlass-bench + +# WGMMA microbenchmarks. +# wgmma_no_tma uses CuTe (CUTLASS) headers; wgmma_verify is standalone. +ALL_BENCHES := wgmma_verify wgmma_no_tma + +# WGMMA requires Hopper (SM90a). +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 \ + -I$(CUTLASS_ROOT)/include \ + -I$(CUTLASS_ROOT)/tools/util/include \ + -DCUTLASS_ARCH_MMA_SM90_SUPPORTED + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu new file mode 100644 index 000000000..1a10e92b7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu @@ -0,0 +1,306 @@ +// ============================================================================= +// wgmma_no_tma.cu +// +// Simplified Hopper WGMMA GEMM copied from CuTe tutorial wgmma_sm90.cu with: +// - TMA removed (smem loads use cp.async, same as SM80 kernels) +// - mbarrier removed (sync uses __syncthreads + cp_async_wait) +// - warp specialization removed (single role, all 128 threads do load+mma) +// - F32 accumulator (like CUTLASS example 48) +// - N=16 tile (smallest WGMMA shape; fast to simulate in GPGPU-Sim) +// +// The key test: cute::gemm() issues wgmma.mma_async PTX through the compiler, +// not via hand-written asm(). This verifies GPGPU-Sim handles compiled WGMMA. +// +// GEMM shape: C[M×N] = A[M×K] * B^T[N×K] (NT layout, column-major) +// MMA tile : m64n16k16 (accumulated over bK=64 → 4 wgmma calls per tile) +// Global : M=64, N=16, K=64 +// +// Build (SM90a required): +// nvcc -gencode=arch=compute_90a,code=sm_90a \ +// -gencode=arch=compute_90a,code=compute_90a \ +// -I${CUTLASS_ROOT}/include -I${CUTLASS_ROOT}/tools/util/include \ +// -std=c++17 -O2 -o wgmma_no_tma wgmma_no_tma.cu -lcudart +// +// Run on real H100: +// ./wgmma_no_tma +// Run under GPGPU-Sim (functional): +// PTX_SIM_MODE_FUNC=1 ./wgmma_no_tma +// ============================================================================= + +#include +#include +#include +#include + +#include +#include + +#include + +#include "cutlass/cluster_launch.hpp" +#include "cutlass/util/print_error.hpp" +#include "cutlass/util/GPU_Clock.hpp" +#include "cutlass/util/helper_cuda.hpp" + +using namespace cute; + +// ============================================================================ +// Shared memory layout struct (same as wgmma_sm90.cu) +// ============================================================================ + +template +struct SharedStorage { + alignas(128) cute::ArrayEngine> A; + alignas(128) cute::ArrayEngine> B; +}; + +// ============================================================================ +// Kernel (adapted from gemm_device in wgmma_sm90.cu) +// +// Changes vs. the original: +// 1. No pipeline prefetch — single stage (bP=1). +// 2. Inner loop: copy → cp_async_fence → cp_async_wait<0> → __syncthreads +// → wgmma → __syncthreads. +// 3. warpgroup_fence_operand / warpgroup_arrive / warpgroup_commit_batch / +// warpgroup_wait stay exactly as in wgmma_sm90.cu. +// ============================================================================ + +template +__global__ static +__launch_bounds__(decltype(size(TiledMma{}))::value) +void gemm_device(ProblemShape shape_MNK, CtaTiler cta_tiler, + TA const* A, AStride dA, ASmemLayout /*sA_layout*/, TiledCopyA copy_a, + TB const* B, BStride dB, BSmemLayout /*sB_layout*/, TiledCopyB copy_b, + TC * C, CStride dC, TiledMma mma) +{ + CUTE_STATIC_ASSERT_V(rank(shape_MNK) == Int<3>{}); + CUTE_STATIC_ASSERT_V(rank(cta_tiler) == Int<3>{}); + CUTE_STATIC_ASSERT_V(size(copy_a) == size(mma)); + CUTE_STATIC_ASSERT_V(size(copy_b) == size(mma)); + static_assert(is_static::value); + static_assert(is_static::value); + + // ---- Full and tiled global tensors ---------------------------------------- + Tensor mA = make_tensor(make_gmem_ptr(A), select<0,2>(shape_MNK), dA); // (M,K) + Tensor mB = make_tensor(make_gmem_ptr(B), select<1,2>(shape_MNK), dB); // (N,K) + Tensor mC = make_tensor(make_gmem_ptr(C), select<0,1>(shape_MNK), dC); // (M,N) + + auto cta_coord = make_coord(blockIdx.x, blockIdx.y, _); + Tensor gA = local_tile(mA, cta_tiler, cta_coord, Step<_1, X,_1>{}); // (BLK_M,BLK_K,k) + Tensor gB = local_tile(mB, cta_tiler, cta_coord, Step< X,_1,_1>{}); // (BLK_N,BLK_K,k) + Tensor gC = local_tile(mC, cta_tiler, cta_coord, Step<_1,_1, X>{}); // (BLK_M,BLK_N) + + // ---- Shared memory -------------------------------------------------------- + extern __shared__ char shared_memory[]; + using SS = SharedStorage; + SS& smem = *reinterpret_cast(shared_memory); + Tensor sA = make_tensor(make_smem_ptr(smem.A.begin()), ASmemLayout{}); // (BLK_M,BLK_K,1) + Tensor sB = make_tensor(make_smem_ptr(smem.B.begin()), BSmemLayout{}); // (BLK_N,BLK_K,1) + + // ---- Copy partitioning (gmem → smem) ------------------------------------- + ThrCopy thr_copy_a = copy_a.get_slice(threadIdx.x); + Tensor tAgA = thr_copy_a.partition_S(gA); // (CPY,CPY_M,CPY_K,k) + Tensor tAsA = thr_copy_a.partition_D(as_position_independent_swizzle_tensor(sA)); + + ThrCopy thr_copy_b = copy_b.get_slice(threadIdx.x); + Tensor tBgB = thr_copy_b.partition_S(gB); // (CPY,CPY_N,CPY_K,k) + Tensor tBsB = thr_copy_b.partition_D(as_position_independent_swizzle_tensor(sB)); + + // ---- MMA partitioning (smem → regs → accum) ------------------------------ + ThrMMA thr_mma = mma.get_slice(threadIdx.x); + Tensor tCsA = thr_mma.partition_A(sA); // (MMA,MMA_M,MMA_K,1) + Tensor tCsB = thr_mma.partition_B(sB); // (MMA,MMA_N,MMA_K,1) + Tensor tCgC = thr_mma.partition_C(gC); // (MMA,MMA_M,MMA_N) + + Tensor tCrA = thr_mma.make_fragment_A(tCsA); + Tensor tCrB = thr_mma.make_fragment_B(tCsB); + Tensor tCrC = thr_mma.make_fragment_C(tCgC); + clear(tCrC); + + // ---- K-tile loop (no pipelining — single smem stage) --------------------- + auto K_TILE_MAX = size<3>(tAgA); + + CUTE_NO_UNROLL + for (int k_tile = 0; k_tile < K_TILE_MAX; ++k_tile) { + // Load this K-tile into smem stage 0 (cp.async, no TMA) + copy(copy_a, tAgA(_,_,_,k_tile), tAsA(_,_,_,0)); + copy(copy_b, tBgB(_,_,_,k_tile), tBsB(_,_,_,0)); + cp_async_fence(); + cp_async_wait<0>(); + __syncthreads(); + + // WGMMA: accumulate D += A * B (generated as wgmma.mma_async PTX by compiler) + warpgroup_fence_operand(tCrC); + warpgroup_arrive(); + cute::gemm(mma, tCrA(_,_,_,0), tCrB(_,_,_,0), tCrC); // issues 4 wgmma PTX calls + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(tCrC); + + __syncthreads(); + } + + // ---- Epilogue: store accumulators to C ----------------------------------- + axpby(TC(1), tCrC, TC(0), tCgC); +} + +// ============================================================================ +// Host launcher (NT layout: A col-major M×K, B col-major N×K, C col-major M×N) +// Adapted from gemm_nt() in wgmma_sm90.cu, with MMA atom changed to +// SM90_64x16x16_F32F16F16_SS (N=16 tile, F32 accumulator). +// ============================================================================ + +void gemm_nt(int m, int n, int k, + float alpha, + cute::half_t const* A, int ldA, // col-major M×K + cute::half_t const* B, int ldB, // col-major N×K (B^T is K×N) + float beta, + float * C, int ldC, // col-major M×N + cudaStream_t stream = 0) +{ + using TA = cute::half_t; + using TB = cute::half_t; + using TC = float; + + auto M = int(m), N = int(n), K = int(k); + auto prob_shape = make_shape(M, N, K); // (M, N, K) + + // NT strides: A col-major (stride-1 in M), B col-major (stride-1 in N) + auto dA = make_stride(Int<1>{}, ldA); // (dM=1, dK=ldA) + auto dB = make_stride(Int<1>{}, ldB); // (dN=1, dK=ldB) + auto dC = make_stride(Int<1>{}, ldC); // (dM=1, dN=ldC) + + // CTA tile sizes + auto bM = Int<64>{}; + auto bN = Int<16>{}; + auto bK = Int<64>{}; // bK/16 = 4 wgmma calls per K-tile + auto bP = Int< 1>{}; // single pipeline stage (no TMA double-buffer) + auto cta_tiler = make_shape(bM, bN, bK); + + // Smem layouts (MN-major = col-major). + // Atom widths for half_t: SW128→64, SW64→32, SW32→16. + // bM=64 fits SW128 (64), bN=16 fits SW32 (16). + auto sA = tile_to_shape(GMMA::Layout_MN_SW128_Atom{}, make_shape(bM, bK, bP)); + auto sB = tile_to_shape(GMMA::Layout_MN_SW32_Atom{}, make_shape(bN, bK, bP)); + + // Copy atoms: SM80 cp.async (NOT TMA) — col-major thr layout + TiledCopy copyA = make_tiled_copy( + Copy_Atom, TA>{}, + Layout>{}, // 16×8 threads, m-major + Layout>{}); // 8×1 values = 16 bytes per thread + TiledCopy copyB = make_tiled_copy( + Copy_Atom, TB>{}, + Layout>{}, + Layout>{}); + + // WGMMA tiled MMA: m64n16k16 F32←F16×F16 (MN-major smem for both A and B) + TiledMMA tiled_mma = make_tiled_mma( + SM90_64x16x16_F32F16F16_SS{}); + + // Shared memory size + int smem_size = int(sizeof(SharedStorage)); + + dim3 dimBlock(size(tiled_mma)); // 128 threads (1 warpgroup) + dim3 dimCluster(1, 1, 1); + dim3 dimGrid(ceil_div(m, int(bM)), ceil_div(n, int(bN))); + + auto kernel_ptr = reinterpret_cast( + &gemm_device); + + CUTE_CHECK_ERROR(cudaFuncSetAttribute( + kernel_ptr, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + + cutlass::ClusterLaunchParams params = {dimGrid, dimBlock, dimCluster, smem_size, stream}; + cutlass::Status status = cutlass::launch_kernel_on_cluster( + params, kernel_ptr, + prob_shape, cta_tiler, + A, dA, sA, copyA, + B, dB, sB, copyB, + C, dC, tiled_mma); + + CUTE_CHECK_LAST(); + if (status != cutlass::Status::kSuccess) + fprintf(stderr, "launch_kernel_on_cluster failed\n"); +} + +// ============================================================================ +// main: allocate, run, verify +// ============================================================================ + +int main() { + cudaDeviceProp props; + int dev; + CUTE_CHECK_ERROR(cudaGetDevice(&dev)); + CUTE_CHECK_ERROR(cudaGetDeviceProperties(&props, dev)); + printf("wgmma_no_tma — %s (SM %d.%d)\n", props.name, props.major, props.minor); + + if (props.major != 9) { + printf("This test requires SM90a (Hopper). Skipping.\n"); + return 0; + } + + // Problem size: keep small for GPGPU-Sim + constexpr int M = 64, N = 16, K = 64; + printf("GEMM: C[%dx%d] = A[%dx%d] * B^T[%dx%d] (NT col-major, FP16→FP32)\n", + M, N, M, K, N, K); + printf("Tile: m64n16k64 (4 wgmma.m64n16k16 calls via cute::gemm per K-tile)\n"); + printf("------------------------------------------------------------\n"); + + // Host allocation (col-major) + thrust::host_vector h_A(M * K), h_B(N * K); + thrust::host_vector h_C(M * N, 0.f); + + // Fill with small integer values (avoids FP16 rounding in reference) + for (int j = 0; j < M*K; ++j) h_A[j] = cute::half_t(float((j % 5) - 2)); // {-2,-1,0,1,2} + for (int j = 0; j < N*K; ++j) h_B[j] = cute::half_t(float((j % 3) - 1)); // {-1,0,1} + + // Device + thrust::device_vector d_A = h_A; + thrust::device_vector d_B = h_B; + thrust::device_vector d_C = h_C; + + // Run WGMMA kernel + gemm_nt(M, N, K, 1.0f, + d_A.data().get(), M, // A: ldA = M (col-major M×K) + d_B.data().get(), N, // B: ldB = N (col-major N×K) + 0.0f, + d_C.data().get(), M); // C: ldC = M (col-major M×N) + + CUTE_CHECK_ERROR(cudaDeviceSynchronize()); + h_C = d_C; + + // ---- CPU reference ------------------------------------------------------- + // C_ref[m][n] = sum_k A[m][k] * B[n][k] + // Col-major indexing: A[m + k*M], B[n + k*N], C[m + n*M] + std::vector ref(M * N, 0.0); + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) + for (int kk = 0; kk < K; ++kk) + ref[m + n*M] += double(float(h_A[m + kk*M])) * double(float(h_B[n + kk*N])); + + // ---- Check --------------------------------------------------------------- + double max_err = 0.0; + int max_m = 0, max_n = 0; + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) { + double e = std::fabs(double(h_C[m + n*M]) - ref[m + n*M]); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + + double ref_mag = std::fabs(ref[max_m + max_n*M]); + double tol = (ref_mag < 1.0) ? 1e-3 : ref_mag * 1e-2; + bool pass = (max_err <= tol); + + printf("Result: %s max_err=%.4e (at C[%d][%d] ref=%.4f sim=%.4f)\n", + pass ? "PASS" : "FAIL", max_err, max_m, max_n, + float(ref[max_m + max_n*M]), h_C[max_m + max_n*M]); + + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu new file mode 100644 index 000000000..842349c85 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu @@ -0,0 +1,980 @@ +// ============================================================================= +// wgmma_verify.cu (v2) +// +// Functional-correctness and timing microbenchmark for wgmma.mma_async (SM90a). +// +// Type / shape combinations tested (all m64n16kK): +// f16 k16 D=f32 swizzle modes 0, 1(128B), 2(64B), 3(32B) +// bf16 k16 D=f32 +// tf32 k8 D=f32 +// e4m3 k32 D=f32 +// e5m2 k32 D=f32 +// s8 k32 D=s32 +// u8 k32 D=s32 +// b1 k256 D=s32 (GPGPU-Sim functional sim only; +// real H100 needs bit-packed smem, not 1 byte/bit) +// +// Smem layout (K-major, no swizzle unless SW!=0): +// T = 16 / E (elements per 128-bit column) +// off = (stride%T + leading%8 * T)*E + (stride/T)*SBO + (leading/8)*LBO +// SBO = 128 always. LBO_A = M*E*8, LBO_B = N*E*8. +// All shapes satisfy M*K*E = 2048 and K*N*E = 512. +// +// Two test cases per type: +// all-ones – A=B=1 → D[m][n] = K (trivially verifiable) +// sequential – small values compared against a host double-precision reference +// +// Usage: +// ./wgmma_verify – real H100 +// PTX_SIM_MODE_FUNC=1 ./wgmma_verify – GPGPU-Sim functional sim +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; // 8 D-registers per thread + +// For all standard types: M*K*E = 2048, K*N*E = 512 (fixed by WGMMA design) +static constexpr int SMEM_A = 2048; +static constexpr int SMEM_B = 512; +static constexpr int SBO = 128; // stride byte offset (always 128 for K-major) + +// b1 (sim convention): 1 byte per bit, K=256 "elements" +static constexpr int B1_K = 256; +static constexpr int B1_SMEM_A = M * B1_K; // 16384 +static constexpr int B1_SMEM_B = B1_K * N; // 4096 + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }\n" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +// K-major smem byte offset (before swizzle). +// For A (M×K): leading=k, stride=m. For B (K×N): leading=k, stride=n. +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int E, int LBO) { + int T = 16 / E; + return (stride % T + (leading % 8) * T) * E + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +// XOR-swizzle (self-inverse) applied to a byte offset. +__host__ __device__ __forceinline__ +int apply_sw(int off, int sw) { + if (!sw) return off; + int s = (sw == 1) ? 128 : (sw == 2) ? 64 : 32; + int b = off / s, i = off % s; + return b * s + (i ^ ((b & 1) ? (s >> 1) : 0)); +} + +__host__ __device__ __forceinline__ +int smem_off_sw(int leading, int stride, int E, int LBO, int sw) { + return apply_sw(smem_off(leading, stride, E, LBO), sw); +} + +// --------------------------------------------------------------------------- +// WGMMA protocol macros +// --------------------------------------------------------------------------- +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") +#define READ_CLK(c) asm volatile("mov.u32 %0,%%clock;\n" : "=r"(c)) + +// =========================================================================== +// Kernels +// =========================================================================== + +// --------------------------------------------------------------------------- +// F16 m64n16k16 D=f32 template on swizzle mode SW (0–3) +// --------------------------------------------------------------------------- +template +__global__ void kernel_f16(const half* A_g, const half* B_g, + float* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 16, E = 2, LBO_A = M*E*8, LBO_B = N*E*8; // 1024, 256 + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off_sw(k, m, E, LBO_A, SW)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off_sw(k, n, E, LBO_B, SW)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, SW); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, SW); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// BF16 m64n16k16 D=f32 +// --------------------------------------------------------------------------- +__global__ void kernel_bf16(const __nv_bfloat16* A_g, const __nv_bfloat16* B_g, + float* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 16, E = 2, LBO_A = M*E*8, LBO_B = N*E*8; + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(__nv_bfloat16*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(__nv_bfloat16*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.bf16.bf16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// TF32 m64n16k8 D=f32 (no trans parameters) +// --------------------------------------------------------------------------- +__global__ void kernel_tf32(const float* A_g, const float* B_g, + float* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 8, E = 4, LBO_A = M*E*8, LBO_B = N*E*8; // 2048, 512 + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(float*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(float*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k8.f32.tf32.tf32 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// FP8 E4M3 m64n16k32 D=f32 (no trans) +// A_g / B_g carry raw fp8 bytes (uint8_t). +// --------------------------------------------------------------------------- +__global__ void kernel_e4m3(const uint8_t* A_g, const uint8_t* B_g, + float* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; // 512, 128 + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(uint8_t*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(uint8_t*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e4m3.e4m3 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// FP8 E5M2 m64n16k32 D=f32 (no trans) +// --------------------------------------------------------------------------- +__global__ void kernel_e5m2(const uint8_t* A_g, const uint8_t* B_g, + float* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(uint8_t*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(uint8_t*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e5m2.e5m2 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// S8 m64n16k32 D=s32 (no scale, no trans) +// --------------------------------------------------------------------------- +__global__ void kernel_s8(const int8_t* A_g, const int8_t* B_g, + int32_t* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(int8_t*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(int8_t*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + int32_t d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.s32.s8.s8 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p; }\n" + : "+r"(d0),"+r"(d1),"+r"(d2),"+r"(d3),"+r"(d4),"+r"(d5),"+r"(d6),"+r"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// U8 m64n16k32 D=s32 +// --------------------------------------------------------------------------- +__global__ void kernel_u8(const uint8_t* A_g, const uint8_t* B_g, + int32_t* D_g, uint32_t* clk_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(uint8_t*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(uint8_t*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + int32_t d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.s32.u8.u8 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p; }\n" + : "+r"(d0),"+r"(d1),"+r"(d2),"+r"(d3),"+r"(d4),"+r"(d5),"+r"(d6),"+r"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// B1 m64n16k256 D=s32 +// +// WARNING — GPGPU-Sim functional sim only. +// The sim reads each "bit" as a separate byte (1 byte per bit element, using +// only the LSB). Real H100 hardware expects bits packed 8-per-byte (256 bits +// = 32 bytes per row of A). This kernel uses 1 byte/bit, so the smem +// buffers are 256× larger than on real hardware. +// --------------------------------------------------------------------------- +__global__ void kernel_b1(const uint8_t* A_g, const uint8_t* B_g, + int32_t* D_g, uint32_t* clk_g) { + __shared__ __align__(128) uint8_t smA[B1_SMEM_A]; // 16384 bytes + __shared__ __align__(128) uint8_t smB[B1_SMEM_B]; // 4096 bytes + const int tid = threadIdx.x; + constexpr int K = B1_K, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; // 512, 128 + + if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + smA[smem_off(k, m, E, LBO_A)] = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + smB[smem_off(k, n, E, LBO_B)] = B_g[i]; + } + __syncthreads(); + if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, 0); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, 0); + int32_t d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k256.s32.b1.b1.and.popc " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p; }\n" + : "+r"(d0),"+r"(d1),"+r"(d2),"+r"(d3),"+r"(d4),"+r"(d5),"+r"(d6),"+r"(d7) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + if (!tid) { uint32_t c; READ_CLK(c); clk_g[2] = c; } + + int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Host: fragment mapping and matrix reassembly +// =========================================================================== + +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static void reassemble_D_f32(const float* flat, float D[M][N]) { + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) D[m][n] = 0.f; + for (int T = 0; T < WGSIZE; ++T) + for (int e = 0; e < D_ELEMS; ++e) { + int row, col; d_frag_pos(T, e, &row, &col); + if (row < M && col < N) D[row][col] = flat[T*D_ELEMS + e]; + } +} + +static void reassemble_D_s32(const int32_t* flat, int32_t D[M][N]) { + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) D[m][n] = 0; + for (int T = 0; T < WGSIZE; ++T) + for (int e = 0; e < D_ELEMS; ++e) { + int row, col; d_frag_pos(T, e, &row, &col); + if (row < M && col < N) D[row][col] = flat[T*D_ELEMS + e]; + } +} + +// =========================================================================== +// Host: result checking and printing +// =========================================================================== + +static bool check_f32(const char* name, + float D_sim[M][N], const double* D_ref_flat, + uint32_t c0, uint32_t c1, uint32_t c2) { + double max_err = 0; int max_m = 0, max_n = 0; + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) { + double e = fabs((double)D_sim[m][n] - D_ref_flat[m*N+n]); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + double ref_mag = fabs(D_ref_flat[max_m*N+max_n]); + double tol = (ref_mag < 1.0) ? 1e-3 : ref_mag * 1e-2; + bool pass = (max_err <= tol); + printf("[%-40s] %s max_err=%.4e fill=%u wgmma=%u cycles\n", + name, pass ? "PASS" : "FAIL", max_err, c1-c0, c2-c1); + if (!pass) + printf(" Worst: D[%d][%d] sim=%.6f ref=%.6f\n", + max_m, max_n, D_sim[max_m][max_n], (float)D_ref_flat[max_m*N+max_n]); + return pass; +} + +static bool check_s32(const char* name, + int32_t D_sim[M][N], const int64_t* D_ref_flat, + uint32_t c0, uint32_t c1, uint32_t c2) { + int64_t max_err = 0; int max_m = 0, max_n = 0; + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) { + int64_t e = llabs((int64_t)D_sim[m][n] - D_ref_flat[m*N+n]); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + bool pass = (max_err == 0); + printf("[%-40s] %s max_err=%lld fill=%u wgmma=%u cycles\n", + name, pass ? "PASS" : "FAIL", (long long)max_err, c1-c0, c2-c1); + if (!pass) + printf(" Worst: D[%d][%d] sim=%d ref=%lld\n", + max_m, max_n, D_sim[max_m][max_n], (long long)D_ref_flat[max_m*N+max_n]); + return pass; +} + +// =========================================================================== +// Generic test runner helpers +// =========================================================================== + +// Run a f32-accumulator test. +// K — K dimension +// bytes_a/b — device memory sizes for A and B +// h_A / h_B — host pointers to device-formatted input data +// A_dbl/B_dbl — double arrays (M*K and K*N) for the reference matmul; +// must already reflect type quantization (e.g. bf16-rounded) +// launch — lambda: (d_A, d_B, d_D, d_clk) → void +static bool run_f32(const char* name, int K, + size_t bytes_a, size_t bytes_b, + const void* h_A, const void* h_B, + const double* A_dbl, const double* B_dbl, + std::function launch) { + void *d_A, *d_B; + float *d_D; + uint32_t *d_clk; + cudaMalloc(&d_A, bytes_a); + cudaMalloc(&d_B, bytes_b); + cudaMalloc(&d_D, WGSIZE * D_ELEMS * sizeof(float)); + cudaMalloc(&d_clk, 3 * sizeof(uint32_t)); + cudaMemcpy(d_A, h_A, bytes_a, cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B, bytes_b, cudaMemcpyHostToDevice); + + launch(d_A, d_B, d_D, d_clk); + cudaError_t err = cudaDeviceSynchronize(); + if (err != cudaSuccess) { + printf("[%s] CUDA error: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); cudaFree(d_clk); + return false; + } + + std::vector flat(WGSIZE * D_ELEMS); + uint32_t clk[3]; + cudaMemcpy(flat.data(), d_D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk, d_clk, 3*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); cudaFree(d_clk); + + static float D_sim[M][N]; + static double D_ref[M*N]; + reassemble_D_f32(flat.data(), D_sim); + + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) { + double s = 0.0; + for (int k = 0; k < K; ++k) s += A_dbl[m*K+k] * B_dbl[k*N+n]; + D_ref[m*N+n] = s; + } + + return check_f32(name, D_sim, D_ref, clk[0], clk[1], clk[2]); +} + +// Run a s32-accumulator test. +// use_and — if true, reference uses AND-accumulate (b1); else multiply-accumulate +static bool run_s32(const char* name, int K, + size_t bytes_a, size_t bytes_b, + const void* h_A, const void* h_B, + const int64_t* A_i64, const int64_t* B_i64, + bool use_and, + std::function launch) { + void *d_A, *d_B; + int32_t *d_D; + uint32_t *d_clk; + cudaMalloc(&d_A, bytes_a); + cudaMalloc(&d_B, bytes_b); + cudaMalloc(&d_D, WGSIZE * D_ELEMS * sizeof(int32_t)); + cudaMalloc(&d_clk, 3 * sizeof(uint32_t)); + cudaMemcpy(d_A, h_A, bytes_a, cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B, bytes_b, cudaMemcpyHostToDevice); + + launch(d_A, d_B, d_D, d_clk); + cudaError_t err = cudaDeviceSynchronize(); + if (err != cudaSuccess) { + printf("[%s] CUDA error: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); cudaFree(d_clk); + return false; + } + + std::vector flat(WGSIZE * D_ELEMS); + uint32_t clk[3]; + cudaMemcpy(flat.data(), d_D, WGSIZE*D_ELEMS*sizeof(int32_t), cudaMemcpyDeviceToHost); + cudaMemcpy(clk, d_clk, 3*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); cudaFree(d_clk); + + static int32_t D_sim[M][N]; + static int64_t D_ref[M*N]; + reassemble_D_s32(flat.data(), D_sim); + + for (int m = 0; m < M; ++m) + for (int n = 0; n < N; ++n) { + int64_t s = 0; + for (int k = 0; k < K; ++k) { + int64_t a = A_i64[m*K+k], b = B_i64[k*N+n]; + s += use_and ? ((a & b) & 1) : (a * b); + } + D_ref[m*N+n] = s; + } + + return check_s32(name, D_sim, D_ref, clk[0], clk[1], clk[2]); +} + +// =========================================================================== +// Host-side type quantization helpers +// =========================================================================== + +// Round float to the nearest bf16-representable value. +static double quant_bf16(double f) { + return (double)__bfloat162float(__float2bfloat16((float)f)); +} + +// Round float to the nearest tf32-representable value (10 mantissa bits). +static double quant_tf32(double f) { + float fp = (float)f; + uint32_t b; memcpy(&b, &fp, 4); + b &= 0xFFFFE000u; // keep sign + 8 exp + top 10 mantissa bits + memcpy(&fp, &b, 4); + return (double)fp; +} + +// Round float to nearest fp8 e4m3 value. +static double quant_e4m3(double f) { + return (double)(float)(__nv_fp8_e4m3)((float)f); +} + +// Round float to nearest fp8 e5m2 value. +static double quant_e5m2(double f) { + return (double)(float)(__nv_fp8_e5m2)((float)f); +} + +// Convert float to raw fp8 e4m3 byte. +static uint8_t float_to_e4m3(float f) { + __nv_fp8_e4m3 v(f); + uint8_t b; memcpy(&b, &v, 1); + return b; +} + +// Convert float to raw fp8 e5m2 byte. +static uint8_t float_to_e5m2(float f) { + __nv_fp8_e5m2 v(f); + uint8_t b; memcpy(&b, &v, 1); + return b; +} + +// =========================================================================== +// Per-type test functions +// =========================================================================== + +// ---- F16 ---- +static bool test_f16(const char* name, int sw, + const float* A_host, const float* B_host) { + constexpr int K = 16; + std::vector A_h(M*K), B_h(K*N); + std::vector A_d(M*K), B_d(K*N); + for (int i = 0; i < M*K; ++i) { + A_h[i] = __float2half(A_host[i]); + A_d[i] = (double)__half2float(A_h[i]); + } + for (int i = 0; i < K*N; ++i) { + B_h[i] = __float2half(B_host[i]); + B_d[i] = (double)__half2float(B_h[i]); + } + auto launch = [sw](const void* dA, const void* dB, float* dD, uint32_t* dC) { + switch (sw) { + case 0: kernel_f16<0><<<1,WGSIZE>>>((const half*)dA,(const half*)dB,dD,dC); break; + case 1: kernel_f16<1><<<1,WGSIZE>>>((const half*)dA,(const half*)dB,dD,dC); break; + case 2: kernel_f16<2><<<1,WGSIZE>>>((const half*)dA,(const half*)dB,dD,dC); break; + case 3: kernel_f16<3><<<1,WGSIZE>>>((const half*)dA,(const half*)dB,dD,dC); break; + } + }; + return run_f32(name, K, M*K*sizeof(half), K*N*sizeof(half), + A_h.data(), B_h.data(), A_d.data(), B_d.data(), launch); +} + +// ---- BF16 ---- +static bool test_bf16(const char* name, + const float* A_host, const float* B_host) { + constexpr int K = 16; + std::vector<__nv_bfloat16> A_b(M*K), B_b(K*N); + std::vector A_d(M*K), B_d(K*N); + for (int i = 0; i < M*K; ++i) { + A_b[i] = __float2bfloat16(A_host[i]); + A_d[i] = quant_bf16(A_host[i]); + } + for (int i = 0; i < K*N; ++i) { + B_b[i] = __float2bfloat16(B_host[i]); + B_d[i] = quant_bf16(B_host[i]); + } + auto launch = [](const void* dA, const void* dB, float* dD, uint32_t* dC) { + kernel_bf16<<<1,WGSIZE>>>((const __nv_bfloat16*)dA,(const __nv_bfloat16*)dB,dD,dC); + }; + return run_f32(name, K, M*K*sizeof(__nv_bfloat16), K*N*sizeof(__nv_bfloat16), + A_b.data(), B_b.data(), A_d.data(), B_d.data(), launch); +} + +// ---- TF32 ---- +static bool test_tf32(const char* name, + const float* A_host, const float* B_host) { + constexpr int K = 8; + std::vector A_f(M*K), B_f(K*N); + std::vector A_d(M*K), B_d(K*N); + for (int i = 0; i < M*K; ++i) { + A_d[i] = quant_tf32(A_host[i]); + A_f[i] = (float)A_d[i]; + } + for (int i = 0; i < K*N; ++i) { + B_d[i] = quant_tf32(B_host[i]); + B_f[i] = (float)B_d[i]; + } + auto launch = [](const void* dA, const void* dB, float* dD, uint32_t* dC) { + kernel_tf32<<<1,WGSIZE>>>((const float*)dA,(const float*)dB,dD,dC); + }; + return run_f32(name, K, M*K*sizeof(float), K*N*sizeof(float), + A_f.data(), B_f.data(), A_d.data(), B_d.data(), launch); +} + +// ---- FP8 E4M3 ---- +static bool test_e4m3(const char* name, + const float* A_host, const float* B_host) { + constexpr int K = 32; + std::vector A_b(M*K), B_b(K*N); + std::vector A_d(M*K), B_d(K*N); + for (int i = 0; i < M*K; ++i) { + A_b[i] = float_to_e4m3(A_host[i]); + A_d[i] = quant_e4m3(A_host[i]); + } + for (int i = 0; i < K*N; ++i) { + B_b[i] = float_to_e4m3(B_host[i]); + B_d[i] = quant_e4m3(B_host[i]); + } + auto launch = [](const void* dA, const void* dB, float* dD, uint32_t* dC) { + kernel_e4m3<<<1,WGSIZE>>>((const uint8_t*)dA,(const uint8_t*)dB,dD,dC); + }; + return run_f32(name, K, M*K, K*N, + A_b.data(), B_b.data(), A_d.data(), B_d.data(), launch); +} + +// ---- FP8 E5M2 ---- +static bool test_e5m2(const char* name, + const float* A_host, const float* B_host) { + constexpr int K = 32; + std::vector A_b(M*K), B_b(K*N); + std::vector A_d(M*K), B_d(K*N); + for (int i = 0; i < M*K; ++i) { + A_b[i] = float_to_e5m2(A_host[i]); + A_d[i] = quant_e5m2(A_host[i]); + } + for (int i = 0; i < K*N; ++i) { + B_b[i] = float_to_e5m2(B_host[i]); + B_d[i] = quant_e5m2(B_host[i]); + } + auto launch = [](const void* dA, const void* dB, float* dD, uint32_t* dC) { + kernel_e5m2<<<1,WGSIZE>>>((const uint8_t*)dA,(const uint8_t*)dB,dD,dC); + }; + return run_f32(name, K, M*K, K*N, + A_b.data(), B_b.data(), A_d.data(), B_d.data(), launch); +} + +// ---- S8 ---- +static bool test_s8(const char* name, + const int8_t* A_host, const int8_t* B_host) { + constexpr int K = 32; + std::vector A_i(M*K), B_i(K*N); + for (int i = 0; i < M*K; ++i) A_i[i] = A_host[i]; + for (int i = 0; i < K*N; ++i) B_i[i] = B_host[i]; + auto launch = [](const void* dA, const void* dB, int32_t* dD, uint32_t* dC) { + kernel_s8<<<1,WGSIZE>>>((const int8_t*)dA,(const int8_t*)dB,dD,dC); + }; + return run_s32(name, K, M*K, K*N, + A_host, B_host, A_i.data(), B_i.data(), false, launch); +} + +// ---- U8 ---- +static bool test_u8(const char* name, + const uint8_t* A_host, const uint8_t* B_host) { + constexpr int K = 32; + std::vector A_i(M*K), B_i(K*N); + for (int i = 0; i < M*K; ++i) A_i[i] = A_host[i]; + for (int i = 0; i < K*N; ++i) B_i[i] = B_host[i]; + auto launch = [](const void* dA, const void* dB, int32_t* dD, uint32_t* dC) { + kernel_u8<<<1,WGSIZE>>>((const uint8_t*)dA,(const uint8_t*)dB,dD,dC); + }; + return run_s32(name, K, M*K, K*N, + A_host, B_host, A_i.data(), B_i.data(), false, launch); +} + +// ---- B1 (sim-only) ---- +static bool test_b1(const char* name, + const uint8_t* A_host, const uint8_t* B_host) { + constexpr int K = B1_K; + std::vector A_i(M*K), B_i(K*N); + for (int i = 0; i < M*K; ++i) A_i[i] = A_host[i] & 1; + for (int i = 0; i < K*N; ++i) B_i[i] = B_host[i] & 1; + auto launch = [](const void* dA, const void* dB, int32_t* dD, uint32_t* dC) { + kernel_b1<<<1,WGSIZE>>>((const uint8_t*)dA,(const uint8_t*)dB,dD,dC); + }; + return run_s32(name, K, (size_t)M*K, (size_t)K*N, + A_host, B_host, A_i.data(), B_i.data(), true, launch); +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_verify v2 — %s (SM %d.%d)\n", + prop.name, prop.major, prop.minor); + } + printf("================================================================\n"); + + bool all_pass = true; + + // ========================================================================= + // F16 m64n16k16 D=f32 (swizzle modes 0–3) + // ========================================================================= + printf("\n--- F16 m64n16k16 D=f32 ---\n"); + { + static float A[M][16], B[16][N]; + // all-ones + for (int m = 0; m < M; ++m) for (int k = 0; k < 16; ++k) A[m][k] = 1.f; + for (int k = 0; k < 16; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1.f; + for (int sw = 0; sw < 4; ++sw) { + char name[64]; snprintf(name, sizeof(name), "f16 all-ones swizzle=%d", sw); + all_pass &= test_f16(name, sw, (float*)A, (float*)B); + } + // sequential + for (int m = 0; m < M; ++m) + for (int k = 0; k < 16; ++k) A[m][k] = (m*16 + k + 1) * 0.1f; + for (int k = 0; k < 16; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (k*N + n + 1) * 0.1f; + for (int sw = 0; sw < 4; ++sw) { + char name[64]; snprintf(name, sizeof(name), "f16 sequential swizzle=%d", sw); + all_pass &= test_f16(name, sw, (float*)A, (float*)B); + } + } + + // ========================================================================= + // BF16 m64n16k16 D=f32 + // ========================================================================= + printf("\n--- BF16 m64n16k16 D=f32 ---\n"); + { + static float A[M][16], B[16][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 16; ++k) A[m][k] = 1.f; + for (int k = 0; k < 16; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1.f; + all_pass &= test_bf16("bf16 all-ones", (float*)A, (float*)B); + + for (int m = 0; m < M; ++m) + for (int k = 0; k < 16; ++k) A[m][k] = (m*16 + k + 1) * 0.1f; + for (int k = 0; k < 16; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (k*N + n + 1) * 0.1f; + all_pass &= test_bf16("bf16 sequential", (float*)A, (float*)B); + } + + // ========================================================================= + // TF32 m64n16k8 D=f32 + // ========================================================================= + printf("\n--- TF32 m64n16k8 D=f32 ---\n"); + { + static float A[M][8], B[8][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 8; ++k) A[m][k] = 1.f; + for (int k = 0; k < 8; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1.f; + all_pass &= test_tf32("tf32 all-ones", (float*)A, (float*)B); + + for (int m = 0; m < M; ++m) + for (int k = 0; k < 8; ++k) A[m][k] = (m*8 + k + 1) * 0.1f; + for (int k = 0; k < 8; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (k*N + n + 1) * 0.1f; + all_pass &= test_tf32("tf32 sequential", (float*)A, (float*)B); + } + + // ========================================================================= + // FP8 E4M3 m64n16k32 D=f32 + // Values {1,2,3,4} are exactly representable in e4m3; no quantisation error. + // ========================================================================= + printf("\n--- FP8 E4M3 m64n16k32 D=f32 ---\n"); + { + static float A[M][32], B[32][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 32; ++k) A[m][k] = 1.f; + for (int k = 0; k < 32; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1.f; + all_pass &= test_e4m3("e4m3 all-ones", (float*)A, (float*)B); + + for (int m = 0; m < M; ++m) + for (int k = 0; k < 32; ++k) A[m][k] = (float)(1 + (m+k) % 4); + for (int k = 0; k < 32; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (float)(1 + (k+n) % 4); + all_pass &= test_e4m3("e4m3 sequential", (float*)A, (float*)B); + } + + // ========================================================================= + // FP8 E5M2 m64n16k32 D=f32 + // ========================================================================= + printf("\n--- FP8 E5M2 m64n16k32 D=f32 ---\n"); + { + static float A[M][32], B[32][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 32; ++k) A[m][k] = 1.f; + for (int k = 0; k < 32; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1.f; + all_pass &= test_e5m2("e5m2 all-ones", (float*)A, (float*)B); + + for (int m = 0; m < M; ++m) + for (int k = 0; k < 32; ++k) A[m][k] = (float)(1 + (m+k) % 4); + for (int k = 0; k < 32; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (float)(1 + (k+n) % 4); + all_pass &= test_e5m2("e5m2 sequential", (float*)A, (float*)B); + } + + // ========================================================================= + // S8 m64n16k32 D=s32 + // ========================================================================= + printf("\n--- S8 m64n16k32 D=s32 ---\n"); + { + static int8_t A[M][32], B[32][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 32; ++k) A[m][k] = 1; + for (int k = 0; k < 32; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1; + all_pass &= test_s8("s8 all-ones", (int8_t*)A, (int8_t*)B); + + // Small signed values to avoid overflow: max sum = 32 * 5 * 5 = 800 < INT32_MAX + for (int m = 0; m < M; ++m) + for (int k = 0; k < 32; ++k) A[m][k] = (int8_t)(1 + (m+k) % 5); + for (int k = 0; k < 32; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (int8_t)(1 + (k+n) % 5); + all_pass &= test_s8("s8 sequential", (int8_t*)A, (int8_t*)B); + } + + // ========================================================================= + // U8 m64n16k32 D=s32 + // ========================================================================= + printf("\n--- U8 m64n16k32 D=s32 ---\n"); + { + static uint8_t A[M][32], B[32][N]; + for (int m = 0; m < M; ++m) for (int k = 0; k < 32; ++k) A[m][k] = 1; + for (int k = 0; k < 32; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1; + all_pass &= test_u8("u8 all-ones", (uint8_t*)A, (uint8_t*)B); + + for (int m = 0; m < M; ++m) + for (int k = 0; k < 32; ++k) A[m][k] = (uint8_t)(1 + (m+k) % 5); + for (int k = 0; k < 32; ++k) + for (int n = 0; n < N; ++n) B[k][n] = (uint8_t)(1 + (k+n) % 5); + all_pass &= test_u8("u8 sequential", (uint8_t*)A, (uint8_t*)B); + } + + // ========================================================================= + // B1 m64n16k256 D=s32 (GPGPU-Sim functional sim only) + // D[m][n] = popcount of positions k where A[m][k]=1 AND B[k][n]=1. + // ========================================================================= + printf("\n--- B1 m64n16k256 D=s32 (sim-only layout) ---\n"); + { + static uint8_t A[M][B1_K], B[B1_K][N]; + // all-ones: every bit = 1 → D[m][n] = K = 256 + for (int m = 0; m < M; ++m) for (int k = 0; k < B1_K; ++k) A[m][k] = 1; + for (int k = 0; k < B1_K; ++k) for (int n = 0; n < N; ++n) B[k][n] = 1; + all_pass &= test_b1("b1 all-ones", (uint8_t*)A, (uint8_t*)B); + + // checkerboard: A[m][k] = k%2, B[k][n] = k%2 + // D[m][n] = #{k : k%2=1 AND k%2=1} = K/2 = 128 + for (int m = 0; m < M; ++m) for (int k = 0; k < B1_K; ++k) A[m][k] = k % 2; + for (int k = 0; k < B1_K; ++k) for (int n = 0; n < N; ++n) B[k][n] = k % 2; + all_pass &= test_b1("b1 checkerboard", (uint8_t*)A, (uint8_t*)B); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/Makefile new file mode 100644 index 000000000..244de21bb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/Makefile @@ -0,0 +1,29 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := wgmma_scale + +# WGMMA requires Hopper (SM90a). +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu new file mode 100644 index 000000000..070395bc0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu @@ -0,0 +1,300 @@ +// ============================================================================= +// wgmma_scale.cu +// +// Tests WGMMA scale / accumulator-init parameters on SM90a. +// +// Two parameter axes: +// +// scaleD (predicate) — controls whether D is used as accumulator: +// scaleD=1 (true): D_out = D_in + A*B +// scaleD=0 (false): D_out = A*B (D_in is ignored) +// Applies to all WGMMA types. Tested here with f16. +// +// scaleA / scaleB — negate A or B operand (fp8 types only): +// scaleA=-1: D_out = (-A)*B = -(A*B) +// scaleB=-1: D_out = A*(-B) = -(A*B) +// both=-1: D_out = (-A)*(-B) = +(A*B) +// Tested with e4m3 m64n16k32. +// +// Tests: +// scaleD=0 (f16) : D pre-init to 50, expect D = K = 16 +// scaleD=1 (f16) : D pre-init to K, expect D = 2K = 32 +// e4m3 scaleA=-1 : all-ones A,B, expect D = -K32 = -32 +// e4m3 scaleB=-1 : all-ones A,B, expect D = -K32 = -32 +// e4m3 scaleA=-1 scaleB=-1: all-ones, expect D = +K32 = +32 +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; // 8 +static constexpr int SBO = 128; +static constexpr int E_F16 = 2; +static constexpr int E_FP8 = 1; +static constexpr int K_F16 = 16; +static constexpr int K_FP8 = 32; + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }\n" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +// =========================================================================== +// scaleD tests (f16) +// =========================================================================== + +// D pre-initialized to init_val; scaleD selects whether D_in is used. +// Template parameter SCALE_D: 1 = accumulate, 0 = ignore D_in. +template +__global__ void kernel_f16_scaled(const half* A_g, const half* B_g, + float init_val, float* D_g) { + constexpr int LBO_A = M * E_F16 * 8; + constexpr int LBO_B = N * E_F16 * 8; + __shared__ __align__(128) char smA[M*K_F16*E_F16], smB[K_F16*N*E_F16]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K_F16; i += WGSIZE) { int m=i/K_F16,k=i%K_F16; *(half*)(smA+smem_off(k,m,E_F16,LBO_A))=A_g[i]; } + for (int i = tid; i < K_F16*N; i += WGSIZE) { int k=i/N,n=i%N; *(half*)(smB+smem_off(k,n,E_F16,LBO_B))=B_g[i]; } + __syncthreads(); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + // Pre-initialize D to init_val to test whether scaleD=0 ignores it. + float d0=init_val,d1=init_val,d2=init_val,d3=init_val; + float d4=init_val,d5=init_val,d6=init_val,d7=init_val; + WGMMA_FENCE; + if constexpr (SCALE_D == 1) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" // p = true → accumulate + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } else { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,0,0;\n" // p = false → D_in ignored + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b = tid * D_ELEMS; + D_g[b]=d0;D_g[b+1]=d1;D_g[b+2]=d2;D_g[b+3]=d3; + D_g[b+4]=d4;D_g[b+5]=d5;D_g[b+6]=d6;D_g[b+7]=d7; +} + +// =========================================================================== +// scaleA / scaleB tests (fp8 e4m3) +// SA, SB: compile-time literal, must be 1 or -1 +// =========================================================================== +template +__global__ void kernel_e4m3_scale(const uint8_t* A_g, const uint8_t* B_g, + float* D_g) { + constexpr int LBO_A = M * E_FP8 * 8; + constexpr int LBO_B = N * E_FP8 * 8; + __shared__ __align__(128) char smA[M*K_FP8*E_FP8], smB[K_FP8*N*E_FP8]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K_FP8; i += WGSIZE) { int m=i/K_FP8,k=i%K_FP8; *(uint8_t*)(smA+smem_off(k,m,E_FP8,LBO_A))=A_g[i]; } + for (int i = tid; i < K_FP8*N; i += WGSIZE) { int k=i/N,n=i%N; *(uint8_t*)(smB+smem_off(k,n,E_FP8,LBO_B))=B_g[i]; } + __syncthreads(); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + // SA and SB are template parameters, so the compiler emits a constant + // literal in the PTX instruction (required by the ISA). + if constexpr (SA == 1 && SB == 1) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e4m3.e4m3 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } else if constexpr (SA == -1 && SB == 1) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e4m3.e4m3 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,-1,1; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } else if constexpr (SA == 1 && SB == -1) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e4m3.e4m3 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,-1; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } else { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k32.f32.e4m3.e4m3 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,-1,-1; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + } + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b = tid * D_ELEMS; + D_g[b]=d0;D_g[b+1]=d1;D_g[b+2]=d2;D_g[b+3]=d3; + D_g[b+4]=d4;D_g[b+5]=d5;D_g[b+6]=d6;D_g[b+7]=d7; +} + +// =========================================================================== +// Host helpers +// =========================================================================== +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static void reassemble(const float* flat, float D[M][N]) { + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) D[m][n] = 0.f; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; d_frag_pos(T, e, &row, &col); + if (row < M && col < N) D[row][col] = flat[T*D_ELEMS + e]; + } +} + +static bool check_uniform(const char* name, float D[M][N], float expected) { + float max_err = 0.f; int max_m = 0, max_n = 0; + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) { + float e = fabsf(D[m][n] - expected); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + float tol = (fabsf(expected) < 1.f) ? 1e-3f : fabsf(expected) * 1e-2f; + bool pass = (max_err <= tol); + printf("[%-42s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + max_m, max_n, D[max_m][max_n], expected); + return pass; +} + +// Convert float to raw fp8 e4m3 byte (same as wgmma_verify) +static uint8_t float_to_e4m3(float f) { + __nv_fp8_e4m3 v(f); uint8_t b; memcpy(&b, &v, 1); return b; +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_scale — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + static float D[M][N]; + + // ------------------------------------------------------------------------- + // scaleD tests (f16) + // ------------------------------------------------------------------------- + printf("\n--- scaleD (f16 m64n16k16) ---\n"); + { + std::vector hA(M*K_F16, __float2half(1.f)); + std::vector hB(K_F16*N, __float2half(1.f)); + half *dA, *dB; float *dD; + cudaMalloc(&dA, hA.size()*sizeof(half)); + cudaMalloc(&dB, hB.size()*sizeof(half)); + cudaMalloc(&dD, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(dA, hA.data(), hA.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(dB, hB.data(), hB.size()*sizeof(half), cudaMemcpyHostToDevice); + + // scaleD=0: D_in=50 is ignored → result = A*B = K_F16 = 16 + kernel_f16_scaled<0><<<1, WGSIZE>>>(dA, dB, 50.f, dD); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE*D_ELEMS); + cudaMemcpy(flat.data(), dD, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + reassemble(flat.data(), D); + all_pass &= check_uniform("f16 scaleD=0 (D_in=50 ignored)", D, (float)K_F16); + + // scaleD=1: D_in=K_F16=16, accumulates → result = 16+16 = 32 + kernel_f16_scaled<1><<<1, WGSIZE>>>(dA, dB, (float)K_F16, dD); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), dD, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + reassemble(flat.data(), D); + all_pass &= check_uniform("f16 scaleD=1 (D_in=K accumulated)", D, (float)(2*K_F16)); + + cudaFree(dA); cudaFree(dB); cudaFree(dD); + } + + // ------------------------------------------------------------------------- + // scaleA / scaleB tests (fp8 e4m3, all-ones A and B) + // K_FP8=32, all-ones: baseline D[m][n] = 32, negated = -32, double-neg = +32 + // ------------------------------------------------------------------------- + printf("\n--- scaleA/scaleB (e4m3 m64n16k32, A=B=1.0) ---\n"); + { + uint8_t raw = float_to_e4m3(1.f); + std::vector hA(M*K_FP8, raw), hB(K_FP8*N, raw); + uint8_t *dA, *dB; float *dD; + cudaMalloc(&dA, hA.size()); + cudaMalloc(&dB, hB.size()); + cudaMalloc(&dD, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(dA, hA.data(), hA.size(), cudaMemcpyHostToDevice); + cudaMemcpy(dB, hB.data(), hB.size(), cudaMemcpyHostToDevice); + std::vector flat(WGSIZE*D_ELEMS); + + auto run_fp8 = [&](auto kern, float expected, const char* name) { + kern<<<1, WGSIZE>>>(dA, dB, dD); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), dD, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + reassemble(flat.data(), D); + return check_uniform(name, D, expected); + }; + + all_pass &= run_fp8(kernel_e4m3_scale<1,1>, (float) K_FP8, "e4m3 scaleA=+1 scaleB=+1 (baseline)"); + all_pass &= run_fp8(kernel_e4m3_scale<-1,1>, (float)-K_FP8, "e4m3 scaleA=-1 scaleB=+1"); + all_pass &= run_fp8(kernel_e4m3_scale<1,-1>, (float)-K_FP8, "e4m3 scaleA=+1 scaleB=-1"); + all_pass &= run_fp8(kernel_e4m3_scale<-1,-1>, (float) K_FP8, "e4m3 scaleA=-1 scaleB=-1 (double-neg)"); + + cudaFree(dA); cudaFree(dB); cudaFree(dD); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/Makefile new file mode 100644 index 000000000..70c78db5d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/Makefile @@ -0,0 +1,29 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := wgmma_shapes + +# WGMMA requires Hopper (SM90a). +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu new file mode 100644 index 000000000..bfbf2837f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu @@ -0,0 +1,248 @@ +// ============================================================================= +// wgmma_shapes.cu +// +// Tests f16 m64nNk16 → f32 for N = 16, 32, 64. +// +// Motivation: different N values produce different D-register counts per thread +// and different fragment layouts. wgmma_verify only covers N=16; this file +// stresses the larger shapes that exercise different simulator code paths. +// +// N=16: 8 D-regs/thread (baseline — same as wgmma_verify) +// N=32: 16 D-regs/thread +// N=64: 32 D-regs/thread +// +// N=128/256 follow the same fragment formula as N=64 (more regs, same layout +// logic); they are covered implicitly by wgmma_no_tma via CUTLASS. +// +// All tests use all-ones A and B. +// Expected D[m][n] = K = 16 for every (m, n) regardless of N. +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include + +static constexpr int M = 64; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int SBO = 128; +static constexpr int E = 2; // bytes per f16 + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }\n" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +// --------------------------------------------------------------------------- +// Kernel: f16 m64n16k16 (8 D-regs — baseline) +// --------------------------------------------------------------------------- +__global__ void kernel_n16(const half* A_g, const half* B_g, float* D_g) { + constexpr int N = 16, D_ELEMS = N/2; + constexpr int LBO_A = M*E*8, LBO_B = N*E*8; + __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } + for (int i = tid; i < K*N; i += WGSIZE) { int k=i/N,n=i%N; *(half*)(smB+smem_off(k,n,E,LBO_B))=B_g[i]; } + __syncthreads(); + uint64_t da=make_gmma_desc(smem_addr(smA),LBO_A,SBO); + uint64_t db=make_gmma_desc(smem_addr(smB),LBO_B,SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + :"l"(da),"l"(db):"memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b=tid*D_ELEMS; + D_g[b]=d0;D_g[b+1]=d1;D_g[b+2]=d2;D_g[b+3]=d3; + D_g[b+4]=d4;D_g[b+5]=d5;D_g[b+6]=d6;D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// Kernel: f16 m64n32k16 (16 D-regs) +// --------------------------------------------------------------------------- +__global__ void kernel_n32(const half* A_g, const half* B_g, float* D_g) { + constexpr int N = 32, D_ELEMS = N/2; + constexpr int LBO_A = M*E*8, LBO_B = N*E*8; + __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } + for (int i = tid; i < K*N; i += WGSIZE) { int k=i/N,n=i%N; *(half*)(smB+smem_off(k,n,E,LBO_B))=B_g[i]; } + __syncthreads(); + uint64_t da=make_gmma_desc(smem_addr(smA),LBO_A,SBO); + uint64_t db=make_gmma_desc(smem_addr(smB),LBO_B,SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + float d8=0,d9=0,d10=0,d11=0,d12=0,d13=0,d14=0,d15=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n32k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15},%16,%17,p,1,1,0,0; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7), + "+f"(d8),"+f"(d9),"+f"(d10),"+f"(d11),"+f"(d12),"+f"(d13),"+f"(d14),"+f"(d15) + :"l"(da),"l"(db):"memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b=tid*D_ELEMS; + D_g[b]=d0;D_g[b+1]=d1;D_g[b+2]=d2;D_g[b+3]=d3; + D_g[b+4]=d4;D_g[b+5]=d5;D_g[b+6]=d6;D_g[b+7]=d7; + D_g[b+8]=d8;D_g[b+9]=d9;D_g[b+10]=d10;D_g[b+11]=d11; + D_g[b+12]=d12;D_g[b+13]=d13;D_g[b+14]=d14;D_g[b+15]=d15; +} + +// --------------------------------------------------------------------------- +// Kernel: f16 m64n64k16 (32 D-regs) +// --------------------------------------------------------------------------- +__global__ void kernel_n64(const half* A_g, const half* B_g, float* D_g) { + constexpr int N = 64, D_ELEMS = N/2; + constexpr int LBO_A = M*E*8, LBO_B = N*E*8; + __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } + for (int i = tid; i < K*N; i += WGSIZE) { int k=i/N,n=i%N; *(half*)(smB+smem_off(k,n,E,LBO_B))=B_g[i]; } + __syncthreads(); + uint64_t da=make_gmma_desc(smem_addr(smA),LBO_A,SBO); + uint64_t db=make_gmma_desc(smem_addr(smB),LBO_B,SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + float d8=0,d9=0,d10=0,d11=0,d12=0,d13=0,d14=0,d15=0; + float d16=0,d17=0,d18=0,d19=0,d20=0,d21=0,d22=0,d23=0; + float d24=0,d25=0,d26=0,d27=0,d28=0,d29=0,d30=0,d31=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n64k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15," + "%16,%17,%18,%19,%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31},%32,%33,p,1,1,0,0; }\n" + :"+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3),"+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7), + "+f"(d8),"+f"(d9),"+f"(d10),"+f"(d11),"+f"(d12),"+f"(d13),"+f"(d14),"+f"(d15), + "+f"(d16),"+f"(d17),"+f"(d18),"+f"(d19),"+f"(d20),"+f"(d21),"+f"(d22),"+f"(d23), + "+f"(d24),"+f"(d25),"+f"(d26),"+f"(d27),"+f"(d28),"+f"(d29),"+f"(d30),"+f"(d31) + :"l"(da),"l"(db):"memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b=tid*D_ELEMS; + D_g[b]=d0;D_g[b+1]=d1;D_g[b+2]=d2;D_g[b+3]=d3; + D_g[b+4]=d4;D_g[b+5]=d5;D_g[b+6]=d6;D_g[b+7]=d7; + D_g[b+8]=d8;D_g[b+9]=d9;D_g[b+10]=d10;D_g[b+11]=d11; + D_g[b+12]=d12;D_g[b+13]=d13;D_g[b+14]=d14;D_g[b+15]=d15; + D_g[b+16]=d16;D_g[b+17]=d17;D_g[b+18]=d18;D_g[b+19]=d19; + D_g[b+20]=d20;D_g[b+21]=d21;D_g[b+22]=d22;D_g[b+23]=d23; + D_g[b+24]=d24;D_g[b+25]=d25;D_g[b+26]=d26;D_g[b+27]=d27; + D_g[b+28]=d28;D_g[b+29]=d29;D_g[b+30]=d30;D_g[b+31]=d31; +} + +// =========================================================================== +// Host helpers +// =========================================================================== + +// Fragment layout formula is the same for all N. +// e ranges 0..(N/2-1); s=e/4 indexes groups of 8 columns. +static void reassemble(const float* flat, float* D, int N) { + const int D_ELEMS = N / 2; + for (int i = 0; i < M*N; i++) D[i] = 0.f; + for (int T = 0; T < WGSIZE; T++) { + int warp = T/32, lane = T%32; + for (int e = 0; e < D_ELEMS; e++) { + int s = e/4, k = e%4; + int row = (lane/4)*2 + k/2 + warp*16; + int col = (lane%4)*2 + k%2 + s*8; + if (row < M && col < N) + D[row*N + col] = flat[T*D_ELEMS + e]; + } + } +} + +static bool check_uniform(const char* name, const float* D, int N, float expected) { + float max_err = 0.f; + int worst = 0; + for (int i = 0; i < M*N; i++) { + float e = fabsf(D[i] - expected); + if (e > max_err) { max_err = e; worst = i; } + } + float tol = (fabsf(expected) < 1.f) ? 1e-3f : fabsf(expected) * 1e-2f; + bool pass = (max_err <= tol); + printf("[%-38s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + worst/N, worst%N, D[worst], expected); + return pass; +} + +struct KernFn { void (*fn)(const half*, const half*, float*); int N; const char* name; }; + +static bool run(KernFn kf) { + const int D_ELEMS = kf.N / 2; + std::vector hA(M*K, __float2half(1.f)); + std::vector hB(K*kf.N, __float2half(1.f)); + half *dA, *dB; float *dD; + cudaMalloc(&dA, M*K*sizeof(half)); + cudaMalloc(&dB, K*kf.N*sizeof(half)); + cudaMalloc(&dD, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(dA, hA.data(), hA.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(dB, hB.data(), hB.size()*sizeof(half), cudaMemcpyHostToDevice); + kf.fn<<<1, WGSIZE>>>(dA, dB, dD); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE*D_ELEMS); + cudaMemcpy(flat.data(), dD, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(dA); cudaFree(dB); cudaFree(dD); + std::vector D(M*kf.N); + reassemble(flat.data(), D.data(), kf.N); + return check_uniform(kf.name, D.data(), kf.N, (float)K); +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_shapes — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + KernFn tests[] = { + { kernel_n16, 16, "f16 m64n16k16 (8 D-regs/thread)" }, + { kernel_n32, 32, "f16 m64n32k16 (16 D-regs/thread)" }, + { kernel_n64, 64, "f16 m64n64k16 (32 D-regs/thread)" }, + }; + for (auto& t : tests) all_pass &= run(t); + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/Makefile new file mode 100644 index 000000000..ec21ffdfd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/Makefile @@ -0,0 +1,30 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# WGMMA sync primitive microbenchmarks. +ALL_BENCHES := wgmma_fence wgmma_sync + +# WGMMA requires Hopper (SM90a). +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu new file mode 100644 index 000000000..6f3504b3c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu @@ -0,0 +1,331 @@ +// ============================================================================= +// wgmma_fence.cu +// +// Tests wgmma.fence.sync.aligned semantics on SM90a. +// +// Three subtests using f16 m64n16k16 → f32: +// +// fence_single – one fence→MMA→commit→wait_group_0→fence cycle. +// Baseline: verifies MMA sees smem data written before fence. +// Expected D[m][n] = 16.0 (K ones × 1×1). +// +// fence_sequence – two back-to-back cycles in one kernel with different smem +// data. The second fence gates a fresh smem write, so the +// second MMA must produce a different result from the first. +// Expected D1[m][n] = 16.0 (A=1, B=1) +// Expected D2[m][n] = 64.0 (A=2, B=2, K × 2×2) +// +// fence_accum – two cycles accumulating into the same D registers. +// Verifies that D is not accidentally reset between cycles +// and that the post-wait fence properly releases D. +// Expected D[m][n] = 32.0 (2 × 16.0) +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; // 8 D-regs per thread +static constexpr int E = 2; // bytes per f16 +static constexpr int SBO = 128; +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int SMEM_A = M * K * E; // 2048 +static constexpr int SMEM_B = K * N * E; // 512 + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }\n" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +// --------------------------------------------------------------------------- +// WGMMA protocol macros +// --------------------------------------------------------------------------- +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +__device__ __forceinline__ void mma_f16( + uint64_t da, uint64_t db, + float& d0, float& d1, float& d2, float& d3, + float& d4, float& d5, float& d6, float& d7) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3), + "+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); +} + +// --------------------------------------------------------------------------- +// Kernel helpers: load A/B globals into K-major smem (no swizzle) +// --------------------------------------------------------------------------- +__device__ void load_smem(char* smA, char* smB, + const half* A_g, const half* B_g) { + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); +} + +// =========================================================================== +// Kernel 1: single fence→MMA→commit→wait_group_0→fence +// =========================================================================== +__global__ void kernel_fence_single(const half* A_g, const half* B_g, float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + load_smem(smA, smB, A_g, B_g); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + + const int b = threadIdx.x * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Kernel 2: two back-to-back fence→MMA→commit→wait→fence sequences. +// Sequence 1 uses A_g1, B_g1 (all 1.0). Sequence 2 uses A_g2, B_g2 (all 2.0). +// Outputs D_g1 and D_g2 separately for independent checking. +// =========================================================================== +__global__ void kernel_fence_sequence(const half* A_g1, const half* B_g1, + const half* A_g2, const half* B_g2, + float* D_g1, float* D_g2) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + + // Sequence 1 + load_smem(smA, smB, A_g1, B_g1); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + const int b = tid * D_ELEMS; + D_g1[b]=d0; D_g1[b+1]=d1; D_g1[b+2]=d2; D_g1[b+3]=d3; + D_g1[b+4]=d4; D_g1[b+5]=d5; D_g1[b+6]=d6; D_g1[b+7]=d7; + + // Overwrite smem with sequence-2 data, then re-fence + __syncthreads(); + load_smem(smA, smB, A_g2, B_g2); + float e0=0,e1=0,e2=0,e3=0,e4=0,e5=0,e6=0,e7=0; + WGMMA_FENCE; + mma_f16(da, db, e0,e1,e2,e3,e4,e5,e6,e7); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + D_g2[b]=e0; D_g2[b+1]=e1; D_g2[b+2]=e2; D_g2[b+3]=e3; + D_g2[b+4]=e4; D_g2[b+5]=e5; D_g2[b+6]=e6; D_g2[b+7]=e7; +} + +// =========================================================================== +// Kernel 3: two accumulating cycles using the same smem. +// The post-wait fence releases D so the second cycle can start immediately. +// Expected: D[m][n] = 32.0 (2 × K × 1 × 1). +// =========================================================================== +__global__ void kernel_fence_accum(const half* A_g, const half* B_g, float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + load_smem(smA, smB, A_g, B_g); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + // Cycle 1 + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; WGMMA_WAIT; + // Single fence serves as end-of-cycle-1 and start-of-cycle-2 + WGMMA_FENCE; + // Cycle 2: accumulate into same D registers + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + + const int b = threadIdx.x * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Host helpers +// =========================================================================== + +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static void reassemble_D(const float* flat, float D[M][N]) { + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) D[m][n] = 0.f; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, &row, &col); + if (row < M && col < N) D[row][col] = flat[T*D_ELEMS + e]; + } +} + +static bool check_uniform(const char* name, float D[M][N], float expected) { + float max_err = 0.f; + int max_m = 0, max_n = 0; + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) { + float e = fabsf(D[m][n] - expected); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + float tol = (fabsf(expected) < 1.f) ? 1e-3f : fabsf(expected) * 1e-2f; + bool pass = (max_err <= tol); + printf("[%-40s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + max_m, max_n, D[max_m][max_n], expected); + return pass; +} + +// Allocate device buffers, copy h_A/h_B, run kernel, retrieve result. +// Returns a flat WGSIZE*D_ELEMS float vector. +static std::vector run1(const std::vector& h_A, + const std::vector& h_B, + void (*kern)(const half*, const half*, float*)) { + half *d_A, *d_B; + float *d_D; + cudaMalloc(&d_A, h_A.size() * sizeof(half)); + cudaMalloc(&d_B, h_B.size() * sizeof(half)); + cudaMalloc(&d_D, WGSIZE * D_ELEMS * sizeof(float)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + kern<<<1, WGSIZE>>>(d_A, d_B, d_D); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE * D_ELEMS); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + return flat; +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_fence — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + + // Host matrices: all-ones (A1=B1=1.0), all-twos (A2=B2=2.0) + std::vector A1(M*K), B1(K*N), A2(M*K), B2(K*N); + for (auto& v : A1) v = __float2half(1.f); + for (auto& v : B1) v = __float2half(1.f); + for (auto& v : A2) v = __float2half(2.f); + for (auto& v : B2) v = __float2half(2.f); + + static float D[M][N]; + + // ------------------------------------------------------------------------- + // Test 1: fence_single + // ------------------------------------------------------------------------- + printf("\n--- fence_single ---\n"); + { + auto flat = run1(A1, B1, kernel_fence_single); + reassemble_D(flat.data(), D); + all_pass &= check_uniform("fence_single", D, (float)K); + } + + // ------------------------------------------------------------------------- + // Test 2: fence_sequence (two separate cycles, independent D) + // ------------------------------------------------------------------------- + printf("\n--- fence_sequence ---\n"); + { + half *dA1, *dB1, *dA2, *dB2; + float *dD1, *dD2; + cudaMalloc(&dA1, M*K*sizeof(half)); cudaMalloc(&dB1, K*N*sizeof(half)); + cudaMalloc(&dA2, M*K*sizeof(half)); cudaMalloc(&dB2, K*N*sizeof(half)); + cudaMalloc(&dD1, WGSIZE*D_ELEMS*sizeof(float)); + cudaMalloc(&dD2, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(dA1, A1.data(), M*K*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(dB1, B1.data(), K*N*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(dA2, A2.data(), M*K*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(dB2, B2.data(), K*N*sizeof(half), cudaMemcpyHostToDevice); + + kernel_fence_sequence<<<1, WGSIZE>>>(dA1, dB1, dA2, dB2, dD1, dD2); + cudaDeviceSynchronize(); + + std::vector flat1(WGSIZE*D_ELEMS), flat2(WGSIZE*D_ELEMS); + cudaMemcpy(flat1.data(), dD1, flat1.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(flat2.data(), dD2, flat2.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(dA1); cudaFree(dB1); cudaFree(dA2); cudaFree(dB2); + cudaFree(dD1); cudaFree(dD2); + + static float D2[M][N]; + reassemble_D(flat1.data(), D); + reassemble_D(flat2.data(), D2); + all_pass &= check_uniform("fence_sequence seq1 (A=1,B=1)", D, (float)K); + all_pass &= check_uniform("fence_sequence seq2 (A=2,B=2)", D2, (float)(K*4)); + } + + // ------------------------------------------------------------------------- + // Test 3: fence_accum (two cycles accumulate into same D) + // ------------------------------------------------------------------------- + printf("\n--- fence_accum ---\n"); + { + auto flat = run1(A1, B1, kernel_fence_accum); + reassemble_D(flat.data(), D); + all_pass &= check_uniform("fence_accum (2 × K)", D, (float)(2*K)); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu new file mode 100644 index 000000000..6cadbd674 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu @@ -0,0 +1,290 @@ +// ============================================================================= +// wgmma_sync.cu +// +// Tests wgmma.commit_group / wgmma.wait_group semantics on SM90a. +// +// Three subtests using f16 m64n16k16 → f32, all-ones inputs: +// +// one_group_two_mmas – two MMA ops in the same committed group. +// Verifies D accumulates both before commit. +// Expected D[m][n] = 32.0 +// +// two_groups_serial – two groups committed and waited one at a time. +// The inter-group fence ends group 0 and starts group 1. +// Expected D[m][n] = 32.0 (accumulated across both groups) +// +// two_groups_pipeline – two groups committed before any wait (software pipeline). +// Tests the group queue: both groups in flight simultaneously +// before a single wait_group 0 drains them. +// Expected D[m][n] = 32.0 +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; +static constexpr int E = 2; +static constexpr int SBO = 128; +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int SMEM_A = M * K * E; // 2048 +static constexpr int SMEM_B = K * N * E; // 512 + +// --------------------------------------------------------------------------- +// Device helpers (identical to wgmma_verify / wgmma_fence) +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }\n" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +__device__ __forceinline__ void mma_f16( + uint64_t da, uint64_t db, + float& d0, float& d1, float& d2, float& d3, + float& d4, float& d5, float& d6, float& d7) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3), + "+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); +} + +__device__ void load_smem(char* smA, char* smB, + const half* A_g, const half* B_g) { + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); +} + +// =========================================================================== +// Kernel 1: two MMA ops in one committed group +// fence → MMA_a → MMA_b → commit_group → wait_group_0 → fence +// Both MMAs accumulate into the same D registers before commit. +// =========================================================================== +__global__ void kernel_one_group_two_mmas(const half* A_g, const half* B_g, + float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + load_smem(smA, smB, A_g, B_g); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // contributes K = 16 + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // contributes K = 16 more + WGMMA_COMMIT; + WGMMA_WAIT; WGMMA_FENCE; + + const int b = threadIdx.x * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Kernel 2: two groups committed and waited serially +// fence → MMA_a → commit → wait → fence → MMA_b → commit → wait → fence +// The inter-group fence acts as both the post-wait fence (releasing D) +// and the pre-MMA fence (signalling smem ready for the next MMA). +// =========================================================================== +__global__ void kernel_two_groups_serial(const half* A_g, const half* B_g, + float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + load_smem(smA, smB, A_g, B_g); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + // Group 0 + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + // Inter-group fence: ends group 0 (D readable) and starts group 1 (smem ready) + WGMMA_FENCE; + // Group 1: accumulates into same D + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; WGMMA_FENCE; + + const int b = threadIdx.x * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Kernel 3: two groups pipelined (both committed before any wait) +// fence → MMA_a → commit_group_0 → MMA_b → commit_group_1 → wait_group_0 → fence +// No fence between the two groups: smem is unchanged so no new fence is needed. +// Tests the group queue: two in-flight groups drained by a single wait. +// =========================================================================== +__global__ void kernel_two_groups_pipeline(const half* A_g, const half* B_g, + float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + load_smem(smA, smB, A_g, B_g); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // into group 0 + WGMMA_COMMIT; // commit group 0 + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // into group 1 (no fence needed) + WGMMA_COMMIT; // commit group 1 + WGMMA_WAIT; WGMMA_FENCE; // drain both groups + + const int b = threadIdx.x * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Host helpers +// =========================================================================== + +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static void reassemble_D(const float* flat, float D[M][N]) { + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) D[m][n] = 0.f; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, &row, &col); + if (row < M && col < N) D[row][col] = flat[T*D_ELEMS + e]; + } +} + +static bool check_uniform(const char* name, float D[M][N], float expected) { + float max_err = 0.f; + int max_m = 0, max_n = 0; + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) { + float e = fabsf(D[m][n] - expected); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + float tol = (fabsf(expected) < 1.f) ? 1e-3f : fabsf(expected) * 1e-2f; + bool pass = (max_err <= tol); + printf("[%-40s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + max_m, max_n, D[max_m][max_n], expected); + return pass; +} + +static std::vector run1(const std::vector& h_A, + const std::vector& h_B, + void (*kern)(const half*, const half*, float*)) { + half *d_A, *d_B; + float *d_D; + cudaMalloc(&d_A, h_A.size()*sizeof(half)); + cudaMalloc(&d_B, h_B.size()*sizeof(half)); + cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + kern<<<1, WGSIZE>>>(d_A, d_B, d_D); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE*D_ELEMS); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + return flat; +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_sync — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + + std::vector A(M*K), B(K*N); + for (auto& v : A) v = __float2half(1.f); + for (auto& v : B) v = __float2half(1.f); + + static float D[M][N]; + const float expected = (float)(2 * K); // two accumulated MMAs × K ones + + printf("\n--- one_group_two_mmas ---\n"); + { + auto flat = run1(A, B, kernel_one_group_two_mmas); + reassemble_D(flat.data(), D); + all_pass &= check_uniform("one_group_two_mmas", D, expected); + } + + printf("\n--- two_groups_serial ---\n"); + { + auto flat = run1(A, B, kernel_two_groups_serial); + reassemble_D(flat.data(), D); + all_pass &= check_uniform("two_groups_serial", D, expected); + } + + printf("\n--- two_groups_pipeline ---\n"); + { + auto flat = run1(A, B, kernel_two_groups_pipeline); + reassemble_D(flat.data(), D); + all_pass &= check_uniform("two_groups_pipeline", D, expected); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} From 0643c0f058f5abc96380878ac2f5ee2640dbb867 Mon Sep 17 00:00:00 2001 From: Ivorchu <41135347+Ivorchu@users.noreply.github.com> Date: Sat, 2 May 2026 02:44:38 -0400 Subject: [PATCH 08/35] replace cp.async TiledCopy with single threaded smem fill --- .../ubench/wgmma/wgmma/wgmma_no_tma.cu | 55 +++++++++---------- .../ubench/wgmma/wgmma/wgmma_verify.cu | 2 +- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu index 1a10e92b7..3ea0303d4 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu @@ -2,8 +2,9 @@ // wgmma_no_tma.cu // // Simplified Hopper WGMMA GEMM copied from CuTe tutorial wgmma_sm90.cu with: -// - TMA removed (smem loads use cp.async, same as SM80 kernels) -// - mbarrier removed (sync uses __syncthreads + cp_async_wait) +// - TMA removed (smem loads use plain ld.global + st.shared via DefaultCopy) +// - mbarrier removed (sync uses __syncthreads only) +// - cp.async removed (not implemented in GPGPU-Sim; DefaultCopy is synchronous) // - warp specialization removed (single role, all 128 threads do load+mma) // - F32 accumulator (like CUTLASS example 48) // - N=16 tile (smallest WGMMA shape; fast to simulate in GPGPU-Sim) @@ -24,7 +25,7 @@ // Run on real H100: // ./wgmma_no_tma // Run under GPGPU-Sim (functional): -// PTX_SIM_MODE_FUNC=1 ./wgmma_no_tma +// PTX_SIM_USE_PTX_FILE=1 ./wgmma_no_tma // ============================================================================= #include @@ -60,8 +61,8 @@ struct SharedStorage { // // Changes vs. the original: // 1. No pipeline prefetch — single stage (bP=1). -// 2. Inner loop: copy → cp_async_fence → cp_async_wait<0> → __syncthreads -// → wgmma → __syncthreads. +// 2. Inner loop: copy → __syncthreads → wgmma → __syncthreads. +// DefaultCopy is synchronous so no cp_async_fence/wait needed. // 3. warpgroup_fence_operand / warpgroup_arrive / warpgroup_commit_batch / // warpgroup_wait stay exactly as in wgmma_sm90.cu. // ============================================================================ @@ -101,14 +102,8 @@ void gemm_device(ProblemShape shape_MNK, CtaTiler cta_tiler, Tensor sA = make_tensor(make_smem_ptr(smem.A.begin()), ASmemLayout{}); // (BLK_M,BLK_K,1) Tensor sB = make_tensor(make_smem_ptr(smem.B.begin()), BSmemLayout{}); // (BLK_N,BLK_K,1) - // ---- Copy partitioning (gmem → smem) ------------------------------------- - ThrCopy thr_copy_a = copy_a.get_slice(threadIdx.x); - Tensor tAgA = thr_copy_a.partition_S(gA); // (CPY,CPY_M,CPY_K,k) - Tensor tAsA = thr_copy_a.partition_D(as_position_independent_swizzle_tensor(sA)); - - ThrCopy thr_copy_b = copy_b.get_slice(threadIdx.x); - Tensor tBgB = thr_copy_b.partition_S(gB); // (CPY,CPY_N,CPY_K,k) - Tensor tBsB = thr_copy_b.partition_D(as_position_independent_swizzle_tensor(sB)); + // copy_a / copy_b not used — replaced by single-threaded fill below. + (void)copy_a; (void)copy_b; // ---- MMA partitioning (smem → regs → accum) ------------------------------ ThrMMA thr_mma = mma.get_slice(threadIdx.x); @@ -122,15 +117,20 @@ void gemm_device(ProblemShape shape_MNK, CtaTiler cta_tiler, clear(tCrC); // ---- K-tile loop (no pipelining — single smem stage) --------------------- - auto K_TILE_MAX = size<3>(tAgA); + auto K_TILE_MAX = size<2>(gA); // gA shape = (BLK_M, BLK_K, num_k_tiles) CUTE_NO_UNROLL for (int k_tile = 0; k_tile < K_TILE_MAX; ++k_tile) { - // Load this K-tile into smem stage 0 (cp.async, no TMA) - copy(copy_a, tAgA(_,_,_,k_tile), tAsA(_,_,_,0)); - copy(copy_b, tBgB(_,_,_,k_tile), tBsB(_,_,_,0)); - cp_async_fence(); - cp_async_wait<0>(); + // Single-threaded smem fill: thread 0 writes each element directly through + // CuTe's swizzled layout indexing — no TiledCopy, no intra-warp conflicts. + if (threadIdx.x == 0) { + for (int m = 0; m < size<0>(sA); ++m) + for (int k = 0; k < size<1>(sA); ++k) + sA(m, k, 0) = gA(m, k, k_tile); + for (int n = 0; n < size<0>(sB); ++n) + for (int k = 0; k < size<1>(sB); ++k) + sB(n, k, 0) = gB(n, k, k_tile); + } __syncthreads(); // WGMMA: accumulate D += A * B (generated as wgmma.mma_async PTX by compiler) @@ -187,13 +187,15 @@ void gemm_nt(int m, int n, int k, auto sA = tile_to_shape(GMMA::Layout_MN_SW128_Atom{}, make_shape(bM, bK, bP)); auto sB = tile_to_shape(GMMA::Layout_MN_SW32_Atom{}, make_shape(bN, bK, bP)); - // Copy atoms: SM80 cp.async (NOT TMA) — col-major thr layout + // Copy atoms: plain ld.global + st.shared via DefaultCopy (NOT cp.async, NOT TMA) + // DefaultCopy is synchronous — no cp_async_fence/wait needed. + // Same thread and value layout as before; CuTe applies the smem swizzle correctly. TiledCopy copyA = make_tiled_copy( - Copy_Atom, TA>{}, + Copy_Atom{}, Layout>{}, // 16×8 threads, m-major - Layout>{}); // 8×1 values = 16 bytes per thread + Layout>{}); // 8×1 values per thread TiledCopy copyB = make_tiled_copy( - Copy_Atom, TB>{}, + Copy_Atom{}, Layout>{}, Layout>{}); @@ -241,11 +243,6 @@ int main() { CUTE_CHECK_ERROR(cudaGetDeviceProperties(&props, dev)); printf("wgmma_no_tma — %s (SM %d.%d)\n", props.name, props.major, props.minor); - if (props.major != 9) { - printf("This test requires SM90a (Hopper). Skipping.\n"); - return 0; - } - // Problem size: keep small for GPGPU-Sim constexpr int M = 64, N = 16, K = 64; printf("GEMM: C[%dx%d] = A[%dx%d] * B^T[%dx%d] (NT col-major, FP16→FP32)\n", @@ -258,7 +255,7 @@ int main() { thrust::host_vector h_C(M * N, 0.f); // Fill with small integer values (avoids FP16 rounding in reference) - for (int j = 0; j < M*K; ++j) h_A[j] = cute::half_t(float((j % 5) - 2)); // {-2,-1,0,1,2} + for (int j = 0; j < M*K; ++j) h_A[j] = cute::half_t(float((j % 5) - 2)); for (int j = 0; j < N*K; ++j) h_B[j] = cute::half_t(float((j % 3) - 1)); // {-1,0,1} // Device diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu index 842349c85..ec86afe1e 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu @@ -446,7 +446,7 @@ __global__ void kernel_b1(const uint8_t* A_g, const uint8_t* B_g, static void d_frag_pos(int T, int e, int* row, int* col) { int warp = T/32, lane = T%32, s = e/4, k = e%4; - *row = (lane/4)*2 + k/2 + warp*16; + *row = lane/4 + (k/2)*8 + warp*16; *col = (lane%4)*2 + k%2 + s*8; } From 529b0d7b2fc9d37821e64a65483fd1579ed43b05 Mon Sep 17 00:00:00 2001 From: Ivorchu <41135347+Ivorchu@users.noreply.github.com> Date: Sun, 3 May 2026 18:54:42 -0400 Subject: [PATCH 09/35] stmatrix tests --- .../ubench/stmatrix/stmatrix_verify/Makefile | 26 ++ .../stmatrix_verify/stmatrix_verify.cu | 278 ++++++++++++++++ .../ubench/wgmma/wgmma/wgmma_no_tma.cu | 17 +- .../ubench/wgmma/wgmma_dtypes/Makefile | 28 ++ .../ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu | 190 +++++++++++ .../ubench/wgmma/wgmma_multik/Makefile | 28 ++ .../ubench/wgmma/wgmma_multik/wgmma_multik.cu | 296 ++++++++++++++++++ 7 files changed, 854 insertions(+), 9 deletions(-) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/stmatrix_verify.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/Makefile b/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/Makefile new file mode 100644 index 000000000..07368a3f1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/Makefile @@ -0,0 +1,26 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := stmatrix_verify + +ARCH ?= sm_90a +CC := nvcc + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/stmatrix_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/stmatrix_verify.cu new file mode 100644 index 000000000..f1cf48174 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/stmatrix/stmatrix_verify/stmatrix_verify.cu @@ -0,0 +1,278 @@ +// ============================================================================= +// stmatrix_verify.cu +// +// Verifies stmatrix.sync.aligned.m8n8.shared.b16 for all Hopper sm_90a variants: +// x1, x2, x4 × non-transposed and .trans +// +// Matrix model: M_m[r][c] = (uint16_t)(m*64 + r*8 + c) (values 0..255) +// +// Non-transposed layout: smem[m*64 + r*8 + c] = M_m[r][c] (row-major) +// Transposed layout: smem[m*64 + c*8 + r] = M_m[r][c] (column-major) +// +// For each test, one warp sets up registers, calls stmatrix, copies smem to +// global memory, then the host verifies every element. +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Device helper: generic-pointer → 32-bit shared-memory address +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_u32(const void *ptr) { + uint32_t a; + asm volatile("{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }" + : "=r"(a) : "l"(ptr)); + return a; +} + +// --------------------------------------------------------------------------- +// stmatrix wrappers (inline asm, one per register count × .trans) +// --------------------------------------------------------------------------- +__device__ __forceinline__ +void stm_x1(uint32_t addr, uint32_t r0) { + asm volatile("stmatrix.sync.aligned.x1.m8n8.shared.b16 [%0], {%1};" + :: "r"(addr), "r"(r0) : "memory"); +} +__device__ __forceinline__ +void stm_x1t(uint32_t addr, uint32_t r0) { + asm volatile("stmatrix.sync.aligned.x1.trans.m8n8.shared.b16 [%0], {%1};" + :: "r"(addr), "r"(r0) : "memory"); +} +__device__ __forceinline__ +void stm_x2(uint32_t addr, uint32_t r0, uint32_t r1) { + asm volatile("stmatrix.sync.aligned.x2.m8n8.shared.b16 [%0], {%1,%2};" + :: "r"(addr), "r"(r0), "r"(r1) : "memory"); +} +__device__ __forceinline__ +void stm_x2t(uint32_t addr, uint32_t r0, uint32_t r1) { + asm volatile("stmatrix.sync.aligned.x2.trans.m8n8.shared.b16 [%0], {%1,%2};" + :: "r"(addr), "r"(r0), "r"(r1) : "memory"); +} +__device__ __forceinline__ +void stm_x4(uint32_t addr, uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3) { + asm volatile("stmatrix.sync.aligned.x4.m8n8.shared.b16 [%0], {%1,%2,%3,%4};" + :: "r"(addr), "r"(r0), "r"(r1), "r"(r2), "r"(r3) : "memory"); +} +__device__ __forceinline__ +void stm_x4t(uint32_t addr, uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3) { + asm volatile("stmatrix.sync.aligned.x4.trans.m8n8.shared.b16 [%0], {%1,%2,%3,%4};" + :: "r"(addr), "r"(r0), "r"(r1), "r"(r2), "r"(r3) : "memory"); +} + +// --------------------------------------------------------------------------- +// Data helpers +// --------------------------------------------------------------------------- + +// Value model: M_m[r][c] = m*64 + r*8 + c +__host__ __device__ __forceinline__ +uint16_t mat_val(int m, int r, int c) { return (uint16_t)(m*64 + r*8 + c); } + +// Non-transposed register for matrix m, thread T: +// lo = M_m[T/4][(T%4)*2], hi = M_m[T/4][(T%4)*2+1] +__device__ __forceinline__ +uint32_t reg_nontrans(int m, int T) { + uint16_t lo = mat_val(m, T/4, (T%4)*2); + uint16_t hi = mat_val(m, T/4, (T%4)*2 + 1); + return (uint32_t)lo | ((uint32_t)hi << 16); +} + +// Transposed register for matrix m, thread T: +// lo = M_m[(T%4)*2][T/4], hi = M_m[(T%4)*2+1][T/4] +__device__ __forceinline__ +uint32_t reg_trans(int m, int T) { + uint16_t lo = mat_val(m, (T%4)*2, T/4); + uint16_t hi = mat_val(m, (T%4)*2 + 1, T/4); + return (uint32_t)lo | ((uint32_t)hi << 16); +} + +// Address register for thread T (for non-transposed and transposed alike): +// Threads 0-7: address provider for matrix 0, row/col (T%8). +// Threads 8-15: address provider for matrix 1. +// Threads 16-23, 24-31: matrices 2, 3. +// Formula: smem_base + (T/8)*128 + (T%8)*16 +__device__ __forceinline__ +uint32_t addr_reg(uint32_t smem_base, int T) { + return smem_base + (T/8)*128 + (T%8)*16; +} + +// --------------------------------------------------------------------------- +// Kernels +// (each launched with 1 block × 32 threads, dynamic shared memory) +// --------------------------------------------------------------------------- + +__global__ void kernel_x1(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[128]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_nontrans(0, T); + stm_x1(addr, r0); + __syncthreads(); + for (int i = T; i < 64; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +__global__ void kernel_x1t(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[128]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_trans(0, T); + stm_x1t(addr, r0); + __syncthreads(); + for (int i = T; i < 64; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +__global__ void kernel_x2(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[256]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_nontrans(0, T); + uint32_t r1 = reg_nontrans(1, T); + stm_x2(addr, r0, r1); + __syncthreads(); + for (int i = T; i < 128; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +__global__ void kernel_x2t(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[256]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_trans(0, T); + uint32_t r1 = reg_trans(1, T); + stm_x2t(addr, r0, r1); + __syncthreads(); + for (int i = T; i < 128; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +__global__ void kernel_x4(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[512]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_nontrans(0, T); + uint32_t r1 = reg_nontrans(1, T); + uint32_t r2 = reg_nontrans(2, T); + uint32_t r3 = reg_nontrans(3, T); + stm_x4(addr, r0, r1, r2, r3); + __syncthreads(); + for (int i = T; i < 256; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +__global__ void kernel_x4t(uint16_t *out) { + __shared__ __align__(128) uint8_t smem_bytes[512]; + int T = threadIdx.x; + uint32_t base = smem_u32(smem_bytes); + uint32_t addr = addr_reg(base, T); + uint32_t r0 = reg_trans(0, T); + uint32_t r1 = reg_trans(1, T); + uint32_t r2 = reg_trans(2, T); + uint32_t r3 = reg_trans(3, T); + stm_x4t(addr, r0, r1, r2, r3); + __syncthreads(); + for (int i = T; i < 256; i += 32) out[i] = ((uint16_t *)smem_bytes)[i]; +} + +// --------------------------------------------------------------------------- +// Host: expected-value helpers +// --------------------------------------------------------------------------- + +// Non-transposed: smem[m*64 + r*8 + c] = M_m[r][c] +static uint16_t expected_nontrans(int num_matrices, int idx) { + int m = idx / 64; + int local = idx % 64; + int r = local / 8; + int c = local % 8; + (void)num_matrices; + return mat_val(m, r, c); +} + +// Transposed: smem[m*64 + c*8 + r] = M_m[r][c] +// → smem[m*64 + i] = M_m[i%8][i/8] +static uint16_t expected_trans(int num_matrices, int idx) { + int m = idx / 64; + int local = idx % 64; + int r = local % 8; // row index + int c = local / 8; // column index + (void)num_matrices; + return mat_val(m, r, c); +} + +// --------------------------------------------------------------------------- +// Host: run one test variant +// --------------------------------------------------------------------------- +typedef void (*kernel_fn)(uint16_t *); + +static bool run_test(const char *name, + kernel_fn kernel, + int num_matrices, + bool is_trans) { + const int nelems = num_matrices * 64; + uint16_t *d_out; + cudaMalloc(&d_out, nelems * sizeof(uint16_t)); + cudaMemset(d_out, 0, nelems * sizeof(uint16_t)); + + kernel<<<1, 32>>>(d_out); + cudaDeviceSynchronize(); + + std::vector h_out(nelems); + cudaMemcpy(h_out.data(), d_out, nelems * sizeof(uint16_t), + cudaMemcpyDeviceToHost); + cudaFree(d_out); + + int errors = 0; + int first_bad = -1; + uint16_t first_got = 0, first_exp = 0; + for (int i = 0; i < nelems; i++) { + uint16_t exp = is_trans ? expected_trans(num_matrices, i) + : expected_nontrans(num_matrices, i); + if (h_out[i] != exp) { + if (errors == 0) { + first_bad = i; + first_got = h_out[i]; + first_exp = exp; + } + errors++; + } + } + + bool pass = (errors == 0); + printf("[%-44s] %s", name, pass ? "PASS" : "FAIL"); + if (!pass) + printf(" first mismatch: smem[%d] = %u, expected %u (%d/%d wrong)", + first_bad, first_got, first_exp, errors, nelems); + printf("\n"); + return pass; +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("stmatrix_verify — %s (SM %d.%d)\n", + prop.name, prop.major, prop.minor); + printf("============================================================\n"); + + bool all_pass = true; + + all_pass &= run_test("x1 non-transposed (1 matrix)", kernel_x1, 1, false); + all_pass &= run_test("x1 transposed (1 matrix)", kernel_x1t, 1, true); + all_pass &= run_test("x2 non-transposed (2 matrices)", kernel_x2, 2, false); + all_pass &= run_test("x2 transposed (2 matrices)", kernel_x2t, 2, true); + all_pass &= run_test("x4 non-transposed (4 matrices)", kernel_x4, 4, false); + all_pass &= run_test("x4 transposed (4 matrices)", kernel_x4t, 4, true); + + printf("============================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu index 3ea0303d4..1a49b5039 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_no_tma.cu @@ -2,9 +2,9 @@ // wgmma_no_tma.cu // // Simplified Hopper WGMMA GEMM copied from CuTe tutorial wgmma_sm90.cu with: -// - TMA removed (smem loads use plain ld.global + st.shared via DefaultCopy) +// - TMA removed (smem filled by single-threaded loop from thread 0) // - mbarrier removed (sync uses __syncthreads only) -// - cp.async removed (not implemented in GPGPU-Sim; DefaultCopy is synchronous) +// - cp_async_fence/wait removed (unimplemented in GPGPU-Sim; not needed with thread-0 fill) // - warp specialization removed (single role, all 128 threads do load+mma) // - F32 accumulator (like CUTLASS example 48) // - N=16 tile (smallest WGMMA shape; fast to simulate in GPGPU-Sim) @@ -61,8 +61,8 @@ struct SharedStorage { // // Changes vs. the original: // 1. No pipeline prefetch — single stage (bP=1). -// 2. Inner loop: copy → __syncthreads → wgmma → __syncthreads. -// DefaultCopy is synchronous so no cp_async_fence/wait needed. +// 2. Inner loop: thread-0 fill → __syncthreads → wgmma → __syncthreads. +// cp_async_fence/wait removed (unimplemented in GPGPU-Sim). // 3. warpgroup_fence_operand / warpgroup_arrive / warpgroup_commit_batch / // warpgroup_wait stay exactly as in wgmma_sm90.cu. // ============================================================================ @@ -187,15 +187,14 @@ void gemm_nt(int m, int n, int k, auto sA = tile_to_shape(GMMA::Layout_MN_SW128_Atom{}, make_shape(bM, bK, bP)); auto sB = tile_to_shape(GMMA::Layout_MN_SW32_Atom{}, make_shape(bN, bK, bP)); - // Copy atoms: plain ld.global + st.shared via DefaultCopy (NOT cp.async, NOT TMA) - // DefaultCopy is synchronous — no cp_async_fence/wait needed. - // Same thread and value layout as before; CuTe applies the smem swizzle correctly. + // Copy atoms: original SM80_CP_ASYNC (not called — smem fill is done by thread 0 loop). + // cp_async_fence/wait removed (unimplemented in GPGPU-Sim; not needed with thread-0 fill). TiledCopy copyA = make_tiled_copy( - Copy_Atom{}, + Copy_Atom, TA>{}, Layout>{}, // 16×8 threads, m-major Layout>{}); // 8×1 values per thread TiledCopy copyB = make_tiled_copy( - Copy_Atom{}, + Copy_Atom, TB>{}, Layout>{}, Layout>{}); diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/Makefile new file mode 100644 index 000000000..01f6b40ea --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/Makefile @@ -0,0 +1,28 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := wgmma_dtypes + +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu new file mode 100644 index 000000000..c76e51d3e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu @@ -0,0 +1,190 @@ +// ============================================================================= +// wgmma_dtypes.cu +// +// Tests WGMMA with f16 accumulator: m64n16k16.f16.f16.f16. +// +// wgmma_verify already covers f16/bf16/tf32/fp8/int8 with f32/s32 accumulators. +// This file adds the f16←f16×f16 variant where D registers are f16x2 packed +// into .b32 (N/4 = 4 registers per thread for N=16, each holding 2 f16 values). +// +// Fragment layout for f16 accumulator is identical to f32 — same d_frag_pos +// formula — but each register slot holds 2 f16 values (low and high halfword). +// +// Two tests (all-ones style, exactly representable in f16): +// A=1, B=1 → D[m][n] = K = 16 +// A=2, B=3 → D[m][n] = 6*K = 96 +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int E = 2; // bytes per f16 +static constexpr int SBO = 128; +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int SMEM_A = M * K * E; // 2048 +static constexpr int SMEM_B = K * N * E; // 512 +static constexpr int D_REGS = N / 4; // 4 f16x2 registers per thread (N=16) + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +// --------------------------------------------------------------------------- +// Kernel: m64n16k16.f16.f16.f16 +// D is 4 f16x2 registers (uint32_t) per thread for N=16. +// --------------------------------------------------------------------------- +__global__ void kernel_f16f16(const half* A_g, const half* B_g, uint32_t* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + uint32_t d0=0, d1=0, d2=0, d3=0; + WGMMA_FENCE; + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f16.f16.f16 " + "{%0,%1,%2,%3},%4,%5,p,1,1,0,0; }\n" + : "+r"(d0),"+r"(d1),"+r"(d2),"+r"(d3) + : "l"(da),"l"(db) : "memory"); + WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + int b = tid * D_REGS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; +} + +// --------------------------------------------------------------------------- +// Host helpers +// --------------------------------------------------------------------------- +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +// Unpack WGSIZE*D_REGS f16x2 registers into M×N float matrix. +// Thread T, register r → scalar elements e=2r (low half) and e=2r+1 (high half). +static void reassemble(const uint32_t* flat, float* D) { + for (int i = 0; i < M*N; i++) D[i] = 0.f; + for (int T = 0; T < WGSIZE; T++) { + for (int r = 0; r < D_REGS; r++) { + uint32_t packed = flat[T * D_REGS + r]; + for (int h = 0; h < 2; h++) { + int e = 2*r + h; + uint16_t bits = (h == 0) ? (uint16_t)(packed & 0xFFFFu) + : (uint16_t)(packed >> 16); + half val; + memcpy(&val, &bits, sizeof(val)); + int row, col; + d_frag_pos(T, e, &row, &col); + if (row < M && col < N) + D[row * N + col] = __half2float(val); + } + } + } +} + +static bool check_uniform(const char* name, const float* D, float expected) { + float max_err = 0.f; + int worst = 0; + for (int i = 0; i < M*N; i++) { + float e = fabsf(D[i] - expected); + if (e > max_err) { max_err = e; worst = i; } + } + float tol = fabsf(expected) * 5e-3f + 0.5f; // generous for f16 precision + bool pass = (max_err <= tol); + printf("[%-40s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + worst/N, worst%N, D[worst], expected); + return pass; +} + +static bool run(const char* name, float a_val, float b_val, float expected) { + std::vector h_A(M*K, __float2half(a_val)); + std::vector h_B(K*N, __float2half(b_val)); + half* d_A; + half* d_B; + uint32_t* d_D; + cudaMalloc(&d_A, h_A.size() * sizeof(half)); + cudaMalloc(&d_B, h_B.size() * sizeof(half)); + cudaMalloc(&d_D, WGSIZE * D_REGS * sizeof(uint32_t)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + kernel_f16f16<<<1, WGSIZE>>>(d_A, d_B, d_D); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE * D_REGS); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + std::vector D(M * N); + reassemble(flat.data(), D.data()); + return check_uniform(name, D.data(), expected); +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_dtypes — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + all_pass &= run("f16←f16×f16 m64n16k16 A=1 B=1", 1.f, 1.f, (float)K); + all_pass &= run("f16←f16×f16 m64n16k16 A=2 B=3", 2.f, 3.f, (float)(2*3*K)); + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/Makefile new file mode 100644 index 000000000..2f918a69f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/Makefile @@ -0,0 +1,28 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := wgmma_multik + +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu new file mode 100644 index 000000000..f6fb953f4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu @@ -0,0 +1,296 @@ +// ============================================================================= +// wgmma_multik.cu +// +// Tests WGMMA D-register accumulation across multiple K-tiles. +// +// The critical property being tested: after each wgmma.mma_async with scaleD=1, +// the D registers hold the running sum. The simulator must preserve the D +// register file across wgmma commits and not reset it between iterations. +// +// Two tests using f16→f32 m64n16k16: +// +// static_4tiles – same smem (A=B=1) used for 4 back-to-back MMAs in one group. +// All 4 contribute K=16 each → D[m][n] = 4*K = 64. +// No smem reload; one commit/wait covers all 4 ops. +// +// varying_4tiles – smem reloaded each tile with A[tile] = (tile+1), B = 1. +// tile 0: A=1 → K*1 = 16 +// tile 1: A=2 → K*2 = 32 +// tile 2: A=3 → K*3 = 48 +// tile 3: A=4 → K*4 = 64 +// Total: K*(1+2+3+4) = 160 +// Each tile is a separate commit/wait group with smem reload. +// +// Exit status: 0 = all PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include + +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; // 8 f32 registers per thread +static constexpr int E = 2; // bytes per f16 +static constexpr int SBO = 128; +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int SMEM_A = M * K * E; // 2048 +static constexpr int SMEM_B = K * N * E; // 512 + +// --------------------------------------------------------------------------- +// Device helpers +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") + +__device__ __forceinline__ void mma_f16( + uint64_t da, uint64_t db, + float& d0, float& d1, float& d2, float& d3, + float& d4, float& d5, float& d6, float& d7) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3), + "+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); +} + +// --------------------------------------------------------------------------- +// Kernel 1: 4 MMAs in one group, constant smem (A=B=1). +// D accumulates across all 4 ops → expected = 4*K = 64. +// --------------------------------------------------------------------------- +__global__ void kernel_static4(const half* A_g, const half* B_g, float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // D += K (D was 0) + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // D += K + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // D += K + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // D += K + WGMMA_COMMIT; + WGMMA_WAIT; + WGMMA_FENCE; + const int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// Kernel 2: 4 tiles with different A values, smem reloaded each tile. +// A_g layout: [4 * M * K], where A_g[tile*M*K + m*K + k] = (half)(tile+1). +// B_g: [K * N] all ones (shared across tiles). +// Expected D[m][n] = K*(1+2+3+4) = 160. +// --------------------------------------------------------------------------- +__global__ void kernel_varying4(const half* A_g, const half* B_g, float* D_g) { + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + + // Load B once — same for all tiles. + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + + // Tile 0: A = 1.0 + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[0*M*K + m*K + k]; + } + __syncthreads(); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + + // Tile 1: A = 2.0 + WGMMA_FENCE; // post-wait: releases D, signals smem can be updated + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[1*M*K + m*K + k]; + } + __syncthreads(); + WGMMA_FENCE; // pre-MMA: signals new smem content visible to wgmma + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + + // Tile 2: A = 3.0 + WGMMA_FENCE; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[2*M*K + m*K + k]; + } + __syncthreads(); + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + + // Tile 3: A = 4.0 + WGMMA_FENCE; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[3*M*K + m*K + k]; + } + __syncthreads(); + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + WGMMA_FENCE; // final fence + + const int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// =========================================================================== +// Host helpers +// =========================================================================== +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static void reassemble(const float* flat, float D[M][N]) { + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) D[m][n] = 0.f; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, &row, &col); + if (row < M && col < N) + D[row][col] = flat[T * D_ELEMS + e]; + } +} + +static bool check_uniform(const char* name, float D[M][N], float expected) { + float max_err = 0.f; + int max_m = 0, max_n = 0; + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) { + float e = fabsf(D[m][n] - expected); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + float tol = fabsf(expected) * 1e-2f; + bool pass = (max_err <= tol); + printf("[%-36s] %s expected=%.1f max_err=%.4e\n", + name, pass ? "PASS" : "FAIL", expected, max_err); + if (!pass) + printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", + max_m, max_n, D[max_m][max_n], expected); + return pass; +} + +// =========================================================================== +// main +// =========================================================================== +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_multik — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + bool all_pass = true; + static float D[M][N]; + + // --- Test 1: static_4tiles --- + printf("\n--- static_4tiles (4 MMAs, same smem A=B=1, expected D=64) ---\n"); + { + std::vector h_A(M*K, __float2half(1.f)); + std::vector h_B(K*N, __float2half(1.f)); + half *d_A, *d_B; float *d_D; + cudaMalloc(&d_A, h_A.size()*sizeof(half)); + cudaMalloc(&d_B, h_B.size()*sizeof(half)); + cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + kernel_static4<<<1, WGSIZE>>>(d_A, d_B, d_D); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE*D_ELEMS); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + reassemble(flat.data(), D); + all_pass &= check_uniform("static_4tiles", D, (float)(4 * K)); + } + + // --- Test 2: varying_4tiles --- + printf("\n--- varying_4tiles (A[tile]=tile+1, B=1, expected D=160) ---\n"); + { + // A_g: [4*M*K] where tile t has all elements = float(t+1) + std::vector h_A(4 * M * K); + for (int t = 0; t < 4; t++) + for (int i = 0; i < M*K; i++) + h_A[t*M*K + i] = __float2half((float)(t + 1)); + std::vector h_B(K*N, __float2half(1.f)); + half *d_A, *d_B; float *d_D; + cudaMalloc(&d_A, h_A.size()*sizeof(half)); + cudaMalloc(&d_B, h_B.size()*sizeof(half)); + cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + kernel_varying4<<<1, WGSIZE>>>(d_A, d_B, d_D); + cudaDeviceSynchronize(); + std::vector flat(WGSIZE*D_ELEMS); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + reassemble(flat.data(), D); + // Expected: K*(1+2+3+4) = 16*10 = 160 + all_pass &= check_uniform("varying_4tiles", D, (float)(K * 10)); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} From 6dc2940844d818237a54910e2925997c4295ee9b Mon Sep 17 00:00:00 2001 From: Ivorchu <41135347+Ivorchu@users.noreply.github.com> Date: Thu, 7 May 2026 15:29:47 -0400 Subject: [PATCH 10/35] wgmma silicon verification benchmark --- .../ubench/wgmma/lat_gmma/.gitignore | 1 + .../ubench/wgmma/lat_gmma/Makefile | 31 ++ .../kernels/lat_gmma_f16e4m3e4m3_64x128x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e4m3_64x16x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e4m3_64x192x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e4m3_64x256x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e4m3_64x32x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e4m3_64x64x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e4m3_64x8x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e4m3_64x96x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e5m2_64x128x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e5m2_64x16x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e5m2_64x192x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e5m2_64x256x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e5m2_64x32x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e5m2_64x64x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e5m2_64x8x32.cu | 5 + .../kernels/lat_gmma_f16e4m3e5m2_64x96x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e4m3_64x128x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e4m3_64x16x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e4m3_64x192x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e4m3_64x256x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e4m3_64x32x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e4m3_64x64x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e4m3_64x8x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e4m3_64x96x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e5m2_64x128x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e5m2_64x16x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e5m2_64x192x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e5m2_64x256x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e5m2_64x32x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e5m2_64x64x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e5m2_64x8x32.cu | 5 + .../kernels/lat_gmma_f16e5m2e5m2_64x96x32.cu | 5 + .../kernels/lat_gmma_f16f16f16_64x128x16.cu | 5 + .../kernels/lat_gmma_f16f16f16_64x16x16.cu | 5 + .../kernels/lat_gmma_f16f16f16_64x192x16.cu | 5 + .../kernels/lat_gmma_f16f16f16_64x256x16.cu | 5 + .../kernels/lat_gmma_f16f16f16_64x32x16.cu | 5 + .../kernels/lat_gmma_f16f16f16_64x64x16.cu | 5 + .../kernels/lat_gmma_f16f16f16_64x8x16.cu | 5 + .../kernels/lat_gmma_f16f16f16_64x96x16.cu | 5 + .../kernels/lat_gmma_f32bf16bf16_64x128x16.cu | 5 + .../kernels/lat_gmma_f32bf16bf16_64x16x16.cu | 5 + .../kernels/lat_gmma_f32bf16bf16_64x192x16.cu | 5 + .../kernels/lat_gmma_f32bf16bf16_64x256x16.cu | 5 + .../kernels/lat_gmma_f32bf16bf16_64x32x16.cu | 5 + .../kernels/lat_gmma_f32bf16bf16_64x64x16.cu | 5 + .../kernels/lat_gmma_f32bf16bf16_64x8x16.cu | 5 + .../kernels/lat_gmma_f32bf16bf16_64x96x16.cu | 5 + .../lat_gmma_f32e4m3e4m3e4m3_64x128x32.cu | 5 + .../lat_gmma_f32e4m3e4m3e4m3_64x16x32.cu | 5 + .../lat_gmma_f32e4m3e4m3e4m3_64x192x32.cu | 5 + .../lat_gmma_f32e4m3e4m3e4m3_64x256x32.cu | 5 + .../lat_gmma_f32e4m3e4m3e4m3_64x32x32.cu | 5 + .../lat_gmma_f32e4m3e4m3e4m3_64x64x32.cu | 5 + .../lat_gmma_f32e4m3e4m3e4m3_64x8x32.cu | 5 + .../lat_gmma_f32e4m3e4m3e4m3_64x96x32.cu | 5 + .../lat_gmma_f32e4m3e5m2e4m3_64x128x32.cu | 5 + .../lat_gmma_f32e4m3e5m2e4m3_64x16x32.cu | 5 + .../lat_gmma_f32e4m3e5m2e4m3_64x192x32.cu | 5 + .../lat_gmma_f32e4m3e5m2e4m3_64x256x32.cu | 5 + .../lat_gmma_f32e4m3e5m2e4m3_64x32x32.cu | 5 + .../lat_gmma_f32e4m3e5m2e4m3_64x64x32.cu | 5 + .../lat_gmma_f32e4m3e5m2e4m3_64x8x32.cu | 5 + .../lat_gmma_f32e4m3e5m2e4m3_64x96x32.cu | 5 + .../lat_gmma_f32e5m2e4m3e5m2_64x128x32.cu | 5 + .../lat_gmma_f32e5m2e4m3e5m2_64x16x32.cu | 5 + .../lat_gmma_f32e5m2e4m3e5m2_64x192x32.cu | 5 + .../lat_gmma_f32e5m2e4m3e5m2_64x256x32.cu | 5 + .../lat_gmma_f32e5m2e4m3e5m2_64x32x32.cu | 5 + .../lat_gmma_f32e5m2e4m3e5m2_64x64x32.cu | 5 + .../lat_gmma_f32e5m2e4m3e5m2_64x8x32.cu | 5 + .../lat_gmma_f32e5m2e4m3e5m2_64x96x32.cu | 5 + .../lat_gmma_f32e5m2e5m2e5m2_64x128x32.cu | 5 + .../lat_gmma_f32e5m2e5m2e5m2_64x16x32.cu | 5 + .../lat_gmma_f32e5m2e5m2e5m2_64x192x32.cu | 5 + .../lat_gmma_f32e5m2e5m2e5m2_64x256x32.cu | 5 + .../lat_gmma_f32e5m2e5m2e5m2_64x32x32.cu | 5 + .../lat_gmma_f32e5m2e5m2e5m2_64x64x32.cu | 5 + .../lat_gmma_f32e5m2e5m2e5m2_64x8x32.cu | 5 + .../lat_gmma_f32e5m2e5m2e5m2_64x96x32.cu | 5 + .../kernels/lat_gmma_f32f16f16_64x128x16.cu | 5 + .../kernels/lat_gmma_f32f16f16_64x16x16.cu | 5 + .../kernels/lat_gmma_f32f16f16_64x192x16.cu | 5 + .../kernels/lat_gmma_f32f16f16_64x256x16.cu | 5 + .../kernels/lat_gmma_f32f16f16_64x32x16.cu | 5 + .../kernels/lat_gmma_f32f16f16_64x64x16.cu | 5 + .../kernels/lat_gmma_f32f16f16_64x8x16.cu | 5 + .../kernels/lat_gmma_f32f16f16_64x96x16.cu | 5 + .../lat_gmma_f32tf32tf32tf32_64x128x8.cu | 5 + .../lat_gmma_f32tf32tf32tf32_64x16x8.cu | 5 + .../lat_gmma_f32tf32tf32tf32_64x192x8.cu | 5 + .../lat_gmma_f32tf32tf32tf32_64x256x8.cu | 5 + .../lat_gmma_f32tf32tf32tf32_64x32x8.cu | 5 + .../lat_gmma_f32tf32tf32tf32_64x64x8.cu | 5 + .../lat_gmma_f32tf32tf32tf32_64x8x8.cu | 5 + .../lat_gmma_f32tf32tf32tf32_64x96x8.cu | 5 + .../kernels/lat_gmma_int32s8s8s8_64x128x32.cu | 5 + .../kernels/lat_gmma_int32s8s8s8_64x16x32.cu | 5 + .../kernels/lat_gmma_int32s8s8s8_64x192x32.cu | 5 + .../kernels/lat_gmma_int32s8s8s8_64x256x32.cu | 5 + .../kernels/lat_gmma_int32s8s8s8_64x32x32.cu | 5 + .../kernels/lat_gmma_int32s8s8s8_64x64x32.cu | 5 + .../kernels/lat_gmma_int32s8s8s8_64x8x32.cu | 5 + .../kernels/lat_gmma_int32s8s8s8_64x96x32.cu | 5 + .../kernels/lat_gmma_int32s8u8s8_64x128x32.cu | 5 + .../kernels/lat_gmma_int32s8u8s8_64x16x32.cu | 5 + .../kernels/lat_gmma_int32s8u8s8_64x192x32.cu | 5 + .../kernels/lat_gmma_int32s8u8s8_64x256x32.cu | 5 + .../kernels/lat_gmma_int32s8u8s8_64x32x32.cu | 5 + .../kernels/lat_gmma_int32s8u8s8_64x64x32.cu | 5 + .../kernels/lat_gmma_int32s8u8s8_64x8x32.cu | 5 + .../kernels/lat_gmma_int32s8u8s8_64x96x32.cu | 5 + .../kernels/lat_gmma_int32u8s8u8_64x128x32.cu | 5 + .../kernels/lat_gmma_int32u8s8u8_64x16x32.cu | 5 + .../kernels/lat_gmma_int32u8s8u8_64x192x32.cu | 5 + .../kernels/lat_gmma_int32u8s8u8_64x256x32.cu | 5 + .../kernels/lat_gmma_int32u8s8u8_64x32x32.cu | 5 + .../kernels/lat_gmma_int32u8s8u8_64x64x32.cu | 5 + .../kernels/lat_gmma_int32u8s8u8_64x8x32.cu | 5 + .../kernels/lat_gmma_int32u8s8u8_64x96x32.cu | 5 + .../kernels/lat_gmma_int32u8u8u8_64x128x32.cu | 5 + .../kernels/lat_gmma_int32u8u8u8_64x16x32.cu | 5 + .../kernels/lat_gmma_int32u8u8u8_64x192x32.cu | 5 + .../kernels/lat_gmma_int32u8u8u8_64x256x32.cu | 5 + .../kernels/lat_gmma_int32u8u8u8_64x32x32.cu | 5 + .../kernels/lat_gmma_int32u8u8u8_64x64x32.cu | 5 + .../kernels/lat_gmma_int32u8u8u8_64x8x32.cu | 5 + .../kernels/lat_gmma_int32u8u8u8_64x96x32.cu | 5 + .../ubench/wgmma/lat_gmma/lat_gmma.cu | 22 + .../ubench/wgmma/lat_gmma/lat_gmma.h | 385 +++++++++++++++ .../ubench/wgmma/lat_gmma/lat_gmma_common.h | 210 +++++++++ .../ubench/wgmma/lat_gmma/lat_gmma_test.cu | 17 + .../ubench/wgmma/wgmma_multik/wgmma_multik.cu | 438 ++++++++++++++++-- 135 files changed, 1712 insertions(+), 32 deletions(-) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/.gitignore create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x128x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x16x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x192x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x256x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x32x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x64x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x8x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x96x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x128x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x16x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x192x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x256x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x32x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x64x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x8x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x96x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x128x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x16x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x192x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x256x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x32x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x64x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x8x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x96x16.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x128x8.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x16x8.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x192x8.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x256x8.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x32x8.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x64x8.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x8x8.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x96x8.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x128x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x16x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x192x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x256x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x32x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x64x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x8x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x96x32.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.h create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_common.h create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_test.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/.gitignore b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/.gitignore new file mode 100644 index 000000000..08f514ebc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/.gitignore @@ -0,0 +1 @@ +include/ diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/Makefile new file mode 100644 index 000000000..3272e6dad --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/Makefile @@ -0,0 +1,31 @@ +# Source files split for parallel compilation +# Use wildcard to automatically include all size-specific breakdown files +SRC = lat_gmma.cu $(wildcard kernels/lat_gmma_*.cu) + +EXE = lat_gmma + +# Add include path for CUTLASS. +# Prefer the bundled include/ directory so no external setup is needed. +INCLUDE += -I./include -I$(GPUAPPS_ROOT)/src/cuda/cutlass-bench/include -I./ + +# GMMA is only supported in sm_90a +ARCH?=sm_90a +# Unset the CUDA_CPPFLAGS which is set based on CUDA version +CUDA_CPPFLAGS= +# Generate code for both sm_90a and compute_90a (SASS and PTX) +HOPPER_CUDA_CPPFLAGS=$(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) + +# CUTLASS cute library requires C++17 +NVCC_FLAGS := $(HOPPER_CUDA_CPPFLAGS) -std=c++17 + +include ../../../common/common.mk + +# A test executable for checking the library and simulator debugging +TEST_SRC = lat_gmma_test.cu +TEST_EXE = lat_gmma_test +test: $(TEST_SRC) + $(CC) $(NVCC_FLAGS) $^ -o $(TEST_EXE) $(INCLUDE) -lcudart --cudart shared + cp $(TEST_EXE) $(BIN_DIR) + +# Append the test to the release target +release: .EXTRA_PREREQS = test \ No newline at end of file diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x128x32.cu new file mode 100644 index 000000000..ec65d5eff --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x128x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 128, 32, "MMA_64x128x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x16x32.cu new file mode 100644 index 000000000..0da1c52bd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x16x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 16, 32, "MMA_64x16x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x192x32.cu new file mode 100644 index 000000000..b227b645d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x192x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 192, 32, "MMA_64x192x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x256x32.cu new file mode 100644 index 000000000..a885466d9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x256x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 256, 32, "MMA_64x256x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x32x32.cu new file mode 100644 index 000000000..cbb28cea9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x32x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 32, 32, "MMA_64x32x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x64x32.cu new file mode 100644 index 000000000..c9687a66c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x64x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 64, 32, "MMA_64x64x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x8x32.cu new file mode 100644 index 000000000..9b445f92a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x8x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 8, 32, "MMA_64x8x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x96x32.cu new file mode 100644 index 000000000..fc4180513 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e4m3_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e4m3_64x96x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, half_t, 64, 96, 32, "MMA_64x96x32_F16E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x128x32.cu new file mode 100644 index 000000000..a02254201 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x128x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 128, 32, "MMA_64x128x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x16x32.cu new file mode 100644 index 000000000..68b9546e7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x16x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 16, 32, "MMA_64x16x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x192x32.cu new file mode 100644 index 000000000..c0f3993b2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x192x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 192, 32, "MMA_64x192x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x256x32.cu new file mode 100644 index 000000000..297053fe7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x256x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 256, 32, "MMA_64x256x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x32x32.cu new file mode 100644 index 000000000..0126fcc9a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x32x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 32, 32, "MMA_64x32x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x64x32.cu new file mode 100644 index 000000000..744a42dfb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x64x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 64, 32, "MMA_64x64x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x8x32.cu new file mode 100644 index 000000000..bfca52f2a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x8x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 8, 32, "MMA_64x8x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x96x32.cu new file mode 100644 index 000000000..d4a580d6f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e4m3e5m2_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e4m3e5m2_64x96x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, half_t, 64, 96, 32, "MMA_64x96x32_F16E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x128x32.cu new file mode 100644 index 000000000..9319c9ee6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x128x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 128, 32, "MMA_64x128x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x16x32.cu new file mode 100644 index 000000000..a1602c766 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x16x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 16, 32, "MMA_64x16x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x192x32.cu new file mode 100644 index 000000000..2db60e978 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x192x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 192, 32, "MMA_64x192x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x256x32.cu new file mode 100644 index 000000000..22e2df853 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x256x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 256, 32, "MMA_64x256x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x32x32.cu new file mode 100644 index 000000000..ec0fdbdef --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x32x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 32, 32, "MMA_64x32x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x64x32.cu new file mode 100644 index 000000000..1101feb0f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x64x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 64, 32, "MMA_64x64x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x8x32.cu new file mode 100644 index 000000000..755c81a33 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x8x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 8, 32, "MMA_64x8x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x96x32.cu new file mode 100644 index 000000000..350d8f5c4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e4m3_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e4m3_64x96x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, half_t, 64, 96, 32, "MMA_64x96x32_F16E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x128x32.cu new file mode 100644 index 000000000..d466a9795 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x128x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 128, 32, "MMA_64x128x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x16x32.cu new file mode 100644 index 000000000..44d9835ad --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x16x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 16, 32, "MMA_64x16x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x192x32.cu new file mode 100644 index 000000000..31db9558a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x192x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 192, 32, "MMA_64x192x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x256x32.cu new file mode 100644 index 000000000..3ddacc999 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x256x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 256, 32, "MMA_64x256x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x32x32.cu new file mode 100644 index 000000000..25d80681d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x32x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 32, 32, "MMA_64x32x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x64x32.cu new file mode 100644 index 000000000..696c59694 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x64x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 64, 32, "MMA_64x64x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x8x32.cu new file mode 100644 index 000000000..1c2c62651 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x8x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 8, 32, "MMA_64x8x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x96x32.cu new file mode 100644 index 000000000..c4b93b17f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16e5m2e5m2_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16e5m2e5m2_64x96x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, half_t, 64, 96, 32, "MMA_64x96x32_F16E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x128x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x128x16.cu new file mode 100644 index 000000000..38d284502 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x128x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x128x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 128, 16, "MMA_64x128x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x16x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x16x16.cu new file mode 100644 index 000000000..40e841ad0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x16x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x16x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 16, 16, "MMA_64x16x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x192x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x192x16.cu new file mode 100644 index 000000000..2200355a9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x192x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x192x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 192, 16, "MMA_64x192x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x256x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x256x16.cu new file mode 100644 index 000000000..057ee4222 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x256x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x256x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 256, 16, "MMA_64x256x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x32x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x32x16.cu new file mode 100644 index 000000000..252e4cde7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x32x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x32x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 32, 16, "MMA_64x32x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x64x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x64x16.cu new file mode 100644 index 000000000..d3d8715bd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x64x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x64x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 64, 16, "MMA_64x64x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x8x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x8x16.cu new file mode 100644 index 000000000..df7d161ba --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x8x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x8x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 8, 16, "MMA_64x8x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x96x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x96x16.cu new file mode 100644 index 000000000..92dc99dc5 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f16f16f16_64x96x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f16f16f16_64x96x16_test() { + TEST_MMA_CONFIG(half_t, half_t, half_t, 64, 96, 16, "MMA_64x96x16_F16F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x128x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x128x16.cu new file mode 100644 index 000000000..58c068112 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x128x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x128x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 128, 16, "MMA_64x128x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x16x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x16x16.cu new file mode 100644 index 000000000..63b5adc61 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x16x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x16x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 16, 16, "MMA_64x16x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x192x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x192x16.cu new file mode 100644 index 000000000..1275b3a16 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x192x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x192x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 192, 16, "MMA_64x192x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x256x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x256x16.cu new file mode 100644 index 000000000..e3aabaa75 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x256x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x256x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 256, 16, "MMA_64x256x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x32x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x32x16.cu new file mode 100644 index 000000000..0d9e07f78 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x32x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x32x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 32, 16, "MMA_64x32x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x64x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x64x16.cu new file mode 100644 index 000000000..14f6fcd8c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x64x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x64x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 64, 16, "MMA_64x64x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x8x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x8x16.cu new file mode 100644 index 000000000..1b0d0e6d2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x8x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x8x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 8, 16, "MMA_64x8x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x96x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x96x16.cu new file mode 100644 index 000000000..62a4331de --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32bf16bf16_64x96x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32bf16bf16_64x96x16_test() { + TEST_MMA_CONFIG(bfloat16_t, bfloat16_t, float, 64, 96, 16, "MMA_64x96x16_F32BF16BF16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x128x32.cu new file mode 100644 index 000000000..71ea82971 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x128x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 128, 32, "MMA_64x128x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x16x32.cu new file mode 100644 index 000000000..6ef3e404c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x16x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 16, 32, "MMA_64x16x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x192x32.cu new file mode 100644 index 000000000..ad1cd2296 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x192x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 192, 32, "MMA_64x192x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x256x32.cu new file mode 100644 index 000000000..000a40d72 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x256x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 256, 32, "MMA_64x256x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x32x32.cu new file mode 100644 index 000000000..522366660 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x32x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 32, 32, "MMA_64x32x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x64x32.cu new file mode 100644 index 000000000..1394b2a40 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x64x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 64, 32, "MMA_64x64x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x8x32.cu new file mode 100644 index 000000000..f39d1456c --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x8x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 8, 32, "MMA_64x8x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x96x32.cu new file mode 100644 index 000000000..fe7242b99 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e4m3e4m3_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e4m3e4m3_64x96x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e4m3_t, float, 64, 96, 32, "MMA_64x96x32_F32E4M3E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x128x32.cu new file mode 100644 index 000000000..dc46a76b6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x128x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 128, 32, "MMA_64x128x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x16x32.cu new file mode 100644 index 000000000..14c139710 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x16x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 16, 32, "MMA_64x16x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x192x32.cu new file mode 100644 index 000000000..3678c5414 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x192x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 192, 32, "MMA_64x192x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x256x32.cu new file mode 100644 index 000000000..39f479742 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x256x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 256, 32, "MMA_64x256x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x32x32.cu new file mode 100644 index 000000000..7607963c4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x32x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 32, 32, "MMA_64x32x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x64x32.cu new file mode 100644 index 000000000..e20b25a04 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x64x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 64, 32, "MMA_64x64x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x8x32.cu new file mode 100644 index 000000000..8b2f9c8d6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x8x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 8, 32, "MMA_64x8x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x96x32.cu new file mode 100644 index 000000000..a8fb10753 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e4m3e5m2e4m3_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e4m3e5m2e4m3_64x96x32_test() { + TEST_MMA_CONFIG(float_e4m3_t, float_e5m2_t, float, 64, 96, 32, "MMA_64x96x32_F32E4M3E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x128x32.cu new file mode 100644 index 000000000..279aa4d37 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x128x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 128, 32, "MMA_64x128x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x16x32.cu new file mode 100644 index 000000000..0a5aaf41a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x16x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 16, 32, "MMA_64x16x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x192x32.cu new file mode 100644 index 000000000..5eac44263 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x192x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 192, 32, "MMA_64x192x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x256x32.cu new file mode 100644 index 000000000..47bc6a3ef --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x256x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 256, 32, "MMA_64x256x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x32x32.cu new file mode 100644 index 000000000..4f0d5856a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x32x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 32, 32, "MMA_64x32x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x64x32.cu new file mode 100644 index 000000000..d8326f3ac --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x64x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 64, 32, "MMA_64x64x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x8x32.cu new file mode 100644 index 000000000..fbec6e6f1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x8x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 8, 32, "MMA_64x8x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x96x32.cu new file mode 100644 index 000000000..1ab9ef53d --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e4m3e5m2_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e4m3e5m2_64x96x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e4m3_t, float, 64, 96, 32, "MMA_64x96x32_F32E5M2E4M3_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x128x32.cu new file mode 100644 index 000000000..b41d328e1 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x128x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 128, 32, "MMA_64x128x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x16x32.cu new file mode 100644 index 000000000..414ea343e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x16x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 16, 32, "MMA_64x16x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x192x32.cu new file mode 100644 index 000000000..53d23cdc9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x192x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 192, 32, "MMA_64x192x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x256x32.cu new file mode 100644 index 000000000..7055d9edc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x256x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 256, 32, "MMA_64x256x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x32x32.cu new file mode 100644 index 000000000..b34f08491 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x32x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 32, 32, "MMA_64x32x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x64x32.cu new file mode 100644 index 000000000..0f1c26687 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x64x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 64, 32, "MMA_64x64x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x8x32.cu new file mode 100644 index 000000000..1450e5f7e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x8x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 8, 32, "MMA_64x8x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x96x32.cu new file mode 100644 index 000000000..6077058b7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32e5m2e5m2e5m2_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32e5m2e5m2e5m2_64x96x32_test() { + TEST_MMA_CONFIG(float_e5m2_t, float_e5m2_t, float, 64, 96, 32, "MMA_64x96x32_F32E5M2E5M2_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x128x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x128x16.cu new file mode 100644 index 000000000..9eaa49ab0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x128x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x128x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 128, 16, "MMA_64x128x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x16x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x16x16.cu new file mode 100644 index 000000000..183878140 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x16x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x16x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 16, 16, "MMA_64x16x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x192x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x192x16.cu new file mode 100644 index 000000000..ea30da004 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x192x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x192x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 192, 16, "MMA_64x192x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x256x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x256x16.cu new file mode 100644 index 000000000..5e82afe2b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x256x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x256x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 256, 16, "MMA_64x256x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x32x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x32x16.cu new file mode 100644 index 000000000..24f208beb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x32x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x32x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 32, 16, "MMA_64x32x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x64x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x64x16.cu new file mode 100644 index 000000000..68a4b96cb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x64x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x64x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 64, 16, "MMA_64x64x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x8x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x8x16.cu new file mode 100644 index 000000000..2f3dab3a4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x8x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x8x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 8, 16, "MMA_64x8x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x96x16.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x96x16.cu new file mode 100644 index 000000000..2b2e8722e --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32f16f16_64x96x16.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32f16f16_64x96x16_test() { + TEST_MMA_CONFIG(half_t, half_t, float, 64, 96, 16, "MMA_64x96x16_F32F16F16_SS"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x128x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x128x8.cu new file mode 100644 index 000000000..158088161 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x128x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x128x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 128, 8, "MMA_64x128x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x16x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x16x8.cu new file mode 100644 index 000000000..fe1492be2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x16x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x16x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 16, 8, "MMA_64x16x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x192x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x192x8.cu new file mode 100644 index 000000000..463e9905b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x192x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x192x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 192, 8, "MMA_64x192x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x256x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x256x8.cu new file mode 100644 index 000000000..53d323884 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x256x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x256x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 256, 8, "MMA_64x256x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x32x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x32x8.cu new file mode 100644 index 000000000..a59c41861 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x32x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x32x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 32, 8, "MMA_64x32x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x64x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x64x8.cu new file mode 100644 index 000000000..ab5d032ca --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x64x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x64x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 64, 8, "MMA_64x64x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x8x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x8x8.cu new file mode 100644 index 000000000..97eb9b4b6 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x8x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x8x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 8, 8, "MMA_64x8x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x96x8.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x96x8.cu new file mode 100644 index 000000000..fca3734c2 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_f32tf32tf32tf32_64x96x8.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_f32tf32tf32tf32_64x96x8_test() { + TEST_MMA_CONFIG(tfloat32_t, tfloat32_t, float, 64, 96, 8, "MMA_64x96x8_F32TF32TF32_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x128x32.cu new file mode 100644 index 000000000..c41a660a4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x128x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 128, 32, "MMA_64x128x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x16x32.cu new file mode 100644 index 000000000..9c3895b45 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x16x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 16, 32, "MMA_64x16x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x192x32.cu new file mode 100644 index 000000000..f4b60f169 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x192x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 192, 32, "MMA_64x192x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x256x32.cu new file mode 100644 index 000000000..c6522cb86 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x256x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 256, 32, "MMA_64x256x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x32x32.cu new file mode 100644 index 000000000..468abc59b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x32x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 32, 32, "MMA_64x32x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x64x32.cu new file mode 100644 index 000000000..9786b4c45 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x64x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 64, 32, "MMA_64x64x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x8x32.cu new file mode 100644 index 000000000..823375478 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x8x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 8, 32, "MMA_64x8x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x96x32.cu new file mode 100644 index 000000000..0104423e3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8s8s8_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8s8s8_64x96x32_test() { + TEST_MMA_CONFIG(int8_t, int8_t, int32_t, 64, 96, 32, "MMA_64x96x32_S32S8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x128x32.cu new file mode 100644 index 000000000..6d8210349 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x128x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 128, 32, "MMA_64x128x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x16x32.cu new file mode 100644 index 000000000..8622cacfb --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x16x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 16, 32, "MMA_64x16x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x192x32.cu new file mode 100644 index 000000000..a4feb009a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x192x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 192, 32, "MMA_64x192x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x256x32.cu new file mode 100644 index 000000000..e9d817de8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x256x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 256, 32, "MMA_64x256x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x32x32.cu new file mode 100644 index 000000000..65ff0d122 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x32x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 32, 32, "MMA_64x32x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x64x32.cu new file mode 100644 index 000000000..48938cac9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x64x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 64, 32, "MMA_64x64x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x8x32.cu new file mode 100644 index 000000000..55d601042 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x8x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 8, 32, "MMA_64x8x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x96x32.cu new file mode 100644 index 000000000..38925b524 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32s8u8s8_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32s8u8s8_64x96x32_test() { + TEST_MMA_CONFIG(int8_t, uint8_t, int32_t, 64, 96, 32, "MMA_64x96x32_S32S8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x128x32.cu new file mode 100644 index 000000000..70d09f174 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x128x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 128, 32, "MMA_64x128x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x16x32.cu new file mode 100644 index 000000000..f260d2d33 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x16x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 16, 32, "MMA_64x16x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x192x32.cu new file mode 100644 index 000000000..93f12eede --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x192x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 192, 32, "MMA_64x192x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x256x32.cu new file mode 100644 index 000000000..e25b2a8b8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x256x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 256, 32, "MMA_64x256x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x32x32.cu new file mode 100644 index 000000000..81329112b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x32x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 32, 32, "MMA_64x32x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x64x32.cu new file mode 100644 index 000000000..7ad6c9175 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x64x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 64, 32, "MMA_64x64x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x8x32.cu new file mode 100644 index 000000000..e6e4fc35a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x8x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 8, 32, "MMA_64x8x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x96x32.cu new file mode 100644 index 000000000..c35ec92ce --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8s8u8_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8s8u8_64x96x32_test() { + TEST_MMA_CONFIG(uint8_t, int8_t, int32_t, 64, 96, 32, "MMA_64x96x32_S32U8S8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x128x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x128x32.cu new file mode 100644 index 000000000..326a3f4dd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x128x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x128x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 128, 32, "MMA_64x128x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x16x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x16x32.cu new file mode 100644 index 000000000..46192dee8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x16x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x16x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 16, 32, "MMA_64x16x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x192x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x192x32.cu new file mode 100644 index 000000000..d26787621 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x192x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x192x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 192, 32, "MMA_64x192x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x256x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x256x32.cu new file mode 100644 index 000000000..9a5962ddd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x256x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x256x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 256, 32, "MMA_64x256x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x32x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x32x32.cu new file mode 100644 index 000000000..bbca73ed8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x32x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x32x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 32, 32, "MMA_64x32x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x64x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x64x32.cu new file mode 100644 index 000000000..9914d67a8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x64x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x64x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 64, 32, "MMA_64x64x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x8x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x8x32.cu new file mode 100644 index 000000000..f2044b708 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x8x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x8x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 8, 32, "MMA_64x8x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x96x32.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x96x32.cu new file mode 100644 index 000000000..d0d84c06b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/kernels/lat_gmma_int32u8u8u8_64x96x32.cu @@ -0,0 +1,5 @@ +#include "lat_gmma_common.h" + +void run_int32u8u8u8_64x96x32_test() { + TEST_MMA_CONFIG(uint8_t, uint8_t, int32_t, 64, 96, 32, "MMA_64x96x32_S32U8U8_SS_TN"); +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.cu new file mode 100644 index 000000000..bcde5cbd0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.cu @@ -0,0 +1,22 @@ +#include +#include +#include "lat_gmma.h" +#include "../../../hw_def/hw_def.h" + +FILE* g_lat_gmma_outfile = NULL; + +int main(int argc, char *argv[]) +{ + initializeDeviceProp(0, argc, argv); + + g_lat_gmma_outfile = fopen("lat_gmma_results.csv", "w"); + if (g_lat_gmma_outfile) + fprintf(g_lat_gmma_outfile, "config,cycles_per_mma\n"); + + run_all_wgmma_latency_tests(); + + if (g_lat_gmma_outfile) + fclose(g_lat_gmma_outfile); + + return 0; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.h b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.h new file mode 100644 index 000000000..ccb2ca7d7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma.h @@ -0,0 +1,385 @@ +/*************************************************************************************************** + * GMMA Latency Microbenchmark for SM90 (Hopper Architecture) + * + * This file provides a comprehensive sweep over all valid WGMMA (Warpgroup Matrix Multiply- + * Accumulate) operations supported by the NVIDIA Hopper architecture (SM90). + * + * The sweep covers: + * + * 1. F32 Accumulator (ElementC = float): + * - FP16 x FP16 -> FP32 + * - BF16 x BF16 -> FP32 + * - TF32 x TF32 -> FP32 + * - E4M3 x E4M3 -> FP32 + * - E4M3 x E5M2 -> FP32 + * - E5M2 x E4M3 -> FP32 + * - E5M2 x E5M2 -> FP32 + * + * 2. F16 Accumulator (ElementC = half_t): + * - FP16 x FP16 -> FP16 + * - E4M3 x E4M3 -> FP16 + * - E4M3 x E5M2 -> FP16 + * - E5M2 x E4M3 -> FP16 + * - E5M2 x E5M2 -> FP16 + * + * 3. INT32 Accumulator (ElementC = int32_t): + * - INT8 x INT8 -> INT32 + * - INT8 x UINT8 -> INT32 + * - UINT8 x INT8 -> INT32 + * - UINT8 x UINT8 -> INT32 + * + * For each data type combination, the sweep tests multiple tile shapes: + * - 64x8, 64x16, 64x32, 64x64, 64x96, 64x128, 64x192, 64x256 + * + * Usage: + * Call run_all_wgmma_latency_tests() to run the comprehensive sweep. + * + * NOTE: This file has been refactored for parallel compilation. Test implementations + * are split across lat_gmma_f32.cu, lat_gmma_f16.cu, and lat_gmma_int32.cu. + * + **************************************************************************************************/ + +#ifndef LAT_GMMA_DEF_H +#define LAT_GMMA_DEF_H + +#include +#include +#include +#include + + +// Function declarations for test suites +// These are defined in separate .cu files for parallel compilation + +// F32 accumulator tests - TF32 x TF32 -> F32 +void run_f32tf32tf32tf32_64x8x8_test(); +void run_f32tf32tf32tf32_64x16x8_test(); +void run_f32tf32tf32tf32_64x32x8_test(); +void run_f32tf32tf32tf32_64x64x8_test(); +void run_f32tf32tf32tf32_64x96x8_test(); +void run_f32tf32tf32tf32_64x128x8_test(); +void run_f32tf32tf32tf32_64x192x8_test(); +void run_f32tf32tf32tf32_64x256x8_test(); + +// F32 accumulator tests - E4M3 x E4M3 -> F32 +void run_f32e4m3e4m3e4m3_64x8x32_test(); +void run_f32e4m3e4m3e4m3_64x16x32_test(); +void run_f32e4m3e4m3e4m3_64x32x32_test(); +void run_f32e4m3e4m3e4m3_64x64x32_test(); +void run_f32e4m3e4m3e4m3_64x96x32_test(); +void run_f32e4m3e4m3e4m3_64x128x32_test(); +void run_f32e4m3e4m3e4m3_64x192x32_test(); +void run_f32e4m3e4m3e4m3_64x256x32_test(); + +// F32 accumulator tests - E4M3 x E5M2 -> F32 +void run_f32e4m3e5m2e4m3_64x8x32_test(); +void run_f32e4m3e5m2e4m3_64x16x32_test(); +void run_f32e4m3e5m2e4m3_64x32x32_test(); +void run_f32e4m3e5m2e4m3_64x64x32_test(); +void run_f32e4m3e5m2e4m3_64x96x32_test(); +void run_f32e4m3e5m2e4m3_64x128x32_test(); +void run_f32e4m3e5m2e4m3_64x192x32_test(); +void run_f32e4m3e5m2e4m3_64x256x32_test(); + +// F32 accumulator tests - E5M2 x E4M3 -> F32 +void run_f32e5m2e4m3e5m2_64x8x32_test(); +void run_f32e5m2e4m3e5m2_64x16x32_test(); +void run_f32e5m2e4m3e5m2_64x32x32_test(); +void run_f32e5m2e4m3e5m2_64x64x32_test(); +void run_f32e5m2e4m3e5m2_64x96x32_test(); +void run_f32e5m2e4m3e5m2_64x128x32_test(); +void run_f32e5m2e4m3e5m2_64x192x32_test(); +void run_f32e5m2e4m3e5m2_64x256x32_test(); + +// F32 accumulator tests - E5M2 x E5M2 -> F32 +void run_f32e5m2e5m2e5m2_64x8x32_test(); +void run_f32e5m2e5m2e5m2_64x16x32_test(); +void run_f32e5m2e5m2e5m2_64x32x32_test(); +void run_f32e5m2e5m2e5m2_64x64x32_test(); +void run_f32e5m2e5m2e5m2_64x96x32_test(); +void run_f32e5m2e5m2e5m2_64x128x32_test(); +void run_f32e5m2e5m2e5m2_64x192x32_test(); +void run_f32e5m2e5m2e5m2_64x256x32_test(); + +// INT32 accumulator tests - INT8 x INT8 -> INT32 +void run_int32s8s8s8_64x8x32_test(); +void run_int32s8s8s8_64x16x32_test(); +void run_int32s8s8s8_64x32x32_test(); +void run_int32s8s8s8_64x64x32_test(); +void run_int32s8s8s8_64x96x32_test(); +void run_int32s8s8s8_64x128x32_test(); +void run_int32s8s8s8_64x192x32_test(); +void run_int32s8s8s8_64x256x32_test(); + +// INT32 accumulator tests - INT8 x UINT8 -> INT32 +void run_int32s8u8s8_64x8x32_test(); +void run_int32s8u8s8_64x16x32_test(); +void run_int32s8u8s8_64x32x32_test(); +void run_int32s8u8s8_64x64x32_test(); +void run_int32s8u8s8_64x96x32_test(); +void run_int32s8u8s8_64x128x32_test(); +void run_int32s8u8s8_64x192x32_test(); +void run_int32s8u8s8_64x256x32_test(); + +// INT32 accumulator tests - UINT8 x INT8 -> INT32 +void run_int32u8s8u8_64x8x32_test(); +void run_int32u8s8u8_64x16x32_test(); +void run_int32u8s8u8_64x32x32_test(); +void run_int32u8s8u8_64x64x32_test(); +void run_int32u8s8u8_64x96x32_test(); +void run_int32u8s8u8_64x128x32_test(); +void run_int32u8s8u8_64x192x32_test(); +void run_int32u8s8u8_64x256x32_test(); + +// INT32 accumulator tests - UINT8 x UINT8 -> INT32 +void run_int32u8u8u8_64x8x32_test(); +void run_int32u8u8u8_64x16x32_test(); +void run_int32u8u8u8_64x32x32_test(); +void run_int32u8u8u8_64x64x32_test(); +void run_int32u8u8u8_64x96x32_test(); +void run_int32u8u8u8_64x128x32_test(); +void run_int32u8u8u8_64x192x32_test(); +void run_int32u8u8u8_64x256x32_test(); + +// F16 accumulator tests (defined in lat_gmma_f16.cu) +// F32 accumulator tests - FP16 x FP16 -> F32 +void run_f32f16f16_64x8x16_test(); +void run_f32f16f16_64x16x16_test(); +void run_f32f16f16_64x32x16_test(); +void run_f32f16f16_64x64x16_test(); +void run_f32f16f16_64x96x16_test(); +void run_f32f16f16_64x128x16_test(); +void run_f32f16f16_64x192x16_test(); +void run_f32f16f16_64x256x16_test(); + +// F32 accumulator tests - BF16 x BF16 -> F32 +void run_f32bf16bf16_64x8x16_test(); +void run_f32bf16bf16_64x16x16_test(); +void run_f32bf16bf16_64x32x16_test(); +void run_f32bf16bf16_64x64x16_test(); +void run_f32bf16bf16_64x96x16_test(); +void run_f32bf16bf16_64x128x16_test(); +void run_f32bf16bf16_64x192x16_test(); +void run_f32bf16bf16_64x256x16_test(); + +// F16 accumulator tests - FP16 x FP16 -> F16 +void run_f16f16f16_64x8x16_test(); +void run_f16f16f16_64x16x16_test(); +void run_f16f16f16_64x32x16_test(); +void run_f16f16f16_64x64x16_test(); +void run_f16f16f16_64x96x16_test(); +void run_f16f16f16_64x128x16_test(); +void run_f16f16f16_64x192x16_test(); +void run_f16f16f16_64x256x16_test(); + +// F16 accumulator tests - E4M3 x E4M3 -> F16 +void run_f16e4m3e4m3_64x8x32_test(); +void run_f16e4m3e4m3_64x16x32_test(); +void run_f16e4m3e4m3_64x32x32_test(); +void run_f16e4m3e4m3_64x64x32_test(); +void run_f16e4m3e4m3_64x96x32_test(); +void run_f16e4m3e4m3_64x128x32_test(); +void run_f16e4m3e4m3_64x192x32_test(); +void run_f16e4m3e4m3_64x256x32_test(); + +// F16 accumulator tests - E4M3 x E5M2 -> F16 +void run_f16e4m3e5m2_64x8x32_test(); +void run_f16e4m3e5m2_64x16x32_test(); +void run_f16e4m3e5m2_64x32x32_test(); +void run_f16e4m3e5m2_64x64x32_test(); +void run_f16e4m3e5m2_64x96x32_test(); +void run_f16e4m3e5m2_64x128x32_test(); +void run_f16e4m3e5m2_64x192x32_test(); +void run_f16e4m3e5m2_64x256x32_test(); + +// F16 accumulator tests - E5M2 x E4M3 -> F16 +void run_f16e5m2e4m3_64x8x32_test(); +void run_f16e5m2e4m3_64x16x32_test(); +void run_f16e5m2e4m3_64x32x32_test(); +void run_f16e5m2e4m3_64x64x32_test(); +void run_f16e5m2e4m3_64x96x32_test(); +void run_f16e5m2e4m3_64x128x32_test(); +void run_f16e5m2e4m3_64x192x32_test(); +void run_f16e5m2e4m3_64x256x32_test(); + +// F16 accumulator tests - E5M2 x E5M2 -> F16 +void run_f16e5m2e5m2_64x8x32_test(); +void run_f16e5m2e5m2_64x16x32_test(); +void run_f16e5m2e5m2_64x32x32_test(); +void run_f16e5m2e5m2_64x64x32_test(); +void run_f16e5m2e5m2_64x96x32_test(); +void run_f16e5m2e5m2_64x128x32_test(); +void run_f16e5m2e5m2_64x192x32_test(); +void run_f16e5m2e5m2_64x256x32_test(); + +void run_f16accumulator_tests() { + run_f16f16f16_64x8x16_test(); + run_f16f16f16_64x16x16_test(); + run_f16f16f16_64x32x16_test(); + run_f16f16f16_64x64x16_test(); + run_f16f16f16_64x96x16_test(); + run_f16f16f16_64x128x16_test(); + run_f16f16f16_64x192x16_test(); + run_f16f16f16_64x256x16_test(); + run_f16e4m3e4m3_64x8x32_test(); + run_f16e4m3e4m3_64x16x32_test(); + run_f16e4m3e4m3_64x32x32_test(); + run_f16e4m3e4m3_64x64x32_test(); + run_f16e4m3e4m3_64x96x32_test(); + run_f16e4m3e4m3_64x128x32_test(); + run_f16e4m3e4m3_64x192x32_test(); + run_f16e4m3e4m3_64x256x32_test(); + run_f16e4m3e5m2_64x8x32_test(); + run_f16e4m3e5m2_64x16x32_test(); + run_f16e4m3e5m2_64x32x32_test(); + run_f16e4m3e5m2_64x64x32_test(); + run_f16e4m3e5m2_64x96x32_test(); + run_f16e4m3e5m2_64x128x32_test(); + run_f16e4m3e5m2_64x192x32_test(); + run_f16e4m3e5m2_64x256x32_test(); + run_f16e5m2e4m3_64x8x32_test(); + run_f16e5m2e4m3_64x16x32_test(); + run_f16e5m2e4m3_64x32x32_test(); + run_f16e5m2e4m3_64x64x32_test(); + run_f16e5m2e4m3_64x96x32_test(); + run_f16e5m2e4m3_64x128x32_test(); + run_f16e5m2e4m3_64x192x32_test(); + run_f16e5m2e4m3_64x256x32_test(); + run_f16e5m2e5m2_64x8x32_test(); + run_f16e5m2e5m2_64x16x32_test(); + run_f16e5m2e5m2_64x32x32_test(); + run_f16e5m2e5m2_64x64x32_test(); + run_f16e5m2e5m2_64x96x32_test(); + run_f16e5m2e5m2_64x128x32_test(); + run_f16e5m2e5m2_64x192x32_test(); + run_f16e5m2e5m2_64x256x32_test(); +} + +void run_f32accumulator_tests() { + run_f32tf32tf32tf32_64x8x8_test(); + run_f32tf32tf32tf32_64x16x8_test(); + run_f32tf32tf32tf32_64x32x8_test(); + run_f32tf32tf32tf32_64x64x8_test(); + run_f32tf32tf32tf32_64x96x8_test(); + run_f32tf32tf32tf32_64x128x8_test(); + run_f32tf32tf32tf32_64x192x8_test(); + run_f32tf32tf32tf32_64x256x8_test(); + run_f32f16f16_64x8x16_test(); + run_f32f16f16_64x16x16_test(); + run_f32f16f16_64x32x16_test(); + run_f32f16f16_64x64x16_test(); + run_f32f16f16_64x96x16_test(); + run_f32f16f16_64x128x16_test(); + run_f32f16f16_64x192x16_test(); + run_f32f16f16_64x256x16_test(); + run_f32bf16bf16_64x8x16_test(); + run_f32bf16bf16_64x16x16_test(); + run_f32bf16bf16_64x32x16_test(); + run_f32bf16bf16_64x64x16_test(); + run_f32bf16bf16_64x96x16_test(); + run_f32bf16bf16_64x128x16_test(); + run_f32bf16bf16_64x192x16_test(); + run_f32bf16bf16_64x256x16_test(); + run_f32e4m3e4m3e4m3_64x8x32_test(); + run_f32e4m3e4m3e4m3_64x16x32_test(); + run_f32e4m3e4m3e4m3_64x32x32_test(); + run_f32e4m3e4m3e4m3_64x64x32_test(); + run_f32e4m3e4m3e4m3_64x96x32_test(); + run_f32e4m3e4m3e4m3_64x128x32_test(); + run_f32e4m3e4m3e4m3_64x192x32_test(); + run_f32e4m3e4m3e4m3_64x256x32_test(); + run_f32e4m3e5m2e4m3_64x8x32_test(); + run_f32e4m3e5m2e4m3_64x16x32_test(); + run_f32e4m3e5m2e4m3_64x32x32_test(); + run_f32e4m3e5m2e4m3_64x64x32_test(); + run_f32e4m3e5m2e4m3_64x96x32_test(); + run_f32e4m3e5m2e4m3_64x128x32_test(); + run_f32e4m3e5m2e4m3_64x192x32_test(); + run_f32e4m3e5m2e4m3_64x256x32_test(); + run_f32e5m2e4m3e5m2_64x8x32_test(); + run_f32e5m2e4m3e5m2_64x16x32_test(); + run_f32e5m2e4m3e5m2_64x32x32_test(); + run_f32e5m2e4m3e5m2_64x64x32_test(); + run_f32e5m2e4m3e5m2_64x96x32_test(); + run_f32e5m2e4m3e5m2_64x128x32_test(); + run_f32e5m2e4m3e5m2_64x192x32_test(); + run_f32e5m2e4m3e5m2_64x256x32_test(); + run_f32e5m2e5m2e5m2_64x8x32_test(); + run_f32e5m2e5m2e5m2_64x16x32_test(); + run_f32e5m2e5m2e5m2_64x32x32_test(); + run_f32e5m2e5m2e5m2_64x64x32_test(); + run_f32e5m2e5m2e5m2_64x96x32_test(); + run_f32e5m2e5m2e5m2_64x128x32_test(); + run_f32e5m2e5m2e5m2_64x192x32_test(); + run_f32e5m2e5m2e5m2_64x256x32_test(); +} + +void run_int32accumulator_tests() { + run_int32s8s8s8_64x8x32_test(); + run_int32s8s8s8_64x16x32_test(); + run_int32s8s8s8_64x32x32_test(); + run_int32s8s8s8_64x64x32_test(); + run_int32s8s8s8_64x96x32_test(); + run_int32s8s8s8_64x128x32_test(); + run_int32s8s8s8_64x192x32_test(); + run_int32s8s8s8_64x256x32_test(); + run_int32s8u8s8_64x8x32_test(); + run_int32s8u8s8_64x16x32_test(); + run_int32s8u8s8_64x32x32_test(); + run_int32s8u8s8_64x64x32_test(); + run_int32s8u8s8_64x96x32_test(); + run_int32s8u8s8_64x128x32_test(); + run_int32s8u8s8_64x192x32_test(); + run_int32s8u8s8_64x256x32_test(); + run_int32u8s8u8_64x8x32_test(); + run_int32u8s8u8_64x16x32_test(); + run_int32u8s8u8_64x32x32_test(); + run_int32u8s8u8_64x64x32_test(); + run_int32u8s8u8_64x96x32_test(); + run_int32u8s8u8_64x128x32_test(); + run_int32u8s8u8_64x192x32_test(); + run_int32u8s8u8_64x256x32_test(); + run_int32u8u8u8_64x8x32_test(); + run_int32u8u8u8_64x16x32_test(); + run_int32u8u8u8_64x32x32_test(); + run_int32u8u8u8_64x64x32_test(); + run_int32u8u8u8_64x96x32_test(); + run_int32u8u8u8_64x128x32_test(); + run_int32u8u8u8_64x192x32_test(); + run_int32u8u8u8_64x256x32_test(); +} + +// ============================================================================ +// Main Test Function - Run All Configurations +// ============================================================================ + +inline void run_all_wgmma_latency_tests() { + printf("\n"); + printf("================================================================================\n"); + printf(" SM90 GMMA Latency Comprehensive Sweep\n"); + printf("================================================================================\n"); + printf("\n"); + + // Run F32 accumulator tests + run_f32accumulator_tests(); + + // Run F16 accumulator tests + run_f16accumulator_tests(); + + // Run INT32 accumulator tests + run_int32accumulator_tests(); + + printf("================================================================================\n"); + printf(" Sweep Complete\n"); + printf("================================================================================\n"); + printf("\n"); +} + +// Legacy function signatures for compatibility +float gmma_latency_ss() { + printf("Running comprehensive WGMMA latency tests...\n"); + run_all_wgmma_latency_tests(); + return 0.0f; +} + +#endif diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_common.h b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_common.h new file mode 100644 index 000000000..53c70f0f4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_common.h @@ -0,0 +1,210 @@ +/*************************************************************************************************** + * GMMA Latency Microbenchmark - Common Definitions + * + * This header contains shared kernel templates and helper macros used by all GMMA latency tests. + * + **************************************************************************************************/ + +#ifndef LAT_GMMA_COMMON_H +#define LAT_GMMA_COMMON_H + +#include +#include +#include +#include +#include +#include + +// Output CSV file; defined in lat_gmma.cu, NULL disables file output. +extern FILE* g_lat_gmma_outfile; + +#include "cute/arch/util.hpp" +#include "../../../hw_def/hw_def.h" + +// CUTLASS cute library headers +#include +#include "cutlass/numeric_types.h" +#include +#include +#include +#include +#include +#include + +using namespace cute; + +// ============================================================================ +// Base Kernel Template +// ============================================================================ + +template< + class ElementA, + class ElementB, + class ElementC, + class TileShape_MNK, + class RepeatTimes = cute::Int<1024> +> +__global__ void wgmma_latency_kernel(uint32_t *startClk, uint32_t *stopClk, uint32_t *checksum) { + int thread_idx = threadIdx.x + blockDim.x * threadIdx.y + threadIdx.z * blockDim.x * blockDim.y ; + int warp_group_idx = __shfl_sync(0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0); + + static constexpr GMMA::Major GmmaMajorA = cute::GMMA::Major::K; + static constexpr GMMA::Major GmmaMajorB = cute::GMMA::Major::K; + + // Create the GMMA operation + auto gmma_op = cute::GMMA::ss_op_selector< + ElementA, ElementB, ElementC, TileShape_MNK, GmmaMajorA, GmmaMajorB>(); + using MMA_Op = decltype(gmma_op); + + // Create the TiledMma based on element types and tile shape + using TiledMma = decltype(cute::make_tiled_mma(gmma_op)); + using MMA_Traits = typename TiledMma::Traits; + TiledMma tiled_mma; + MMA_Traits traits; + + // Create the fragment A, B, C + // Define the smem layouts using GMMA layout helpers for K-major + // Using Layout_K_INTER_Atom which has minimal swizzling (Swizzle<0,4,3> = identity) + // Layout_K_INTER_Atom_Bits has shape (8, 128 bits) = (8, 4) for 32-bit elements + constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape(GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape(GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int{}))); + + // Allocate shared memory with proper size (using type aliases for constexpr evaluation) + // There will be warnings as CUDA cannot determine if the sharedmem size is static + // But for this ubench case, since we are only measuring GMMA instruction, + // we can ignore the warnings on shmem allocation. + // CUTLASS bypass this by passing the shmem size during kernel launch via + // sizeof(typename GemmKernel::SharedStorage), but this means we have to + // rewrite this function as part of a Class to access the SharedStorage type + // defined with the M,N,K shape. + __shared__ ElementA smem_A[cosize_v]; + __shared__ ElementB smem_B[cosize_v]; + + // Create the layout objects for tensor construction + SmemLayoutA sA_layout{}; + SmemLayoutB sB_layout{}; + + + // Create the tensors with GMMA-compatible layouts + Tensor sA = make_tensor(make_smem_ptr(smem_A), sA_layout); // (BLK_M, BLK_K, PIPE) + Tensor sB = make_tensor(make_smem_ptr(smem_B), sB_layout); // (BLK_N, BLK_K, PIPE) + constexpr int MmaWarpGroups = size(TiledMma{}) / cutlass::NumThreadsPerWarpGroup; + Layout warp_group_thread_layout = make_layout(Int{}, + Int{}); + auto thread_mma = tiled_mma.get_slice(warp_group_thread_layout(warp_group_idx)); + + Tensor tCsA = thread_mma.partition_A(sA); // (MMA,MMA_M,MMA_K,PIPE) + Tensor tCsB = thread_mma.partition_B(sB); // (MMA,MMA_N,MMA_K,PIPE) + + // Allocate "fragments/descriptors" + Tensor tCrA = thread_mma.make_fragment_A(tCsA); // (MMA,MMA_M,MMA_K,PIPE) + Tensor tCrB = thread_mma.make_fragment_B(tCsB); // (MMA,MMA_N,MMA_K,PIPE) + + // Get fragment registers for accumulator with MN size + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape_MNK{})); + + __syncthreads(); + + // Start timing (only thread 0) + uint32_t start = 0; + if (thread_idx == 0) { + asm volatile("mov.u32 %0, %%clock;" : "=r"(start) :: "memory"); + } + __syncthreads(); + + // Fence accumulator operands + warpgroup_fence_operand(accum); + + // Arrive and execute WGMMA + warpgroup_arrive(); + constexpr int repeat_times = static_value(); + + #pragma unroll + for (int j = 0; j < repeat_times; j++) { + // Call the fma method + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + } + // Wait for WGMMA to complete + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + __syncthreads(); + + // Stop timing + uint32_t stop = 0; + if (thread_idx == 0) { + asm volatile("mov.u32 %0, %%clock;" : "=r"(stop) :: "memory"); + } + + // Write results + if (thread_idx == 0) { + startClk[blockIdx.x] = start; + stopClk[blockIdx.x] = stop; + + // Compute checksum to prevent optimization + uint32_t sum = reinterpret_cast(accum.data())[0]; + // Simple checksum over accumulator + checksum[blockIdx.x] = sum; + } +} + +// ============================================================================ +// Host Function Template +// ============================================================================ + +template> +float run_wgmma_latency_test_typed() { + // Allocate device memory + uint32_t *startClk_g, *stopClk_g, *checksum_g; + gpuErrchk(cudaMalloc(&startClk_g, sizeof(uint32_t))); + gpuErrchk(cudaMalloc(&stopClk_g, sizeof(uint32_t))); + gpuErrchk(cudaMalloc(&checksum_g, sizeof(uint32_t))); + + // Launch kernel with 128 threads (warpgroup size) + dim3 grid(1); + dim3 block(128); + wgmma_latency_kernel<<>>(startClk_g, stopClk_g, checksum_g); + + gpuErrchk(cudaPeekAtLastError()); + gpuErrchk(cudaDeviceSynchronize()); + + // Copy results back + uint32_t startClk, stopClk, checksum; + gpuErrchk(cudaMemcpy(&startClk, startClk_g, sizeof(uint32_t), cudaMemcpyDeviceToHost)); + gpuErrchk(cudaMemcpy(&stopClk, stopClk_g, sizeof(uint32_t), cudaMemcpyDeviceToHost)); + gpuErrchk(cudaMemcpy(&checksum, checksum_g, sizeof(uint32_t), cudaMemcpyDeviceToHost)); + + // Calculate latency + constexpr int repeat_times = static_value(); + float latency = ((float)(stopClk - startClk)) / ((float)repeat_times); + + // Cleanup + cudaFree(startClk_g); + cudaFree(stopClk_g); + cudaFree(checksum_g); + + return latency; +} + +// ============================================================================ +// Helper Macro for Testing +// ============================================================================ + +#define TEST_MMA_CONFIG(EA, EB, EC, M, N, K, DESC) \ + do { \ + try { \ + using TileShape = decltype(make_shape(Int{}, Int{}, Int{})); \ + float lat = run_wgmma_latency_test_typed(); \ + printf("%-50s: %6.2f cycles\n", DESC, lat); \ + if (g_lat_gmma_outfile) fprintf(g_lat_gmma_outfile, "%s,%.2f\n", DESC, lat); \ + } catch (...) { \ + printf("%-50s: FAILED\n", DESC); \ + if (g_lat_gmma_outfile) fprintf(g_lat_gmma_outfile, "%s,FAILED\n", DESC); \ + } \ + } while(0) + +#endif // LAT_GMMA_COMMON_H diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_test.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_test.cu new file mode 100644 index 000000000..a216f65a8 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/lat_gmma/lat_gmma_test.cu @@ -0,0 +1,17 @@ +#include +#include "lat_gmma_common.h" + +int main() +{ + // A simple test kernel to check if the library is working + try { + using TileShape = decltype(make_shape(cute::Int<64>{}, cute::Int<256>{}, cute::Int<16>{})); + // Repeat once for the test + using RepeatTimes = cute::Int<1>; + float lat = run_wgmma_latency_test_typed(); + printf("MMA_64x256x16_F32F16F16_SS: %6.2f cycles\n", lat); + } catch (...) { + printf("MMA_64x256x16_F32F16F16_SS: FAILED\n"); + } + return 0; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu index f6fb953f4..c95b6102f 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu @@ -7,20 +7,26 @@ // the D registers hold the running sum. The simulator must preserve the D // register file across wgmma commits and not reset it between iterations. // -// Two tests using f16→f32 m64n16k16: +// Two tests using f16->f32 m64n16k16: // -// static_4tiles – same smem (A=B=1) used for 4 back-to-back MMAs in one group. -// All 4 contribute K=16 each → D[m][n] = 4*K = 64. +// static_4tiles - same smem (A=B=1) used for 4 back-to-back MMAs in one group. +// All 4 contribute K=16 each -> D[m][n] = 4*K = 64. // No smem reload; one commit/wait covers all 4 ops. // -// varying_4tiles – smem reloaded each tile with A[tile] = (tile+1), B = 1. -// tile 0: A=1 → K*1 = 16 -// tile 1: A=2 → K*2 = 32 -// tile 2: A=3 → K*3 = 48 -// tile 3: A=4 → K*4 = 64 +// varying_4tiles - smem reloaded each tile with A[tile] = (tile+1), B = 1. +// tile 0: A=1 -> K*1 = 16 +// tile 1: A=2 -> K*2 = 32 +// tile 2: A=3 -> K*3 = 48 +// tile 3: A=4 -> K*4 = 64 // Total: K*(1+2+3+4) = 160 // Each tile is a separate commit/wait group with smem reload. // +// Dumps for silicon-vs-sim comparison: +// smem_A / smem_B : raw smem content before WGMMA (verifies descriptor layout) +// D registers : final D values (verifies WGMMA computation) +// Per-tile D snap : intermediate D after each tile in varying_4tiles +// Clock timing : %clock delta around WGMMA ops +// // Exit status: 0 = all PASS, 1 = any FAIL. // ============================================================================= @@ -92,9 +98,14 @@ __device__ __forceinline__ void mma_f16( // --------------------------------------------------------------------------- // Kernel 1: 4 MMAs in one group, constant smem (A=B=1). -// D accumulates across all 4 ops → expected = 4*K = 64. +// D accumulates across all 4 ops -> expected = 4*K = 64. +// Dumps: smem A/B (before WGMMA), D registers (after), clock timing. // --------------------------------------------------------------------------- -__global__ void kernel_static4(const half* A_g, const half* B_g, float* D_g) { +__global__ void kernel_static4( + const half* A_g, const half* B_g, + float* D_g, uint32_t* clk_out, + half* smA_raw, half* smB_raw) +{ __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; for (int i = tid; i < M*K; i += WGSIZE) { @@ -106,9 +117,22 @@ __global__ void kernel_static4(const half* A_g, const half* B_g, float* D_g) { *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; } __syncthreads(); + + // Dump raw smem content (physical byte order) before WGMMA + for (int i = tid; i < M*K; i += WGSIZE) + smA_raw[i] = *(const half*)(smA + i * E); + for (int i = tid; i < K*N; i += WGSIZE) + smB_raw[i] = *(const half*)(smB + i * E); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t_start = 0, t_stop = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_start) :: "memory"); + __syncthreads(); + WGMMA_FENCE; mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // D += K (D was 0) mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); // D += K @@ -117,6 +141,11 @@ __global__ void kernel_static4(const half* A_g, const half* B_g, float* D_g) { WGMMA_COMMIT; WGMMA_WAIT; WGMMA_FENCE; + + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_stop) :: "memory"); + if (tid == 0) { clk_out[0] = t_start; clk_out[1] = t_stop; } + const int b = tid * D_ELEMS; D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; @@ -127,12 +156,23 @@ __global__ void kernel_static4(const half* A_g, const half* B_g, float* D_g) { // A_g layout: [4 * M * K], where A_g[tile*M*K + m*K + k] = (half)(tile+1). // B_g: [K * N] all ones (shared across tiles). // Expected D[m][n] = K*(1+2+3+4) = 160. +// +// Dumps: +// smA_raw/smB_raw : physical smem content of tile 0 (before first WGMMA) +// D_tiles : D register snapshot after each of the 4 tiles +// (4 * WGSIZE * D_ELEMS floats, tile-major) +// clk_out : clock_start/stop bracketing all 4 tiles // --------------------------------------------------------------------------- -__global__ void kernel_varying4(const half* A_g, const half* B_g, float* D_g) { +__global__ void kernel_varying4( + const half* A_g, const half* B_g, + float* D_g, uint32_t* clk_out, + half* smA_raw, half* smB_raw, + float* D_tiles) +{ __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - // Load B once — same for all tiles. + // Load B once - same for all tiles. for (int i = tid; i < K*N; i += WGSIZE) { int k = i/N, n = i%N; *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; @@ -144,14 +184,32 @@ __global__ void kernel_varying4(const half* A_g, const half* B_g, float* D_g) { *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[0*M*K + m*K + k]; } __syncthreads(); + + // Dump tile 0 smem (physical byte order) before any WGMMA + for (int i = tid; i < M*K; i += WGSIZE) + smA_raw[i] = *(const half*)(smA + i * E); + for (int i = tid; i < K*N; i += WGSIZE) + smB_raw[i] = *(const half*)(smB + i * E); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t_start = 0, t_stop = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_start) :: "memory"); + __syncthreads(); + WGMMA_FENCE; mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; WGMMA_WAIT; + // Snapshot D after tile 0 (expected each element = K*1 = 16) + { const int b = tid * D_ELEMS, off = 0 * WGSIZE * D_ELEMS; + D_tiles[off+b]=d0; D_tiles[off+b+1]=d1; D_tiles[off+b+2]=d2; D_tiles[off+b+3]=d3; + D_tiles[off+b+4]=d4; D_tiles[off+b+5]=d5; D_tiles[off+b+6]=d6; D_tiles[off+b+7]=d7; } + // Tile 1: A = 2.0 WGMMA_FENCE; // post-wait: releases D, signals smem can be updated for (int i = tid; i < M*K; i += WGSIZE) { @@ -164,6 +222,11 @@ __global__ void kernel_varying4(const half* A_g, const half* B_g, float* D_g) { WGMMA_COMMIT; WGMMA_WAIT; + // Snapshot D after tile 1 (expected each element = K*(1+2) = 48) + { const int b = tid * D_ELEMS, off = 1 * WGSIZE * D_ELEMS; + D_tiles[off+b]=d0; D_tiles[off+b+1]=d1; D_tiles[off+b+2]=d2; D_tiles[off+b+3]=d3; + D_tiles[off+b+4]=d4; D_tiles[off+b+5]=d5; D_tiles[off+b+6]=d6; D_tiles[off+b+7]=d7; } + // Tile 2: A = 3.0 WGMMA_FENCE; for (int i = tid; i < M*K; i += WGSIZE) { @@ -176,6 +239,11 @@ __global__ void kernel_varying4(const half* A_g, const half* B_g, float* D_g) { WGMMA_COMMIT; WGMMA_WAIT; + // Snapshot D after tile 2 (expected each element = K*(1+2+3) = 96) + { const int b = tid * D_ELEMS, off = 2 * WGSIZE * D_ELEMS; + D_tiles[off+b]=d0; D_tiles[off+b+1]=d1; D_tiles[off+b+2]=d2; D_tiles[off+b+3]=d3; + D_tiles[off+b+4]=d4; D_tiles[off+b+5]=d5; D_tiles[off+b+6]=d6; D_tiles[off+b+7]=d7; } + // Tile 3: A = 4.0 WGMMA_FENCE; for (int i = tid; i < M*K; i += WGSIZE) { @@ -189,6 +257,110 @@ __global__ void kernel_varying4(const half* A_g, const half* B_g, float* D_g) { WGMMA_WAIT; WGMMA_FENCE; // final fence + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_stop) :: "memory"); + if (tid == 0) { clk_out[0] = t_start; clk_out[1] = t_stop; } + + // Snapshot D after tile 3 (final, expected each element = K*(1+2+3+4) = 160) + { const int b = tid * D_ELEMS, off = 3 * WGSIZE * D_ELEMS; + D_tiles[off+b]=d0; D_tiles[off+b+1]=d1; D_tiles[off+b+2]=d2; D_tiles[off+b+3]=d3; + D_tiles[off+b+4]=d4; D_tiles[off+b+5]=d5; D_tiles[off+b+6]=d6; D_tiles[off+b+7]=d7; } + + const int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// Kernel 3: Fence overhead. +// Measures the cost of wgmma.fence.sync.aligned in isolation. +// N_FENCE consecutive FENCEs; no WGMMA ops. +// cycles_per_fence = (stop - start) / N_FENCE +// --------------------------------------------------------------------------- +static constexpr int N_FENCE = 256; + +__global__ void kernel_fence_overhead(uint32_t* clk_out) { + const int tid = threadIdx.x; + uint32_t t_start = 0, t_stop = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_start) :: "memory"); + __syncthreads(); + #pragma unroll + for (int i = 0; i < N_FENCE; i++) + WGMMA_FENCE; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_stop) :: "memory"); + if (tid == 0) { clk_out[0] = t_start; clk_out[1] = t_stop; } +} + +// --------------------------------------------------------------------------- +// Kernel 4: Commit+Wait barrier overhead (no MMA ops). +// Commits an empty wgmma group (no ops issued) and waits on it. +// This measures the pure synchronization barrier cost of COMMIT+WAIT. +// N_BARRIER iterations; cycles_per_barrier = (stop - start) / N_BARRIER. +// --------------------------------------------------------------------------- +static constexpr int N_BARRIER = 256; + +__global__ void kernel_commit_wait_overhead(uint32_t* clk_out) { + const int tid = threadIdx.x; + uint32_t t_start = 0, t_stop = 0; + WGMMA_FENCE; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_start) :: "memory"); + __syncthreads(); + #pragma unroll + for (int i = 0; i < N_BARRIER; i++) { + WGMMA_COMMIT; + WGMMA_WAIT; + } + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_stop) :: "memory"); + if (tid == 0) { clk_out[0] = t_start; clk_out[1] = t_stop; } +} + +// --------------------------------------------------------------------------- +// Kernel 5: Single-shape MMA throughput (m64n16k16 f32<-f16xf16). +// N_MMA back-to-back MMA ops in one commit group. +// Includes smem setup; clock brackets fence..commit_wait..fence. +// cycles_per_mma = (stop - start) / N_MMA (includes fence+commit+wait amortised). +// --------------------------------------------------------------------------- +static constexpr int N_MMA = 1024; + +__global__ void kernel_mma_throughput( + const half* A_g, const half* B_g, float* D_g, uint32_t* clk_out) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + for (int i = tid; i < M*K; i += WGSIZE) { + int m = i/K, k = i%K; + *(half*)(smA + smem_off(k, m, E, LBO_A)) = A_g[i]; + } + for (int i = tid; i < K*N; i += WGSIZE) { + int k = i/N, n = i%N; + *(half*)(smB + smem_off(k, n, E, LBO_B)) = B_g[i]; + } + __syncthreads(); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t_start = 0, t_stop = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_start) :: "memory"); + __syncthreads(); + + WGMMA_FENCE; + #pragma unroll 8 + for (int i = 0; i < N_MMA; i++) + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT; + WGMMA_FENCE; + + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t_stop) :: "memory"); + if (tid == 0) { clk_out[0] = t_start; clk_out[1] = t_stop; } + const int b = tid * D_ELEMS; D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; @@ -215,7 +387,7 @@ static void reassemble(const float* flat, float D[M][N]) { } } -static bool check_uniform(const char* name, float D[M][N], float expected) { +static bool check_uniform(const char* name, float D[M][N], float expected, FILE* outfile = NULL) { float max_err = 0.f; int max_m = 0, max_n = 0; for (int m = 0; m < M; m++) @@ -230,18 +402,113 @@ static bool check_uniform(const char* name, float D[M][N], float expected) { if (!pass) printf(" Worst: D[%d][%d] = %.6f expected = %.6f\n", max_m, max_n, D[max_m][max_n], expected); + if (outfile) + fprintf(outfile, "%s,%s,%.1f,%.4e\n", name, pass ? "PASS" : "FAIL", expected, max_err); return pass; } +// Write a comprehensive dump file for silicon-vs-sim diff comparison. +// smA_raw/smB_raw: raw physical smem content (M*K and K*N fp16 values). +// flat : per-thread D registers (WGSIZE*D_ELEMS floats). +// D : reassembled D[M][N] matrix. +// D_tiles : per-tile D snapshots (num_tiles * WGSIZE*D_ELEMS), NULL if none. +// num_tiles : number of tile snapshots in D_tiles. +// t_start/stop : raw %clock values. +static void dump_results( + const char* filename, const char* test_name, + const half* smA_raw, const half* smB_raw, + const float* flat, float D[M][N], + const float* D_tiles, int num_tiles, + uint32_t t_start, uint32_t t_stop, + int num_mma_ops) +{ + FILE* f = fopen(filename, "w"); + if (!f) { fprintf(stderr, "cannot open %s\n", filename); return; } + + fprintf(f, "=== %s ===\n", test_name); + fprintf(f, "[timing]\n"); + fprintf(f, "clock_start=%u\n", t_start); + fprintf(f, "clock_stop=%u\n", t_stop); + fprintf(f, "total_cycles=%u\n", t_stop - t_start); + if (num_mma_ops > 0) + fprintf(f, "cycles_per_mma=%.2f\n", (float)(t_stop - t_start) / num_mma_ops); + + // Physical smem layout — what the WGMMA descriptor sees + fprintf(f, "\n[smem_A physical M=%d K=%d halfwords]\n", M, K); + for (int i = 0; i < M*K; i++) + fprintf(f, "%d %.6f\n", i, __half2float(smA_raw[i])); + + fprintf(f, "\n[smem_B physical K=%d N=%d halfwords]\n", K, N); + for (int i = 0; i < K*N; i++) + fprintf(f, "%d %.6f\n", i, __half2float(smB_raw[i])); + + // Logical smem layout via smem_off mapping (shows which (k,m)/(k,n) -> physical index) + fprintf(f, "\n[smem_A logical k m phys_idx value]\n"); + for (int k = 0; k < K; k++) + for (int m = 0; m < M; m++) { + int off = smem_off(k, m, E, LBO_A); + int idx = off / E; + fprintf(f, "%d %d %d %.6f\n", k, m, idx, __half2float(smA_raw[idx])); + } + + fprintf(f, "\n[smem_B logical k n phys_idx value]\n"); + for (int k = 0; k < K; k++) + for (int n = 0; n < N; n++) { + int off = smem_off(k, n, E, LBO_B); + int idx = off / E; + fprintf(f, "%d %d %d %.6f\n", k, n, idx, __half2float(smB_raw[idx])); + } + + // Per-tile D snapshots (if provided) + if (D_tiles && num_tiles > 0) { + static const float tile_expected[] = {16.f, 48.f, 96.f, 160.f}; + for (int t = 0; t < num_tiles; t++) { + fprintf(f, "\n[D_tile_%d thread elem value expected=%.1f]\n", + t, (t < 4) ? tile_expected[t] : 0.f); + const float* snap = D_tiles + t * WGSIZE * D_ELEMS; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) + fprintf(f, "%d %d %.6f\n", T, e, snap[T*D_ELEMS + e]); + } + } + + // Final D registers (per-thread, raw) + fprintf(f, "\n[D_registers thread elem value]\n"); + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) + fprintf(f, "%d %d %.6f\n", T, e, flat[T*D_ELEMS + e]); + + // Thread-to-matrix mapping + D matrix + fprintf(f, "\n[thread_to_matrix thread elem row col]\n"); + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, &row, &col); + fprintf(f, "%d %d %d %d\n", T, e, row, col); + } + + fprintf(f, "\n[D_matrix row col value]\n"); + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) + fprintf(f, "%d %d %.6f\n", m, n, D[m][n]); + + fclose(f); + printf(" -> dumped to %s\n", filename); +} + // =========================================================================== // main // =========================================================================== int main() { cudaDeviceProp prop; cudaGetDeviceProperties(&prop, 0); - printf("wgmma_multik — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("wgmma_multik - %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); printf("================================================================\n"); + FILE* outfile = fopen("wgmma_multik_results.csv", "w"); + if (outfile) + fprintf(outfile, "test,status,expected,max_err\n"); + bool all_pass = true; static float D[M][N]; @@ -250,46 +517,153 @@ int main() { { std::vector h_A(M*K, __float2half(1.f)); std::vector h_B(K*N, __float2half(1.f)); - half *d_A, *d_B; float *d_D; - cudaMalloc(&d_A, h_A.size()*sizeof(half)); - cudaMalloc(&d_B, h_B.size()*sizeof(half)); - cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + half *d_A, *d_B; float *d_D; uint32_t *d_clk; + half *d_smA_raw, *d_smB_raw; + cudaMalloc(&d_A, h_A.size()*sizeof(half)); + cudaMalloc(&d_B, h_B.size()*sizeof(half)); + cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + cudaMalloc(&d_clk, 2*sizeof(uint32_t)); + cudaMalloc(&d_smA_raw, M*K*sizeof(half)); + cudaMalloc(&d_smB_raw, K*N*sizeof(half)); cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); - kernel_static4<<<1, WGSIZE>>>(d_A, d_B, d_D); + + kernel_static4<<<1, WGSIZE>>>(d_A, d_B, d_D, d_clk, d_smA_raw, d_smB_raw); cudaDeviceSynchronize(); - std::vector flat(WGSIZE*D_ELEMS); - cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + + std::vector flat(WGSIZE*D_ELEMS); + std::vector clk(2); + std::vector smA_raw(M*K), smB_raw(K*N); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), d_clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaMemcpy(smA_raw.data(), d_smA_raw, M*K*sizeof(half), cudaMemcpyDeviceToHost); + cudaMemcpy(smB_raw.data(), d_smB_raw, K*N*sizeof(half), cudaMemcpyDeviceToHost); cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + cudaFree(d_clk); cudaFree(d_smA_raw); cudaFree(d_smB_raw); + reassemble(flat.data(), D); - all_pass &= check_uniform("static_4tiles", D, (float)(4 * K)); + all_pass &= check_uniform("static_4tiles", D, (float)(4 * K), outfile); + + printf(" clock: start=%u stop=%u cycles=%u (%.2f cyc/mma)\n", + clk[0], clk[1], clk[1]-clk[0], (float)(clk[1]-clk[0])/4.f); + + dump_results("wgmma_multik_static4.txt", "static_4tiles", + smA_raw.data(), smB_raw.data(), + flat.data(), D, + nullptr, 0, + clk[0], clk[1], 4); } // --- Test 2: varying_4tiles --- printf("\n--- varying_4tiles (A[tile]=tile+1, B=1, expected D=160) ---\n"); { - // A_g: [4*M*K] where tile t has all elements = float(t+1) std::vector h_A(4 * M * K); for (int t = 0; t < 4; t++) for (int i = 0; i < M*K; i++) h_A[t*M*K + i] = __float2half((float)(t + 1)); std::vector h_B(K*N, __float2half(1.f)); - half *d_A, *d_B; float *d_D; - cudaMalloc(&d_A, h_A.size()*sizeof(half)); - cudaMalloc(&d_B, h_B.size()*sizeof(half)); - cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + half *d_A, *d_B; float *d_D; uint32_t *d_clk; + half *d_smA_raw, *d_smB_raw; + float *d_D_tiles; + cudaMalloc(&d_A, h_A.size()*sizeof(half)); + cudaMalloc(&d_B, h_B.size()*sizeof(half)); + cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + cudaMalloc(&d_clk, 2*sizeof(uint32_t)); + cudaMalloc(&d_smA_raw, M*K*sizeof(half)); + cudaMalloc(&d_smB_raw, K*N*sizeof(half)); + cudaMalloc(&d_D_tiles, 4*WGSIZE*D_ELEMS*sizeof(float)); cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); - kernel_varying4<<<1, WGSIZE>>>(d_A, d_B, d_D); + + kernel_varying4<<<1, WGSIZE>>>(d_A, d_B, d_D, d_clk, d_smA_raw, d_smB_raw, d_D_tiles); cudaDeviceSynchronize(); - std::vector flat(WGSIZE*D_ELEMS); - cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + + std::vector flat(WGSIZE*D_ELEMS); + std::vector clk(2); + std::vector smA_raw(M*K), smB_raw(K*N); + std::vector D_tiles(4*WGSIZE*D_ELEMS); + cudaMemcpy(flat.data(), d_D, flat.size()*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), d_clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaMemcpy(smA_raw.data(), d_smA_raw, M*K*sizeof(half), cudaMemcpyDeviceToHost); + cudaMemcpy(smB_raw.data(), d_smB_raw, K*N*sizeof(half), cudaMemcpyDeviceToHost); + cudaMemcpy(D_tiles.data(), d_D_tiles, D_tiles.size()*sizeof(float), cudaMemcpyDeviceToHost); cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); + cudaFree(d_clk); cudaFree(d_smA_raw); cudaFree(d_smB_raw); cudaFree(d_D_tiles); + reassemble(flat.data(), D); - // Expected: K*(1+2+3+4) = 16*10 = 160 - all_pass &= check_uniform("varying_4tiles", D, (float)(K * 10)); + all_pass &= check_uniform("varying_4tiles", D, (float)(K * 10), outfile); + + printf(" clock: start=%u stop=%u cycles=%u (%.2f cyc/mma)\n", + clk[0], clk[1], clk[1]-clk[0], (float)(clk[1]-clk[0])/4.f); + + dump_results("wgmma_multik_varying4.txt", "varying_4tiles", + smA_raw.data(), smB_raw.data(), + flat.data(), D, + D_tiles.data(), 4, + clk[0], clk[1], 4); + } + + // --- Timing breakdown: fence overhead --- + printf("\n--- fence_overhead (N=%d WGMMA_FENCEs, no MMA) ---\n", N_FENCE); + { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2*sizeof(uint32_t)); + kernel_fence_overhead<<<1, WGSIZE>>>(d_clk); + cudaDeviceSynchronize(); + std::vector clk(2); + cudaMemcpy(clk.data(), d_clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + float cyc_per_fence = (float)(clk[1]-clk[0]) / N_FENCE; + printf(" clock: start=%u stop=%u total=%u cyc/fence=%.2f\n", + clk[0], clk[1], clk[1]-clk[0], cyc_per_fence); + if (outfile) + fprintf(outfile, "fence_overhead,TIMING,N/A,cyc_per_fence=%.2f\n", cyc_per_fence); + } + + // --- Timing breakdown: commit+wait barrier overhead (no MMA ops) --- + printf("\n--- commit_wait_overhead (N=%d empty COMMIT+WAIT, no MMA) ---\n", N_BARRIER); + { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2*sizeof(uint32_t)); + kernel_commit_wait_overhead<<<1, WGSIZE>>>(d_clk); + cudaDeviceSynchronize(); + std::vector clk(2); + cudaMemcpy(clk.data(), d_clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + float cyc_per_barrier = (float)(clk[1]-clk[0]) / N_BARRIER; + printf(" clock: start=%u stop=%u total=%u cyc/barrier=%.2f\n", + clk[0], clk[1], clk[1]-clk[0], cyc_per_barrier); + if (outfile) + fprintf(outfile, "commit_wait_overhead,TIMING,N/A,cyc_per_barrier=%.2f\n", cyc_per_barrier); + } + + // --- Timing: MMA throughput (m64n16k16 f32<-f16xf16, N=%d ops) --- + printf("\n--- mma_throughput_m64n16k16 (N=%d MMAs, A=B=1) ---\n", N_MMA); + { + std::vector h_A(M*K, __float2half(1.f)); + std::vector h_B(K*N, __float2half(1.f)); + half *d_A, *d_B; float *d_D; uint32_t *d_clk; + cudaMalloc(&d_A, h_A.size()*sizeof(half)); + cudaMalloc(&d_B, h_B.size()*sizeof(half)); + cudaMalloc(&d_D, WGSIZE*D_ELEMS*sizeof(float)); + cudaMalloc(&d_clk, 2*sizeof(uint32_t)); + cudaMemcpy(d_A, h_A.data(), h_A.size()*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B.data(), h_B.size()*sizeof(half), cudaMemcpyHostToDevice); + kernel_mma_throughput<<<1, WGSIZE>>>(d_A, d_B, d_D, d_clk); + cudaDeviceSynchronize(); + std::vector clk(2); + cudaMemcpy(clk.data(), d_clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_D); cudaFree(d_clk); + float cyc_per_mma = (float)(clk[1]-clk[0]) / N_MMA; + printf(" clock: start=%u stop=%u total=%u cyc/mma=%.2f\n", + clk[0], clk[1], clk[1]-clk[0], cyc_per_mma); + if (outfile) + fprintf(outfile, "mma_throughput_m64n16k16,TIMING,N/A,cyc_per_mma=%.2f\n", cyc_per_mma); } + if (outfile) + fclose(outfile); + printf("\n================================================================\n"); printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); return all_pass ? 0 : 1; From 0d7aa0b1a9e995cfe57c0161c874ea1291a1da6c Mon Sep 17 00:00:00 2001 From: Ivorchu <41135347+Ivorchu@users.noreply.github.com> Date: Fri, 8 May 2026 20:54:03 -0400 Subject: [PATCH 11/35] wgmma sweep and rs verify --- .../ubench/wgmma/wgmma_rs_verify/.gitignore | 1 + .../ubench/wgmma/wgmma_rs_verify/Makefile | 12 + .../wgmma/wgmma_rs_verify/wgmma_rs_verify.cu | 481 +++++++++ .../ubench/wgmma/wgmma_sweep/.gitignore | 1 + .../ubench/wgmma/wgmma_sweep/Makefile | 12 + .../ubench/wgmma/wgmma_sweep/wgmma_sweep.cu | 951 ++++++++++++++++++ 6 files changed, 1458 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/.gitignore create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/.gitignore create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/wgmma_sweep.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/.gitignore b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/.gitignore new file mode 100644 index 000000000..08f514ebc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/.gitignore @@ -0,0 +1 @@ +include/ diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile new file mode 100644 index 000000000..3867f9231 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile @@ -0,0 +1,12 @@ +SRC = wgmma_rs_verify.cu +EXE = wgmma_rs_verify + +INCLUDE += -I../lat_gmma/include -I$(GPUAPPS_ROOT)/src/cuda/cutlass-bench/include -I./ + +ARCH ?= sm_90a +CUDA_CPPFLAGS= +HOPPER_CUDA_CPPFLAGS=$(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) + +NVCC_FLAGS := $(HOPPER_CUDA_CPPFLAGS) -std=c++17 + +include ../../../common/common.mk diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu new file mode 100644 index 000000000..16414ad8f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu @@ -0,0 +1,481 @@ +// ============================================================================= +// wgmma_rs_verify.cu — RS WGMMA correctness + A-register layout dump +// +// RS WGMMA = "Register-Source" WGMMA: A operand loaded into registers via +// ldmatrix before the wgmma.mma_async instruction. B still uses smem descriptor. +// This is distinct from SS WGMMA (both A and B from smem descriptors). +// +// Purpose: everything the simulator needs to implement RS WGMMA. +// +// PHASE 1 — RS correctness, column test +// A=1, B[k][n]=n => D[m][n] = K*n +// Verifies that RS WGMMA produces correct D and that the D-register +// position formula (d_frag_pos) is the same as for SS WGMMA. +// Covers: F32F16, F16F16, F32TF32, F32BF16 × N=8,16,32,64,96,128,192,256 +// Output: wgmma_rs_col_N.txt +// +// PHASE 2 — RS correctness, row test +// A[m][k]=m, B=1 => D[m][n] = K*m +// Jointly verifies: (a) ldmatrix correctly loaded A from smem, and +// (b) RS WGMMA computed the right product. +// If any ldmatrix address or swizzle is wrong, A values are misread +// and D deviates from K*m. +// Same shapes as Phase 1. +// Output: wgmma_rs_row_N.txt +// +// PHASE 3 — A register layout dump (NO WGMMA) +// A[m][k] = m*K + k (unique per matrix element) +// Loads smem_A into rA via ldmatrix, then dumps rA directly to global +// memory WITHOUT issuing WGMMA. For each (thread T, register element e): +// decoded_m = round(val) / K +// decoded_k = round(val) % K +// This gives the full (T, e) -> (m, k) mapping for the A register fragment — +// exactly what the simulator needs to implement RS WGMMA functional mode. +// Covers: F32F16 N=16 (K=16), F32TF32 N=16 (K=8) +// BF16 uses same element size / ldmatrix as F16; its layout is identical. +// Output: wgmma_rs_areg_N.txt +// +// Build: make (see Makefile) +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cute/arch/util.hpp" +#include +#include "cutlass/numeric_types.h" +#include +#include +#include +#include +#include +#include +#include // SM75_U32x4_LDSM_N (ldmatrix) + +using namespace cute; + +static constexpr int WGSIZE = 128; + +// --------------------------------------------------------------------------- +// Kernel 1 & 2: RS correctness (col test and row test) +// COL_MODE=true : A=1, B[k][n]=n => D[m][n] = K*n +// COL_MODE=false : A[m][k]=m, B=1 => D[m][n] = K*m +// --------------------------------------------------------------------------- +template +__global__ void kernel_rs_verify(EC* D_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + static constexpr int M_val = (int)get<0>(TileShape{}); + static constexpr int N_val = (int)get<1>(TileShape{}); + static constexpr int K_val = (int)get<2>(TileShape{}); + + auto rs_op = GMMA::rs_op_selector(); + using TiledMma = decltype(make_tiled_mma(rs_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + + if constexpr (COL_MODE) { + // A=1 everywhere + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_A[i] = EA(1.0f); + // B[k][n] = n (sB logical is (N, K, PIPE)) + for (int i = tid; i < N_val * K_val; i += WGSIZE) { + int n = i % N_val, k = i / N_val; + sB(n, k, 0) = EB(float(n)); + } + } else { + // A[m][k] = m (row index) + for (int i = tid; i < M_val * K_val; i += WGSIZE) { + int m = i / K_val, k = i % K_val; + sA(m, k, 0) = EA(float(m)); + } + // B=1 everywhere + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(1.0f); + } + __syncthreads(); + + // --- Load A from smem into registers via ldmatrix --- + // make_tiled_copy_A derives the right thread-to-smem mapping from the + // RS TiledMma's A tile layout (get_layoutA_TV). + // SM75_U32x4_LDSM_N = ldmatrix.sync.aligned.m8n8.x4.shared.b16 + auto tiled_copy_A = make_tiled_copy_A( + Copy_Atom{}, tiled_mma); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + + // For RS: make_fragment_A returns a REGISTER tensor (not a smem view). + // Use single-stage partition so tCrA is rank-3 (MMA, MMA_M, MMA_K), + // matching rank-3 output of partition_S(sA(_,_,0)) after retile_D. + auto tCsA_mma = thread_mma.partition_A(sA); + auto tCrA = thread_mma.make_fragment_A(tCsA_mma(_,_,_,0)); // rank-3, stage 0 + + auto smem_thr_copy_A = tiled_copy_A.get_thread_slice(tid); + auto tCsA_copy = smem_thr_copy_A.partition_S(sA(_,_,0)); // smem source (stage 0) + auto tCrA_view = smem_thr_copy_A.retile_D(tCrA); // register dest view + copy(tiled_copy_A, tCsA_copy, tCrA_view); // execute ldmatrix + + // --- Issue RS WGMMA --- + auto tCsB = thread_mma.partition_B(sB); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + warpgroup_fence_operand(accum); + warpgroup_arrive(); + cute::gemm(tiled_mma, tCrA, tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + const int D_ELEMS = (int)size(accum); + EC* base = D_out + tid * D_ELEMS; + for (int e = 0; e < D_ELEMS; e++) + base[e] = accum(e); +} + +// --------------------------------------------------------------------------- +// Kernel 3: A register layout dump — no WGMMA issued +// Fill A[m][k] = m*K+k (unique per element), load via ldmatrix, +// dump raw rA register values to global memory. +// Host decodes: m = round(val)/K, k = round(val)%K. +// --------------------------------------------------------------------------- +template +__global__ void kernel_rs_areg_dump(float* A_out, int* A_reg_size_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + static constexpr int M_val = (int)get<0>(TileShape{}); + static constexpr int K_val = (int)get<2>(TileShape{}); + + auto rs_op = GMMA::rs_op_selector(); + using TiledMma = decltype(make_tiled_mma(rs_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + + // A[m][k] = m*K + k (unique per matrix element) + for (int i = tid; i < M_val * K_val; i += WGSIZE) { + int m = i / K_val, k = i % K_val; + sA(m, k, 0) = EA(float(m * K_val + k)); + } + // B unused but smem needs to be valid to avoid issues + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(0.0f); + __syncthreads(); + + auto tiled_copy_A = make_tiled_copy_A( + Copy_Atom{}, tiled_mma); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + + auto tCsA_mma = thread_mma.partition_A(sA); + auto tCrA = thread_mma.make_fragment_A(tCsA_mma(_,_,_,0)); // rank-3, stage 0 + + auto smem_thr_copy_A = tiled_copy_A.get_thread_slice(tid); + auto tCsA_copy = smem_thr_copy_A.partition_S(sA(_,_,0)); + auto tCrA_view = smem_thr_copy_A.retile_D(tCrA); + copy(tiled_copy_A, tCsA_copy, tCrA_view); // ldmatrix → registers + + // Dump: no WGMMA, just write register values to global memory + const int A_regs = (int)size(tCrA); + if (tid == 0) *A_reg_size_out = A_regs; + for (int e = 0; e < A_regs; e++) + A_out[tid * A_regs + e] = (float)tCrA(e); +} + +// --------------------------------------------------------------------------- +// Host helpers +// --------------------------------------------------------------------------- + +// D-register position: (thread T, accumulator element e) -> (row, col) +// Valid for F32/S32 accumulator at all N. F16 accumulator uses same formula. +static void d_frag_pos(int T, int e, int /*N*/, int* row, int* col) { + int warp = T / 32, lane = T % 32; + int g = e / 4, k = e % 4; + *row = (lane / 4) * 2 + (k / 2) + warp * 16; + *col = (lane % 4) * 2 + (k % 2) + g * 8; +} + +template +static void run_verify(const char* dtype_tag, const char* name, int N, int K) { + const int D_ELEMS = N / 2; + const int total = WGSIZE * D_ELEMS; + std::vector h_D(total, EC(0)); + EC* d_D; + cudaMalloc(&d_D, total * sizeof(EC)); + cudaMemset(d_D, 0, total * sizeof(EC)); + + kernel_rs_verify<<<1, WGSIZE>>>(d_D); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" RS-%-3s %-50s KERNEL ERROR: %s\n", + COL_MODE ? "Col" : "Row", name, cudaGetErrorString(err)); + cudaFree(d_D); return; + } + cudaDeviceSynchronize(); + cudaMemcpy(h_D.data(), d_D, total * sizeof(EC), cudaMemcpyDeviceToHost); + cudaFree(d_D); + + char fname[128]; + snprintf(fname, sizeof(fname), "wgmma_rs_%s_%sN%d.txt", + COL_MODE ? "col" : "row", dtype_tag, N); + FILE* f = fopen(fname, "w"); + if (f) { + if (COL_MODE) + fprintf(f, "# %s RS: A=1 B[k][n]=n expected D[m][n]=K*n (K=%d)\n" + "# thread elem row col expected actual match\n", name, K); + else + fprintf(f, "# %s RS: A[m][k]=m B=1 expected D[m][n]=K*m (K=%d)\n" + "# thread elem row col expected actual match\n", name, K); + } + + int errors = 0, checked = 0; + for (int T = 0; T < WGSIZE; T++) { + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, N, &row, &col); + if (row >= 64 || col >= N) continue; + float expected = COL_MODE ? (float)(K * col) : (float)(K * row); + float actual = (float)h_D[T * D_ELEMS + e]; + bool ok = (fabsf(actual - expected) < 0.5f); + if (!ok) errors++; + checked++; + if (f) + fprintf(f, "%d %d %d %d %.1f %.1f %s\n", + T, e, row, col, expected, actual, ok ? "OK" : "FAIL"); + } + } + if (f) { fprintf(f, "# errors=%d / %d checked\n", errors, checked); fclose(f); } + + printf(" RS-%-3s %-50s errors=%d/%d -> %s\n", + COL_MODE ? "Col" : "Row", name, errors, checked, + errors == 0 ? "PASS" : "FAIL"); +} + +// A register layout dump: loads smem_A via ldmatrix, writes raw values. +// Produces (T, e) -> (m, k) mapping for the simulator to reference. +template +static void run_areg_dump(const char* dtype_tag, const char* name, int K) { + // A_reg_size per thread is K*M/(128) / (sizeof_EA_in_regs) — query from kernel + // Allocate conservatively: max is K*64/128 = K/2 per thread (for 2-byte types) + // For F16 K=16: 8 elements per thread. For TF32 K=8: 4 per thread. + const int MAX_REGS = 16; + const int total = WGSIZE * MAX_REGS; + std::vector h_A(total, 0.0f); + float* d_A; + int* d_cnt; + cudaMalloc(&d_A, total * sizeof(float)); + cudaMalloc(&d_cnt, sizeof(int)); + cudaMemset(d_A, 0, total * sizeof(float)); + cudaMemset(d_cnt, 0, sizeof(int)); + + kernel_rs_areg_dump<<<1, WGSIZE>>>(d_A, d_cnt); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" RS-AregDump %-46s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_A); cudaFree(d_cnt); return; + } + cudaDeviceSynchronize(); + cudaMemcpy(h_A.data(), d_A, total * sizeof(float), cudaMemcpyDeviceToHost); + int h_cnt = 0; + cudaMemcpy(&h_cnt, d_cnt, sizeof(int), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_cnt); + + if (h_cnt == 0 || h_cnt > MAX_REGS) { + printf(" RS-AregDump %-46s unexpected A_reg_size=%d\n", name, h_cnt); + return; + } + + char fname[128]; + snprintf(fname, sizeof(fname), "wgmma_rs_areg_%sN%d.txt", dtype_tag, + (int)get<1>(TileShape{})); + FILE* f = fopen(fname, "w"); + if (f) { + fprintf(f, "# %s A[m][k]=m*K+k loaded via ldmatrix -> rA registers\n", name); + fprintf(f, "# A_reg_size per thread = %d\n", h_cnt); + fprintf(f, "# K=%d (so decoded_m = round(val)/%d, decoded_k = round(val)%%%d)\n", + K, K, K); + fprintf(f, "# thread elem raw_value decoded_m decoded_k\n"); + } + + int decode_errors = 0; + for (int T = 0; T < WGSIZE; T++) { + for (int e = 0; e < h_cnt; e++) { + float val = h_A[T * MAX_REGS + e]; + int iv = (int)roundf(val); + int m = iv / K; + int k = iv % K; + bool ok = (m >= 0 && m < 64 && k >= 0 && k < K); + if (!ok) decode_errors++; + if (f) + fprintf(f, "%3d %2d %8.3f %2d %2d%s\n", + T, e, val, ok ? m : -1, ok ? k : -1, + ok ? "" : " [DECODE_ERROR]"); + } + } + if (f) { + fprintf(f, "# decode_errors=%d / %d entries\n", + decode_errors, WGSIZE * h_cnt); + fclose(f); + } + + printf(" RS-AregDump %-46s A_reg_size=%d decode_errors=%d/%d -> %s\n", + name, h_cnt, decode_errors, WGSIZE * h_cnt, + decode_errors == 0 ? "OK" : "DECODE_ERROR"); +} + +// --------------------------------------------------------------------------- +// Convenience macros +// --------------------------------------------------------------------------- +#define SHAPE(M,N,K) decltype(make_shape(Int{}, Int{}, Int{})) + +#define RUN_COL(EA, EB, EC, M, N, K, TAG, LABEL) \ + run_verify(TAG, LABEL, N, K) + +#define RUN_ROW(EA, EB, EC, M, N, K, TAG, LABEL) \ + run_verify(TAG, LABEL, N, K) + +#define RUN_DUMP(EA, EB, EC, M, N, K, TAG, LABEL) \ + run_areg_dump(TAG, LABEL, K) + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_rs_verify - %s (SM %d.%d)\n\n", prop.name, prop.major, prop.minor); + + // ----------------------------------------------------------------------- + // Phase 1: RS correctness — column test (A=1, B[k][n]=n => D=K*n) + // Verifies RS WGMMA computes correct D and d_frag_pos formula matches SS. + // ----------------------------------------------------------------------- + printf("=== Phase 1: RS column test (A=1, B[k][n]=n => D[m][n]=K*n) ===\n"); + printf(" %-55s errors/checked\n", "shape"); + + printf("\n -- F32 accumulator, F16 inputs RS (K=16) --\n"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32F16", "F32F16_m64n8k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16", "F32F16_m64n16k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32F16", "F32F16_m64n32k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16", "F32F16_m64n64k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32F16", "F32F16_m64n96k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32F16", "F32F16_m64n128k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32F16", "F32F16_m64n192k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32F16", "F32F16_m64n256k16"); + + printf("\n -- F16 accumulator, F16 inputs RS (K=16) --\n"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 8, 16, "F16F16", "F16F16_m64n8k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 16, 16, "F16F16", "F16F16_m64n16k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 32, 16, "F16F16", "F16F16_m64n32k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 64, 16, "F16F16", "F16F16_m64n64k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 96, 16, "F16F16", "F16F16_m64n96k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 128, 16, "F16F16", "F16F16_m64n128k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 192, 16, "F16F16", "F16F16_m64n192k16"); + RUN_COL(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 256, 16, "F16F16", "F16F16_m64n256k16"); + + printf("\n -- F32 accumulator, TF32 inputs RS (K=8) --\n"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 8, 8, "F32TF32", "F32TF32_m64n8k8"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "F32TF32", "F32TF32_m64n16k8"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 32, 8, "F32TF32", "F32TF32_m64n32k8"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 64, 8, "F32TF32", "F32TF32_m64n64k8"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 128, 8, "F32TF32", "F32TF32_m64n128k8"); + RUN_COL(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 256, 8, "F32TF32", "F32TF32_m64n256k8"); + + printf("\n -- F32 accumulator, BF16 inputs RS (K=16) --\n"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 8, 16, "F32BF16", "F32BF16_m64n8k16"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 16, 16, "F32BF16", "F32BF16_m64n16k16"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 32, 16, "F32BF16", "F32BF16_m64n32k16"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 64, 16, "F32BF16", "F32BF16_m64n64k16"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 128, 16, "F32BF16", "F32BF16_m64n128k16"); + RUN_COL(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 256, 16, "F32BF16", "F32BF16_m64n256k16"); + + // ----------------------------------------------------------------------- + // Phase 2: RS correctness — row test (A[m][k]=m, B=1 => D=K*m) + // Jointly verifies ldmatrix loaded A correctly AND RS WGMMA computed D. + // ----------------------------------------------------------------------- + printf("\n=== Phase 2: RS row test (A[m][k]=m, B=1 => D[m][n]=K*m) ===\n"); + printf(" %-55s errors/checked\n", "shape"); + + printf("\n -- F32 accumulator, F16 inputs RS (K=16) --\n"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32F16", "F32F16_m64n8k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16", "F32F16_m64n16k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32F16", "F32F16_m64n32k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16", "F32F16_m64n64k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32F16", "F32F16_m64n96k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32F16", "F32F16_m64n128k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32F16", "F32F16_m64n192k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32F16", "F32F16_m64n256k16"); + + printf("\n -- F16 accumulator, F16 inputs RS (K=16) --\n"); + RUN_ROW(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 16, 16, "F16F16", "F16F16_m64n16k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 64, 16, "F16F16", "F16F16_m64n64k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 128, 16, "F16F16", "F16F16_m64n128k16"); + RUN_ROW(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 256, 16, "F16F16", "F16F16_m64n256k16"); + + printf("\n -- F32 accumulator, TF32 inputs RS (K=8) --\n"); + RUN_ROW(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "F32TF32", "F32TF32_m64n16k8"); + RUN_ROW(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 64, 8, "F32TF32", "F32TF32_m64n64k8"); + RUN_ROW(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 128, 8, "F32TF32", "F32TF32_m64n128k8"); + + printf("\n -- F32 accumulator, BF16 inputs RS (K=16) --\n"); + RUN_ROW(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 16, 16, "F32BF16", "F32BF16_m64n16k16"); + RUN_ROW(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 64, 16, "F32BF16", "F32BF16_m64n64k16"); + RUN_ROW(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 256, 16, "F32BF16", "F32BF16_m64n256k16"); + + // ----------------------------------------------------------------------- + // Phase 3: A register layout dump — the key data for simulator implementation + // Shows which A[m][k] each (thread, register_element) holds after ldmatrix. + // BF16 uses same element size and ldmatrix layout as F16 (K=16, 2-byte); + // its mapping is implied by the F32F16 dump. + // ----------------------------------------------------------------------- + printf("\n=== Phase 3: A register layout dump (A[m][k]=m*K+k, no WGMMA) ===\n"); + printf(" Decodes (thread T, register element e) -> (row m, col k) in A tile.\n"); + printf(" This is the reference for simulator RS WGMMA functional implementation.\n\n"); + + // F16 K=16: primary case, covers F16 and BF16 (same 2-byte ldmatrix) + RUN_DUMP(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16", "F32F16_m64n16k16"); + RUN_DUMP(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16", "F32F16_m64n64k16"); + + // TF32 K=8: 4-byte elements, different ldmatrix pattern + RUN_DUMP(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "F32TF32", "F32TF32_m64n16k8"); + + printf("\n A register layout dumps -> wgmma_rs_areg_N.txt\n"); + printf(" Format: thread elem raw_value decoded_m decoded_k\n"); + + printf("\nDone.\n"); + return 0; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/.gitignore b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/.gitignore new file mode 100644 index 000000000..08f514ebc --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/.gitignore @@ -0,0 +1 @@ +include/ diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile new file mode 100644 index 000000000..30010cd5f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile @@ -0,0 +1,12 @@ +SRC = wgmma_sweep.cu +EXE = wgmma_sweep + +INCLUDE += -I../lat_gmma/include -I$(GPUAPPS_ROOT)/src/cuda/cutlass-bench/include -I./ + +ARCH ?= sm_90a +CUDA_CPPFLAGS= +HOPPER_CUDA_CPPFLAGS=$(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) + +NVCC_FLAGS := $(HOPPER_CUDA_CPPFLAGS) -std=c++17 + +include ../../../common/common.mk diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/wgmma_sweep.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/wgmma_sweep.cu new file mode 100644 index 000000000..c2f656eca --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/wgmma_sweep.cu @@ -0,0 +1,951 @@ +// ============================================================================= +// wgmma_sweep.cu — Combined WGMMA timing + D-fragment / smem layout sweep +// +// Fills ALL measurement gaps left by lat_gmma and wgmma_multik benchmarks. +// +// PHASE 0 — Shape-independent overhead (run once) +// kernel_fence_overhead: 256 FENCEs → cyc_per_fence +// kernel_commit_wait_overhead: 256 empty COMMIT+WAITs → cyc_per_barrier +// +// PHASE 1 — SS timing sweep (smem-A, smem-B) +// kernel_mma_throughput: 1024 back-to-back SS MMAs → cyc_per_mma +// Derived true latency: +// lat = cyc_per_mma*N_MMA - 2*cyc_fence - (N_MMA-1)*II - cyc_barrier +// Covers all 6 dtypes × 9 N values (N=8,16,32,64,96,128,192,224,256): +// F32F16, F32TF32, F32BF16, F32E4M3, F16F16, S32S8 +// Output: wgmma_sweep_timing.csv +// +// PHASE 1b — Pipelined 2-group overlap (SS, F32F16, all N) +// Issues N_MMA/2 MMAs + COMMIT + N_MMA/2 MMAs + COMMIT + WAIT<1> + WAIT<0>. +// gap_after_wait1 = clock(after WAIT<0>) - clock(after WAIT<1>). +// If gap ≈ 0: group 2 completed during group 1's latency tail → overlap confirmed. +// Output: wgmma_sweep_pipeline.csv +// +// PHASE 2 — D-fragment layout verification (all N) +// A=1, B[k][n]=n → D[m][n]=K*n. Checks d_frag_pos column formula. +// F32F16 (F32 accum) : N=8..256 — primary +// F16F16 (F16 accum) : N=8..256 — verifies F16 accumulator column mapping +// S32S8 (S32 accum) : N=8..256 — verifies int accumulator column mapping +// B[k][n] = n%64 (int8 overflow guard), expected K*(col%64) +// Output: wgmma_sweep_layout_colN.txt per shape +// +// PHASE 3 — smem_A layout verification (selected N and K values) +// A[m][k]=m, B[k][n]=1 → D[m][n]=K*m. Verifies smem_A row layout +// AND d_frag_pos row formula jointly. +// F16 K=16: N=16,32,64,128,256 (K-interleaved for 2-byte elements) +// TF32 K=8 : N=16,32,64,128 (K-interleaved for 4-byte elements) +// S8 K=32: N=16,32,64 (K-interleaved for 1-byte elements) +// Output: wgmma_sweep_layout_rowN.txt per shape +// +// PHASE 4 — RS timing sweep (register-A, smem-B) +// kernel_mma_throughput_rs: 1024 back-to-back RS MMAs → cyc_per_mma_rs +// A register fragment filled with 1s (values immaterial for timing). +// RS is supported for F32{F16,BF16,TF32} and F16F16 dtypes. +// Covers all 4 RS dtypes × 9 N values. +// Output: appended to wgmma_sweep_timing.csv (section marked RS) +// +// Build: make (see Makefile) +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cute/arch/util.hpp" +#include +#include "cutlass/numeric_types.h" +#include +#include +#include +#include +#include + +using namespace cute; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- +static constexpr int WGSIZE = 128; // one warpgroup +static constexpr int N_FENCE = 256; +static constexpr int N_BARRIER = 256; +static constexpr int N_MMA = 1024; +static constexpr int N_HALF = N_MMA / 2; + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT0 asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory") +#define WGMMA_WAIT1 asm volatile("wgmma.wait_group.sync.aligned 1;\n" ::: "memory") + +// --------------------------------------------------------------------------- +// Kernel 1: Fence overhead (shape-independent) +// --------------------------------------------------------------------------- +__global__ void kernel_fence_overhead(uint32_t* clk_out) { + const int tid = threadIdx.x; + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t0) :: "memory"); + __syncthreads(); + #pragma unroll + for (int i = 0; i < N_FENCE; i++) WGMMA_FENCE; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t1) :: "memory"); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } +} + +// --------------------------------------------------------------------------- +// Kernel 2: Commit+wait overhead with EMPTY group (shape-independent) +// --------------------------------------------------------------------------- +__global__ void kernel_commit_wait_overhead(uint32_t* clk_out) { + const int tid = threadIdx.x; + uint32_t t0 = 0, t1 = 0; + WGMMA_FENCE; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t0) :: "memory"); + __syncthreads(); + #pragma unroll + for (int i = 0; i < N_BARRIER; i++) { WGMMA_COMMIT; WGMMA_WAIT0; } + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t1) :: "memory"); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } +} + +// --------------------------------------------------------------------------- +// Kernel 3: SS MMA throughput timing +// FENCE + N_MMA*MMA + COMMIT + WAIT + FENCE +// cyc_per_mma = total / N_MMA +// --------------------------------------------------------------------------- +template +__global__ void kernel_mma_throughput(uint32_t* clk_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + auto gmma_op = GMMA::ss_op_selector(); + using TiledMma = decltype(make_tiled_mma(gmma_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_A[i] = EA(1.0f); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(1.0f); + __syncthreads(); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + auto tCsA = thread_mma.partition_A(sA); + auto tCsB = thread_mma.partition_B(sB); + auto tCrA = thread_mma.make_fragment_A(tCsA); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t0) :: "memory"); + __syncthreads(); + + warpgroup_fence_operand(accum); + warpgroup_arrive(); + #pragma unroll 8 + for (int j = 0; j < N_MMA; j++) + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t1) :: "memory"); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + if (tid == 0 && accum(0) == -9999.f) printf("x"); // prevent DCE +} + +// --------------------------------------------------------------------------- +// Kernel 4: RS MMA throughput timing (register-A, smem-B) +// Same structure as kernel_mma_throughput but uses rs_op_selector. +// A register fragment is pre-filled with zeros (values immaterial for timing). +// --------------------------------------------------------------------------- +template +__global__ void kernel_mma_throughput_rs(uint32_t* clk_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + auto gmma_op = GMMA::rs_op_selector(); + using TiledMma = decltype(make_tiled_mma(gmma_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_A[i] = EA(1.0f); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(1.0f); + __syncthreads(); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + + // For RS: partition_A gives smem shape; make_fragment_A returns a REGISTER + // tensor (FrgTypeA is a value type for RS atoms, not a smem-pointer wrapper). + // clear() zeros the registers — values don't affect timing. + auto tCsA = thread_mma.partition_A(sA); + auto tCrA = thread_mma.make_fragment_A(tCsA); + clear(tCrA); + + auto tCsB = thread_mma.partition_B(sB); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t0) :: "memory"); + __syncthreads(); + + warpgroup_fence_operand(accum); + warpgroup_arrive(); + #pragma unroll 8 + for (int j = 0; j < N_MMA; j++) + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t1) :: "memory"); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + if (tid == 0 && accum(0) == -9999.f) printf("x"); +} + +// --------------------------------------------------------------------------- +// Kernel 5: Pipelined 2-group SS timing +// Issues two committed groups of N_HALF MMAs each: +// FENCE + N_HALF*MMA + COMMIT + FENCE + N_HALF*MMA + COMMIT +// + WAIT<1> [wait for group 1, group 2 still pending] +// + WAIT<0> [wait for group 2] +// Outputs 3 clock values: [t_before, t_after_wait1, t_after_wait0]. +// gap_after_wait1 = clk[2]-clk[1]: if small → group 2 completed during +// group 1's latency tail (true overlap). +// --------------------------------------------------------------------------- +template +__global__ void kernel_mma_pipelined(uint32_t* clk_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + auto gmma_op = GMMA::ss_op_selector(); + using TiledMma = decltype(make_tiled_mma(gmma_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_A[i] = EA(1.0f); + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(1.0f); + __syncthreads(); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + auto tCsA = thread_mma.partition_A(sA); + auto tCsB = thread_mma.partition_B(sB); + auto tCrA = thread_mma.make_fragment_A(tCsA); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + uint32_t t0 = 0, t1 = 0, t2 = 0; + __syncthreads(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t0) :: "memory"); + __syncthreads(); + + warpgroup_fence_operand(accum); + + // Group 1: N_HALF MMAs + warpgroup_arrive(); + #pragma unroll 8 + for (int j = 0; j < N_HALF; j++) + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + + // Group 2: N_HALF more MMAs (overlaps with group 1's latency tail) + warpgroup_arrive(); + #pragma unroll 8 + for (int j = 0; j < N_HALF; j++) + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + + // Wait for group 1 only (1 group still pending); group 2 may be done already + warpgroup_wait<1>(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t1) :: "memory"); + + // Wait for group 2 + warpgroup_wait<0>(); + if (tid == 0) asm volatile("mov.u32 %0, %%clock;" : "=r"(t2) :: "memory"); + + warpgroup_fence_operand(accum); + + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; clk_out[2] = t2; } + if (tid == 0 && accum(0) == -9999.f) printf("x"); +} + +// --------------------------------------------------------------------------- +// Kernel 6: D-fragment column layout verification +// A[m][k]=1, B[k][n]=(n%FILL_MOD if FILL_MOD>0 else n) => D[m][n]=K*(n or n%FILL_MOD) +// Verifies d_frag_pos column formula. FILL_MOD=64 for int8 B to avoid overflow. +// --------------------------------------------------------------------------- +template +__global__ void kernel_mma_layout_col(EC* D_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + static constexpr int N_val = (int)get<1>(TileShape{}); + static constexpr int K_val = (int)get<2>(TileShape{}); + + auto gmma_op = GMMA::ss_op_selector(); + using TiledMma = decltype(make_tiled_mma(gmma_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_A[i] = EA(1.0f); + for (int i = tid; i < N_val * K_val; i += WGSIZE) { + int n = i % N_val, k = i / N_val; + int n_fill = (FILL_MOD > 0) ? (n % FILL_MOD) : n; + sB(n, k, 0) = EB(float(n_fill)); + } + __syncthreads(); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + auto tCsA = thread_mma.partition_A(sA); + auto tCsB = thread_mma.partition_B(sB); + auto tCrA = thread_mma.make_fragment_A(tCsA); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + warpgroup_fence_operand(accum); + warpgroup_arrive(); + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + const int D_ELEMS = (int)size(accum); + EC* base = D_out + tid * D_ELEMS; + for (int e = 0; e < D_ELEMS; e++) + base[e] = accum(e); +} + +// --------------------------------------------------------------------------- +// Kernel 7: smem_A row layout verification +// A[m][k]=m, B[k][n]=1 => D[m][n]=K*m +// Verifies d_frag_pos row formula AND smem_A K-interleaved swizzled layout. +// --------------------------------------------------------------------------- +template +__global__ void kernel_mma_layout_row(EC* D_out) { + const int tid = threadIdx.x; + const int wg = tid / cutlass::NumThreadsPerWarpGroup; + + static constexpr int M_val = (int)get<0>(TileShape{}); + static constexpr int N_val = (int)get<1>(TileShape{}); + static constexpr int K_val = (int)get<2>(TileShape{}); + + auto gmma_op = GMMA::ss_op_selector(); + using TiledMma = decltype(make_tiled_mma(gmma_op)); + TiledMma tiled_mma; + + static constexpr int PIPE = 1; + using SmemLayoutA = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}))); + using SmemLayoutB = decltype(tile_to_shape( + GMMA::Layout_K_INTER_Atom{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}))); + + __shared__ EA smem_A[cosize_v]; + __shared__ EB smem_B[cosize_v]; + + Tensor sA = make_tensor(make_smem_ptr(smem_A), SmemLayoutA{}); + Tensor sB = make_tensor(make_smem_ptr(smem_B), SmemLayoutB{}); + + for (int i = tid; i < M_val * K_val; i += WGSIZE) { + int m = i / K_val, k = i % K_val; + sA(m, k, 0) = EA(float(m)); + } + for (int i = tid; i < (int)cosize_v; i += WGSIZE) + smem_B[i] = EB(1.0f); + __syncthreads(); + + Layout wg_layout = make_layout(Int<1>{}, Int{}); + auto thread_mma = tiled_mma.get_slice(wg_layout(wg)); + auto tCsA = thread_mma.partition_A(sA); + auto tCsB = thread_mma.partition_B(sB); + auto tCrA = thread_mma.make_fragment_A(tCsA); + auto tCrB = thread_mma.make_fragment_B(tCsB); + auto accum = partition_fragment_C(tiled_mma, take<0,2>(TileShape{})); + clear(accum); + + warpgroup_fence_operand(accum); + warpgroup_arrive(); + cute::gemm(tiled_mma, tCrA(_,_,_,0), tCrB(_,_,_,0), accum); + warpgroup_commit_batch(); + warpgroup_wait<0>(); + warpgroup_fence_operand(accum); + + const int D_ELEMS = (int)size(accum); + EC* base = D_out + tid * D_ELEMS; + for (int e = 0; e < D_ELEMS; e++) + base[e] = accum(e); +} + +// --------------------------------------------------------------------------- +// Host helpers +// --------------------------------------------------------------------------- + +// d_frag_pos: maps (thread T, accumulator element e, tile width N) → (row, col) +// Valid for f32/s32 accumulator at all N. For f16 accumulator, the spatial +// mapping is identical (same warpgroup thread layout), just different precision. +// elems_per_group=4 for f32/s32 and for f16 (each f16 element is 1 per slot). +static void d_frag_pos(int T, int e, int /*N*/, int* row, int* col) { + int warp = T / 32, lane = T % 32; + int g = e / 4, k = e % 4; + *row = (lane / 4) * 2 + (k / 2) + warp * 16; + *col = (lane % 4) * 2 + (k % 2) + g * 8; +} + +static float measure_fence_overhead() { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2 * sizeof(uint32_t)); + kernel_fence_overhead<<<1, WGSIZE>>>(d_clk); + cudaDeviceSynchronize(); + uint32_t clk[2]; + cudaMemcpy(clk, d_clk, 2 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + return (float)(clk[1] - clk[0]) / N_FENCE; +} + +static float measure_barrier_overhead() { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2 * sizeof(uint32_t)); + kernel_commit_wait_overhead<<<1, WGSIZE>>>(d_clk); + cudaDeviceSynchronize(); + uint32_t clk[2]; + cudaMemcpy(clk, d_clk, 2 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + return (float)(clk[1] - clk[0]) / N_BARRIER; +} + +// Known II values from lat_gmma silicon run (H100, 2026-05-08). +// N=224 is interpolated (II=112); verify with this benchmark. +static int table_II(int N) { + switch (N) { + case 8: return 18; case 16: return 20; case 32: return 24; + case 64: return 32; case 96: return 48; case 128: return 64; + case 192: return 96; case 224: return 112; case 256: return 128; + default: return N / 2; + } +} + +// SS timing: calls kernel_mma_throughput, derives latency +template +static void run_timing(const char* name, int N, float cyc_fence, + float cyc_barrier, FILE* csv) { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2 * sizeof(uint32_t)); + kernel_mma_throughput<<<1, WGSIZE>>>(d_clk); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" %-60s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_clk); return; + } + cudaDeviceSynchronize(); + uint32_t clk[2]; + cudaMemcpy(clk, d_clk, 2 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + + float cyc_mma = (float)(clk[1] - clk[0]) / N_MMA; + int II = table_II(N); + float lat = cyc_mma * N_MMA - 2.0f * cyc_fence + - (float)(N_MMA - 1) * II - cyc_barrier; + printf(" %-60s cyc/mma=%7.3f II=%3d lat=%7.3f\n", name, cyc_mma, II, lat); + if (csv) + fprintf(csv, "%s,SS,%.3f,%.3f,%.3f,%d,%.3f\n", + name, cyc_fence, cyc_barrier, cyc_mma, II, lat); +} + +// RS timing: calls kernel_mma_throughput_rs, derives latency +template +static void run_timing_rs(const char* name, int N, float cyc_fence, + float cyc_barrier, FILE* csv) { + uint32_t *d_clk; + cudaMalloc(&d_clk, 2 * sizeof(uint32_t)); + kernel_mma_throughput_rs<<<1, WGSIZE>>>(d_clk); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" RS %-57s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_clk); return; + } + cudaDeviceSynchronize(); + uint32_t clk[2]; + cudaMemcpy(clk, d_clk, 2 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + + float cyc_mma = (float)(clk[1] - clk[0]) / N_MMA; + int II = table_II(N); + float lat = cyc_mma * N_MMA - 2.0f * cyc_fence + - (float)(N_MMA - 1) * II - cyc_barrier; + printf(" RS %-57s cyc/mma=%7.3f II=%3d lat=%7.3f\n", name, cyc_mma, II, lat); + if (csv) + fprintf(csv, "%s,RS,%.3f,%.3f,%.3f,%d,%.3f\n", + name, cyc_fence, cyc_barrier, cyc_mma, II, lat); +} + +// Pipelined 2-group timing: reports total cyc/mma and gap between WAIT<1> and WAIT<0> +template +static void run_timing_pipelined(const char* name, int N, FILE* csv) { + uint32_t *d_clk; + cudaMalloc(&d_clk, 3 * sizeof(uint32_t)); + kernel_mma_pipelined<<<1, WGSIZE>>>(d_clk); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" Pipe %-55s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_clk); return; + } + cudaDeviceSynchronize(); + uint32_t clk[3]; + cudaMemcpy(clk, d_clk, 3 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_clk); + + float cyc_total = (float)(clk[2] - clk[0]) / N_MMA; + float gap_wait1 = (float)(clk[2] - clk[1]); // cycles from WAIT<1> to WAIT<0> + printf(" Pipe %-55s cyc/mma=%7.3f gap_after_wait1=%7.1f%s\n", + name, cyc_total, gap_wait1, gap_wait1 < 5.0f ? " [OVERLAP]" : ""); + if (csv) + fprintf(csv, "%s,%.3f,%.1f\n", name, cyc_total, gap_wait1); +} + +// Layout column verification: A=1, B[k][n]=n (or n%FILL_MOD) => D=K*n (or K*(n%FILL_MOD)) +template +static void run_layout_col(const char* tag, const char* name, int N, int K) { + const int D_ELEMS = N / 2; // universal: f32, f16, s32 accumulators all = N/2 + const int total = WGSIZE * D_ELEMS; + std::vector h_D(total, EC(0)); + EC* d_D; + cudaMalloc(&d_D, total * sizeof(EC)); + cudaMemset(d_D, 0, total * sizeof(EC)); + + kernel_mma_layout_col<<<1, WGSIZE>>>(d_D); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" ColLayout %-50s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_D); return; + } + cudaDeviceSynchronize(); + cudaMemcpy(h_D.data(), d_D, total * sizeof(EC), cudaMemcpyDeviceToHost); + cudaFree(d_D); + + char fname[128]; + snprintf(fname, sizeof(fname), "wgmma_sweep_layout_col%sN%d.txt", tag, N); + FILE* f = fopen(fname, "w"); + if (f) { + if (FILL_MOD > 0) + fprintf(f, "# %s A=1 B[k][n]=n%%64 expected D[m][n]=K*(n%%64) (K=%d)\n" + "# thread elem row col expected actual match\n", name, K); + else + fprintf(f, "# %s A=1 B[k][n]=n expected D[m][n]=K*n (K=%d)\n" + "# thread elem row col expected actual match\n", name, K); + } + + int errors = 0, checked = 0; + for (int T = 0; T < WGSIZE; T++) { + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, N, &row, &col); + if (row >= 64 || col >= N) continue; + int col_fill = (FILL_MOD > 0) ? (col % FILL_MOD) : col; + float expected = (float)(K * col_fill); + float actual = (float)h_D[T * D_ELEMS + e]; + bool ok = (fabsf(actual - expected) < 0.5f); + if (!ok) errors++; + checked++; + if (f) + fprintf(f, "%d %d %d %d %.1f %.1f %s\n", + T, e, row, col, expected, actual, ok ? "OK" : "FAIL"); + } + } + if (f) { fprintf(f, "# errors=%d / %d checked\n", errors, checked); fclose(f); } + printf(" ColLayout%-6s %-48s errors=%d/%d -> %s\n", + tag, name, errors, checked, errors == 0 ? "PASS" : "FAIL"); +} + +// Layout row verification: A[m][k]=m, B=1 => D=K*m +template +static void run_layout_row(const char* tag, const char* name, int N, int K) { + const int D_ELEMS = N / 2; + const int total = WGSIZE * D_ELEMS; + std::vector h_D(total, EC(0)); + EC* d_D; + cudaMalloc(&d_D, total * sizeof(EC)); + cudaMemset(d_D, 0, total * sizeof(EC)); + + kernel_mma_layout_row<<<1, WGSIZE>>>(d_D); + cudaError_t err = cudaPeekAtLastError(); + if (err != cudaSuccess) { + printf(" RowLayout %-50s KERNEL ERROR: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_D); return; + } + cudaDeviceSynchronize(); + cudaMemcpy(h_D.data(), d_D, total * sizeof(EC), cudaMemcpyDeviceToHost); + cudaFree(d_D); + + char fname[128]; + snprintf(fname, sizeof(fname), "wgmma_sweep_layout_row%sN%d.txt", tag, N); + FILE* f = fopen(fname, "w"); + if (f) + fprintf(f, "# %s A[m][k]=m B=1 expected D[m][n]=K*m (K=%d)\n" + "# thread elem row col expected actual match\n", name, K); + + int errors = 0, checked = 0; + for (int T = 0; T < WGSIZE; T++) { + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, N, &row, &col); + if (row >= 64 || col >= N) continue; + float expected = (float)(K * row); + float actual = (float)h_D[T * D_ELEMS + e]; + bool ok = (fabsf(actual - expected) < 0.5f); + if (!ok) errors++; + checked++; + if (f) + fprintf(f, "%d %d %d %d %.1f %.1f %s\n", + T, e, row, col, expected, actual, ok ? "OK" : "FAIL"); + } + } + if (f) { fprintf(f, "# errors=%d / %d checked\n", errors, checked); fclose(f); } + printf(" RowLayout%-6s %-48s errors=%d/%d -> %s\n", + tag, name, errors, checked, errors == 0 ? "PASS" : "FAIL"); +} + +// --------------------------------------------------------------------------- +// Convenience macros +// --------------------------------------------------------------------------- +#define SHAPE(M,N,K) decltype(make_shape(Int{}, Int{}, Int{})) + +#define RUN_TIMING(EA, EB, EC, M, N, K, LABEL) \ + run_timing(LABEL, N, cyc_fence, cyc_barrier, timing_csv) + +#define RUN_TIMING_RS(EA, EB, EC, M, N, K, LABEL) \ + run_timing_rs(LABEL, N, cyc_fence, cyc_barrier, timing_csv) + +#define RUN_PIPELINED(EA, EB, EC, M, N, K, LABEL) \ + run_timing_pipelined(LABEL, N, pipeline_csv) + +#define RUN_COL_LAYOUT(EA, EB, EC, M, N, K, TAG, LABEL) \ + run_layout_col(TAG, LABEL, N, K) + +#define RUN_COL_LAYOUT_MOD(EA, EB, EC, M, N, K, MOD, TAG, LABEL) \ + run_layout_col(TAG, LABEL, N, K) + +#define RUN_ROW_LAYOUT(EA, EB, EC, M, N, K, TAG, LABEL) \ + run_layout_row(TAG, LABEL, N, K) + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_sweep - %s (SM %d.%d)\n\n", prop.name, prop.major, prop.minor); + + // ----------------------------------------------------------------------- + // Phase 0: Shape-independent overhead + // ----------------------------------------------------------------------- + printf("=== Phase 0: Overhead kernels (shape-independent) ===\n"); + float cyc_fence = measure_fence_overhead(); + float cyc_barrier = measure_barrier_overhead(); + printf(" cyc_per_fence = %.3f (256 x wgmma.fence.sync.aligned)\n", cyc_fence); + printf(" cyc_per_barrier = %.3f (256 x COMMIT+WAIT empty group)\n\n", cyc_barrier); + + FILE* timing_csv = fopen("wgmma_sweep_timing.csv", "w"); + if (timing_csv) + fprintf(timing_csv, "shape,mode,cyc_fence,cyc_barrier,cyc_mma,II_table,derived_lat\n"); + + // ----------------------------------------------------------------------- + // Phase 1: SS timing sweep — all dtypes × all N + // ----------------------------------------------------------------------- + printf("=== Phase 1: SS timing sweep ===\n"); + printf(" %-60s cyc/mma II derived_lat\n", "shape"); + + printf("\n -- F32 accumulator, F16 inputs (K=16) --\n"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32F16_m64n8k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16_m64n16k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32F16_m64n32k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16_m64n64k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32F16_m64n96k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32F16_m64n128k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32F16_m64n192k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 224, 16, "F32F16_m64n224k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32F16_m64n256k16"); + + printf("\n -- F32 accumulator, TF32 inputs (K=8) --\n"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 8, 8, "F32TF32_m64n8k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "F32TF32_m64n16k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 32, 8, "F32TF32_m64n32k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 64, 8, "F32TF32_m64n64k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 96, 8, "F32TF32_m64n96k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 128, 8, "F32TF32_m64n128k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 192, 8, "F32TF32_m64n192k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 224, 8, "F32TF32_m64n224k8"); + RUN_TIMING(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 256, 8, "F32TF32_m64n256k8"); + + printf("\n -- F32 accumulator, BF16 inputs (K=16) --\n"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 8, 16, "F32BF16_m64n8k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 16, 16, "F32BF16_m64n16k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 32, 16, "F32BF16_m64n32k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 64, 16, "F32BF16_m64n64k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 96, 16, "F32BF16_m64n96k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 128, 16, "F32BF16_m64n128k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 192, 16, "F32BF16_m64n192k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 224, 16, "F32BF16_m64n224k16"); + RUN_TIMING(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 256, 16, "F32BF16_m64n256k16"); + + printf("\n -- F32 accumulator, FP8-E4M3 inputs (K=32) --\n"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 8, 32, "F32E4M3_m64n8k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 16, 32, "F32E4M3_m64n16k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 32, 32, "F32E4M3_m64n32k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 64, 32, "F32E4M3_m64n64k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 96, 32, "F32E4M3_m64n96k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 128, 32, "F32E4M3_m64n128k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 192, 32, "F32E4M3_m64n192k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 224, 32, "F32E4M3_m64n224k32"); + RUN_TIMING(cutlass::float_e4m3_t, cutlass::float_e4m3_t, float, 64, 256, 32, "F32E4M3_m64n256k32"); + + printf("\n -- F16 accumulator, F16 inputs (K=16) --\n"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 8, 16, "F16F16_m64n8k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 16, 16, "F16F16_m64n16k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 32, 16, "F16F16_m64n32k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 64, 16, "F16F16_m64n64k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 96, 16, "F16F16_m64n96k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 128, 16, "F16F16_m64n128k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 192, 16, "F16F16_m64n192k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 224, 16, "F16F16_m64n224k16"); + RUN_TIMING(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 256, 16, "F16F16_m64n256k16"); + + printf("\n -- S32 accumulator, S8 inputs (K=32) --\n"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 8, 32, "S32S8_m64n8k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 16, 32, "S32S8_m64n16k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 32, 32, "S32S8_m64n32k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 64, 32, "S32S8_m64n64k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 96, 32, "S32S8_m64n96k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 128, 32, "S32S8_m64n128k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 192, 32, "S32S8_m64n192k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 224, 32, "S32S8_m64n224k32"); + RUN_TIMING(int8_t, int8_t, int32_t, 64, 256, 32, "S32S8_m64n256k32"); + + // ----------------------------------------------------------------------- + // Phase 1b: Pipelined 2-group overlap (F32F16, all N) + // ----------------------------------------------------------------------- + printf("\n=== Phase 1b: Pipelined 2-group overlap (F32F16) ===\n"); + printf(" gap_after_wait1 ≈ 0 means group 2 completed during group 1's latency tail.\n"); + printf(" %-60s cyc/mma gap_after_wait1\n", "shape"); + + FILE* pipeline_csv = fopen("wgmma_sweep_pipeline.csv", "w"); + if (pipeline_csv) + fprintf(pipeline_csv, "shape,cyc_per_mma,gap_after_wait1_cycles\n"); + + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32F16_m64n8k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16_m64n16k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32F16_m64n32k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16_m64n64k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32F16_m64n96k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32F16_m64n128k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32F16_m64n192k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 224, 16, "F32F16_m64n224k16"); + RUN_PIPELINED(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32F16_m64n256k16"); + + if (pipeline_csv) fclose(pipeline_csv); + printf(" Pipelined results -> wgmma_sweep_pipeline.csv\n"); + + // ----------------------------------------------------------------------- + // Phase 4: RS timing sweep (before layout phases to group timing output) + // ----------------------------------------------------------------------- + printf("\n=== Phase 4: RS timing sweep (register-A, smem-B) ===\n"); + printf(" %-60s cyc/mma II derived_lat\n", "shape"); + + printf("\n -- F32 accumulator, F16 inputs RS (K=16) --\n"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32F16_m64n8k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32F16_m64n16k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32F16_m64n32k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32F16_m64n64k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32F16_m64n96k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32F16_m64n128k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32F16_m64n192k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 224, 16, "F32F16_m64n224k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32F16_m64n256k16"); + + printf("\n -- F32 accumulator, TF32 inputs RS (K=8) --\n"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 8, 8, "F32TF32_m64n8k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "F32TF32_m64n16k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 32, 8, "F32TF32_m64n32k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 64, 8, "F32TF32_m64n64k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 96, 8, "F32TF32_m64n96k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 128, 8, "F32TF32_m64n128k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 192, 8, "F32TF32_m64n192k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 224, 8, "F32TF32_m64n224k8"); + RUN_TIMING_RS(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 256, 8, "F32TF32_m64n256k8"); + + printf("\n -- F32 accumulator, BF16 inputs RS (K=16) --\n"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 8, 16, "F32BF16_m64n8k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 16, 16, "F32BF16_m64n16k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 32, 16, "F32BF16_m64n32k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 64, 16, "F32BF16_m64n64k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 96, 16, "F32BF16_m64n96k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 128, 16, "F32BF16_m64n128k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 192, 16, "F32BF16_m64n192k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 224, 16, "F32BF16_m64n224k16"); + RUN_TIMING_RS(cutlass::bfloat16_t, cutlass::bfloat16_t, float, 64, 256, 16, "F32BF16_m64n256k16"); + + printf("\n -- F16 accumulator, F16 inputs RS (K=16) --\n"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 8, 16, "F16F16_m64n8k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 16, 16, "F16F16_m64n16k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 32, 16, "F16F16_m64n32k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 64, 16, "F16F16_m64n64k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 96, 16, "F16F16_m64n96k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 128, 16, "F16F16_m64n128k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 192, 16, "F16F16_m64n192k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 224, 16, "F16F16_m64n224k16"); + RUN_TIMING_RS(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 256, 16, "F16F16_m64n256k16"); + + if (timing_csv) fclose(timing_csv); + printf("\n Timing results -> wgmma_sweep_timing.csv\n"); + + // ----------------------------------------------------------------------- + // Phase 2: D-fragment column layout — all N, three accumulators + // ----------------------------------------------------------------------- + printf("\n=== Phase 2: D-fragment column layout (B[k][n]=n => D=K*n) ===\n"); + printf(" Verifies d_frag_pos column formula for all N and three accumulator types.\n"); + printf(" S32S8 uses B[k][n]=n%%64 (int8 overflow guard), expected K*(col%%64).\n"); + + printf("\n -- F32 accumulator (F16 inputs) --\n"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 8, 16, "F32", "F32F16_m64n8k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F32", "F32F16_m64n16k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F32", "F32F16_m64n32k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F32", "F32F16_m64n64k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 96, 16, "F32", "F32F16_m64n96k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F32", "F32F16_m64n128k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 192, 16, "F32", "F32F16_m64n192k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 224, 16, "F32", "F32F16_m64n224k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F32", "F32F16_m64n256k16"); + + printf("\n -- F16 accumulator (F16 inputs) — D_ELEMS=N/2, same d_frag_pos --\n"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 8, 16, "F16", "F16F16_m64n8k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 16, 16, "F16", "F16F16_m64n16k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 32, 16, "F16", "F16F16_m64n32k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 64, 16, "F16", "F16F16_m64n64k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 96, 16, "F16", "F16F16_m64n96k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 128, 16, "F16", "F16F16_m64n128k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 192, 16, "F16", "F16F16_m64n192k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 224, 16, "F16", "F16F16_m64n224k16"); + RUN_COL_LAYOUT(cutlass::half_t, cutlass::half_t, cutlass::half_t, 64, 256, 16, "F16", "F16F16_m64n256k16"); + + printf("\n -- S32 accumulator (S8 inputs) — B[k][n]=n%%64, expected K*(col%%64) --\n"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 8, 32, 64, "S32", "S32S8_m64n8k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 16, 32, 64, "S32", "S32S8_m64n16k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 32, 32, 64, "S32", "S32S8_m64n32k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 64, 32, 64, "S32", "S32S8_m64n64k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 96, 32, 64, "S32", "S32S8_m64n96k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 128, 32, 64, "S32", "S32S8_m64n128k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 192, 32, 64, "S32", "S32S8_m64n192k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 224, 32, 64, "S32", "S32S8_m64n224k32"); + RUN_COL_LAYOUT_MOD(int8_t, int8_t, int32_t, 64, 256, 32, 64, "S32", "S32S8_m64n256k32"); + + printf("\n Column layout dumps -> wgmma_sweep_layout_colN.txt\n"); + + // ----------------------------------------------------------------------- + // Phase 3: smem_A row layout — three K values + // A[m][k]=m, B=1 => D=K*m. Independently verifies both d_frag_pos row + // formula AND smem_A K-interleaved physical layout. + // ----------------------------------------------------------------------- + printf("\n=== Phase 3: smem_A row layout (A[m][k]=m, B=1 => D=K*m) ===\n"); + printf(" Verifies smem_A layout for different K (element size) values.\n"); + + printf("\n -- F16 inputs K=16 (2-byte elements, Swizzle<0,4,3>) --\n"); + RUN_ROW_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 16, 16, "F16", "F32F16_m64n16k16"); + RUN_ROW_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 32, 16, "F16", "F32F16_m64n32k16"); + RUN_ROW_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 64, 16, "F16", "F32F16_m64n64k16"); + RUN_ROW_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 128, 16, "F16", "F32F16_m64n128k16"); + RUN_ROW_LAYOUT(cutlass::half_t, cutlass::half_t, float, 64, 256, 16, "F16", "F32F16_m64n256k16"); + + printf("\n -- TF32 inputs K=8 (4-byte elements) — different K interleaving --\n"); + RUN_ROW_LAYOUT(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 16, 8, "TF32", "F32TF32_m64n16k8"); + RUN_ROW_LAYOUT(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 32, 8, "TF32", "F32TF32_m64n32k8"); + RUN_ROW_LAYOUT(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 64, 8, "TF32", "F32TF32_m64n64k8"); + RUN_ROW_LAYOUT(cutlass::tfloat32_t, cutlass::tfloat32_t, float, 64, 128, 8, "TF32", "F32TF32_m64n128k8"); + + printf("\n -- S8 inputs K=32 (1-byte elements) — different K interleaving --\n"); + RUN_ROW_LAYOUT(int8_t, int8_t, int32_t, 64, 16, 32, "S8", "S32S8_m64n16k32"); + RUN_ROW_LAYOUT(int8_t, int8_t, int32_t, 64, 32, 32, "S8", "S32S8_m64n32k32"); + RUN_ROW_LAYOUT(int8_t, int8_t, int32_t, 64, 64, 32, "S8", "S32S8_m64n64k32"); + + printf("\n Row layout dumps -> wgmma_sweep_layout_rowN.txt\n"); + + printf("\nDone.\n"); + return 0; +} From 92148e1dde9adf8bf620850208f455b9387a2006 Mon Sep 17 00:00:00 2001 From: Ivorchu <41135347+Ivorchu@users.noreply.github.com> Date: Mon, 11 May 2026 23:29:36 -0400 Subject: [PATCH 12/35] fix d_frag_pos row formula in wgmma_rs_verify --- .../ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu index 16414ad8f..331d65004 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/wgmma_rs_verify.cu @@ -226,7 +226,7 @@ __global__ void kernel_rs_areg_dump(float* A_out, int* A_reg_size_out) { static void d_frag_pos(int T, int e, int /*N*/, int* row, int* col) { int warp = T / 32, lane = T % 32; int g = e / 4, k = e % 4; - *row = (lane / 4) * 2 + (k / 2) + warp * 16; + *row = (lane / 4) + (k / 2) * 8 + warp * 16; *col = (lane % 4) * 2 + (k % 2) + g * 8; } From 522f770aca0089b8a6559939d457eafe988a22cc Mon Sep 17 00:00:00 2001 From: sam-pty Date: Fri, 15 May 2026 21:45:49 +0800 Subject: [PATCH 13/35] Add grid dependency control semantic test --- .../gpgpusim_tests/griddepcontrol/Makefile | 44 +++ .../griddepcontrol_semantics.cu | 324 ++++++++++++++++++ 2 files changed, 368 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/griddepcontrol_semantics.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/Makefile b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/Makefile new file mode 100644 index 000000000..e20ac26ab --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/Makefile @@ -0,0 +1,44 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin + +CC := nvcc +EXE := gpgpusim_griddepcontrol_semantics +SRC := griddepcontrol_semantics.cu + +# Programmatic dependent launch / grid dependency control is Hopper-era CUDA +# functionality. Keep the default narrow, but allow callers to override ARCH. +ARCH ?= sm_90 + +CUDA_CPPFLAGS = +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +CUDA_SRC_FILES := $(filter %.cu, $(SRC)) +CUDA_LIB_FILES := $(CUDA_SRC_FILES:.cu=.a) + +.DEFAULT_GOAL := all + +all: $(BIN_DIR)/$(EXE) + +$(BIN_DIR): + mkdir -p $@ + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(BIN_DIR)/$(EXE): $(CUDA_LIB_FILES) | $(BIN_DIR) + $(CC) $(NVCC_FLAGS) $^ -o $@ -lcudart --cudart shared + +run: $(BIN_DIR)/$(EXE) + $(BIN_DIR)/$(EXE) + +ptx: $(BIN_DIR)/$(EXE) + cuobjdump -ptx $(BIN_DIR)/$(EXE) | tee $(EXE).ptx + +sass: $(BIN_DIR)/$(EXE) + cuobjdump -sass $(BIN_DIR)/$(EXE) | tee $(EXE).sass + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(EXE) $(CUDA_LIB_FILES) + rm -f $(BIN_DIR)/$(EXE) + +.PHONY: all clean run ptx sass diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/griddepcontrol_semantics.cu b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/griddepcontrol_semantics.cu new file mode 100644 index 000000000..7a04624e3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/griddepcontrol/griddepcontrol_semantics.cu @@ -0,0 +1,324 @@ +// griddepcontrol_semantics.cu +// Semantic tests for CUDA Programmatic Dependent Launch / grid dependency +// control. +// +// Covers: +// 1. Explicit trigger smoke test +// 2. Implicit trigger smoke test +// 3. Large memory visibility: pre-trigger and post-trigger writes +// 4. Uneven CTA trigger timing: dependent launch must wait for all primary +// CTAs to trigger +// 5. No-attribute control: without programmatic stream serialization, normal +// stream ordering applies +// +// Notes for simulator bring-up: +// - overlap_seen is intentionally a soft observation, not a hard requirement. +// - Correctness is based on data visibility after +// cudaGridDependencySynchronize(). +// - The memory-visibility checks intentionally do not rely on __threadfence() +// around primary writes. + +#include + +#include +#include +#include + +#define CHECK_CUDA(call) \ + do { \ + cudaError_t err__ = (call); \ + if (err__ != cudaSuccess) { \ + fprintf(stderr, "%s:%d: CUDA error: %s\n", \ + __FILE__, __LINE__, cudaGetErrorString(err__)); \ + std::exit(1); \ + } \ + } while (0) + +static constexpr int BLOCKS = 2; +static constexpr int THREADS = 32; +static constexpr int WORDS_PER_BLOCK = 256; +static constexpr int PRE_SPIN = 200; +static constexpr int POST_TRIGGER_SPIN = 200; +static constexpr int LONG_SPIN = 4000; + +// Device-global witness flags. +// secondary_started: set by secondary as soon as it begins executing +// overlap_seen: set by primary if it sees secondary_started before +// finishing +__device__ volatile int secondary_started; +__device__ volatile int overlap_seen; + +__device__ __forceinline__ void burn_cycles(int n) { + volatile int x = threadIdx.x + blockIdx.x; + for (int i = 0; i < n; ++i) { + x = x * 1664525 + 1013904223; + } +} + +__device__ __forceinline__ int pre_pattern(int block, int i) { + return 0x10000000 ^ (block * 0x1009) ^ i; +} + +__device__ __forceinline__ int post_pattern(int block, int i) { + return 0x20000000 ^ (block * 0x2003) ^ i; +} + +__device__ __forceinline__ void write_pre_region(int *buf) { + int base = blockIdx.x * WORDS_PER_BLOCK; + for (int i = threadIdx.x; i < WORDS_PER_BLOCK / 2; i += blockDim.x) { + buf[base + i] = pre_pattern(blockIdx.x, i); + } +} + +__device__ __forceinline__ void write_post_region(int *buf) { + int base = blockIdx.x * WORDS_PER_BLOCK; + for (int i = WORDS_PER_BLOCK / 2 + threadIdx.x; i < WORDS_PER_BLOCK; + i += blockDim.x) { + buf[base + i] = post_pattern(blockIdx.x, i); + } +} + +__device__ __forceinline__ void observe_secondary_and_finish(int *done) { + for (int i = 0; i < POST_TRIGGER_SPIN; ++i) { + if (threadIdx.x == 0 && secondary_started) { + overlap_seen = 1; + } + burn_cycles(1); + } + + if (threadIdx.x == 0) { + done[blockIdx.x] = 1; + } +} + +// Explicit primary: write first region, trigger dependent launch eligibility, +// then continue with post-trigger writes. +__global__ void primary_explicit_memory(int *done, int *buf) { + burn_cycles(PRE_SPIN); + write_pre_region(buf); + + if (threadIdx.x == 0) { + cudaTriggerProgrammaticLaunchCompletion(); + } + + write_post_region(buf); + observe_secondary_and_finish(done); +} + +// Implicit primary: no explicit trigger; release is implicit at primary +// completion. +__global__ void primary_implicit_memory(int *done, int *buf) { + burn_cycles(PRE_SPIN); + write_pre_region(buf); + write_post_region(buf); + observe_secondary_and_finish(done); +} + +// Uneven explicit trigger timing: +// block 0 writes pre, triggers early, then spins and writes post +// block 1 writes pre, spins first, triggers late, then writes post +// The secondary may start only after all primary CTAs have reached +// trigger/completion eligibility. +__global__ void primary_explicit_uneven(int *done, int *buf) { + write_pre_region(buf); + + if (blockIdx.x == 0) { + if (threadIdx.x == 0) { + cudaTriggerProgrammaticLaunchCompletion(); + } + burn_cycles(LONG_SPIN); + } else { + burn_cycles(LONG_SPIN); + if (threadIdx.x == 0) { + cudaTriggerProgrammaticLaunchCompletion(); + } + } + + write_post_region(buf); + observe_secondary_and_finish(done); +} + +__global__ void secondary_check_full(const int *done, + const int *buf, + int *errors, + int *presync_counter) { + if (threadIdx.x == 0 && blockIdx.x == 0) { + secondary_started = 1; + } + + if (threadIdx.x == 0) { + atomicAdd(presync_counter, 1); + } + + // The key operation under test. + cudaGridDependencySynchronize(); + + // Check original-style per-primary-CTA completion witness. + int tid = blockIdx.x * blockDim.x + threadIdx.x; + if (tid < BLOCKS) { + if (done[tid] != 1) { + atomicAdd(errors, 1); + } + } + + // Check larger memory visibility, including pre-trigger and post-trigger + // writes. + int total_words = BLOCKS * WORDS_PER_BLOCK; + for (int idx = tid; idx < total_words; idx += gridDim.x * blockDim.x) { + int b = idx / WORDS_PER_BLOCK; + int i = idx % WORDS_PER_BLOCK; + int expected = + (i < WORDS_PER_BLOCK / 2) ? pre_pattern(b, i) : post_pattern(b, i); + if (buf[idx] != expected) { + atomicAdd(errors, 1); + } + } +} + +static void reset_device_flags() { + int zero = 0; + CHECK_CUDA(cudaMemcpyToSymbol(secondary_started, &zero, sizeof(int))); + CHECK_CUDA(cudaMemcpyToSymbol(overlap_seen, &zero, sizeof(int))); +} + +static int copy_overlap_seen_to_host() { + int h_overlap = 0; + CHECK_CUDA(cudaMemcpyFromSymbol(&h_overlap, overlap_seen, sizeof(int))); + return h_overlap; +} + +static int copy_secondary_started_to_host() { + int h_started = 0; + CHECK_CUDA(cudaMemcpyFromSymbol(&h_started, secondary_started, sizeof(int))); + return h_started; +} + +enum class PrimaryKind { + ExplicitMemory, + ImplicitMemory, + ExplicitUneven, +}; + +static void launch_primary(PrimaryKind kind, + int *d_done, + int *d_buf, + cudaStream_t stream) { + switch (kind) { + case PrimaryKind::ExplicitMemory: + primary_explicit_memory<<>>(d_done, + d_buf); + break; + case PrimaryKind::ImplicitMemory: + primary_implicit_memory<<>>(d_done, + d_buf); + break; + case PrimaryKind::ExplicitUneven: + primary_explicit_uneven<<>>(d_done, + d_buf); + break; + } + CHECK_CUDA(cudaGetLastError()); +} + +static void run_test(const char *name, + PrimaryKind kind, + bool enable_pdl_attribute) { + int *d_done = nullptr; + int *d_buf = nullptr; + int *d_errors = nullptr; + int *d_presync = nullptr; + + const int total_words = BLOCKS * WORDS_PER_BLOCK; + + CHECK_CUDA(cudaMalloc(&d_done, BLOCKS * sizeof(int))); + CHECK_CUDA(cudaMalloc(&d_buf, total_words * sizeof(int))); + CHECK_CUDA(cudaMalloc(&d_errors, sizeof(int))); + CHECK_CUDA(cudaMalloc(&d_presync, sizeof(int))); + + CHECK_CUDA(cudaMemset(d_done, 0, BLOCKS * sizeof(int))); + CHECK_CUDA(cudaMemset(d_buf, 0, total_words * sizeof(int))); + CHECK_CUDA(cudaMemset(d_errors, 0, sizeof(int))); + CHECK_CUDA(cudaMemset(d_presync, 0, sizeof(int))); + + reset_device_flags(); + + cudaStream_t stream; + CHECK_CUDA(cudaStreamCreate(&stream)); + + cudaLaunchAttribute attr{}; + attr.id = cudaLaunchAttributeProgrammaticStreamSerialization; + attr.val.programmaticStreamSerializationAllowed = + enable_pdl_attribute ? 1 : 0; + + cudaLaunchConfig_t cfg{}; + cfg.gridDim = dim3(1); + cfg.blockDim = dim3(128); + cfg.dynamicSmemBytes = 0; + cfg.stream = stream; + cfg.attrs = enable_pdl_attribute ? &attr : nullptr; + cfg.numAttrs = enable_pdl_attribute ? 1 : 0; + + launch_primary(kind, d_done, d_buf, stream); + + CHECK_CUDA(cudaLaunchKernelEx(&cfg, secondary_check_full, d_done, d_buf, + d_errors, d_presync)); + CHECK_CUDA(cudaStreamSynchronize(stream)); + + int h_errors = -1; + int h_presync = -1; + int h_overlap = copy_overlap_seen_to_host(); + int h_secondary_started = copy_secondary_started_to_host(); + std::vector h_done(BLOCKS, 0); + + CHECK_CUDA(cudaMemcpy(&h_errors, d_errors, sizeof(int), + cudaMemcpyDeviceToHost)); + CHECK_CUDA(cudaMemcpy(&h_presync, d_presync, sizeof(int), + cudaMemcpyDeviceToHost)); + CHECK_CUDA(cudaMemcpy(h_done.data(), d_done, BLOCKS * sizeof(int), + cudaMemcpyDeviceToHost)); + + bool all_done = true; + for (int i = 0; i < BLOCKS; ++i) { + if (h_done[i] != 1) { + all_done = false; + break; + } + } + + printf("[%s] pdl_attr=%d, secondary_started=%d, presync_counter=%d, " + "overlap_seen=%d, errors=%d, all_done=%s\n", + name, enable_pdl_attribute ? 1 : 0, h_secondary_started, h_presync, + h_overlap, h_errors, all_done ? "true" : "false"); + + CHECK_CUDA(cudaStreamDestroy(stream)); + CHECK_CUDA(cudaFree(d_done)); + CHECK_CUDA(cudaFree(d_buf)); + CHECK_CUDA(cudaFree(d_errors)); + CHECK_CUDA(cudaFree(d_presync)); + + // Hard correctness requirements: + // 1. dependent reads after cudaGridDependencySynchronize() must be correct + // 2. all primary blocks must eventually finish + if (h_errors != 0 || !all_done) { + fprintf(stderr, "[%s] FAILED: semantic correctness check failed\n", + name); + std::exit(2); + } + + // Soft observations: + // - explicit PDL cases may show overlap_seen=1, but this is + // scheduling/resource dependent. + // - no_attr_control is expected to have overlap_seen=0 under normal + // stream serialization, but correctness should not depend on asserting + // that in this portable test. +} + +int main() { + run_test("explicit_basic_memory", PrimaryKind::ExplicitMemory, true); + run_test("explicit_uneven_trigger", PrimaryKind::ExplicitUneven, true); + run_test("implicit_memory", PrimaryKind::ImplicitMemory, true); + run_test("no_attribute_control", PrimaryKind::ExplicitMemory, false); + + printf("PASS\n"); + return 0; +} From bcb56134a39a55836fa732993e8d1b42bad65beb Mon Sep 17 00:00:00 2001 From: Weili An Date: Tue, 9 Jun 2026 13:28:02 -0400 Subject: [PATCH 14/35] Remove unneeded gpgpusim unit test files --- .../ubench/gpgpusim_tests/README.md | 17 - .../gpgpusim_tests/fp_lowp_cvt/Makefile | 48 - .../gpgpusim_tests/fp_lowp_cvt/README.md | 26 - .../lowp_cvt_diff/kernels_sm120.ptx | 295 -- .../lowp_cvt_runtime_hw_sim_diff.cu | 2364 ----------------- .../lowp_cvt_runtime_hw_sim_diff_rs.cu | 748 ------ .../fp_lowp_cvt/ptx_datatype_cases/README.md | 14 - .../declaration_param_bf16_not_allowed.expect | 2 - .../declaration_param_bf16_not_allowed.ptx | 7 - ...eclaration_param_bf16x2_not_allowed.expect | 2 - .../declaration_param_bf16x2_not_allowed.ptx | 7 - .../declaration_param_e2m1_not_allowed.expect | 2 - .../declaration_param_e2m1_not_allowed.ptx | 7 - ...eclaration_param_e2m1x2_not_allowed.expect | 2 - .../declaration_param_e2m1x2_not_allowed.ptx | 7 - ...eclaration_param_e2m1x4_not_allowed.expect | 2 - .../declaration_param_e2m1x4_not_allowed.ptx | 7 - .../declaration_param_e2m3_not_allowed.expect | 2 - .../declaration_param_e2m3_not_allowed.ptx | 7 - ...eclaration_param_e2m3x2_not_allowed.expect | 2 - .../declaration_param_e2m3x2_not_allowed.ptx | 7 - ...eclaration_param_e2m3x4_not_allowed.expect | 2 - .../declaration_param_e2m3x4_not_allowed.ptx | 7 - .../declaration_param_e3m2_not_allowed.expect | 2 - .../declaration_param_e3m2_not_allowed.ptx | 7 - ...eclaration_param_e3m2x2_not_allowed.expect | 2 - .../declaration_param_e3m2x2_not_allowed.ptx | 7 - ...eclaration_param_e3m2x4_not_allowed.expect | 2 - .../declaration_param_e3m2x4_not_allowed.ptx | 7 - .../declaration_param_e4m3_not_allowed.expect | 2 - .../declaration_param_e4m3_not_allowed.ptx | 7 - ...eclaration_param_e4m3x2_not_allowed.expect | 2 - .../declaration_param_e4m3x2_not_allowed.ptx | 7 - ...eclaration_param_e4m3x4_not_allowed.expect | 2 - .../declaration_param_e4m3x4_not_allowed.ptx | 7 - .../declaration_param_e5m2_not_allowed.expect | 2 - .../declaration_param_e5m2_not_allowed.ptx | 7 - ...eclaration_param_e5m2x2_not_allowed.expect | 2 - .../declaration_param_e5m2x2_not_allowed.ptx | 7 - ...eclaration_param_e5m2x4_not_allowed.expect | 2 - .../declaration_param_e5m2x4_not_allowed.ptx | 7 - ...declaration_param_f16x2_not_allowed.expect | 1 - .../declaration_param_f16x2_not_allowed.ptx | 7 - ...declaration_param_f32x2_not_allowed.expect | 2 - .../declaration_param_f32x2_not_allowed.ptx | 7 - ...declaration_param_s16x2_not_allowed.expect | 2 - .../declaration_param_s16x2_not_allowed.ptx | 7 - .../declaration_param_s2f6_not_allowed.expect | 2 - .../declaration_param_s2f6_not_allowed.ptx | 7 - ...eclaration_param_s2f6x2_not_allowed.expect | 2 - .../declaration_param_s2f6x2_not_allowed.ptx | 7 - .../declaration_param_tf32_not_allowed.expect | 2 - .../declaration_param_tf32_not_allowed.ptx | 7 - ...declaration_param_u16x2_not_allowed.expect | 2 - .../declaration_param_u16x2_not_allowed.ptx | 7 - ...declaration_param_ue4m3_not_allowed.expect | 2 - .../declaration_param_ue4m3_not_allowed.ptx | 7 - ...declaration_param_ue8m0_not_allowed.expect | 2 - .../declaration_param_ue8m0_not_allowed.ptx | 7 - ...claration_param_ue8m0x2_not_allowed.expect | 2 - .../declaration_param_ue8m0x2_not_allowed.ptx | 7 - .../declaration_reg_bf16_not_allowed.expect | 2 - .../fail/declaration_reg_bf16_not_allowed.ptx | 8 - .../declaration_reg_bf16x2_not_allowed.expect | 2 - .../declaration_reg_bf16x2_not_allowed.ptx | 8 - .../declaration_reg_e2m1_not_allowed.expect | 2 - .../fail/declaration_reg_e2m1_not_allowed.ptx | 8 - .../declaration_reg_e2m1x2_not_allowed.expect | 2 - .../declaration_reg_e2m1x2_not_allowed.ptx | 8 - .../declaration_reg_e2m1x4_not_allowed.expect | 2 - .../declaration_reg_e2m1x4_not_allowed.ptx | 8 - .../declaration_reg_e2m3_not_allowed.expect | 2 - .../fail/declaration_reg_e2m3_not_allowed.ptx | 8 - .../declaration_reg_e2m3x2_not_allowed.expect | 2 - .../declaration_reg_e2m3x2_not_allowed.ptx | 8 - .../declaration_reg_e2m3x4_not_allowed.expect | 2 - .../declaration_reg_e2m3x4_not_allowed.ptx | 8 - .../declaration_reg_e3m2_not_allowed.expect | 2 - .../fail/declaration_reg_e3m2_not_allowed.ptx | 8 - .../declaration_reg_e3m2x2_not_allowed.expect | 2 - .../declaration_reg_e3m2x2_not_allowed.ptx | 8 - .../declaration_reg_e3m2x4_not_allowed.expect | 2 - .../declaration_reg_e3m2x4_not_allowed.ptx | 8 - .../declaration_reg_e4m3_not_allowed.expect | 2 - .../fail/declaration_reg_e4m3_not_allowed.ptx | 8 - .../declaration_reg_e4m3x2_not_allowed.expect | 2 - .../declaration_reg_e4m3x2_not_allowed.ptx | 8 - .../declaration_reg_e4m3x4_not_allowed.expect | 2 - .../declaration_reg_e4m3x4_not_allowed.ptx | 8 - .../declaration_reg_e5m2_not_allowed.expect | 2 - .../fail/declaration_reg_e5m2_not_allowed.ptx | 8 - .../declaration_reg_e5m2x2_not_allowed.expect | 2 - .../declaration_reg_e5m2x2_not_allowed.ptx | 8 - .../declaration_reg_e5m2x4_not_allowed.expect | 2 - .../declaration_reg_e5m2x4_not_allowed.ptx | 8 - .../declaration_reg_f32x2_not_allowed.expect | 2 - .../declaration_reg_f32x2_not_allowed.ptx | 8 - .../declaration_reg_s16x2_not_allowed.expect | 2 - .../declaration_reg_s16x2_not_allowed.ptx | 8 - .../declaration_reg_s2f6_not_allowed.expect | 2 - .../fail/declaration_reg_s2f6_not_allowed.ptx | 8 - .../declaration_reg_s2f6x2_not_allowed.expect | 2 - .../declaration_reg_s2f6x2_not_allowed.ptx | 8 - .../declaration_reg_tf32_not_allowed.expect | 2 - .../fail/declaration_reg_tf32_not_allowed.ptx | 8 - .../declaration_reg_u16x2_not_allowed.expect | 2 - .../declaration_reg_u16x2_not_allowed.ptx | 8 - .../declaration_reg_ue4m3_not_allowed.expect | 2 - .../declaration_reg_ue4m3_not_allowed.ptx | 8 - .../declaration_reg_ue8m0_not_allowed.expect | 2 - .../declaration_reg_ue8m0_not_allowed.ptx | 8 - ...declaration_reg_ue8m0x2_not_allowed.expect | 2 - .../declaration_reg_ue8m0x2_not_allowed.ptx | 8 - .../initializer_f16_brace_not_allowed.expect | 1 - .../initializer_f16_brace_not_allowed.ptx | 8 - .../fail/initializer_f16_not_allowed.expect | 1 - .../fail/initializer_f16_not_allowed.ptx | 8 - ...initializer_f16x2_brace_not_allowed.expect | 1 - .../initializer_f16x2_brace_not_allowed.ptx | 8 - .../fail/initializer_f16x2_not_allowed.expect | 1 - .../fail/initializer_f16x2_not_allowed.ptx | 8 - .../initializer_pred_brace_not_allowed.expect | 1 - .../initializer_pred_brace_not_allowed.ptx | 8 - .../fail/initializer_pred_not_allowed.expect | 1 - .../fail/initializer_pred_not_allowed.ptx | 8 - ...uction_cvt_bf16_f16_ftz_not_allowed.expect | 1 - ...struction_cvt_bf16_f16_ftz_not_allowed.ptx | 11 - ...ction_cvt_bf16_f16_sm80_not_allowed.expect | 1 - ...truction_cvt_bf16_f16_sm80_not_allowed.ptx | 11 - ...16_f32_missing_rounding_not_allowed.expect | 1 - ..._bf16_f32_missing_rounding_not_allowed.ptx | 11 - ...uction_cvt_bf16_f32_sat_not_allowed.expect | 1 - ...struction_cvt_bf16_f32_sat_not_allowed.ptx | 11 - ...t_bf16_f32_satfinite_rm_not_allowed.expect | 1 - ..._cvt_bf16_f32_satfinite_rm_not_allowed.ptx | 11 - ...ction_cvt_bf16_f32_sm75_not_allowed.expect | 1 - ...truction_cvt_bf16_f32_sm75_not_allowed.ptx | 11 - ...ction_cvt_bf16_s32_sm80_not_allowed.expect | 1 - ...truction_cvt_bf16_s32_sm80_not_allowed.ptx | 11 - ...t_bf16x2_f32_single_src_not_allowed.expect | 2 - ..._cvt_bf16x2_f32_single_src_not_allowed.ptx | 11 - ...f16x2_ue8m0x2_satfinite_not_allowed.expect | 1 - ...t_bf16x2_ue8m0x2_satfinite_not_allowed.ptx | 11 - ...on_cvt_e2m1x2_dst_sm100_not_allowed.expect | 1 - ...ction_cvt_e2m1x2_dst_sm100_not_allowed.ptx | 11 - ...on_cvt_e2m1x2_f32_sm120_not_allowed.expect | 1 - ...ction_cvt_e2m1x2_f32_sm120_not_allowed.ptx | 12 - ...on_cvt_e2m1x2_src_sm100_not_allowed.expect | 1 - ...ction_cvt_e2m1x2_src_sm100_not_allowed.ptx | 11 - ...uction_cvt_e2m3x2_sm100_not_allowed.expect | 1 - ...struction_cvt_e2m3x2_sm100_not_allowed.ptx | 11 - ...ruction_cvt_e4m3x2_sm80_not_allowed.expect | 1 - ...nstruction_cvt_e4m3x2_sm80_not_allowed.ptx | 12 - ...ction_cvt_f16_bf16_sm80_not_allowed.expect | 1 - ...truction_cvt_f16_bf16_sm80_not_allowed.ptx | 11 - ...16_f32_missing_rounding_not_allowed.expect | 1 - ...t_f16_f32_missing_rounding_not_allowed.ptx | 11 - ...vt_f16x2_f32_single_src_not_allowed.expect | 2 - ...n_cvt_f16x2_f32_single_src_not_allowed.ptx | 11 - ...tion_cvt_f16x2_f32_sm75_not_allowed.expect | 1 - ...ruction_cvt_f16x2_f32_sm75_not_allowed.ptx | 11 - ...ction_cvt_f16x2_ue8m0x2_not_allowed.expect | 2 - ...truction_cvt_f16x2_ue8m0x2_not_allowed.ptx | 10 - ...n_cvt_f32_bf16_ftz_sm80_not_allowed.expect | 1 - ...tion_cvt_f32_bf16_ftz_sm80_not_allowed.ptx | 11 - ...32_s32_missing_rounding_not_allowed.expect | 1 - ...t_f32_s32_missing_rounding_not_allowed.ptx | 11 - ...nstruction_cvt_f32_tf32_not_allowed.expect | 2 - .../instruction_cvt_f32_tf32_not_allowed.ptx | 11 - ...struction_cvt_f32x2_dst_not_allowed.expect | 1 - .../instruction_cvt_f32x2_dst_not_allowed.ptx | 10 - ...struction_cvt_f32x2_src_not_allowed.expect | 1 - .../instruction_cvt_f32x2_src_not_allowed.ptx | 10 - .../instruction_cvt_pred_not_allowed.expect | 2 - .../fail/instruction_cvt_pred_not_allowed.ptx | 10 - .../instruction_cvt_requires_two_types.expect | 1 - .../instruction_cvt_requires_two_types.ptx | 10 - ...nstruction_cvt_rs_sm100_not_allowed.expect | 1 - .../instruction_cvt_rs_sm100_not_allowed.ptx | 11 - ...struction_cvt_rs_sm100f_not_allowed.expect | 1 - .../instruction_cvt_rs_sm100f_not_allowed.ptx | 11 - ...struction_cvt_rs_sm110a_not_allowed.expect | 1 - .../instruction_cvt_rs_sm110a_not_allowed.ptx | 13 - ...struction_cvt_rs_sm120a_not_allowed.expect | 1 - .../instruction_cvt_rs_sm120a_not_allowed.ptx | 11 - ...struction_cvt_rs_sm120f_not_allowed.expect | 1 - .../instruction_cvt_rs_sm120f_not_allowed.ptx | 11 - ...f16x2_missing_satfinite_not_allowed.expect | 2 - ...2_bf16x2_missing_satfinite_not_allowed.ptx | 11 - ..._operand_without_scaled_not_allowed.expect | 2 - ...6x2_operand_without_scaled_not_allowed.ptx | 11 - ..._scaled_without_operand_not_allowed.expect | 2 - ...6x2_scaled_without_operand_not_allowed.ptx | 11 - ...vt_s2f6x2_bf16x2_sm100f_not_allowed.expect | 1 - ...n_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx | 11 - ...vt_s2f6x2_bf16x2_sm120f_not_allowed.expect | 1 - ...n_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx | 11 - ...uction_cvt_s2f6x2_sm100_not_allowed.expect | 1 - ...struction_cvt_s2f6x2_sm100_not_allowed.ptx | 11 - ...uction_cvt_s32_bf16_sat_not_allowed.expect | 1 - ...struction_cvt_s32_bf16_sat_not_allowed.ptx | 11 - ...ction_cvt_s32_bf16_sm80_not_allowed.expect | 1 - ...truction_cvt_s32_bf16_sm80_not_allowed.ptx | 11 - ...ruction_cvt_s32_s32_sat_not_allowed.expect | 1 - ...nstruction_cvt_s32_s32_sat_not_allowed.ptx | 11 - ...ruction_cvt_scalar_relu_not_allowed.expect | 1 - ...nstruction_cvt_scalar_relu_not_allowed.ptx | 11 - ...on_cvt_scalar_satfinite_not_allowed.expect | 1 - ...ction_cvt_scalar_satfinite_not_allowed.ptx | 11 - ...32_f32_missing_rounding_not_allowed.expect | 2 - ..._tf32_f32_missing_rounding_not_allowed.ptx | 11 - ...on_cvt_tf32_f32_rn_sm89_not_allowed.expect | 1 - ...ction_cvt_tf32_f32_rn_sm89_not_allowed.ptx | 11 - ...n_cvt_tf32_f32_rna_relu_not_allowed.expect | 1 - ...tion_cvt_tf32_f32_rna_relu_not_allowed.ptx | 11 - ...2_f32_satfinite_rn_sm90_not_allowed.expect | 1 - ...tf32_f32_satfinite_rn_sm90_not_allowed.ptx | 11 - ...nstruction_cvt_tf32_u32_not_allowed.expect | 2 - .../instruction_cvt_tf32_u32_not_allowed.ptx | 10 - .../instruction_cvt_u16x2_not_allowed.expect | 2 - .../instruction_cvt_u16x2_not_allowed.ptx | 10 - ...struction_cvt_ue4m3_f32_not_allowed.expect | 1 - .../instruction_cvt_ue4m3_f32_not_allowed.ptx | 11 - ...bf16x2_missing_rounding_not_allowed.expect | 2 - ...x2_bf16x2_missing_rounding_not_allowed.ptx | 11 - ...ction_cvt_ue8m0x2_f16x2_not_allowed.expect | 2 - ...truction_cvt_ue8m0x2_f16x2_not_allowed.ptx | 10 - ...on_cvt_ue8m0x2_f32_relu_not_allowed.expect | 1 - ...ction_cvt_ue8m0x2_f32_relu_not_allowed.ptx | 11 - ...n_cvt_ue8m0x2_f32_sm120_not_allowed.expect | 1 - ...tion_cvt_ue8m0x2_f32_sm120_not_allowed.ptx | 12 - ...tion_cvt_ue8m0x2_scaled_not_allowed.expect | 1 - ...ruction_cvt_ue8m0x2_scaled_not_allowed.ptx | 12 - ...128_pack_v2_mixed_width_not_allowed.expect | 1 - ...v_b128_pack_v2_mixed_width_not_allowed.ptx | 12 - ...ruction_mov_b128_scalar_not_allowed.expect | 1 - ...nstruction_mov_b128_scalar_not_allowed.ptx | 11 - ...uction_mov_bb128_scalar_not_allowed.expect | 1 - ...struction_mov_bb128_scalar_not_allowed.ptx | 11 - .../instruction_mov_bf16_not_allowed.expect | 2 - .../fail/instruction_mov_bf16_not_allowed.ptx | 9 - .../instruction_mov_f16_not_allowed.expect | 2 - .../fail/instruction_mov_f16_not_allowed.ptx | 9 - .../fail/unsupported_token_s32x1.expect | 1 - .../fail/unsupported_token_s32x1.ptx | 9 - .../fail/unsupported_token_u32x1.expect | 1 - .../fail/unsupported_token_u32x1.ptx | 9 - .../pass/all_new_instruction_types.ptx | 32 - ...on_param_b32_for_f16x2_storage_allowed.ptx | 7 - .../pass/declaration_reg_b128_allowed.ptx | 12 - .../pass/declaration_reg_f16x2_allowed.ptx | 8 - .../pass/declaration_underlying_bit_types.ptx | 16 - .../instruction_cvt_bf16_f16_sm90_allowed.ptx | 11 - ...on_cvt_bf16_f32_satfinite_relu_allowed.ptx | 11 - .../instruction_cvt_bf16_s32_sm90_allowed.ptx | 11 - ...nstruction_cvt_bf16x2_f32_pair_allowed.ptx | 11 - .../instruction_cvt_bf16x2_s2f6x2_allowed.ptx | 11 - ...ction_cvt_bf16x2_s2f6x2_sm103a_allowed.ptx | 11 - ...ction_cvt_bf16x2_s2f6x2_sm121a_allowed.ptx | 11 - ...instruction_cvt_bf16x2_ue8m0x2_allowed.ptx | 11 - ...truction_cvt_e2m1x2_f32_sm100a_allowed.ptx | 12 - ...truction_cvt_e2m1x2_f32_sm100f_allowed.ptx | 12 - ...truction_cvt_e2m1x2_f32_sm103a_allowed.ptx | 12 - ...truction_cvt_e2m1x2_f32_sm120a_allowed.ptx | 12 - ...truction_cvt_e2m1x2_f32_sm120f_allowed.ptx | 12 - ...truction_cvt_e2m1x2_f32_sm121a_allowed.ptx | 12 - ...truction_cvt_e2m1x2_f32_sm121f_allowed.ptx | 12 - .../instruction_cvt_e2m1x4_f32_rs_allowed.ptx | 12 - .../instruction_cvt_e2m3x4_f32_rs_allowed.ptx | 11 - .../instruction_cvt_e3m2x4_f32_rs_allowed.ptx | 11 - .../instruction_cvt_e4m3x4_f32_rs_allowed.ptx | 11 - .../instruction_cvt_e5m2x4_f32_rs_allowed.ptx | 11 - .../instruction_cvt_f16_bf16_sm90_allowed.ptx | 11 - ...ion_cvt_f16_f32_satfinite_relu_allowed.ptx | 11 - ...uction_cvt_f16x2_e2m1x2_sm100a_allowed.ptx | 11 - ...uction_cvt_f16x2_e2m3x2_sm100a_allowed.ptx | 11 - ...instruction_cvt_f16x2_f32_pair_allowed.ptx | 11 - ...truction_cvt_f32_bf16_ftz_sm90_allowed.ptx | 11 - .../instruction_cvt_rs_sm103a_allowed.ptx | 11 - .../instruction_cvt_s2f6x2_bf16x2_allowed.ptx | 11 - ...ction_cvt_s2f6x2_bf16x2_scaled_allowed.ptx | 11 - ...ction_cvt_s2f6x2_bf16x2_sm103a_allowed.ptx | 11 - ...ction_cvt_s2f6x2_bf16x2_sm120a_allowed.ptx | 11 - ...ction_cvt_s2f6x2_bf16x2_sm121a_allowed.ptx | 11 - .../instruction_cvt_s32_bf16_sm90_allowed.ptx | 11 - .../instruction_cvt_s32_f32_sat_allowed.ptx | 11 - ...on_cvt_tf32_f32_satfinite_relu_allowed.ptx | 11 - ...instruction_cvt_tf32_roundtrip_allowed.ptx | 11 - .../instruction_cvt_u32_f64_sat_allowed.ptx | 11 - ...instruction_cvt_ue8m0x2_bf16x2_allowed.ptx | 11 - ...uction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx | 11 - ...8m0x2_bf16x2_without_satfinite_allowed.ptx | 11 - ...ruction_cvt_ue8m0x2_f32_sm100a_allowed.ptx | 12 - ...ruction_cvt_ue8m0x2_f32_sm100f_allowed.ptx | 12 - ...ruction_cvt_ue8m0x2_f32_sm103f_allowed.ptx | 12 - ...ruction_cvt_ue8m0x2_f32_sm110f_allowed.ptx | 12 - ...ruction_cvt_ue8m0x2_f32_sm120a_allowed.ptx | 12 - ...ruction_cvt_ue8m0x2_f32_sm120f_allowed.ptx | 12 - ...ruction_cvt_ue8m0x2_f32_sm121a_allowed.ptx | 12 - .../instruction_mov_b128_pack_allowed.ptx | 11 - .../instruction_mov_b128_pack_v2_allowed.ptx | 11 - ...struction_mov_b128_pack_v2_u64_allowed.ptx | 11 - .../instruction_mov_b128_scalar_allowed.ptx | 10 - .../instruction_mov_b128_unpack_allowed.ptx | 11 - ...instruction_mov_b128_unpack_v2_allowed.ptx | 11 - ...uction_mov_b128_unpack_v4_sink_allowed.ptx | 11 - ...ruction_mov_b128_unpack_v4_u32_allowed.ptx | 11 - .../run_lowp_cvt_ptxas_crosscheck.sh | 174 -- .../gpgpusim_tests/tma_tensormap/Makefile | 50 - .../gpgpusim_tests/tma_tensormap/README.md | 22 - .../tma_tensor_test_REGULAR_LOAD_64_64.txt | 8 - .../tma_tensor_test_UTMALDG_64_64.txt | 8 - .../expected/tma_tensor_test_UTMAPF_64_64.txt | 8 - .../tma_tensor_test_UTMAREDG_64_64.txt | 8 - .../tma_tensor_test_UTMASTG_50_50.txt | 8 - .../tma_tensor_test_UTMASTG_64_64.txt | 8 - .../tma_tensormap/tma_tensor_benchmark.cu | 415 --- 317 files changed, 6209 deletions(-) delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/README.md delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/README.md delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_diff/kernels_sm120.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/README.md delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.expect delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/all_new_instruction_types.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_param_b32_for_f16x2_storage_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_b128_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_f16x2_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_underlying_bit_types.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f16_sm90_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f32_satfinite_relu_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_s32_sm90_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_f32_pair_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm103a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm121a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_ue8m0x2_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100f_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm103a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120f_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121f_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x4_f32_rs_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m3x4_f32_rs_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e3m2x4_f32_rs_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e4m3x4_f32_rs_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e5m2x4_f32_rs_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_bf16_sm90_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_f32_satfinite_relu_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m1x2_sm100a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m3x2_sm100a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_f32_pair_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f32_bf16_ftz_sm90_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_rs_sm103a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_scaled_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm103a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm120a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm121a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_bf16_sm90_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_f32_sat_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_f32_satfinite_relu_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_roundtrip_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_u32_f64_sat_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_without_satfinite_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100f_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm103f_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm110f_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120f_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm121a_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_u64_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_scalar_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v2_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_sink_allowed.ptx delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_u32_allowed.ptx delete mode 100755 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/run_lowp_cvt_ptxas_crosscheck.sh delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/Makefile delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/README.md delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_REGULAR_LOAD_64_64.txt delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMALDG_64_64.txt delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAPF_64_64.txt delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAREDG_64_64.txt delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_50_50.txt delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_64_64.txt delete mode 100644 src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/tma_tensor_benchmark.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/README.md b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/README.md deleted file mode 100644 index 8609fa268..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# GPGPU-Sim Standalone Tests - -This directory collects standalone CUDA/PTX tests that were originally kept -with GPGPU-Sim feature branches. They are placed under `GPU_Microbenchmark` so -the app-collection build can compile them with the same flow as the other -microbenchmarks. - -The tests here intentionally avoid linking against GPGPU-Sim internals. C++ -unit tests that need simulator headers or libraries should stay in the -GPGPU-Sim source tree. - -## Test Groups - -- `fp_lowp_cvt`: runtime low-precision conversion kernels and PTX cases used - for `ptxas` cross-checking. -- `tma_tensormap`: standalone TMA tensor-map benchmark plus expected output - fixtures from the simulator branch. diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile deleted file mode 100644 index f3376e39d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -BASE_DIR := $(shell pwd) -BIN_DIR := $(BASE_DIR)/../../../bin - -CC := nvcc - -LOWP_ARCH ?= sm_120a -LOWP_RS_ARCH ?= sm_100a -LOWP_COMPUTE = compute_$(subst sm_,,$(LOWP_ARCH)) -LOWP_RS_COMPUTE = compute_$(subst sm_,,$(LOWP_RS_ARCH)) -LOWP_CODE ?= $(LOWP_COMPUTE) -LOWP_RS_CODE ?= $(LOWP_RS_COMPUTE) -CUDA_CPPFLAGS = - -LOWP_EXE := lowp_cvt_runtime_hw_sim_diff -LOWP_RS_EXE := lowp_cvt_runtime_hw_sim_diff_rs - -# Keep the default artifact PTX-only: these microbenchmarks intentionally cover -# low-precision PTX forms that can be ahead of CUDA 12.x host ptxas support. -LOWP_NVCC_FLAGS := -gencode=arch=$(LOWP_COMPUTE),code=$(LOWP_CODE) -std=c++17 -O2 -LOWP_RS_NVCC_FLAGS := -gencode=arch=$(LOWP_RS_COMPUTE),code=$(LOWP_RS_CODE) -std=c++17 -O2 - -.DEFAULT_GOAL := all - -all: $(BIN_DIR)/$(LOWP_EXE) $(BIN_DIR)/$(LOWP_RS_EXE) - -$(BIN_DIR): - mkdir -p $@ - -$(LOWP_EXE).a: $(LOWP_EXE).cu - $(CC) $(LOWP_NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ - -$(LOWP_RS_EXE).a: $(LOWP_RS_EXE).cu - $(CC) $(LOWP_RS_NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ - -$(BIN_DIR)/$(LOWP_EXE): $(LOWP_EXE).a | $(BIN_DIR) - $(CC) $(LOWP_NVCC_FLAGS) $< -o $@ -lcudart --cudart shared - -$(BIN_DIR)/$(LOWP_RS_EXE): $(LOWP_RS_EXE).a | $(BIN_DIR) - $(CC) $(LOWP_RS_NVCC_FLAGS) $< -o $@ -lcudart --cudart shared - -ptxas-crosscheck: - ./run_lowp_cvt_ptxas_crosscheck.sh ptx_datatype_cases - -clean: - rm -f *.o *.a *.ptx *.ptxas *.sass $(LOWP_EXE) $(LOWP_RS_EXE) - rm -f $(BIN_DIR)/$(LOWP_EXE) $(BIN_DIR)/$(LOWP_RS_EXE) - -.PHONY: all clean ptxas-crosscheck diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/README.md b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/README.md deleted file mode 100644 index e9c1becff..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# FP Low-Precision Conversion Tests - -This folder contains standalone CUDA/PTX low-precision conversion tests from -the GPGPU-Sim `fp-work` branch. - -## Build - -```sh -make -``` - -The default build creates these binaries under `GPU_Microbenchmark/bin`: - -- `lowp_cvt_runtime_hw_sim_diff` -- `lowp_cvt_runtime_hw_sim_diff_rs` - -Default architectures are `LOWP_ARCH=sm_120a` and -`LOWP_RS_ARCH=sm_100a`. Override them on the make command line if needed. - -## PTX Cross-Check - -```sh -make ptxas-crosscheck -``` - -This target runs the imported PTX accept/reject cases through `ptxas`. diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_diff/kernels_sm120.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_diff/kernels_sm120.ptx deleted file mode 100644 index 376d5dd92..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_diff/kernels_sm120.ptx +++ /dev/null @@ -1,295 +0,0 @@ -.version 9.1 -.target sm_120a -.address_size 64 - -.visible .entry k_f16x2_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<4>; - .reg .f32 %f<3>; - .reg .b32 %r<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - add.u64 %rd2, %rd1, 4; - ld.global.f32 %f1, [%rd2]; - cvt.rn.f16x2.f32 %r0, %f0, %f1; - st.global.u32 [%rd0], %r0; - ret; -} - -.visible .entry k_f16x2_rz( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<4>; - .reg .f32 %f<3>; - .reg .b32 %r<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - add.u64 %rd2, %rd1, 4; - ld.global.f32 %f1, [%rd2]; - cvt.rz.f16x2.f32 %r0, %f0, %f1; - st.global.u32 [%rd0], %r0; - ret; -} - -.visible .entry k_bf16x2_rz( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<4>; - .reg .f32 %f<3>; - .reg .b32 %r<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - add.u64 %rd2, %rd1, 4; - ld.global.f32 %f1, [%rd2]; - cvt.rz.bf16x2.f32 %r0, %f0, %f1; - st.global.u32 [%rd0], %r0; - ret; -} - -.visible .entry k_bf16x2_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<4>; - .reg .f32 %f<3>; - .reg .b32 %r<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - add.u64 %rd2, %rd1, 4; - ld.global.f32 %f1, [%rd2]; - cvt.rn.bf16x2.f32 %r0, %f0, %f1; - st.global.u32 [%rd0], %r0; - ret; -} - -.visible .entry k_e4m3x2_satfinite_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<4>; - .reg .f32 %f<3>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - add.u64 %rd2, %rd1, 4; - ld.global.f32 %f1, [%rd2]; - cvt.rn.satfinite.e4m3x2.f32 %h0, %f0, %f1; - st.global.u16 [%rd0], %h0; - ret; -} - -.visible .entry k_e5m2x2_satfinite_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<4>; - .reg .f32 %f<3>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - add.u64 %rd2, %rd1, 4; - ld.global.f32 %f1, [%rd2]; - cvt.rn.satfinite.e5m2x2.f32 %h0, %f0, %f1; - st.global.u16 [%rd0], %h0; - ret; -} - -.visible .entry k_e2m3x2_satfinite_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<4>; - .reg .f32 %f<3>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - add.u64 %rd2, %rd1, 4; - ld.global.f32 %f1, [%rd2]; - cvt.rn.satfinite.e2m3x2.f32 %h0, %f0, %f1; - st.global.u16 [%rd0], %h0; - ret; -} - -.visible .entry k_e3m2x2_satfinite_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<4>; - .reg .f32 %f<3>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - add.u64 %rd2, %rd1, 4; - ld.global.f32 %f1, [%rd2]; - cvt.rn.satfinite.e3m2x2.f32 %h0, %f0, %f1; - st.global.u16 [%rd0], %h0; - ret; -} - -.visible .entry k_ue8m0x2_satfinite_rz( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<4>; - .reg .f32 %f<3>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - add.u64 %rd2, %rd1, 4; - ld.global.f32 %f1, [%rd2]; - cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; - st.global.u16 [%rd0], %h0; - ret; -} - -.visible .entry k_s2f6x2_satfinite_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<4>; - .reg .f32 %f<3>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - add.u64 %rd2, %rd1, 4; - ld.global.f32 %f1, [%rd2]; - cvt.rn.satfinite.s2f6x2.f32 %h0, %f0, %f1; - st.global.u16 [%rd0], %h0; - ret; -} - -.visible .entry k_tf32_satfinite_relu_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<3>; - .reg .f32 %f<2>; - .reg .b32 %r<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - cvt.rn.satfinite.relu.tf32.f32 %r0, %f0; - st.global.u32 [%rd0], %r0; - ret; -} - -.visible .entry k_tf32_rz( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<3>; - .reg .f32 %f<2>; - .reg .b32 %r<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - cvt.rz.tf32.f32 %r0, %f0; - st.global.u32 [%rd0], %r0; - ret; -} - -.visible .entry k_bf16_satfinite_relu_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<3>; - .reg .f32 %f<2>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - cvt.rn.satfinite.relu.bf16.f32 %h0, %f0; - st.global.b16 [%rd0], %h0; - ret; -} - -.visible .entry k_bf16_rz( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<3>; - .reg .f32 %f<2>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - cvt.rz.bf16.f32 %h0, %f0; - st.global.b16 [%rd0], %h0; - ret; -} - -.visible .entry k_f16_satfinite_relu_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<3>; - .reg .f32 %f<2>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - cvt.rn.satfinite.relu.f16.f32 %h0, %f0; - st.global.b16 [%rd0], %h0; - ret; -} - -.visible .entry k_f16_rz( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<3>; - .reg .f32 %f<2>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - cvt.rz.f16.f32 %h0, %f0; - st.global.b16 [%rd0], %h0; - ret; -} - -.visible .entry k_f16_sat_rn( - .param .u64 out_ptr, - .param .u64 in_ptr -) { - .reg .u64 %rd<3>; - .reg .f32 %f<2>; - .reg .b16 %h<2>; - - ld.param.u64 %rd0, [out_ptr]; - ld.param.u64 %rd1, [in_ptr]; - ld.global.f32 %f0, [%rd1]; - cvt.rn.sat.f16.f32 %h0, %f0; - st.global.b16 [%rd0], %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu deleted file mode 100644 index be4247245..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff.cu +++ /dev/null @@ -1,2364 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef LOWP_CVT_HAS_UE8M0X2_BF16X2_RELU -#define LOWP_CVT_HAS_UE8M0X2_BF16X2_RELU 0 -#endif - -namespace { - -#define CUDACHK(call) \ - do { \ - cudaError_t _err = (call); \ - if (_err != cudaSuccess) { \ - std::cerr << "CUDA error: " << cudaGetErrorString(_err) << " @ " \ - << #call << "\n"; \ - std::exit(1); \ - } \ - } while (0) - -float bits_to_f32(uint32_t bits) { - float out = 0.0f; - std::memcpy(&out, &bits, sizeof(out)); - return out; -} - -uint32_t f32_to_bits(float v) { - uint32_t bits = 0u; - std::memcpy(&bits, &v, sizeof(bits)); - return bits; -} - -uint16_t bf16_bits_from_f32(float v) { - return static_cast(f32_to_bits(v) >> 16); -} - -double bits_to_f64(uint64_t bits) { - double out = 0.0; - std::memcpy(&out, &bits, sizeof(out)); - return out; -} - -enum class minifloat_family_t { - kE4M3, - kE5M2, - kE2M1, - kE2M3, - kE3M2, - kUE8M0, -}; - -struct minifloat_entry_t { - uint16_t code; - float value; -}; - -unsigned minifloat_code_limit(minifloat_family_t fmt) { - switch (fmt) { - case minifloat_family_t::kE4M3: - case minifloat_family_t::kE5M2: - case minifloat_family_t::kUE8M0: - return 256u; - case minifloat_family_t::kE2M1: - return 16u; - case minifloat_family_t::kE2M3: - case minifloat_family_t::kE3M2: - return 64u; - } - return 0u; -} - -float decode_minifloat_code(minifloat_family_t fmt, uint16_t code) { - if (fmt == minifloat_family_t::kUE8M0) { - if (code == 0xffu) return std::numeric_limits::quiet_NaN(); - return std::ldexp(1.0f, static_cast(code) - 127); - } - - int exp_bits = 0; - int mant_bits = 0; - int bias = 0; - bool has_inf = false; - switch (fmt) { - case minifloat_family_t::kE4M3: - exp_bits = 4; - mant_bits = 3; - bias = 7; - has_inf = false; - break; - case minifloat_family_t::kE5M2: - exp_bits = 5; - mant_bits = 2; - bias = 15; - has_inf = true; - break; - case minifloat_family_t::kE2M1: - exp_bits = 2; - mant_bits = 1; - bias = 1; - has_inf = false; - break; - case minifloat_family_t::kE2M3: - exp_bits = 2; - mant_bits = 3; - bias = 1; - has_inf = false; - break; - case minifloat_family_t::kE3M2: - exp_bits = 3; - mant_bits = 2; - bias = 3; - has_inf = false; - break; - case minifloat_family_t::kUE8M0: - break; - } - - const int sign_shift = exp_bits + mant_bits; - const bool neg = ((code >> sign_shift) & 0x1u) != 0u; - const int exp_mask = (1 << exp_bits) - 1; - const int mant_mask = (1 << mant_bits) - 1; - const int exp = (code >> mant_bits) & exp_mask; - const int mant = code & mant_mask; - - if (fmt == minifloat_family_t::kE4M3 && (code == 0x7fu || code == 0xffu)) { - return std::numeric_limits::quiet_NaN(); - } - if (has_inf && exp == exp_mask) { - if (mant == 0) { - return neg ? -std::numeric_limits::infinity() - : std::numeric_limits::infinity(); - } - return std::numeric_limits::quiet_NaN(); - } - - float value = 0.0f; - if (exp == 0) { - if (mant != 0) { - value = std::ldexp(static_cast(mant), 1 - bias - mant_bits); - } - } else { - const int sig = (1 << mant_bits) | mant; - value = std::ldexp(static_cast(sig), exp - bias - mant_bits); - } - return neg ? -value : value; -} - -std::vector collect_minifloat_midpoint_ties(minifloat_family_t fmt, - size_t max_count) { - std::vector entries; - entries.reserve(minifloat_code_limit(fmt)); - for (unsigned code = 0; code < minifloat_code_limit(fmt); ++code) { - const float value = decode_minifloat_code(fmt, static_cast(code)); - if (!std::isfinite(value)) continue; - entries.push_back({static_cast(code), value}); - } - - std::sort(entries.begin(), entries.end(), - [](const minifloat_entry_t &a, const minifloat_entry_t &b) { - if (a.value < b.value) return true; - if (a.value > b.value) return false; - return a.code < b.code; - }); - - std::vector out; - out.reserve(max_count); - for (size_t i = 0; i + 1 < entries.size() && out.size() < max_count; ++i) { - const float lo = entries[i].value; - const float hi = entries[i + 1].value; - if (!(lo < hi)) continue; - const long double midpoint = - (static_cast(lo) + static_cast(hi)) * 0.5L; - if (!std::isfinite(static_cast(midpoint))) continue; - const float mid_f32 = static_cast(midpoint); - if (!std::isfinite(mid_f32)) continue; - if (static_cast(mid_f32) != midpoint) continue; - out.push_back(mid_f32); - } - return out; -} - -std::vector build_minifloat_midpoint_probe_values() { - const minifloat_family_t families[] = { - minifloat_family_t::kE4M3, minifloat_family_t::kE5M2, - minifloat_family_t::kE2M1, minifloat_family_t::kE2M3, - minifloat_family_t::kE3M2, minifloat_family_t::kUE8M0, - }; - - std::vector out; - for (minifloat_family_t fmt : families) { - const std::vector ties = collect_minifloat_midpoint_ties(fmt, 12); - for (float tie : ties) { - out.push_back(tie); - const float down = - std::nextafter(tie, -std::numeric_limits::infinity()); - const float up = - std::nextafter(tie, std::numeric_limits::infinity()); - if (std::isfinite(down)) out.push_back(down); - if (std::isfinite(up)) out.push_back(up); - } - } - - std::sort(out.begin(), out.end(), - [](float a, float b) { return f32_to_bits(a) < f32_to_bits(b); }); - out.erase(std::unique(out.begin(), out.end(), - [](float a, float b) { - return f32_to_bits(a) == f32_to_bits(b); - }), - out.end()); - return out; -} - -std::vector edge_f32_patterns() { - return { - 0x00000000u, 0x80000000u, 0x00000001u, 0x80000001u, 0x007fffffu, - 0x807fffffu, 0x00800000u, 0x80800000u, 0x00800001u, 0x80800001u, - 0x3f7fffffu, 0x3f800000u, 0x3f800001u, 0xbf7fffffu, 0xbf800000u, - 0xbf800001u, 0x7f7fffffu, 0xff7fffffu, 0x7f800000u, 0xff800000u, - 0x7fc00000u, 0xffc00000u, 0x7fa00001u, 0xffa00001u, - }; -} - -std::vector build_scalar_inputs() { - std::vector out; - out.reserve(384); - const std::vector edges = edge_f32_patterns(); - for (size_t i = 0; i < edges.size(); ++i) { - out.push_back(bits_to_f32(edges[i])); - } - std::mt19937 rng(0x40f8b37du); - while (out.size() < 384) out.push_back(bits_to_f32(rng())); - return out; -} - -void build_pair_inputs(std::vector &a, std::vector &b) { - a.clear(); - b.clear(); - a.reserve(448); - b.reserve(448); - - const std::vector edges = edge_f32_patterns(); - for (size_t i = 0; i + 1 < edges.size(); i += 2) { - a.push_back(bits_to_f32(edges[i])); - b.push_back(bits_to_f32(edges[i + 1])); - } - - const std::vector midpoint_probes = build_minifloat_midpoint_probe_values(); - for (size_t i = 0; i < midpoint_probes.size(); i += 2) { - const float lhs = midpoint_probes[i]; - const float rhs = - (i + 1 < midpoint_probes.size()) ? midpoint_probes[i + 1] : lhs; - a.push_back(lhs); - b.push_back(rhs); - } - - std::mt19937 rng(0x7b91d00du); - while (a.size() < 384) { - a.push_back(bits_to_f32(rng())); - b.push_back(bits_to_f32(rng())); - } -} - -std::vector build_bf16_inputs() { - std::vector out = { - 0x0000u, 0x8000u, 0x0001u, 0x8001u, 0x007fu, 0x807fu, 0x0080u, 0x8080u, - 0x3f80u, 0xbf80u, 0x7f80u, 0xff80u, 0x7fffu, - }; - - const float int_edges[] = { - -65536.0f, -32768.0f, -32767.0f, -256.0f, -255.0f, -128.0f, -127.5f, - -127.0f, -1.5f, -1.0f, -0.5f, 0.0f, 0.5f, 1.0f, - 1.5f, 127.0f, 127.5f, 128.0f, 255.0f, 256.0f, 32767.0f, - 32768.0f, 65535.0f, 65536.0f, - }; - for (float v : int_edges) out.push_back(bf16_bits_from_f32(v)); - - out.reserve(384); - std::mt19937 rng(0x13572468u); - while (out.size() < 384) out.push_back(static_cast(rng())); - return out; -} - -std::vector build_f16_inputs() { - std::vector out = { - 0x0000u, 0x8000u, 0x0001u, 0x8001u, 0x03ffu, 0x83ffu, 0x0400u, - 0x8400u, 0x3c00u, 0xbc00u, 0x7bffu, 0xfbffu, 0x7c00u, 0xfc00u, - 0x7e00u, 0xfe00u, - }; - out.reserve(384); - std::mt19937 rng(0x97531bdcu); - while (out.size() < 384) out.push_back(static_cast(rng())); - return out; -} - -std::vector build_s32_inputs() { - std::vector out = { - 0, 1, -1, 2, -2, 127, -127, 255, -255, 1024, -1024, - std::numeric_limits::max(), - std::numeric_limits::min(), - 0x00ffffff, static_cast(0xff000001u), - }; - out.reserve(384); - std::mt19937 rng(0x2468ace0u); - while (out.size() < 384) out.push_back(static_cast(rng())); - return out; -} - -std::vector build_u32_inputs() { - std::vector out = { - 0u, 1u, 2u, 255u, 65535u, 65536u, 0x00ffffffu, 0x7fffffffu, 0xffffffffu, - }; - out.reserve(384); - std::mt19937 rng(0x10293847u); - while (out.size() < 384) out.push_back(rng()); - return out; -} - -std::vector build_f64_inputs() { - std::vector out; - const std::vector edges = { - 0x0000000000000000ull, 0x8000000000000000ull, 0x0000000000000001ull, - 0x8000000000000001ull, 0x000fffffffffffffull, 0x800fffffffffffffull, - 0x0010000000000000ull, 0x8010000000000000ull, 0x3ff0000000000000ull, - 0xbff0000000000000ull, 0x3fefffffffffffffull, 0xbfefffffffffffffull, - 0x7fefffffffffffffull, 0xffefffffffffffffull, 0x7ff0000000000000ull, - 0xfff0000000000000ull, 0x7ff8000000000000ull, 0xfff8000000000000ull, - 0x7ff4000000000001ull, 0xfff4000000000001ull, - }; - out.reserve(384); - for (size_t i = 0; i < edges.size(); ++i) out.push_back(bits_to_f64(edges[i])); - std::mt19937_64 rng(0x1122334455667788ull); - while (out.size() < 384) out.push_back(bits_to_f64(rng())); - return out; -} - -std::vector build_e2m1x2_inputs_exhaustive() { - std::vector out; - out.reserve(256); - for (unsigned packed = 0; packed <= 0xff; ++packed) { - out.push_back(static_cast(packed)); - } - return out; -} - -std::vector build_e2m3x2_inputs_exhaustive() { - std::vector out; - out.reserve(64u * 64u); - for (unsigned hi = 0; hi < 64; ++hi) { - for (unsigned lo = 0; lo < 64; ++lo) { - out.push_back(static_cast((hi << 8) | lo)); - } - } - return out; -} - -std::vector build_e3m2x2_inputs_exhaustive() { - std::vector out; - out.reserve(64u * 64u); - for (unsigned hi = 0; hi < 64; ++hi) { - for (unsigned lo = 0; lo < 64; ++lo) { - out.push_back(static_cast((hi << 8) | lo)); - } - } - return out; -} - -std::vector build_u16_inputs_exhaustive() { - std::vector out; - out.reserve(1u << 16); - for (unsigned packed = 0; packed <= 0xffff; ++packed) { - out.push_back(static_cast(packed)); - } - return out; -} - -std::vector build_scale_inputs(size_t count, uint32_t seed) { - std::vector out; - out.reserve(count); - - // Packed ue8m0x2 scale-factors; include notable edge patterns and random. - const uint16_t edge_scales[] = { - 0x0000u, 0x0101u, 0x7f7fu, 0xfefeu, 0x0080u, - 0x8000u, 0x7f00u, 0x007fu, 0xff00u, 0x00ffu, - }; - for (size_t i = 0; i < sizeof(edge_scales) / sizeof(edge_scales[0]) && - out.size() < count; - ++i) { - out.push_back(edge_scales[i]); - } - - std::mt19937 rng(seed); - while (out.size() < count) out.push_back(static_cast(rng())); - return out; -} - -void build_s2f6_relu_focus_inputs(std::vector &a, std::vector &b, - std::vector &scale) { - a.clear(); - b.clear(); - scale.clear(); - - const uint32_t a_bits[] = { - 0xff800000u, 0x7fc00000u, 0x80000000u, 0xbf800000u, 0x00000000u, - 0x7f800000u, 0x7f7fffffu, 0xff7fffffu, - }; - const uint32_t b_bits[] = { - 0x7f800000u, 0xffc00000u, 0x00000000u, 0x3f800000u, 0x80000000u, - 0xbf800000u, 0xff800000u, 0x7f800000u, - }; - const uint16_t scale_words[] = { - 0x0000u, 0x0000u, 0x7f7fu, 0x0101u, 0x8000u, 0x0080u, 0x00ffu, 0xff00u, - }; - - const size_t n = sizeof(a_bits) / sizeof(a_bits[0]); - for (size_t i = 0; i < n; ++i) { - a.push_back(bits_to_f32(a_bits[i])); - b.push_back(bits_to_f32(b_bits[i])); - scale.push_back(scale_words[i]); - } -} - -__global__ void k_f16_rz(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rz.f16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_f16_rm(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rm.f16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_f16_rp(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rp.f16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_f16_sat_rn(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.sat.f16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_f16_satfinite_relu_rn(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.relu.f16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_bf16_rz(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rz.bf16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_bf16_rm(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rm.bf16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_bf16_rp(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rp.bf16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_bf16_satfinite_relu_rn(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.relu.bf16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_tf32_rz(uint32_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rz.tf32.f32 %0, %1;" : "=r"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_tf32_rna(uint32_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rna.tf32.f32 %0, %1;" : "=r"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_tf32_satfinite_relu_rn(uint32_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rn.satfinite.relu.tf32.f32 %0, %1;" : "=r"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_f32_bf16_ftz_rn(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - float d = 0.0f; - asm volatile("cvt.rn.ftz.f32.bf16 %0, %1;" : "=f"(d) : "h"(in[i])); - out[i] = __float_as_uint(d); -} - -__global__ void k_f16_bf16_rn(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.f16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_f64_bf16(uint64_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - double d = 0.0; - asm volatile("cvt.f64.bf16 %0, %1;" : "=d"(d) : "h"(in[i])); - out[i] = __double_as_longlong(d); -} - -__global__ void k_s32_bf16_rzi(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rzi.s32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_u32_bf16_rzi(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rzi.u32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_s32_bf16_rni(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rni.s32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_s32_bf16_rmi(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rmi.s32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_s32_bf16_rpi(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rpi.s32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_u32_bf16_rni(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rni.u32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_u32_bf16_rmi(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rmi.u32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_u32_bf16_rpi(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rpi.u32.bf16 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_s16_bf16_rzi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - short d = 0; - asm volatile("cvt.rzi.s16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_s16_bf16_rni(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - short d = 0; - asm volatile("cvt.rni.s16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_s16_bf16_rmi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - short d = 0; - asm volatile("cvt.rmi.s16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_s16_bf16_rpi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - short d = 0; - asm volatile("cvt.rpi.s16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_u16_bf16_rzi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rzi.u16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_u16_bf16_rni(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rni.u16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_u16_bf16_rmi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rmi.u16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_u16_bf16_rpi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rpi.u16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_s8_bf16_rzi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - short d = 0; - asm volatile("{ .reg .s8 t; cvt.rzi.s8.bf16 t, %1; cvt.s16.s8 %0, t; }" - : "=h"(d) - : "h"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_s8_bf16_rni(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - short d = 0; - asm volatile("{ .reg .s8 t; cvt.rni.s8.bf16 t, %1; cvt.s16.s8 %0, t; }" - : "=h"(d) - : "h"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_s8_bf16_rmi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - short d = 0; - asm volatile("{ .reg .s8 t; cvt.rmi.s8.bf16 t, %1; cvt.s16.s8 %0, t; }" - : "=h"(d) - : "h"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_s8_bf16_rpi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - short d = 0; - asm volatile("{ .reg .s8 t; cvt.rpi.s8.bf16 t, %1; cvt.s16.s8 %0, t; }" - : "=h"(d) - : "h"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_u8_bf16_rzi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("{ .reg .u8 t; cvt.rzi.u8.bf16 t, %1; cvt.u16.u8 %0, t; }" - : "=h"(d) - : "h"(in[i])); - out[i] = d; -} - -__global__ void k_u8_bf16_rni(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("{ .reg .u8 t; cvt.rni.u8.bf16 t, %1; cvt.u16.u8 %0, t; }" - : "=h"(d) - : "h"(in[i])); - out[i] = d; -} - -__global__ void k_u8_bf16_rmi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("{ .reg .u8 t; cvt.rmi.u8.bf16 t, %1; cvt.u16.u8 %0, t; }" - : "=h"(d) - : "h"(in[i])); - out[i] = d; -} - -__global__ void k_u8_bf16_rpi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("{ .reg .u8 t; cvt.rpi.u8.bf16 t, %1; cvt.u16.u8 %0, t; }" - : "=h"(d) - : "h"(in[i])); - out[i] = d; -} - -__global__ void k_s64_bf16_rzi(uint64_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - long long d = 0; - asm volatile("cvt.rzi.s64.bf16 %0, %1;" : "=l"(d) : "h"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_u64_bf16_rzi(uint64_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned long long d = 0; - asm volatile("cvt.rzi.u64.bf16 %0, %1;" : "=l"(d) : "h"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_s64_f32_rzi(uint64_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - long long d = 0; - asm volatile("cvt.rzi.s64.f32 %0, %1;" : "=l"(d) : "f"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_u64_f32_rzi(uint64_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned long long d = 0; - asm volatile("cvt.rzi.u64.f32 %0, %1;" : "=l"(d) : "f"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_s32_f32_rzi_sat(uint32_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rzi.sat.s32.f32 %0, %1;" : "=r"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_u32_f32_rzi_sat(uint32_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rzi.sat.u32.f32 %0, %1;" : "=r"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_s32_f64_rzi_sat(uint32_t *out, const double *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rzi.sat.s32.f64 %0, %1;" : "=r"(d) : "d"(in[i])); - out[i] = d; -} - -__global__ void k_u32_f64_rzi_sat(uint32_t *out, const double *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rzi.sat.u32.f64 %0, %1;" : "=r"(d) : "d"(in[i])); - out[i] = d; -} - -__global__ void k_s16_f32_rzi_sat(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - short d = 0; - asm volatile("cvt.rzi.sat.s16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_u16_f32_rzi_sat(uint16_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rzi.sat.u16.f32 %0, %1;" : "=h"(d) : "f"(in[i])); - out[i] = d; -} - -__global__ void k_s64_f32_rzi_sat(uint64_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - long long d = 0; - asm volatile("cvt.rzi.sat.s64.f32 %0, %1;" : "=l"(d) : "f"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_u64_f32_rzi_sat(uint64_t *out, const float *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned long long d = 0; - asm volatile("cvt.rzi.sat.u64.f32 %0, %1;" : "=l"(d) : "f"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_s16_f64_rzi_sat(uint16_t *out, const double *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - short d = 0; - asm volatile("cvt.rzi.sat.s16.f64 %0, %1;" : "=h"(d) : "d"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_u16_f64_rzi_sat(uint16_t *out, const double *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rzi.sat.u16.f64 %0, %1;" : "=h"(d) : "d"(in[i])); - out[i] = d; -} - -__global__ void k_s64_f64_rzi_sat(uint64_t *out, const double *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - long long d = 0; - asm volatile("cvt.rzi.sat.s64.f64 %0, %1;" : "=l"(d) : "d"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_u64_f64_rzi_sat(uint64_t *out, const double *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned long long d = 0; - asm volatile("cvt.rzi.sat.u64.f64 %0, %1;" : "=l"(d) : "d"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_s64_f64_rzi(uint64_t *out, const double *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - long long d = 0; - asm volatile("cvt.rzi.s64.f64 %0, %1;" : "=l"(d) : "d"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_u64_f64_rzi(uint64_t *out, const double *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned long long d = 0; - asm volatile("cvt.rzi.u64.f64 %0, %1;" : "=l"(d) : "d"(in[i])); - out[i] = static_cast(d); -} - -__global__ void k_bf16_f16_rn(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.bf16.f16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_bf16_s32_rn(uint16_t *out, const int32_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.bf16.s32 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_bf16_u32_rz(uint16_t *out, const uint32_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rz.bf16.u32 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_bf16_f64_rn(uint16_t *out, const double *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.bf16.f64 %0, %1;" : "=h"(d) : "d"(in[i])); - out[i] = d; -} - -__global__ void k_bf16_bf16(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.bf16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_bf16_bf16_rzi(uint16_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rzi.bf16.bf16 %0, %1;" : "=h"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_f16x2_rn(uint32_t *out, const float *a, const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;" : "=r"(d) : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_bf16x2_rz(uint32_t *out, const float *a, const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rz.bf16x2.f32 %0, %1, %2;" : "=r"(d) : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_e2m1x2_satfinite_rn(uint16_t *out, const float *a, - const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("{ .reg .b8 t; " - "cvt.rn.satfinite.e2m1x2.f32 t, %1, %2; " - "cvt.u16.u8 %0, t; }" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_e4m3x2_satfinite_rn(uint16_t *out, const float *a, - const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_e5m2x2_satfinite_rn(uint16_t *out, const float *a, - const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e5m2x2.f32 %0, %1, %2;" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_e2m3x2_satfinite_rn(uint16_t *out, const float *a, - const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e2m3x2.f32 %0, %1, %2;" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_e3m2x2_satfinite_rn(uint16_t *out, const float *a, - const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e3m2x2.f32 %0, %1, %2;" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_f16x2_e4m3x2_rn(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rn.f16x2.e4m3x2 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_f16x2_e5m2x2_rn(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rn.f16x2.e5m2x2 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_f16x2_e2m1x2_rn(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("{ .reg .b8 t; " - "cvt.u8.u16 t, %1; " - "cvt.rn.f16x2.e2m1x2 %0, t; }" - : "=r"(d) - : "h"(in[i])); - out[i] = d; -} - -__global__ void k_f16x2_e2m3x2_rn(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rn.f16x2.e2m3x2 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_f16x2_e3m2x2_rn(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rn.f16x2.e3m2x2 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_ue8m0x2_satfinite_rz(uint16_t *out, const float *a, - const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rz.satfinite.ue8m0x2.f32 %0, %1, %2;" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_ue8m0x2_rz(uint16_t *out, const float *a, const float *b, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rz.ue8m0x2.f32 %0, %1, %2;" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_ue8m0x2_satfinite_rp(uint16_t *out, const float *a, - const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rp.satfinite.ue8m0x2.f32 %0, %1, %2;" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_ue8m0x2_rp(uint16_t *out, const float *a, const float *b, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rp.ue8m0x2.f32 %0, %1, %2;" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_s2f6x2_satfinite_rn(uint16_t *out, const float *a, - const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.s2f6x2.f32 %0, %1, %2;" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_s2f6x2_satfinite_relu_rn(uint16_t *out, const float *a, - const float *b, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.relu.s2f6x2.f32 %0, %1, %2;" - : "=h"(d) - : "f"(a[i]), "f"(b[i])); - out[i] = d; -} - -__global__ void k_e4m3x2_f16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e4m3x2.f16x2 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_e5m2x2_f16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e5m2x2.f16x2 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_e2m1x2_f16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("{ .reg .b8 t; " - "cvt.rn.satfinite.e2m1x2.f16x2 t, %1; " - "cvt.u16.u8 %0, t; }" - : "=h"(d) - : "r"(in[i])); - out[i] = d; -} - -__global__ void k_e2m3x2_f16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e2m3x2.f16x2 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_e3m2x2_f16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e3m2x2.f16x2 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_e4m3x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_e5m2x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e5m2x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_e2m1x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("{ .reg .b8 t; " - "cvt.rn.satfinite.e2m1x2.bf16x2 t, %1; " - "cvt.u16.u8 %0, t; }" - : "=h"(d) - : "r"(in[i])); - out[i] = d; -} - -__global__ void k_e2m3x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e2m3x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_e3m2x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.e3m2x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_ue8m0x2_bf16x2_satfinite_rz(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rz.satfinite.ue8m0x2.bf16x2 %0, %1;" - : "=h"(d) - : "r"(in[i])); - out[i] = d; -} - -__global__ void k_ue8m0x2_bf16x2_rz(uint16_t *out, const uint32_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rz.ue8m0x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -__global__ void k_ue8m0x2_bf16x2_satfinite_rp(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rp.satfinite.ue8m0x2.bf16x2 %0, %1;" - : "=h"(d) - : "r"(in[i])); - out[i] = d; -} - -__global__ void k_ue8m0x2_bf16x2_rp(uint16_t *out, const uint32_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rp.ue8m0x2.bf16x2 %0, %1;" : "=h"(d) : "r"(in[i])); - out[i] = d; -} - -#if LOWP_CVT_HAS_UE8M0X2_BF16X2_RELU -__global__ void k_ue8m0x2_bf16x2_satfinite_relu_rz(uint16_t *out, - const uint32_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rz.satfinite.relu.ue8m0x2.bf16x2 %0, %1;" - : "=h"(d) - : "r"(in[i])); - out[i] = d; -} -#endif - -__global__ void k_bf16x2_ue8m0x2_rn(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rn.bf16x2.ue8m0x2 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_s2f6x2_bf16x2_satfinite_rn(uint16_t *out, const uint32_t *in, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.s2f6x2.bf16x2 %0, %1;" - : "=h"(d) - : "r"(in[i])); - out[i] = d; -} - -__global__ void k_s2f6x2_bf16x2_satfinite_relu_rn(uint16_t *out, - const uint32_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.relu.s2f6x2.bf16x2 %0, %1;" - : "=h"(d) - : "r"(in[i])); - out[i] = d; -} - -__global__ void k_bf16x2_s2f6x2_rn(uint32_t *out, const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rn.bf16x2.s2f6x2 %0, %1;" : "=r"(d) : "h"(in[i])); - out[i] = d; -} - -__global__ void k_bf16x2_s2f6x2_satfinite_relu_rn(uint32_t *out, - const uint16_t *in, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rn.satfinite.relu.bf16x2.s2f6x2 %0, %1;" - : "=r"(d) - : "h"(in[i])); - out[i] = d; -} - -__global__ void k_s2f6x2_f32_sc(uint16_t *out, const float *a, const float *b, - const uint16_t *scale, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rn.satfinite.scaled::n2::ue8m0.s2f6x2.f32 %0, %1, %2, %3;" - : "=h"(d) - : "f"(a[i]), "f"(b[i]), "h"(scale[i])); - out[i] = d; -} - -__global__ void k_s2f6x2_f32_sc_relu(uint16_t *out, const float *a, - const float *b, const uint16_t *scale, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile( - "cvt.rn.satfinite.relu.scaled::n2::ue8m0.s2f6x2.f32 %0, %1, %2, %3;" - : "=h"(d) - : "f"(a[i]), "f"(b[i]), "h"(scale[i])); - out[i] = d; -} - -__global__ void k_s2f6x2_bf16x2_sc(uint16_t *out, const uint32_t *in, - const uint16_t *scale, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile( - "cvt.rn.satfinite.scaled::n2::ue8m0.s2f6x2.bf16x2 %0, %1, %2;" - : "=h"(d) - : "r"(in[i]), "h"(scale[i])); - out[i] = d; -} - -__global__ void k_s2f6x2_bf16x2_sc_relu(uint16_t *out, const uint32_t *in, - const uint16_t *scale, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile( - "cvt.rn.satfinite.relu.scaled::n2::ue8m0.s2f6x2.bf16x2 %0, %1, %2;" - : "=h"(d) - : "r"(in[i]), "h"(scale[i])); - out[i] = d; -} - -__global__ void k_bf16x2_s2f6x2_sc(uint32_t *out, const uint16_t *in, - const uint16_t *scale, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile( - "cvt.rn.satfinite.scaled::n2::ue8m0.bf16x2.s2f6x2 %0, %1, %2;" - : "=r"(d) - : "h"(in[i]), "h"(scale[i])); - out[i] = d; -} - -__global__ void k_bf16x2_s2f6x2_sc_relu(uint32_t *out, const uint16_t *in, - const uint16_t *scale, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile( - "cvt.rn.satfinite.relu.scaled::n2::ue8m0.bf16x2.s2f6x2 %0, %1, %2;" - : "=r"(d) - : "h"(in[i]), "h"(scale[i])); - out[i] = d; -} - -template -void run_unary_kernel(const std::vector &in, std::vector &out, - LaunchFn launch_kernel) { - TIn *d_in = nullptr; - TOut *d_out = nullptr; - const int n = static_cast(in.size()); - CUDACHK(cudaMalloc(&d_in, sizeof(TIn) * in.size())); - CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * in.size())); - CUDACHK(cudaMemcpy(d_in, in.data(), sizeof(TIn) * in.size(), - cudaMemcpyHostToDevice)); - const int block = 128; - const int grid = (n + block - 1) / block; - launch_kernel(grid, block, d_out, d_in, n); - CUDACHK(cudaDeviceSynchronize()); - out.resize(in.size()); - CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), - cudaMemcpyDeviceToHost)); - CUDACHK(cudaFree(d_out)); - CUDACHK(cudaFree(d_in)); -} - -template -void run_pair_kernel(const std::vector &a, const std::vector &b, - std::vector &out, LaunchFn launch_kernel) { - float *d_a = nullptr; - float *d_b = nullptr; - TOut *d_out = nullptr; - const int n = static_cast(a.size()); - CUDACHK(cudaMalloc(&d_a, sizeof(float) * a.size())); - CUDACHK(cudaMalloc(&d_b, sizeof(float) * b.size())); - CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * a.size())); - CUDACHK(cudaMemcpy(d_a, a.data(), sizeof(float) * a.size(), - cudaMemcpyHostToDevice)); - CUDACHK(cudaMemcpy(d_b, b.data(), sizeof(float) * b.size(), - cudaMemcpyHostToDevice)); - const int block = 128; - const int grid = (n + block - 1) / block; - launch_kernel(grid, block, d_out, d_a, d_b, n); - CUDACHK(cudaDeviceSynchronize()); - out.resize(a.size()); - CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), - cudaMemcpyDeviceToHost)); - CUDACHK(cudaFree(d_out)); - CUDACHK(cudaFree(d_b)); - CUDACHK(cudaFree(d_a)); -} - -template -void run_unary_kernel_with_scale(const std::vector &in, - const std::vector &scale, - std::vector &out, - LaunchFn launch_kernel) { - if (in.size() != scale.size()) { - std::cerr << "input/scale size mismatch\n"; - std::exit(1); - } - TIn *d_in = nullptr; - TScale *d_scale = nullptr; - TOut *d_out = nullptr; - const int n = static_cast(in.size()); - CUDACHK(cudaMalloc(&d_in, sizeof(TIn) * in.size())); - CUDACHK(cudaMalloc(&d_scale, sizeof(TScale) * scale.size())); - CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * in.size())); - CUDACHK( - cudaMemcpy(d_in, in.data(), sizeof(TIn) * in.size(), cudaMemcpyHostToDevice)); - CUDACHK(cudaMemcpy(d_scale, scale.data(), sizeof(TScale) * scale.size(), - cudaMemcpyHostToDevice)); - const int block = 128; - const int grid = (n + block - 1) / block; - launch_kernel(grid, block, d_out, d_in, d_scale, n); - CUDACHK(cudaDeviceSynchronize()); - out.resize(in.size()); - CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), - cudaMemcpyDeviceToHost)); - CUDACHK(cudaFree(d_out)); - CUDACHK(cudaFree(d_scale)); - CUDACHK(cudaFree(d_in)); -} - -template -void run_pair_kernel_with_scale(const std::vector &a, - const std::vector &b, - const std::vector &scale, - std::vector &out, - LaunchFn launch_kernel) { - if (a.size() != b.size() || a.size() != scale.size()) { - std::cerr << "pair/scale size mismatch\n"; - std::exit(1); - } - float *d_a = nullptr; - float *d_b = nullptr; - uint16_t *d_scale = nullptr; - TOut *d_out = nullptr; - const int n = static_cast(a.size()); - CUDACHK(cudaMalloc(&d_a, sizeof(float) * a.size())); - CUDACHK(cudaMalloc(&d_b, sizeof(float) * b.size())); - CUDACHK(cudaMalloc(&d_scale, sizeof(uint16_t) * scale.size())); - CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * a.size())); - CUDACHK(cudaMemcpy(d_a, a.data(), sizeof(float) * a.size(), - cudaMemcpyHostToDevice)); - CUDACHK(cudaMemcpy(d_b, b.data(), sizeof(float) * b.size(), - cudaMemcpyHostToDevice)); - CUDACHK(cudaMemcpy(d_scale, scale.data(), sizeof(uint16_t) * scale.size(), - cudaMemcpyHostToDevice)); - const int block = 128; - const int grid = (n + block - 1) / block; - launch_kernel(grid, block, d_out, d_a, d_b, d_scale, n); - CUDACHK(cudaDeviceSynchronize()); - out.resize(a.size()); - CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), - cudaMemcpyDeviceToHost)); - CUDACHK(cudaFree(d_out)); - CUDACHK(cudaFree(d_scale)); - CUDACHK(cudaFree(d_b)); - CUDACHK(cudaFree(d_a)); -} - -template -void dump_hex(std::ofstream &ofs, const std::string &label, - const std::vector &vals, unsigned width) { - for (size_t i = 0; i < vals.size(); ++i) { - ofs << label << " " << i << " 0x" << std::hex << std::setw(width) - << std::setfill('0') - << static_cast(vals[i]) << std::dec << "\n"; - } -} - -int run_and_dump(const char *out_path) { - const std::vector scalars = build_scalar_inputs(); - std::vector pair_a, pair_b; - build_pair_inputs(pair_a, pair_b); - const std::vector bf16_inputs = build_bf16_inputs(); - const std::vector f16_inputs = build_f16_inputs(); - const std::vector s32_inputs = build_s32_inputs(); - const std::vector u32_inputs = build_u32_inputs(); - const std::vector f64_inputs = build_f64_inputs(); - const std::vector e2m1x2_inputs_exhaustive = - build_e2m1x2_inputs_exhaustive(); - const std::vector e2m3x2_inputs_exhaustive = - build_e2m3x2_inputs_exhaustive(); - const std::vector e3m2x2_inputs_exhaustive = - build_e3m2x2_inputs_exhaustive(); - const std::vector u16_inputs_exhaustive = - build_u16_inputs_exhaustive(); - const std::vector pair_scales = - build_scale_inputs(pair_a.size(), 0x4f2a9c13u); - std::vector s2f6_relu_focus_a, s2f6_relu_focus_b; - std::vector s2f6_relu_focus_scales; - build_s2f6_relu_focus_inputs(s2f6_relu_focus_a, s2f6_relu_focus_b, - s2f6_relu_focus_scales); - const std::vector exhaustive_scales = - build_scale_inputs(u16_inputs_exhaustive.size(), 0x6be03491u); - - std::vector out_u16; - std::vector out_u32; - std::vector out_u64; - std::vector tmp_u16; - std::vector tmp_u32; - - std::ofstream ofs(out_path, std::ios::out | std::ios::trunc); - if (!ofs) { - std::cerr << "failed to open output file: " << out_path << "\n"; - return 1; - } - - run_unary_kernel( - scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, - int n) { k_f16_rz<<>>(d_out, d_in, n); }); - dump_hex(ofs, "k_f16_rz", out_u16, 4); - run_unary_kernel( - scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, - int n) { k_f16_rm<<>>(d_out, d_in, n); }); - dump_hex(ofs, "k_f16_rm", out_u16, 4); - run_unary_kernel( - scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, - int n) { k_f16_rp<<>>(d_out, d_in, n); }); - dump_hex(ofs, "k_f16_rp", out_u16, 4); - run_unary_kernel( - scalars, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_in, int n) { - k_f16_sat_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16_sat_rn", out_u16, 4); - run_unary_kernel( - scalars, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_in, int n) { - k_f16_satfinite_relu_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16_satfinite_relu_rn", out_u16, 4); - - run_unary_kernel( - scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, - int n) { k_bf16_rz<<>>(d_out, d_in, n); }); - dump_hex(ofs, "k_bf16_rz", out_u16, 4); - run_unary_kernel( - scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, - int n) { k_bf16_rm<<>>(d_out, d_in, n); }); - dump_hex(ofs, "k_bf16_rm", out_u16, 4); - run_unary_kernel( - scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, - int n) { k_bf16_rp<<>>(d_out, d_in, n); }); - dump_hex(ofs, "k_bf16_rp", out_u16, 4); - run_unary_kernel( - scalars, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_in, int n) { - k_bf16_satfinite_relu_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16_satfinite_relu_rn", out_u16, 4); - - run_unary_kernel( - scalars, out_u32, [](int grid, int block, uint32_t *d_out, const float *d_in, - int n) { k_tf32_rz<<>>(d_out, d_in, n); }); - dump_hex(ofs, "k_tf32_rz", out_u32, 8); - run_unary_kernel( - scalars, out_u32, [](int grid, int block, uint32_t *d_out, const float *d_in, - int n) { k_tf32_rna<<>>(d_out, d_in, n); }); - dump_hex(ofs, "k_tf32_rna", out_u32, 8); - run_unary_kernel( - scalars, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_in, int n) { - k_tf32_satfinite_relu_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_tf32_satfinite_relu_rn", out_u32, 8); - - run_unary_kernel( - bf16_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f32_bf16_ftz_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f32_bf16_ftz_rn", out_u32, 8); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_f16_bf16_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16_bf16_rn", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u64, - [](int grid, int block, uint64_t *d_out, const uint16_t *d_in, int n) { - k_f64_bf16<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f64_bf16", out_u64, 16); - run_unary_kernel( - bf16_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_s32_bf16_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s32_bf16_rzi", out_u32, 8); - run_unary_kernel( - bf16_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_u32_bf16_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u32_bf16_rzi", out_u32, 8); - run_unary_kernel( - bf16_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_s32_bf16_rni<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s32_bf16_rni", out_u32, 8); - run_unary_kernel( - bf16_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_s32_bf16_rmi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s32_bf16_rmi", out_u32, 8); - run_unary_kernel( - bf16_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_s32_bf16_rpi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s32_bf16_rpi", out_u32, 8); - run_unary_kernel( - bf16_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_u32_bf16_rni<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u32_bf16_rni", out_u32, 8); - run_unary_kernel( - bf16_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_u32_bf16_rmi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u32_bf16_rmi", out_u32, 8); - run_unary_kernel( - bf16_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_u32_bf16_rpi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u32_bf16_rpi", out_u32, 8); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_s16_bf16_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s16_bf16_rzi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_s16_bf16_rni<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s16_bf16_rni", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_s16_bf16_rmi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s16_bf16_rmi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_s16_bf16_rpi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s16_bf16_rpi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_u16_bf16_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u16_bf16_rzi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_u16_bf16_rni<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u16_bf16_rni", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_u16_bf16_rmi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u16_bf16_rmi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_u16_bf16_rpi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u16_bf16_rpi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_s8_bf16_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s8_bf16_rzi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_s8_bf16_rni<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s8_bf16_rni", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_s8_bf16_rmi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s8_bf16_rmi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_s8_bf16_rpi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s8_bf16_rpi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_u8_bf16_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u8_bf16_rzi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_u8_bf16_rni<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u8_bf16_rni", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_u8_bf16_rmi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u8_bf16_rmi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_u8_bf16_rpi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u8_bf16_rpi", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u64, - [](int grid, int block, uint64_t *d_out, const uint16_t *d_in, int n) { - k_s64_bf16_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s64_bf16_rzi", out_u64, 16); - run_unary_kernel( - bf16_inputs, out_u64, - [](int grid, int block, uint64_t *d_out, const uint16_t *d_in, int n) { - k_u64_bf16_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u64_bf16_rzi", out_u64, 16); - run_unary_kernel( - scalars, out_u64, [](int grid, int block, uint64_t *d_out, const float *d_in, - int n) { - k_s64_f32_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s64_f32_rzi", out_u64, 16); - run_unary_kernel( - scalars, out_u64, [](int grid, int block, uint64_t *d_out, const float *d_in, - int n) { - k_u64_f32_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u64_f32_rzi", out_u64, 16); - run_unary_kernel( - scalars, out_u32, [](int grid, int block, uint32_t *d_out, const float *d_in, - int n) { - k_s32_f32_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s32_f32_rzi_sat", out_u32, 8); - run_unary_kernel( - scalars, out_u32, [](int grid, int block, uint32_t *d_out, const float *d_in, - int n) { - k_u32_f32_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u32_f32_rzi_sat", out_u32, 8); - run_unary_kernel( - f64_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const double *d_in, int n) { - k_s32_f64_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s32_f64_rzi_sat", out_u32, 8); - run_unary_kernel( - f64_inputs, out_u32, - [](int grid, int block, uint32_t *d_out, const double *d_in, int n) { - k_u32_f64_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u32_f64_rzi_sat", out_u32, 8); - run_unary_kernel( - scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, - int n) { - k_s16_f32_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s16_f32_rzi_sat", out_u16, 4); - run_unary_kernel( - scalars, out_u16, [](int grid, int block, uint16_t *d_out, const float *d_in, - int n) { - k_u16_f32_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u16_f32_rzi_sat", out_u16, 4); - run_unary_kernel( - scalars, out_u64, [](int grid, int block, uint64_t *d_out, const float *d_in, - int n) { - k_s64_f32_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s64_f32_rzi_sat", out_u64, 16); - run_unary_kernel( - scalars, out_u64, [](int grid, int block, uint64_t *d_out, const float *d_in, - int n) { - k_u64_f32_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u64_f32_rzi_sat", out_u64, 16); - run_unary_kernel( - f64_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const double *d_in, int n) { - k_s16_f64_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s16_f64_rzi_sat", out_u16, 4); - run_unary_kernel( - f64_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const double *d_in, int n) { - k_u16_f64_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u16_f64_rzi_sat", out_u16, 4); - run_unary_kernel( - f64_inputs, out_u64, - [](int grid, int block, uint64_t *d_out, const double *d_in, int n) { - k_s64_f64_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s64_f64_rzi", out_u64, 16); - run_unary_kernel( - f64_inputs, out_u64, - [](int grid, int block, uint64_t *d_out, const double *d_in, int n) { - k_u64_f64_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u64_f64_rzi", out_u64, 16); - run_unary_kernel( - f64_inputs, out_u64, - [](int grid, int block, uint64_t *d_out, const double *d_in, int n) { - k_s64_f64_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s64_f64_rzi_sat", out_u64, 16); - run_unary_kernel( - f64_inputs, out_u64, - [](int grid, int block, uint64_t *d_out, const double *d_in, int n) { - k_u64_f64_rzi_sat<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_u64_f64_rzi_sat", out_u64, 16); - - run_unary_kernel( - f16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_bf16_f16_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16_f16_rn", out_u16, 4); - run_unary_kernel( - s32_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const int32_t *d_in, int n) { - k_bf16_s32_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16_s32_rn", out_u16, 4); - run_unary_kernel( - u32_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_bf16_u32_rz<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16_u32_rz", out_u16, 4); - run_unary_kernel( - f64_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const double *d_in, int n) { - k_bf16_f64_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16_f64_rn", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_bf16_bf16<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16_bf16", out_u16, 4); - run_unary_kernel( - bf16_inputs, out_u16, - [](int grid, int block, uint16_t *d_out, const uint16_t *d_in, int n) { - k_bf16_bf16_rzi<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16_bf16_rzi", out_u16, 4); - - run_pair_kernel( - pair_a, pair_b, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, const float *d_b, - int n) { k_f16x2_rn<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_f16x2_rn", out_u32, 8); - const std::vector f16x2_pairs = out_u32; - run_pair_kernel( - pair_a, pair_b, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, const float *d_b, - int n) { k_bf16x2_rz<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_bf16x2_rz", out_u32, 8); - const std::vector bf16x2_pairs = out_u32; - - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_e4m3x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_e4m3x2_satfinite_rn", out_u16, 4); - const std::vector e4m3x2_pairs = out_u16; - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_e5m2x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_e5m2x2_satfinite_rn", out_u16, 4); - const std::vector e5m2x2_pairs = out_u16; - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_e2m1x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_e2m1x2_satfinite_rn", out_u16, 4); - const std::vector e2m1x2_pairs = out_u16; - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_e2m3x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_e2m3x2_satfinite_rn", out_u16, 4); - const std::vector e2m3x2_pairs = out_u16; - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_e3m2x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_e3m2x2_satfinite_rn", out_u16, 4); - const std::vector e3m2x2_pairs = out_u16; - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_ue8m0x2_satfinite_rz<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_ue8m0x2_satfinite_rz", out_u16, 4); - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_ue8m0x2_rz<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_ue8m0x2_rz", out_u16, 4); - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_ue8m0x2_satfinite_rp<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_ue8m0x2_satfinite_rp", out_u16, 4); - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_ue8m0x2_rp<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_ue8m0x2_rp", out_u16, 4); - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_s2f6x2_satfinite_rn<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_s2f6x2_satfinite_rn", out_u16, 4); - run_pair_kernel( - pair_a, pair_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_s2f6x2_satfinite_relu_rn<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_s2f6x2_satfinite_relu_rn", out_u16, 4); - run_pair_kernel_with_scale( - pair_a, pair_b, pair_scales, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, - const float *d_b, const uint16_t *d_scale, int n) { - k_s2f6x2_f32_sc<<>>(d_out, d_a, d_b, d_scale, n); - }); - dump_hex(ofs, "k_s2f6x2_f32_satfinite_scaled_rn", out_u16, 4); - run_pair_kernel_with_scale( - pair_a, pair_b, pair_scales, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, - const float *d_b, const uint16_t *d_scale, int n) { - k_s2f6x2_f32_sc_relu<<>>(d_out, d_a, d_b, d_scale, n); - }); - dump_hex(ofs, "k_s2f6x2_f32_satfinite_relu_scaled_rn", out_u16, 4); - run_pair_kernel( - s2f6_relu_focus_a, s2f6_relu_focus_b, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, const float *d_b, - int n) { k_s2f6x2_satfinite_relu_rn<<>>(d_out, d_a, d_b, n); }); - dump_hex(ofs, "k_s2f6x2_satfinite_relu_rn_focus", out_u16, 4); - run_pair_kernel_with_scale( - s2f6_relu_focus_a, s2f6_relu_focus_b, s2f6_relu_focus_scales, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, - const float *d_b, const uint16_t *d_scale, int n) { - k_s2f6x2_f32_sc_relu<<>>(d_out, d_a, d_b, d_scale, n); - }); - dump_hex(ofs, "k_s2f6x2_f32_satfinite_relu_scaled_rn_focus", out_u16, 4); - run_pair_kernel( - s2f6_relu_focus_a, s2f6_relu_focus_b, tmp_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, const float *d_b, - int n) { k_bf16x2_rz<<>>(d_out, d_a, d_b, n); }); - const std::vector s2f6_relu_focus_bf16x2 = tmp_u32; - run_unary_kernel( - s2f6_relu_focus_bf16x2, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_s2f6x2_bf16x2_satfinite_relu_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_relu_rn_focus", tmp_u16, 4); - run_unary_kernel_with_scale( - s2f6_relu_focus_bf16x2, s2f6_relu_focus_scales, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, - const uint16_t *d_scale, int n) { - k_s2f6x2_bf16x2_sc_relu<<>>(d_out, d_in, d_scale, n); - }); - dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_relu_scaled_rn_focus", tmp_u16, 4); - - run_unary_kernel( - e4m3x2_pairs, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f16x2_e4m3x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16x2_e4m3x2_rn", tmp_u32, 8); - run_unary_kernel( - e5m2x2_pairs, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f16x2_e5m2x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16x2_e5m2x2_rn", tmp_u32, 8); - run_unary_kernel( - e2m1x2_pairs, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f16x2_e2m1x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16x2_e2m1x2_rn", tmp_u32, 8); - run_unary_kernel( - e2m3x2_pairs, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f16x2_e2m3x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16x2_e2m3x2_rn", tmp_u32, 8); - run_unary_kernel( - e3m2x2_pairs, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f16x2_e3m2x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16x2_e3m2x2_rn", tmp_u32, 8); - - run_unary_kernel( - f16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e4m3x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e4m3x2_f16x2_satfinite_rn", tmp_u16, 4); - run_unary_kernel( - f16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e5m2x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e5m2x2_f16x2_satfinite_rn", tmp_u16, 4); - run_unary_kernel( - f16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e2m1x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e2m1x2_f16x2_satfinite_rn", tmp_u16, 4); - run_unary_kernel( - f16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e2m3x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e2m3x2_f16x2_satfinite_rn", tmp_u16, 4); - run_unary_kernel( - f16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e3m2x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e3m2x2_f16x2_satfinite_rn", tmp_u16, 4); - - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e4m3x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e4m3x2_bf16x2_satfinite_rn", tmp_u16, 4); - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e5m2x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e5m2x2_bf16x2_satfinite_rn", tmp_u16, 4); - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e2m1x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e2m1x2_bf16x2_satfinite_rn", tmp_u16, 4); - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e2m3x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e2m3x2_bf16x2_satfinite_rn", tmp_u16, 4); - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e3m2x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e3m2x2_bf16x2_satfinite_rn", tmp_u16, 4); - - run_unary_kernel( - u16_inputs_exhaustive, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f16x2_e4m3x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16x2_e4m3x2_rn_exhaustive", tmp_u32, 8); - run_unary_kernel( - tmp_u32, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e4m3x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e4m3x2_f16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, - 4); - - run_unary_kernel( - u16_inputs_exhaustive, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f16x2_e5m2x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16x2_e5m2x2_rn_exhaustive", tmp_u32, 8); - run_unary_kernel( - tmp_u32, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e5m2x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e5m2x2_f16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, - 4); - - run_unary_kernel( - e2m1x2_inputs_exhaustive, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f16x2_e2m1x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16x2_e2m1x2_rn_exhaustive", tmp_u32, 8); - run_unary_kernel( - tmp_u32, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e2m1x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e2m1x2_f16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, - 4); - - run_unary_kernel( - e2m3x2_inputs_exhaustive, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f16x2_e2m3x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16x2_e2m3x2_rn_exhaustive", tmp_u32, 8); - run_unary_kernel( - tmp_u32, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e2m3x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e2m3x2_f16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, - 4); - - run_unary_kernel( - e3m2x2_inputs_exhaustive, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_f16x2_e3m2x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_f16x2_e3m2x2_rn_exhaustive", tmp_u32, 8); - run_unary_kernel( - tmp_u32, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_e3m2x2_f16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_e3m2x2_f16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, - 4); - - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_ue8m0x2_bf16x2_satfinite_rz<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_ue8m0x2_bf16x2_satfinite_rz", tmp_u16, 4); - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_ue8m0x2_bf16x2_rz<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_ue8m0x2_bf16x2_rz", tmp_u16, 4); - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_ue8m0x2_bf16x2_satfinite_rp<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_ue8m0x2_bf16x2_satfinite_rp", tmp_u16, 4); - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_ue8m0x2_bf16x2_rp<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_ue8m0x2_bf16x2_rp", tmp_u16, 4); -#if LOWP_CVT_HAS_UE8M0X2_BF16X2_RELU - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_ue8m0x2_bf16x2_satfinite_relu_rz<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_ue8m0x2_bf16x2_satfinite_relu_rz", tmp_u16, 4); -#endif - run_unary_kernel( - tmp_u16, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_bf16x2_ue8m0x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16x2_ue8m0x2_rn", tmp_u32, 8); - - run_unary_kernel( - u16_inputs_exhaustive, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_bf16x2_ue8m0x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16x2_ue8m0x2_rn_exhaustive", tmp_u32, 8); - run_unary_kernel( - tmp_u32, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_ue8m0x2_bf16x2_satfinite_rz<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_ue8m0x2_bf16x2_satfinite_rz_roundtrip_exhaustive", tmp_u16, - 4); - - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_s2f6x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_rn", tmp_u16, 4); - const std::vector s2f6_pairs = tmp_u16; - run_unary_kernel( - bf16x2_pairs, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_s2f6x2_bf16x2_satfinite_relu_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_relu_rn", tmp_u16, 4); - run_unary_kernel( - s2f6_pairs, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_bf16x2_s2f6x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16x2_s2f6x2_rn", tmp_u32, 8); - run_unary_kernel( - s2f6_pairs, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_bf16x2_s2f6x2_satfinite_relu_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16x2_s2f6x2_satfinite_relu_rn", tmp_u32, 8); - run_unary_kernel_with_scale( - bf16x2_pairs, pair_scales, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, - const uint16_t *d_scale, int n) { - k_s2f6x2_bf16x2_sc<<>>(d_out, d_in, d_scale, n); - }); - dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_scaled_rn", tmp_u16, 4); - run_unary_kernel_with_scale( - bf16x2_pairs, pair_scales, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, - const uint16_t *d_scale, int n) { - k_s2f6x2_bf16x2_sc_relu<<>>(d_out, d_in, d_scale, n); - }); - dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_relu_scaled_rn", tmp_u16, 4); - run_unary_kernel_with_scale( - s2f6_pairs, pair_scales, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, - const uint16_t *d_scale, int n) { - k_bf16x2_s2f6x2_sc<<>>(d_out, d_in, d_scale, n); - }); - dump_hex(ofs, "k_bf16x2_s2f6x2_satfinite_scaled_rn", tmp_u32, 8); - run_unary_kernel_with_scale( - s2f6_pairs, pair_scales, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, - const uint16_t *d_scale, int n) { - k_bf16x2_s2f6x2_sc_relu<<>>(d_out, d_in, d_scale, n); - }); - dump_hex(ofs, "k_bf16x2_s2f6x2_satfinite_relu_scaled_rn", tmp_u32, 8); - - run_unary_kernel( - u16_inputs_exhaustive, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, int n) { - k_bf16x2_s2f6x2_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_bf16x2_s2f6x2_rn_exhaustive", tmp_u32, 8); - run_unary_kernel( - tmp_u32, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, int n) { - k_s2f6x2_bf16x2_satfinite_rn<<>>(d_out, d_in, n); - }); - dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_rn_roundtrip_exhaustive", tmp_u16, - 4); - run_unary_kernel_with_scale( - u16_inputs_exhaustive, exhaustive_scales, tmp_u32, - [](int grid, int block, uint32_t *d_out, const uint16_t *d_in, - const uint16_t *d_scale, int n) { - k_bf16x2_s2f6x2_sc<<>>(d_out, d_in, d_scale, n); - }); - dump_hex(ofs, "k_bf16x2_s2f6x2_satfinite_scaled_rn_exhaustive", tmp_u32, 8); - run_unary_kernel_with_scale( - tmp_u32, exhaustive_scales, tmp_u16, - [](int grid, int block, uint16_t *d_out, const uint32_t *d_in, - const uint16_t *d_scale, int n) { - k_s2f6x2_bf16x2_sc<<>>(d_out, d_in, d_scale, n); - }); - dump_hex(ofs, "k_s2f6x2_bf16x2_satfinite_scaled_rn_roundtrip_exhaustive", - tmp_u16, 4); - - return 0; -} - -} // namespace - -int main(int argc, char **argv) { - if (argc != 3 || std::string(argv[1]) != "--output") { - std::cerr << "usage: " << argv[0] << " --output \n"; - return 1; - } - return run_and_dump(argv[2]); -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu deleted file mode 100644 index f8873d205..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/lowp_cvt_runtime_hw_sim_diff_rs.cu +++ /dev/null @@ -1,748 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - -#define CUDACHK(call) \ - do { \ - cudaError_t _err = (call); \ - if (_err != cudaSuccess) { \ - std::cerr << "CUDA error: " << cudaGetErrorString(_err) << " @ " \ - << #call << "\n"; \ - std::exit(1); \ - } \ - } while (0) - -float bits_to_f32(uint32_t bits) { - float out = 0.0f; - std::memcpy(&out, &bits, sizeof(out)); - return out; -} - -enum class rs_minifloat_family_t { - kE4M3, - kE5M2, - kE2M1, - kE2M3, - kE3M2, -}; - -struct rs_minifloat_entry_t { - uint16_t code; - float value; -}; - -unsigned rs_minifloat_code_limit(rs_minifloat_family_t fmt) { - switch (fmt) { - case rs_minifloat_family_t::kE4M3: - case rs_minifloat_family_t::kE5M2: - return 256u; - case rs_minifloat_family_t::kE2M1: - return 16u; - case rs_minifloat_family_t::kE2M3: - case rs_minifloat_family_t::kE3M2: - return 64u; - } - return 0u; -} - -float decode_rs_minifloat_code(rs_minifloat_family_t fmt, uint16_t code) { - int exp_bits = 0; - int mant_bits = 0; - int bias = 0; - bool has_inf = false; - switch (fmt) { - case rs_minifloat_family_t::kE4M3: - exp_bits = 4; - mant_bits = 3; - bias = 7; - has_inf = false; - break; - case rs_minifloat_family_t::kE5M2: - exp_bits = 5; - mant_bits = 2; - bias = 15; - has_inf = true; - break; - case rs_minifloat_family_t::kE2M1: - exp_bits = 2; - mant_bits = 1; - bias = 1; - has_inf = false; - break; - case rs_minifloat_family_t::kE2M3: - exp_bits = 2; - mant_bits = 3; - bias = 1; - has_inf = false; - break; - case rs_minifloat_family_t::kE3M2: - exp_bits = 3; - mant_bits = 2; - bias = 3; - has_inf = false; - break; - } - - const int sign_shift = exp_bits + mant_bits; - const bool neg = ((code >> sign_shift) & 0x1u) != 0u; - const int exp_mask = (1 << exp_bits) - 1; - const int mant_mask = (1 << mant_bits) - 1; - const int exp = (code >> mant_bits) & exp_mask; - const int mant = code & mant_mask; - - if (fmt == rs_minifloat_family_t::kE4M3 && - (code == 0x7fu || code == 0xffu)) { - return std::numeric_limits::quiet_NaN(); - } - if (has_inf && exp == exp_mask) { - if (mant == 0) { - return neg ? -std::numeric_limits::infinity() - : std::numeric_limits::infinity(); - } - return std::numeric_limits::quiet_NaN(); - } - - float value = 0.0f; - if (exp == 0) { - if (mant != 0) { - value = std::ldexp(static_cast(mant), 1 - bias - mant_bits); - } - } else { - const int sig = (1 << mant_bits) | mant; - value = std::ldexp(static_cast(sig), exp - bias - mant_bits); - } - return neg ? -value : value; -} - -std::vector collect_rs_midpoint_ties(rs_minifloat_family_t fmt, - size_t max_count) { - std::vector entries; - entries.reserve(rs_minifloat_code_limit(fmt)); - for (unsigned code = 0; code < rs_minifloat_code_limit(fmt); ++code) { - const float value = decode_rs_minifloat_code(fmt, static_cast(code)); - if (!std::isfinite(value)) continue; - entries.push_back({static_cast(code), value}); - } - - std::sort(entries.begin(), entries.end(), - [](const rs_minifloat_entry_t &a, const rs_minifloat_entry_t &b) { - if (a.value < b.value) return true; - if (a.value > b.value) return false; - return a.code < b.code; - }); - - std::vector out; - out.reserve(max_count); - for (size_t i = 0; i + 1 < entries.size() && out.size() < max_count; ++i) { - const float lo = entries[i].value; - const float hi = entries[i + 1].value; - if (!(lo < hi)) continue; - const long double midpoint = - (static_cast(lo) + static_cast(hi)) * 0.5L; - if (!std::isfinite(static_cast(midpoint))) continue; - const float mid_f32 = static_cast(midpoint); - if (!std::isfinite(mid_f32)) continue; - if (static_cast(mid_f32) != midpoint) continue; - out.push_back(mid_f32); - } - return out; -} - -std::vector edge_f32_patterns() { - return { - 0x00000000u, 0x80000000u, 0x00000001u, 0x80000001u, 0x007fffffu, - 0x807fffffu, 0x00800000u, 0x80800000u, 0x00800001u, 0x80800001u, - 0x3f7fffffu, 0x3f800000u, 0x3f800001u, 0xbf7fffffu, 0xbf800000u, - 0xbf800001u, 0x7f7fffffu, 0xff7fffffu, 0x7f800000u, 0xff800000u, - 0x7fc00000u, 0xffc00000u, 0x7fa00001u, 0xffa00001u, - }; -} - -void build_pair_rs_inputs(std::vector &a, std::vector &b, - std::vector &rbits) { - a.clear(); - b.clear(); - rbits.clear(); - - std::mt19937 rng(0x84f0a313u); - - const std::vector edges = edge_f32_patterns(); - for (size_t i = 0; i + 1 < edges.size(); i += 2) { - a.push_back(bits_to_f32(edges[i])); - b.push_back(bits_to_f32(edges[i + 1])); - - const uint32_t ra = rng() & 0x1fffu; - const uint32_t rb = rng() & 0x1fffu; - rbits.push_back((ra << 16) | rb); - } - - const uint32_t manual_rbits[] = { - 0x00000000u, 0x00010001u, 0x1fff1fffu, 0x1fff0000u, - 0x00001fffu, 0x12340789u, 0x15550aaau, - }; - for (uint32_t r : manual_rbits) { - a.push_back(bits_to_f32(rng())); - b.push_back(bits_to_f32(rng())); - rbits.push_back(static_cast(((r >> 16) & 0x1fffu) << 16) | - (r & 0x1fffu)); - } - - // Deterministic tie-focused probes for .rs pair conversions. - const uint32_t midpoint_bits[] = { - 0x3f801000u, 0xbf801000u, 0x3f001000u, 0xbf001000u, - }; - const uint16_t tie_rbits[] = {0x0fffu, 0x1000u}; - for (uint32_t bits : midpoint_bits) { - const float x = bits_to_f32(bits); - for (uint16_t r : tie_rbits) { - a.push_back(x); - b.push_back(x); - rbits.push_back((static_cast(r) << 16) | r); - } - } - - while (a.size() < 1024) { - a.push_back(bits_to_f32(rng())); - b.push_back(bits_to_f32(rng())); - - const uint32_t ra = rng() & 0x1fffu; - const uint32_t rb = rng() & 0x1fffu; - rbits.push_back((ra << 16) | rb); - } -} - -void build_quad_rs_inputs(std::vector &a, std::vector &b, - std::vector &e, std::vector &f, - std::vector &rbits) { - a.clear(); - b.clear(); - e.clear(); - f.clear(); - rbits.clear(); - - std::mt19937 rng(0x2f08bc41u); - - const std::vector edges = edge_f32_patterns(); - for (size_t i = 0; i + 3 < edges.size(); i += 4) { - a.push_back(bits_to_f32(edges[i])); - b.push_back(bits_to_f32(edges[i + 1])); - e.push_back(bits_to_f32(edges[i + 2])); - f.push_back(bits_to_f32(edges[i + 3])); - rbits.push_back(rng()); - } - - const uint32_t manual_rbits[] = { - 0x00000000u, 0xffffffffu, 0x01020304u, 0x11223344u, - 0x89abcdefu, 0xfedcba98u, - }; - for (uint32_t r : manual_rbits) { - a.push_back(bits_to_f32(rng())); - b.push_back(bits_to_f32(rng())); - e.push_back(bits_to_f32(rng())); - f.push_back(bits_to_f32(rng())); - rbits.push_back(r); - } - - // Deterministic midpoint/tie probes for each minifloat x4 .rs family. - const rs_minifloat_family_t families[] = { - rs_minifloat_family_t::kE4M3, rs_minifloat_family_t::kE5M2, - rs_minifloat_family_t::kE2M1, rs_minifloat_family_t::kE2M3, - rs_minifloat_family_t::kE3M2, - }; - for (rs_minifloat_family_t fmt : families) { - const std::vector ties = collect_rs_midpoint_ties(fmt, 8); - for (float tie : ties) { - a.push_back(tie); - b.push_back(tie); - e.push_back(tie); - f.push_back(tie); - rbits.push_back(0x7f7f7f7fu); - - a.push_back(tie); - b.push_back(tie); - e.push_back(tie); - f.push_back(tie); - rbits.push_back(0x80808080u); - - const float down = - std::nextafter(tie, -std::numeric_limits::infinity()); - const float up = - std::nextafter(tie, std::numeric_limits::infinity()); - if (std::isfinite(down) && std::isfinite(up)) { - a.push_back(down); - b.push_back(tie); - e.push_back(up); - f.push_back(tie); - rbits.push_back(0x7f80807fu); - } - } - } - - while (a.size() < 768) { - a.push_back(bits_to_f32(rng())); - b.push_back(bits_to_f32(rng())); - e.push_back(bits_to_f32(rng())); - f.push_back(bits_to_f32(rng())); - rbits.push_back(rng()); - } -} - -__global__ void k_f16x2_rs(uint32_t *out, const float *a, const float *b, - const uint32_t *rbits, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.f16x2.f32 %0, %1, %2, %3;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_f16x2_satfinite_relu_rs(uint32_t *out, const float *a, - const float *b, - const uint32_t *rbits, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.satfinite.relu.f16x2.f32 %0, %1, %2, %3;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_bf16x2_rs(uint32_t *out, const float *a, const float *b, - const uint32_t *rbits, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.bf16x2.f32 %0, %1, %2, %3;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_bf16x2_satfinite_relu_rs(uint32_t *out, const float *a, - const float *b, - const uint32_t *rbits, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.satfinite.relu.bf16x2.f32 %0, %1, %2, %3;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_e4m3x4_satfinite_rs(uint32_t *out, const float *a, - const float *b, const float *e, - const float *f, const uint32_t *rbits, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.satfinite.e4m3x4.f32 %0, {%1, %2, %3, %4}, %5;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_e4m3x4_satfinite_relu_rs(uint32_t *out, const float *a, - const float *b, const float *e, - const float *f, - const uint32_t *rbits, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.satfinite.relu.e4m3x4.f32 %0, {%1, %2, %3, %4}, %5;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_e5m2x4_satfinite_rs(uint32_t *out, const float *a, - const float *b, const float *e, - const float *f, const uint32_t *rbits, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.satfinite.e5m2x4.f32 %0, {%1, %2, %3, %4}, %5;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_e5m2x4_satfinite_relu_rs(uint32_t *out, const float *a, - const float *b, const float *e, - const float *f, - const uint32_t *rbits, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.satfinite.relu.e5m2x4.f32 %0, {%1, %2, %3, %4}, %5;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_e2m1x4_satfinite_rs(uint16_t *out, const float *a, - const float *b, const float *e, - const float *f, const uint32_t *rbits, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rs.satfinite.e2m1x4.f32 %0, {%1, %2, %3, %4}, %5;" - : "=h"(d) - : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_e2m1x4_satfinite_relu_rs(uint16_t *out, const float *a, - const float *b, const float *e, - const float *f, - const uint32_t *rbits, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned short d = 0; - asm volatile("cvt.rs.satfinite.relu.e2m1x4.f32 %0, {%1, %2, %3, %4}, %5;" - : "=h"(d) - : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_e2m3x4_satfinite_rs(uint32_t *out, const float *a, - const float *b, const float *e, - const float *f, const uint32_t *rbits, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.satfinite.e2m3x4.f32 %0, {%1, %2, %3, %4}, %5;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_e2m3x4_satfinite_relu_rs(uint32_t *out, const float *a, - const float *b, const float *e, - const float *f, - const uint32_t *rbits, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.satfinite.relu.e2m3x4.f32 %0, {%1, %2, %3, %4}, %5;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_e3m2x4_satfinite_rs(uint32_t *out, const float *a, - const float *b, const float *e, - const float *f, const uint32_t *rbits, - int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.satfinite.e3m2x4.f32 %0, {%1, %2, %3, %4}, %5;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); - out[i] = d; -} - -__global__ void k_e3m2x4_satfinite_relu_rs(uint32_t *out, const float *a, - const float *b, const float *e, - const float *f, - const uint32_t *rbits, int n) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - unsigned d = 0; - asm volatile("cvt.rs.satfinite.relu.e3m2x4.f32 %0, {%1, %2, %3, %4}, %5;" - : "=r"(d) - : "f"(a[i]), "f"(b[i]), "f"(e[i]), "f"(f[i]), "r"(rbits[i])); - out[i] = d; -} - -template -void run_pair_kernel_with_rbits(const std::vector &a, - const std::vector &b, - const std::vector &rbits, - std::vector &out, - LaunchFn launch_kernel) { - float *d_a = nullptr; - float *d_b = nullptr; - uint32_t *d_rbits = nullptr; - TOut *d_out = nullptr; - const int n = static_cast(a.size()); - - CUDACHK(cudaMalloc(&d_a, sizeof(float) * a.size())); - CUDACHK(cudaMalloc(&d_b, sizeof(float) * b.size())); - CUDACHK(cudaMalloc(&d_rbits, sizeof(uint32_t) * rbits.size())); - CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * a.size())); - - CUDACHK(cudaMemcpy(d_a, a.data(), sizeof(float) * a.size(), - cudaMemcpyHostToDevice)); - CUDACHK(cudaMemcpy(d_b, b.data(), sizeof(float) * b.size(), - cudaMemcpyHostToDevice)); - CUDACHK(cudaMemcpy(d_rbits, rbits.data(), sizeof(uint32_t) * rbits.size(), - cudaMemcpyHostToDevice)); - - const int block = 128; - const int grid = (n + block - 1) / block; - launch_kernel(grid, block, d_out, d_a, d_b, d_rbits, n); - CUDACHK(cudaDeviceSynchronize()); - - out.resize(a.size()); - CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), - cudaMemcpyDeviceToHost)); - - CUDACHK(cudaFree(d_out)); - CUDACHK(cudaFree(d_rbits)); - CUDACHK(cudaFree(d_b)); - CUDACHK(cudaFree(d_a)); -} - -template -void run_quad_kernel_with_rbits(const std::vector &a, - const std::vector &b, - const std::vector &e, - const std::vector &f, - const std::vector &rbits, - std::vector &out, - LaunchFn launch_kernel) { - float *d_a = nullptr; - float *d_b = nullptr; - float *d_e = nullptr; - float *d_f = nullptr; - uint32_t *d_rbits = nullptr; - TOut *d_out = nullptr; - const int n = static_cast(a.size()); - - CUDACHK(cudaMalloc(&d_a, sizeof(float) * a.size())); - CUDACHK(cudaMalloc(&d_b, sizeof(float) * b.size())); - CUDACHK(cudaMalloc(&d_e, sizeof(float) * e.size())); - CUDACHK(cudaMalloc(&d_f, sizeof(float) * f.size())); - CUDACHK(cudaMalloc(&d_rbits, sizeof(uint32_t) * rbits.size())); - CUDACHK(cudaMalloc(&d_out, sizeof(TOut) * a.size())); - - CUDACHK(cudaMemcpy(d_a, a.data(), sizeof(float) * a.size(), - cudaMemcpyHostToDevice)); - CUDACHK(cudaMemcpy(d_b, b.data(), sizeof(float) * b.size(), - cudaMemcpyHostToDevice)); - CUDACHK(cudaMemcpy(d_e, e.data(), sizeof(float) * e.size(), - cudaMemcpyHostToDevice)); - CUDACHK(cudaMemcpy(d_f, f.data(), sizeof(float) * f.size(), - cudaMemcpyHostToDevice)); - CUDACHK(cudaMemcpy(d_rbits, rbits.data(), sizeof(uint32_t) * rbits.size(), - cudaMemcpyHostToDevice)); - - const int block = 128; - const int grid = (n + block - 1) / block; - launch_kernel(grid, block, d_out, d_a, d_b, d_e, d_f, d_rbits, n); - CUDACHK(cudaDeviceSynchronize()); - - out.resize(a.size()); - CUDACHK(cudaMemcpy(out.data(), d_out, sizeof(TOut) * out.size(), - cudaMemcpyDeviceToHost)); - - CUDACHK(cudaFree(d_out)); - CUDACHK(cudaFree(d_rbits)); - CUDACHK(cudaFree(d_f)); - CUDACHK(cudaFree(d_e)); - CUDACHK(cudaFree(d_b)); - CUDACHK(cudaFree(d_a)); -} - -template -void dump_hex(std::ofstream &ofs, const std::string &label, - const std::vector &vals, unsigned width) { - for (size_t i = 0; i < vals.size(); ++i) { - ofs << label << " " << i << " 0x" << std::hex << std::setw(width) - << std::setfill('0') - << static_cast(vals[i]) << std::dec << "\n"; - } -} - -int run_and_dump(const char *out_path) { - std::vector pair_a, pair_b; - std::vector pair_rbits; - build_pair_rs_inputs(pair_a, pair_b, pair_rbits); - - std::vector quad_a, quad_b, quad_e, quad_f; - std::vector quad_rbits; - build_quad_rs_inputs(quad_a, quad_b, quad_e, quad_f, quad_rbits); - - std::vector out_u16; - std::vector out_u32; - - std::ofstream ofs(out_path, std::ios::out | std::ios::trunc); - if (!ofs) { - std::cerr << "failed to open output file: " << out_path << "\n"; - return 1; - } - - run_pair_kernel_with_rbits( - pair_a, pair_b, pair_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const uint32_t *d_rbits, int n) { - k_f16x2_rs<<>>(d_out, d_a, d_b, d_rbits, n); - }); - bool all_zero = true; - for (size_t i = 0; i < out_u32.size(); ++i) { - if (out_u32[i] != 0u) { - all_zero = false; - break; - } - } - if (all_zero) { - std::cerr << "RS_FEATURE_UNAVAILABLE: k_f16x2_rs produced all-zero output" - << "\n"; - return 3; - } - dump_hex(ofs, "k_f16x2_rs", out_u32, 8); - - run_pair_kernel_with_rbits( - pair_a, pair_b, pair_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const uint32_t *d_rbits, int n) { - k_f16x2_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_rbits, - n); - }); - dump_hex(ofs, "k_f16x2_satfinite_relu_rs", out_u32, 8); - - run_pair_kernel_with_rbits( - pair_a, pair_b, pair_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const uint32_t *d_rbits, int n) { - k_bf16x2_rs<<>>(d_out, d_a, d_b, d_rbits, n); - }); - dump_hex(ofs, "k_bf16x2_rs", out_u32, 8); - - run_pair_kernel_with_rbits( - pair_a, pair_b, pair_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const uint32_t *d_rbits, int n) { - k_bf16x2_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_rbits, - n); - }); - dump_hex(ofs, "k_bf16x2_satfinite_relu_rs", out_u32, 8); - - run_quad_kernel_with_rbits( - quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const float *d_e, const float *d_f, - const uint32_t *d_rbits, int n) { - k_e4m3x4_satfinite_rs<<>>(d_out, d_a, d_b, d_e, d_f, - d_rbits, n); - }); - dump_hex(ofs, "k_e4m3x4_satfinite_rs", out_u32, 8); - run_quad_kernel_with_rbits( - quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const float *d_e, const float *d_f, - const uint32_t *d_rbits, int n) { - k_e4m3x4_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_e, d_f, - d_rbits, n); - }); - dump_hex(ofs, "k_e4m3x4_satfinite_relu_rs", out_u32, 8); - - run_quad_kernel_with_rbits( - quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const float *d_e, const float *d_f, - const uint32_t *d_rbits, int n) { - k_e5m2x4_satfinite_rs<<>>(d_out, d_a, d_b, d_e, d_f, - d_rbits, n); - }); - dump_hex(ofs, "k_e5m2x4_satfinite_rs", out_u32, 8); - run_quad_kernel_with_rbits( - quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const float *d_e, const float *d_f, - const uint32_t *d_rbits, int n) { - k_e5m2x4_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_e, d_f, - d_rbits, n); - }); - dump_hex(ofs, "k_e5m2x4_satfinite_relu_rs", out_u32, 8); - - run_quad_kernel_with_rbits( - quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, - const float *d_b, const float *d_e, const float *d_f, - const uint32_t *d_rbits, int n) { - k_e2m1x4_satfinite_rs<<>>(d_out, d_a, d_b, d_e, d_f, - d_rbits, n); - }); - dump_hex(ofs, "k_e2m1x4_satfinite_rs", out_u16, 4); - run_quad_kernel_with_rbits( - quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u16, - [](int grid, int block, uint16_t *d_out, const float *d_a, - const float *d_b, const float *d_e, const float *d_f, - const uint32_t *d_rbits, int n) { - k_e2m1x4_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_e, d_f, - d_rbits, n); - }); - dump_hex(ofs, "k_e2m1x4_satfinite_relu_rs", out_u16, 4); - - run_quad_kernel_with_rbits( - quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const float *d_e, const float *d_f, - const uint32_t *d_rbits, int n) { - k_e2m3x4_satfinite_rs<<>>(d_out, d_a, d_b, d_e, d_f, - d_rbits, n); - }); - dump_hex(ofs, "k_e2m3x4_satfinite_rs", out_u32, 8); - run_quad_kernel_with_rbits( - quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const float *d_e, const float *d_f, - const uint32_t *d_rbits, int n) { - k_e2m3x4_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_e, d_f, - d_rbits, n); - }); - dump_hex(ofs, "k_e2m3x4_satfinite_relu_rs", out_u32, 8); - - run_quad_kernel_with_rbits( - quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const float *d_e, const float *d_f, - const uint32_t *d_rbits, int n) { - k_e3m2x4_satfinite_rs<<>>(d_out, d_a, d_b, d_e, d_f, - d_rbits, n); - }); - dump_hex(ofs, "k_e3m2x4_satfinite_rs", out_u32, 8); - run_quad_kernel_with_rbits( - quad_a, quad_b, quad_e, quad_f, quad_rbits, out_u32, - [](int grid, int block, uint32_t *d_out, const float *d_a, - const float *d_b, const float *d_e, const float *d_f, - const uint32_t *d_rbits, int n) { - k_e3m2x4_satfinite_relu_rs<<>>(d_out, d_a, d_b, d_e, d_f, - d_rbits, n); - }); - dump_hex(ofs, "k_e3m2x4_satfinite_relu_rs", out_u32, 8); - - return 0; -} - -} // namespace - -int main(int argc, char **argv) { - if (argc != 3 || std::string(argv[1]) != "--output") { - std::cerr << "usage: " << argv[0] << " --output \n"; - return 1; - } - return run_and_dump(argv[2]); -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/README.md b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/README.md deleted file mode 100644 index bc8e104d7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# PTX Datatype Parser Smoke Cases - -`run_ptx_datatype_parser_smoke.sh` validates lexer/parser + IR construction only. -Does not execute PTX instructions, so `instructions.cc:cvt_impl()` runtime -bounds checking for `g_cvt_fn[src_fmt][dst_fmt]` cannot be exercised here. - -The cvt bounds guard remains in runtime code to protect execution paths that -use datatype format IDs beyond the legacy conversion table dimensions. - -`run_lowp_cvt_ptxas_crosscheck.sh` includes a spec-tracking gate for -`cvt.*.ue8m0x2.bf16x2` with `.relu`. By default -`LOWP_CVT_SPEC_TRACK_UE8M0X2_BF16X2_RELU=auto` (track current toolchain -behavior); set `LOWP_CVT_SPEC_TRACK_UE8M0X2_BF16X2_RELU=ptxas_accepts` when -toolchains begin accepting the form. diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.ptx deleted file mode 100644 index 964b028a6..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_bf16_not_allowed(.param .bf16 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.ptx deleted file mode 100644 index a61ca6199..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_bf16x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_bf16x2_not_allowed(.param .bf16x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.ptx deleted file mode 100644 index 59548fd27..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e2m1_not_allowed(.param .e2m1 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.ptx deleted file mode 100644 index 03917b7d3..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e2m1x2_not_allowed(.param .e2m1x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.ptx deleted file mode 100644 index 03fe0839c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m1x4_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e2m1x4_not_allowed(.param .e2m1x4 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.ptx deleted file mode 100644 index 88412e00b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e2m3_not_allowed(.param .e2m3 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.ptx deleted file mode 100644 index 683b918c7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e2m3x2_not_allowed(.param .e2m3x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.ptx deleted file mode 100644 index 3c006fe78..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e2m3x4_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e2m3x4_not_allowed(.param .e2m3x4 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.ptx deleted file mode 100644 index de7bf36f1..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e3m2_not_allowed(.param .e3m2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.ptx deleted file mode 100644 index 9db04a5f2..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e3m2x2_not_allowed(.param .e3m2x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.ptx deleted file mode 100644 index aa28891ca..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e3m2x4_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e3m2x4_not_allowed(.param .e3m2x4 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.ptx deleted file mode 100644 index a6c89a5c4..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e4m3_not_allowed(.param .e4m3 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.ptx deleted file mode 100644 index d5f16ad39..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e4m3x2_not_allowed(.param .e4m3x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.ptx deleted file mode 100644 index 7772ce13b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e4m3x4_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e4m3x4_not_allowed(.param .e4m3x4 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.ptx deleted file mode 100644 index 547d5c8b4..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e5m2_not_allowed(.param .e5m2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.ptx deleted file mode 100644 index dade9ba4f..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e5m2x2_not_allowed(.param .e5m2x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.ptx deleted file mode 100644 index bf76505b2..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_e5m2x4_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_e5m2x4_not_allowed(.param .e5m2x4 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.expect deleted file mode 100644 index aee0192e4..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -datatype ".f16x2" cannot be used in .param declarations; use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.ptx deleted file mode 100644 index afb973b77..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f16x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_f16x2_not_allowed(.param .f16x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.expect deleted file mode 100644 index 1e942bed4..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b64" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.ptx deleted file mode 100644 index 61908015f..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_f32x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_f32x2_not_allowed(.param .f32x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.ptx deleted file mode 100644 index c6d8ae3dd..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s16x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_s16x2_not_allowed(.param .s16x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.ptx deleted file mode 100644 index 17b67ba5b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_s2f6_not_allowed(.param .s2f6 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.ptx deleted file mode 100644 index 07622f91d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_s2f6x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_s2f6x2_not_allowed(.param .s2f6x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.ptx deleted file mode 100644 index d352545dc..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_tf32_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_tf32_not_allowed(.param .tf32 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.ptx deleted file mode 100644 index 80b83a672..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_u16x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_u16x2_not_allowed(.param .u16x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.ptx deleted file mode 100644 index dce1d62c7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue4m3_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_ue4m3_not_allowed(.param .ue4m3 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.ptx deleted file mode 100644 index 001d8f540..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_ue8m0_not_allowed(.param .ue8m0 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.ptx deleted file mode 100644 index cbc81ee0b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_param_ue8m0x2_not_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_ue8m0x2_not_allowed(.param .ue8m0x2 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.ptx deleted file mode 100644 index 57866e700..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_bf16_not_allowed() { - .reg .bf16 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.ptx deleted file mode 100644 index d2530e948..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_bf16x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_bf16x2_not_allowed() { - .reg .bf16x2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.ptx deleted file mode 100644 index 5f7d93375..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e2m1_not_allowed() { - .reg .e2m1 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.ptx deleted file mode 100644 index e53289fc3..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e2m1x2_not_allowed() { - .reg .e2m1x2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.ptx deleted file mode 100644 index 496cd925c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m1x4_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e2m1x4_not_allowed() { - .reg .e2m1x4 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.ptx deleted file mode 100644 index 7483aa0db..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e2m3_not_allowed() { - .reg .e2m3 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.ptx deleted file mode 100644 index 9856e3d53..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e2m3x2_not_allowed() { - .reg .e2m3x2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.ptx deleted file mode 100644 index e71d70b02..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e2m3x4_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e2m3x4_not_allowed() { - .reg .e2m3x4 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.ptx deleted file mode 100644 index 8736a7840..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e3m2_not_allowed() { - .reg .e3m2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.ptx deleted file mode 100644 index e38893d83..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e3m2x2_not_allowed() { - .reg .e3m2x2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.ptx deleted file mode 100644 index 7e3a1172b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e3m2x4_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e3m2x4_not_allowed() { - .reg .e3m2x4 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.ptx deleted file mode 100644 index 48eae06d3..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e4m3_not_allowed() { - .reg .e4m3 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.ptx deleted file mode 100644 index 716827cbe..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e4m3x2_not_allowed() { - .reg .e4m3x2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.ptx deleted file mode 100644 index d8dbfb07e..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e4m3x4_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e4m3x4_not_allowed() { - .reg .e4m3x4 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.ptx deleted file mode 100644 index 8bc5a699a..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e5m2_not_allowed() { - .reg .e5m2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.ptx deleted file mode 100644 index 31d73358e..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e5m2x2_not_allowed() { - .reg .e5m2x2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.ptx deleted file mode 100644 index 7c83ac280..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_e5m2x4_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_e5m2x4_not_allowed() { - .reg .e5m2x4 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.expect deleted file mode 100644 index 1e942bed4..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b64" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.ptx deleted file mode 100644 index 623974f41..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_f32x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_f32x2_not_allowed() { - .reg .f32x2 %d0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.ptx deleted file mode 100644 index 68401ca6c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s16x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_s16x2_not_allowed() { - .reg .s16x2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.ptx deleted file mode 100644 index cd8974de8..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_s2f6_not_allowed() { - .reg .s2f6 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.ptx deleted file mode 100644 index 39a20c848..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_s2f6x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_s2f6x2_not_allowed() { - .reg .s2f6x2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.ptx deleted file mode 100644 index 340ea8208..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_tf32_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_tf32_not_allowed() { - .reg .tf32 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.expect deleted file mode 100644 index bf0ed7010..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b32" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.ptx deleted file mode 100644 index 9f61e6a23..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_u16x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_u16x2_not_allowed() { - .reg .u16x2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.ptx deleted file mode 100644 index 8204785be..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue4m3_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_ue4m3_not_allowed() { - .reg .ue4m3 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.expect deleted file mode 100644 index ed2cbdf84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b8" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.ptx deleted file mode 100644 index 8f1a807b3..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_ue8m0_not_allowed() { - .reg .ue8m0 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.expect deleted file mode 100644 index 8a09ac699..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -non-fundamental instruction-only datatype -use ".b16" for storage declarations diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.ptx deleted file mode 100644 index 50674d5ba..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/declaration_reg_ue8m0x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_ue8m0x2_not_allowed() { - .reg .ue8m0x2 %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.expect deleted file mode 100644 index 97b500494..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -declaration initializers are not supported for datatype ".f16" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.ptx deleted file mode 100644 index 13fae918a..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_brace_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry initializer_f16_brace_not_allowed() { - .reg .f16 %h0 = {0}; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.expect deleted file mode 100644 index 97b500494..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -declaration initializers are not supported for datatype ".f16" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.ptx deleted file mode 100644 index cf52c08dc..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry initializer_f16_not_allowed() { - .reg .f16 %h0 = 0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.expect deleted file mode 100644 index a45fde33c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -declaration initializers are not supported for datatype ".f16x2" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.ptx deleted file mode 100644 index 516b6ace6..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_brace_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry initializer_f16x2_brace_not_allowed() { - .reg .f16x2 %r0 = {0}; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.expect deleted file mode 100644 index a45fde33c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -declaration initializers are not supported for datatype ".f16x2" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.ptx deleted file mode 100644 index d19272d2e..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_f16x2_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry initializer_f16x2_not_allowed() { - .reg .f16x2 %r0 = 0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.expect deleted file mode 100644 index 0e6e35910..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -declaration initializers are not supported for datatype ".pred" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.ptx deleted file mode 100644 index 791f17631..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_brace_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry initializer_pred_brace_not_allowed() { - .reg .pred %p0 = {0}; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.expect deleted file mode 100644 index 0e6e35910..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -declaration initializers are not supported for datatype ".pred" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.ptx deleted file mode 100644 index 3797a84e6..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/initializer_pred_not_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry initializer_pred_not_allowed() { - .reg .pred %p0 = 0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.expect deleted file mode 100644 index 979531235..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -does not allow modifier ".ftz" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.ptx deleted file mode 100644 index 1fc32d9d1..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_ftz_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_bf16_f16_ftz_not_allowed() { - .reg .b16 %h0; - .reg .b16 %h1; - - cvt.rn.ftz.bf16.f16 %h0, %h1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.expect deleted file mode 100644 index a2fbb57f2..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.ptx deleted file mode 100644 index b7530c178..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f16_sm80_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_80 -.address_size 64 - -.visible .entry instruction_cvt_bf16_f16_sm80_not_allowed() { - .reg .b16 %h0; - .reg .b16 %h1; - - cvt.rn.bf16.f16 %h0, %h1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.expect deleted file mode 100644 index a9a8eae8d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires a floating-point rounding modifier in {.rn, .rz, .rm, .rp} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.ptx deleted file mode 100644 index b6aad08df..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_missing_rounding_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_bf16_f32_missing_rounding_not_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - - cvt.bf16.f32 %h0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.expect deleted file mode 100644 index c383bd768..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -allows modifier ".sat" only for floating-point destination types {.f16, .f32, .f64} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.ptx deleted file mode 100644 index 0fa02bac7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sat_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_120 -.address_size 64 - -.visible .entry instruction_cvt_bf16_f32_sat_not_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - - cvt.rn.sat.bf16.f32 %h0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.expect deleted file mode 100644 index 34a42399c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -with modifiers ".satfinite" or ".relu" requires rounding modifier ".rn" or ".rz" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.ptx deleted file mode 100644 index 9befe5494..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_satfinite_rm_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_120 -.address_size 64 - -.visible .entry instruction_cvt_bf16_f32_satfinite_rm_not_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - - cvt.rm.satfinite.bf16.f32 %h0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.expect deleted file mode 100644 index 50a346065..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_80" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.ptx deleted file mode 100644 index 60beec746..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_f32_sm75_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_75 -.address_size 64 - -.visible .entry instruction_cvt_bf16_f32_sm75_not_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - - cvt.rn.bf16.f32 %h0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.expect deleted file mode 100644 index a2fbb57f2..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.ptx deleted file mode 100644 index 34db9bcf0..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16_s32_sm80_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_80 -.address_size 64 - -.visible .entry instruction_cvt_bf16_s32_sm80_not_allowed() { - .reg .b16 %h0; - .reg .s32 %r0; - - cvt.rn.bf16.s32 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.expect deleted file mode 100644 index 759efe784..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" datatype combination "BF16X2_TYPE.F32_TYPE" -requires operand shape "d, a, b" or "d, a, b, rbits" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.ptx deleted file mode 100644 index ce44c747b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_f32_single_src_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_bf16x2_f32_single_src_not_allowed() { - .reg .b32 %r0; - .reg .f32 %f0; - - cvt.rz.bf16x2.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.expect deleted file mode 100644 index 0d36b3f06..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -does not allow modifier ".satfinite" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.ptx deleted file mode 100644 index db4aaef74..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_bf16x2_ue8m0x2_satfinite_not_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.satfinite.bf16x2.ue8m0x2 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.expect deleted file mode 100644 index 6c2280c3d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires an ".target sm_100a" (or newer arch-a) target or ".target sm_100f" (or newer arch-f) target for datatype destination "E2M1X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.ptx deleted file mode 100644 index 4b730968c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_dst_sm100_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_e2m1x2_dst_sm100_not_allowed() { - .reg .b8 %b0; - .reg .b32 %r0; - - cvt.rn.satfinite.e2m1x2.f16x2 %b0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.expect deleted file mode 100644 index f085c3721..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires an ".target sm_100a" (or newer arch-a) target or ".target sm_100f" (or newer arch-f) target diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.ptx deleted file mode 100644 index ffe0525e3..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_120 -.address_size 64 - -.visible .entry instruction_cvt_e2m1x2_f32_sm120_not_allowed() { - .reg .b8 %b0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.expect deleted file mode 100644 index ee57e96b5..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires an ".target sm_100a" (or newer arch-a) target or ".target sm_100f" (or newer arch-f) target for datatype source "E2M1X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.ptx deleted file mode 100644 index 72f4c1cea..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m1x2_src_sm100_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_e2m1x2_src_sm100_not_allowed() { - .reg .b8 %b0; - .reg .b32 %r0; - - cvt.rn.f16x2.e2m1x2 %r0, %b0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.expect deleted file mode 100644 index 52709d518..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires an ".target sm_100a" (or newer arch-a) target or ".target sm_100f" (or newer arch-f) target for datatype source "E2M3X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.ptx deleted file mode 100644 index 7613e344b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e2m3x2_sm100_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_e2m3x2_sm100_not_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.f16x2.e2m3x2 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.expect deleted file mode 100644 index fb2759887..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_89" or higher for datatype destination "E4M3X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.ptx deleted file mode 100644 index 06e51d33a..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_e4m3x2_sm80_not_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_80 -.address_size 64 - -.visible .entry instruction_cvt_e4m3x2_sm80_not_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rn.satfinite.e4m3x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.expect deleted file mode 100644 index a2fbb57f2..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.ptx deleted file mode 100644 index 96e615a82..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_bf16_sm80_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_80 -.address_size 64 - -.visible .entry instruction_cvt_f16_bf16_sm80_not_allowed() { - .reg .b16 %h0; - .reg .b16 %h1; - - cvt.rn.f16.bf16 %h0, %h1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.expect deleted file mode 100644 index a9a8eae8d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires a floating-point rounding modifier in {.rn, .rz, .rm, .rp} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.ptx deleted file mode 100644 index 4bc06f445..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16_f32_missing_rounding_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_f16_f32_missing_rounding_not_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - - cvt.f16.f32 %h0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.expect deleted file mode 100644 index 1c86804d8..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" datatype combination "F16X2_TYPE.F32_TYPE" -requires operand shape "d, a, b" or "d, a, b, rbits" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.ptx deleted file mode 100644 index 34ef10edd..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_single_src_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_f16x2_f32_single_src_not_allowed() { - .reg .b32 %r0; - .reg .f32 %f0; - - cvt.rn.f16x2.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.expect deleted file mode 100644 index 50a346065..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_80" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.ptx deleted file mode 100644 index 4ebe2e2fa..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_f32_sm75_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_75 -.address_size 64 - -.visible .entry instruction_cvt_f16x2_f32_sm75_not_allowed() { - .reg .b32 %r0; - .reg .f32 %f0, %f1; - - cvt.rn.f16x2.f32 %r0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.expect deleted file mode 100644 index 5009bf073..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" does not support datatype combination -F16X2_TYPE.UE8M0X2_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.ptx deleted file mode 100644 index 22ed7b173..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f16x2_ue8m0x2_not_allowed.ptx +++ /dev/null @@ -1,10 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_f16x2_ue8m0x2_not_allowed() { - .reg .b32 %r0; - .reg .b16 %h0; - cvt.rn.f16x2.ue8m0x2 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.expect deleted file mode 100644 index a2fbb57f2..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.ptx deleted file mode 100644 index 91e1b917b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_80 -.address_size 64 - -.visible .entry instruction_cvt_f32_bf16_ftz_sm80_not_allowed() { - .reg .f32 %f0; - .reg .b16 %h0; - - cvt.rn.ftz.f32.bf16 %f0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.expect deleted file mode 100644 index a9a8eae8d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires a floating-point rounding modifier in {.rn, .rz, .rm, .rp} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.ptx deleted file mode 100644 index 90f098dc7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_s32_missing_rounding_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_f32_s32_missing_rounding_not_allowed() { - .reg .f32 %f0; - .reg .s32 %s0; - - cvt.f32.s32 %f0, %s0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.expect deleted file mode 100644 index 8e5841f5d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" does not support datatype combination -F32_TYPE.TF32_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.ptx deleted file mode 100644 index 9ac99951d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32_tf32_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_f32_tf32_not_allowed() { - .reg .f32 %f0; - .reg .b32 %r0; - - cvt.rn.f32.tf32 %f0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.expect deleted file mode 100644 index 524a955e7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -instruction "cvt" does not support datatype combination "F32X2_TYPE.F32_TYPE" per PTX ISA diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.ptx deleted file mode 100644 index a4504b9d5..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_dst_not_allowed.ptx +++ /dev/null @@ -1,10 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_f32x2_dst_not_allowed() { - .reg .b64 %d0; - .reg .f32 %f0; - cvt.f32x2.f32 %d0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.expect deleted file mode 100644 index c6e48c4c0..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -instruction "cvt" does not support datatype combination "F32_TYPE.F32X2_TYPE" per PTX ISA diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.ptx deleted file mode 100644 index e49a901a8..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_f32x2_src_not_allowed.ptx +++ /dev/null @@ -1,10 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_f32x2_src_not_allowed() { - .reg .f32 %f0; - .reg .b64 %d0; - cvt.f32.f32x2 %f0, %d0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.expect deleted file mode 100644 index bbcf66a01..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" does not support datatype combination -PRED_TYPE.U32_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.ptx deleted file mode 100644 index 237934210..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_pred_not_allowed.ptx +++ /dev/null @@ -1,10 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_pred_not_allowed() { - .reg .pred %p0; - .reg .u32 %u0; - cvt.pred.u32 %p0, %u0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.expect deleted file mode 100644 index ad9846e6d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.expect +++ /dev/null @@ -1 +0,0 @@ -instruction "cvt" requires exactly two datatypes ., found 1 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.ptx deleted file mode 100644 index c3c1a0c4a..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_requires_two_types.ptx +++ /dev/null @@ -1,10 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_requires_two_types() { - .reg .u32 %u0; - .reg .s32 %s0; - cvt.u32 %u0, %s0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.expect deleted file mode 100644 index c1e03a0aa..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_100a" or ".target sm_103a" for rounding modifier ".rs" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.ptx deleted file mode 100644 index 58f8c4aa7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_rs_sm100_not_allowed() { - .reg .b32 %r0, %r1; - .reg .f32 %f0, %f1; - - cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.expect deleted file mode 100644 index c1e03a0aa..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_100a" or ".target sm_103a" for rounding modifier ".rs" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.ptx deleted file mode 100644 index c81f294b4..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm100f_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100f -.address_size 64 - -.visible .entry instruction_cvt_rs_sm100f_not_allowed() { - .reg .b32 %r0, %r1; - .reg .f32 %f0, %f1; - - cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.expect deleted file mode 100644 index c1e03a0aa..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_100a" or ".target sm_103a" for rounding modifier ".rs" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.ptx deleted file mode 100644 index b587d5a37..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm110a_not_allowed.ptx +++ /dev/null @@ -1,13 +0,0 @@ -.version 9.1 -.target sm_110a -.address_size 64 - -.visible .entry instruction_cvt_rs_sm110a_not_allowed() { - .reg .b32 %r0; - .reg .f32 %f0; - .reg .f32 %f1; - .reg .b32 %r1; - - cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.expect deleted file mode 100644 index c1e03a0aa..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_100a" or ".target sm_103a" for rounding modifier ".rs" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.ptx deleted file mode 100644 index 2e9d27501..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120a_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_120a -.address_size 64 - -.visible .entry instruction_cvt_rs_sm120a_not_allowed() { - .reg .b32 %r0, %r1; - .reg .f32 %f0, %f1; - - cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.expect deleted file mode 100644 index c1e03a0aa..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_100a" or ".target sm_103a" for rounding modifier ".rs" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.ptx deleted file mode 100644 index 42cfad069..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_rs_sm120f_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_120f -.address_size 64 - -.visible .entry instruction_cvt_rs_sm120f_not_allowed() { - .reg .b32 %r0, %r1; - .reg .f32 %f0, %f1; - - cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.expect deleted file mode 100644 index fbf1f0128..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" datatype combination "S2F6X2_TYPE.BF16X2_TYPE" -requires ".satfinite" modifier diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.ptx deleted file mode 100644 index b1a72724d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.s2f6x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.expect deleted file mode 100644 index 899e8cf13..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" datatype combination "S2F6X2_TYPE.BF16X2_TYPE" -requires ".scaled::n2::ue8m0" qualifier and scale-factor operand to appear together diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.ptx deleted file mode 100644 index 299ab9908..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed() { - .reg .b16 %h0, %h1; - .reg .b32 %r0; - - cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0, %h1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.expect deleted file mode 100644 index 899e8cf13..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" datatype combination "S2F6X2_TYPE.BF16X2_TYPE" -requires ".scaled::n2::ue8m0" qualifier and scale-factor operand to appear together diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.ptx deleted file mode 100644 index 25fa29256..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.satfinite.scaled::n2::ue8m0.s2f6x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.expect deleted file mode 100644 index 924b7f25e..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires an ".target sm_100a/sm_101a/sm_103a/sm_110a/sm_120a/sm_121a" target for datatype destination "S2F6X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx deleted file mode 100644 index e58ace0bc..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100f -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.expect deleted file mode 100644 index 924b7f25e..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires an ".target sm_100a/sm_101a/sm_103a/sm_110a/sm_120a/sm_121a" target for datatype destination "S2F6X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx deleted file mode 100644 index a17478691..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_120f -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.expect deleted file mode 100644 index 924b7f25e..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires an ".target sm_100a/sm_101a/sm_103a/sm_110a/sm_120a/sm_121a" target for datatype destination "S2F6X2_TYPE" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.ptx deleted file mode 100644 index 21366a9cb..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s2f6x2_sm100_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_sm100_not_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.expect deleted file mode 100644 index 3f36dd30a..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -allows modifier ".sat" only for floating-point destination types {.f16, .f32, .f64} or integer destination types with source {.f16, .f32, .f64} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.ptx deleted file mode 100644 index 6a6b6dca1..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sat_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_90 -.address_size 64 - -.visible .entry instruction_cvt_s32_bf16_sat_not_allowed() { - .reg .s32 %r0; - .reg .b16 %h0; - - cvt.rzi.sat.s32.bf16 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.expect deleted file mode 100644 index a2fbb57f2..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.ptx deleted file mode 100644 index b9e71675a..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_bf16_sm80_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_80 -.address_size 64 - -.visible .entry instruction_cvt_s32_bf16_sm80_not_allowed() { - .reg .s32 %r0; - .reg .b16 %h0; - - cvt.rzi.s32.bf16 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.expect deleted file mode 100644 index 3f36dd30a..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -allows modifier ".sat" only for floating-point destination types {.f16, .f32, .f64} or integer destination types with source {.f16, .f32, .f64} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.ptx deleted file mode 100644 index 1629c9ef7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_s32_s32_sat_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_90 -.address_size 64 - -.visible .entry instruction_cvt_s32_s32_sat_not_allowed() { - .reg .s32 %r0; - .reg .s32 %r1; - - cvt.sat.s32.s32 %r0, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.expect deleted file mode 100644 index 68ae1fd84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -allows modifiers ".satfinite" and ".relu" only for destination {.f16, .bf16} with source .f32 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.ptx deleted file mode 100644 index 06e34235f..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_relu_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_scalar_relu_not_allowed() { - .reg .s32 %r0; - .reg .f32 %f0; - - cvt.rni.relu.s32.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.expect deleted file mode 100644 index 68ae1fd84..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -allows modifiers ".satfinite" and ".relu" only for destination {.f16, .bf16} with source .f32 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.ptx deleted file mode 100644 index 787134c31..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_scalar_satfinite_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_scalar_satfinite_not_allowed() { - .reg .s32 %r0; - .reg .f32 %f0; - - cvt.rni.satfinite.s32.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.expect deleted file mode 100644 index 2dc5a8fb4..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" datatype combination "TF32_TYPE.F32_TYPE" -requires a rounding modifier diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.ptx deleted file mode 100644 index 54a937815..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_missing_rounding_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_tf32_f32_missing_rounding_not_allowed() { - .reg .b32 %r0; - .reg .f32 %f0; - - cvt.tf32.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.expect deleted file mode 100644 index a2fbb57f2..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_90" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.ptx deleted file mode 100644 index 9cbe1eb37..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rn_sm89_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_89 -.address_size 64 - -.visible .entry instruction_cvt_tf32_f32_rn_sm89_not_allowed() { - .reg .b32 %r0; - .reg .f32 %f0; - - cvt.rn.tf32.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.expect deleted file mode 100644 index 91af51d18..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -does not allow modifier combination ".relu" with rounding ".rna" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.ptx deleted file mode 100644 index 3fecb3c7f..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_tf32_f32_rna_relu_not_allowed() { - .reg .b32 %r0; - .reg .f32 %f0; - - cvt.rna.relu.tf32.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.expect deleted file mode 100644 index 5ecd319f3..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires ".target sm_100" or higher diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.ptx deleted file mode 100644 index f48243604..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_90 -.address_size 64 - -.visible .entry instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed() { - .reg .b32 %r0; - .reg .f32 %f0; - - cvt.rn.satfinite.tf32.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.expect deleted file mode 100644 index 845d16fea..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" does not support datatype combination -TF32_TYPE.U32_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.ptx deleted file mode 100644 index 3b2d3b6cf..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_tf32_u32_not_allowed.ptx +++ /dev/null @@ -1,10 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_tf32_u32_not_allowed() { - .reg .b32 %r0; - .reg .u32 %u0; - cvt.tf32.u32 %r0, %u0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.expect deleted file mode 100644 index 191081fed..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" does not support datatype combination -U16X2_TYPE.S32_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.ptx deleted file mode 100644 index 228273ccf..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_u16x2_not_allowed.ptx +++ /dev/null @@ -1,10 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_u16x2_not_allowed() { - .reg .b32 %r0; - .reg .s32 %s0; - cvt.u16x2.s32 %r0, %s0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.expect deleted file mode 100644 index 2339eef9c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -instruction "cvt" does not support datatype combination diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.ptx deleted file mode 100644 index 331235b1b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue4m3_f32_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_120a -.address_size 64 - -.visible .entry instruction_cvt_ue4m3_f32_not_allowed() { - .reg .b8 %b0; - .reg .f32 %f0; - - cvt.rn.satfinite.ue4m3.f32 %b0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.expect deleted file mode 100644 index c0b0d5bfa..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" datatype combination "UE8M0X2_TYPE.BF16X2_TYPE" -requires a rounding modifier diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.ptx deleted file mode 100644 index 18d20da69..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.satfinite.ue8m0x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.expect deleted file mode 100644 index 1bdfb0b11..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "cvt" does not support datatype combination -UE8M0X2_TYPE.F16X2_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.ptx deleted file mode 100644 index 915d1fa3c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f16x2_not_allowed.ptx +++ /dev/null @@ -1,10 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_f16x2_not_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - cvt.rz.ue8m0x2.f16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.expect deleted file mode 100644 index a4e52a0a2..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -does not allow modifier ".relu" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.ptx deleted file mode 100644 index 4176a68b2..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_110f -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_f32_relu_not_allowed() { - .reg .f32 %f<3>; - .reg .b16 %h0; - - cvt.rz.satfinite.relu.ue8m0x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.expect deleted file mode 100644 index f085c3721..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -requires an ".target sm_100a" (or newer arch-a) target or ".target sm_100f" (or newer arch-f) target diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.ptx deleted file mode 100644 index 722336440..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_120 -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_f32_sm120_not_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.expect deleted file mode 100644 index fb009a1c6..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -does not allow modifier ".scaled::n2::ue8m0" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.ptx deleted file mode 100644 index 79f0b4f2f..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_cvt_ue8m0x2_scaled_not_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_scaled_not_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - .reg .f32 %f0, %f1; - - cvt.rz.scaled::n2::ue8m0.ue8m0x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.expect deleted file mode 100644 index a707d2e85..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -instruction "mov" datatype ".b128/.bb128" requires operand shapes diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.ptx deleted file mode 100644 index 49df0b963..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_pack_v2_mixed_width_not_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_b128_pack_v2_mixed_width_not_allowed() { - .reg .b128 %q0; - .reg .b64 %d0; - .reg .b32 %r0; - - mov.b128 %q0, {%d0, %r0}; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.expect deleted file mode 100644 index a707d2e85..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -instruction "mov" datatype ".b128/.bb128" requires operand shapes diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.ptx deleted file mode 100644 index 3b7dd5c23..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_b128_scalar_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_b128_scalar_not_allowed() { - .reg .b128 %q0; - .reg .b32 %r0, %r1, %r2; - - mov.b128 %q0, {%r0, %r1, %r2}; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.expect deleted file mode 100644 index a707d2e85..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.expect +++ /dev/null @@ -1 +0,0 @@ -instruction "mov" datatype ".b128/.bb128" requires operand shapes diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.ptx deleted file mode 100644 index 60a63d26f..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bb128_scalar_not_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_bb128_scalar_not_allowed() { - .reg .bb128 %q0; - .reg .b32 %r0, %r1, %r2; - - mov.bb128 %q0, {%r0, %r1, %r2}; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.expect deleted file mode 100644 index 395433de9..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "mov" does not support datatype -BF16_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.ptx deleted file mode 100644 index eb43d3310..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_bf16_not_allowed.ptx +++ /dev/null @@ -1,9 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_bf16_not_allowed() { - .reg .b16 %h0, %h1; - mov.bf16 %h0, %h1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.expect deleted file mode 100644 index b1bc1f01c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.expect +++ /dev/null @@ -1,2 +0,0 @@ -instruction "mov" does not support datatype -F16_TYPE diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.ptx deleted file mode 100644 index 6b1af2fd4..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/instruction_mov_f16_not_allowed.ptx +++ /dev/null @@ -1,9 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_f16_not_allowed() { - .reg .b16 %h0, %h1; - mov.f16 %h0, %h1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.expect deleted file mode 100644 index fa930aec0..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.expect +++ /dev/null @@ -1 +0,0 @@ -unsupported PTX datatype token ".s32x1" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.ptx deleted file mode 100644 index 29590683d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_s32x1.ptx +++ /dev/null @@ -1,9 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry unsupported_token_s32x1() { - .reg .b32 %r0, %r1; - mov.s32x1 %r0, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.expect b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.expect deleted file mode 100644 index c92cc8b79..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.expect +++ /dev/null @@ -1 +0,0 @@ -unsupported PTX datatype token ".u32x1" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.ptx deleted file mode 100644 index d1370ce53..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/fail/unsupported_token_u32x1.ptx +++ /dev/null @@ -1,9 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry unsupported_token_u32x1() { - .reg .b32 %r0, %r1; - mov.u32x1 %r0, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/all_new_instruction_types.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/all_new_instruction_types.ptx deleted file mode 100644 index 9fd5e3d22..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/all_new_instruction_types.ptx +++ /dev/null @@ -1,32 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry all_new_instruction_types() { - .reg .pred %p0, %p1; - .reg .b16 %h0, %h1; - .reg .b32 %r0, %r1; - .reg .b64 %d0, %d1; - .reg .s32 %s0, %s1; - .reg .u32 %u0, %u1; - .reg .f32 %f0, %f1; - .reg .f64 %fd0, %fd1; - - mov.pred %p0, %p1; - mov.b16 %h0, %h1; - mov.b32 %r0, %r1; - mov.b64 %d0, %d1; - mov.s32 %s0, %s1; - mov.u32 %u0, %u1; - mov.f32 %f0, %f1; - mov.f64 %fd0, %fd1; - - cvt.u32.s32 %u0, %s0; - cvt.s32.u32 %s0, %u0; - cvt.rn.f32.s32 %f0, %s0; - cvt.f64.f32 %fd0, %f0; - cvt.rn.f16.f32 %h0, %f0; - cvt.rn.bf16.f32 %h0, %f0; - - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_param_b32_for_f16x2_storage_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_param_b32_for_f16x2_storage_allowed.ptx deleted file mode 100644 index 2ae08b936..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_param_b32_for_f16x2_storage_allowed.ptx +++ /dev/null @@ -1,7 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_param_b32_for_f16x2_storage_allowed(.param .b32 p0) { - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_b128_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_b128_allowed.ptx deleted file mode 100644 index 4e9b2fccb..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_b128_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_b128_allowed() { - .reg .b128 %q0; - .reg .b32 %r0, %r1, %r2, %r3; - - mov.b128 %q0, {%r0, %r1, %r2, %r3}; - mov.b128 {%r0, %r1, %r2, %r3}, %q0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_f16x2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_f16x2_allowed.ptx deleted file mode 100644 index c8c5f17db..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_reg_f16x2_allowed.ptx +++ /dev/null @@ -1,8 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_reg_f16x2_allowed() { - .reg .f16x2 %r0, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_underlying_bit_types.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_underlying_bit_types.ptx deleted file mode 100644 index 47cc7a9dc..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/declaration_underlying_bit_types.ptx +++ /dev/null @@ -1,16 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry declaration_underlying_bit_types() { - .reg .b8 %b0, %b1; - .reg .b16 %h0, %h1; - .reg .b32 %r0, %r1; - .reg .b64 %d0, %d1; - .reg .b128 %q0; - - mov.b16 %h0, %h1; - mov.b32 %r0, %r1; - mov.b64 %d0, %d1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f16_sm90_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f16_sm90_allowed.ptx deleted file mode 100644 index e65c07e09..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f16_sm90_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_90 -.address_size 64 - -.visible .entry instruction_cvt_bf16_f16_sm90_allowed() { - .reg .b16 %h0; - .reg .b16 %h1; - - cvt.rn.bf16.f16 %h0, %h1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f32_satfinite_relu_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f32_satfinite_relu_allowed.ptx deleted file mode 100644 index dffaf4164..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_f32_satfinite_relu_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_120 -.address_size 64 - -.visible .entry instruction_cvt_bf16_f32_satfinite_relu_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - - cvt.rn.satfinite.relu.bf16.f32 %h0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_s32_sm90_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_s32_sm90_allowed.ptx deleted file mode 100644 index ca33c74af..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16_s32_sm90_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_90 -.address_size 64 - -.visible .entry instruction_cvt_bf16_s32_sm90_allowed() { - .reg .b16 %h0; - .reg .s32 %r0; - - cvt.rn.bf16.s32 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_f32_pair_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_f32_pair_allowed.ptx deleted file mode 100644 index 1cad380a8..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_f32_pair_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_bf16x2_f32_pair_allowed() { - .reg .b32 %r0; - .reg .f32 %f0, %f1; - - cvt.rz.bf16x2.f32 %r0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_allowed.ptx deleted file mode 100644 index fdd8e24a5..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_bf16x2_s2f6x2_allowed() { - .reg .b32 %r0; - .reg .b16 %h0; - - cvt.rn.bf16x2.s2f6x2 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm103a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm103a_allowed.ptx deleted file mode 100644 index 10ee8e316..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm103a_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_103a -.address_size 64 - -.visible .entry instruction_cvt_bf16x2_s2f6x2_sm103a_allowed() { - .reg .b32 %r0; - .reg .b16 %h0; - - cvt.rn.bf16x2.s2f6x2 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm121a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm121a_allowed.ptx deleted file mode 100644 index eeac3f0b8..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_s2f6x2_sm121a_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_121a -.address_size 64 - -.visible .entry instruction_cvt_bf16x2_s2f6x2_sm121a_allowed() { - .reg .b32 %r0; - .reg .b16 %h0; - - cvt.rn.bf16x2.s2f6x2 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_ue8m0x2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_ue8m0x2_allowed.ptx deleted file mode 100644 index 945fe46da..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_bf16x2_ue8m0x2_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_bf16x2_ue8m0x2_allowed() { - .reg .b32 %r0; - .reg .b16 %h0; - - cvt.rn.bf16x2.ue8m0x2 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100a_allowed.ptx deleted file mode 100644 index 5817036ea..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100a_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_e2m1x2_f32_sm100a_allowed() { - .reg .b8 %b0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100f_allowed.ptx deleted file mode 100644 index 2776a2a9a..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm100f_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_100f -.address_size 64 - -.visible .entry instruction_cvt_e2m1x2_f32_sm100f_allowed() { - .reg .b8 %b0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm103a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm103a_allowed.ptx deleted file mode 100644 index 36a539b10..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm103a_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_103a -.address_size 64 - -.visible .entry instruction_cvt_e2m1x2_f32_sm103a_allowed() { - .reg .b8 %b0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120a_allowed.ptx deleted file mode 100644 index 5395515eb..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120a_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_120a -.address_size 64 - -.visible .entry instruction_cvt_e2m1x2_f32_sm120a_allowed() { - .reg .b8 %b0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120f_allowed.ptx deleted file mode 100644 index 0c1e71d6b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm120f_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_120f -.address_size 64 - -.visible .entry instruction_cvt_e2m1x2_f32_sm120f_allowed() { - .reg .b8 %b0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121a_allowed.ptx deleted file mode 100644 index 001de537b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121a_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_121a -.address_size 64 - -.visible .entry instruction_cvt_e2m1x2_f32_sm121a_allowed() { - .reg .b8 %b0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121f_allowed.ptx deleted file mode 100644 index b9a36f978..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x2_f32_sm121f_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_121f -.address_size 64 - -.visible .entry instruction_cvt_e2m1x2_f32_sm121f_allowed() { - .reg .b8 %b0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rn.satfinite.e2m1x2.f32 %b0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x4_f32_rs_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x4_f32_rs_allowed.ptx deleted file mode 100644 index 290232e8e..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m1x4_f32_rs_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_e2m1x4_f32_rs_allowed() { - .reg .b16 %h0; - .reg .f32 %f0, %f1, %f2, %f3; - .reg .b32 %r0; - - cvt.rs.satfinite.e2m1x4.f32 %h0, {%f0, %f1, %f2, %f3}, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m3x4_f32_rs_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m3x4_f32_rs_allowed.ptx deleted file mode 100644 index 909131eb7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e2m3x4_f32_rs_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_e2m3x4_f32_rs_allowed() { - .reg .b32 %r0, %r1; - .reg .f32 %f0, %f1, %f2, %f3; - - cvt.rs.satfinite.e2m3x4.f32 %r0, {%f0, %f1, %f2, %f3}, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e3m2x4_f32_rs_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e3m2x4_f32_rs_allowed.ptx deleted file mode 100644 index e16bcfcfb..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e3m2x4_f32_rs_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_e3m2x4_f32_rs_allowed() { - .reg .b32 %r0, %r1; - .reg .f32 %f0, %f1, %f2, %f3; - - cvt.rs.satfinite.e3m2x4.f32 %r0, {%f0, %f1, %f2, %f3}, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e4m3x4_f32_rs_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e4m3x4_f32_rs_allowed.ptx deleted file mode 100644 index 7b074bf48..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e4m3x4_f32_rs_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_e4m3x4_f32_rs_allowed() { - .reg .b32 %r0, %r1; - .reg .f32 %f0, %f1, %f2, %f3; - - cvt.rs.satfinite.e4m3x4.f32 %r0, {%f0, %f1, %f2, %f3}, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e5m2x4_f32_rs_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e5m2x4_f32_rs_allowed.ptx deleted file mode 100644 index 58b38a4ce..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_e5m2x4_f32_rs_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_e5m2x4_f32_rs_allowed() { - .reg .b32 %r0, %r1; - .reg .f32 %f0, %f1, %f2, %f3; - - cvt.rs.satfinite.e5m2x4.f32 %r0, {%f0, %f1, %f2, %f3}, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_bf16_sm90_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_bf16_sm90_allowed.ptx deleted file mode 100644 index b2629cbee..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_bf16_sm90_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_90 -.address_size 64 - -.visible .entry instruction_cvt_f16_bf16_sm90_allowed() { - .reg .b16 %h0; - .reg .b16 %h1; - - cvt.rn.f16.bf16 %h0, %h1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_f32_satfinite_relu_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_f32_satfinite_relu_allowed.ptx deleted file mode 100644 index 2fcc9451f..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16_f32_satfinite_relu_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_120 -.address_size 64 - -.visible .entry instruction_cvt_f16_f32_satfinite_relu_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - - cvt.rn.satfinite.relu.f16.f32 %h0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m1x2_sm100a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m1x2_sm100a_allowed.ptx deleted file mode 100644 index 7b358719c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m1x2_sm100a_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_f16x2_e2m1x2_sm100a_allowed() { - .reg .b8 %b0; - .reg .b32 %r0; - - cvt.rn.f16x2.e2m1x2 %r0, %b0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m3x2_sm100a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m3x2_sm100a_allowed.ptx deleted file mode 100644 index 6c3acd631..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_e2m3x2_sm100a_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_f16x2_e2m3x2_sm100a_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.f16x2.e2m3x2 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_f32_pair_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_f32_pair_allowed.ptx deleted file mode 100644 index 998e2d44d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f16x2_f32_pair_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_f16x2_f32_pair_allowed() { - .reg .b32 %r0; - .reg .f32 %f0, %f1; - - cvt.rn.f16x2.f32 %r0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f32_bf16_ftz_sm90_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f32_bf16_ftz_sm90_allowed.ptx deleted file mode 100644 index c7dad4186..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_f32_bf16_ftz_sm90_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_90 -.address_size 64 - -.visible .entry instruction_cvt_f32_bf16_ftz_sm90_allowed() { - .reg .f32 %f0; - .reg .b16 %h0; - - cvt.rn.ftz.f32.bf16 %f0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_rs_sm103a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_rs_sm103a_allowed.ptx deleted file mode 100644 index e8fec2072..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_rs_sm103a_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_103a -.address_size 64 - -.visible .entry instruction_cvt_rs_sm103a_allowed() { - .reg .b32 %r0, %r1; - .reg .f32 %f0, %f1; - - cvt.rs.f16x2.f32 %r0, %f0, %f1, %r1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_allowed.ptx deleted file mode 100644 index 258ad0662..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_bf16x2_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_scaled_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_scaled_allowed.ptx deleted file mode 100644 index c8755a6d1..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_scaled_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_bf16x2_scaled_allowed() { - .reg .b16 %h0, %h1; - .reg .b32 %r0; - - cvt.rn.satfinite.scaled::n2::ue8m0.s2f6x2.bf16x2 %h0, %r0, %h1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm103a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm103a_allowed.ptx deleted file mode 100644 index 722918a82..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm103a_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_103a -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_bf16x2_sm103a_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm120a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm120a_allowed.ptx deleted file mode 100644 index 849fda4ff..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm120a_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_120a -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_bf16x2_sm120a_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm121a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm121a_allowed.ptx deleted file mode 100644 index 50ec72018..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s2f6x2_bf16x2_sm121a_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_121a -.address_size 64 - -.visible .entry instruction_cvt_s2f6x2_bf16x2_sm121a_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rn.satfinite.s2f6x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_bf16_sm90_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_bf16_sm90_allowed.ptx deleted file mode 100644 index a0113c74d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_bf16_sm90_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_90 -.address_size 64 - -.visible .entry instruction_cvt_s32_bf16_sm90_allowed() { - .reg .s32 %r0; - .reg .b16 %h0; - - cvt.rzi.s32.bf16 %r0, %h0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_f32_sat_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_f32_sat_allowed.ptx deleted file mode 100644 index 7559b90b8..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_s32_f32_sat_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_90 -.address_size 64 - -.visible .entry instruction_cvt_s32_f32_sat_allowed() { - .reg .s32 %r0; - .reg .f32 %f0; - - cvt.rzi.sat.s32.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_f32_satfinite_relu_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_f32_satfinite_relu_allowed.ptx deleted file mode 100644 index be2fde912..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_f32_satfinite_relu_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_tf32_f32_satfinite_relu_allowed() { - .reg .b32 %r0; - .reg .f32 %f0; - - cvt.rn.satfinite.relu.tf32.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_roundtrip_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_roundtrip_allowed.ptx deleted file mode 100644 index b7664f656..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_tf32_roundtrip_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_cvt_tf32_roundtrip_allowed() { - .reg .f32 %f0; - .reg .b32 %r0; - - cvt.rna.tf32.f32 %r0, %f0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_u32_f64_sat_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_u32_f64_sat_allowed.ptx deleted file mode 100644 index 0b9eaacc7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_u32_f64_sat_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_90 -.address_size 64 - -.visible .entry instruction_cvt_u32_f64_sat_allowed() { - .reg .u32 %r0; - .reg .f64 %fd0; - - cvt.rzi.sat.u32.f64 %r0, %fd0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_allowed.ptx deleted file mode 100644 index 8b4046cc5..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_bf16x2_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rz.satfinite.ue8m0x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx deleted file mode 100644 index 654a8de5b..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_bf16x2_relu_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rz.satfinite.relu.ue8m0x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_without_satfinite_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_without_satfinite_allowed.ptx deleted file mode 100644 index dc0c0cece..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_bf16x2_without_satfinite_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_bf16x2_without_satfinite_allowed() { - .reg .b16 %h0; - .reg .b32 %r0; - - cvt.rz.ue8m0x2.bf16x2 %h0, %r0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100a_allowed.ptx deleted file mode 100644 index 939e4a856..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100a_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_100a -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_f32_sm100a_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100f_allowed.ptx deleted file mode 100644 index b5c82a27d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm100f_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_100f -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_f32_sm100f_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm103f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm103f_allowed.ptx deleted file mode 100644 index ec99ca1b4..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm103f_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_103f -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_f32_sm103f_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm110f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm110f_allowed.ptx deleted file mode 100644 index 02d6a0d07..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm110f_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_110f -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_f32_sm110f_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120a_allowed.ptx deleted file mode 100644 index 46999edc6..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120a_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_120a -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_f32_sm120a_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120f_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120f_allowed.ptx deleted file mode 100644 index 431ec0f2c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm120f_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_120f -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_f32_sm120f_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm121a_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm121a_allowed.ptx deleted file mode 100644 index d157676b9..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_cvt_ue8m0x2_f32_sm121a_allowed.ptx +++ /dev/null @@ -1,12 +0,0 @@ -.version 9.1 -.target sm_121a -.address_size 64 - -.visible .entry instruction_cvt_ue8m0x2_f32_sm121a_allowed() { - .reg .b16 %h0; - .reg .f32 %f0; - .reg .f32 %f1; - - cvt.rz.satfinite.ue8m0x2.f32 %h0, %f0, %f1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_allowed.ptx deleted file mode 100644 index f2fd64a41..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_b128_pack_allowed() { - .reg .b128 %q0; - .reg .b32 %r0, %r1, %r2, %r3; - - mov.b128 %q0, {%r0, %r1, %r2, %r3}; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_allowed.ptx deleted file mode 100644 index 652df6487..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_b128_pack_v2_allowed() { - .reg .b128 %q0; - .reg .b64 %d0, %d1; - - mov.b128 %q0, {%d0, %d1}; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_u64_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_u64_allowed.ptx deleted file mode 100644 index c29306eb9..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_pack_v2_u64_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_b128_pack_v2_u64_allowed() { - .reg .b128 %q0; - .reg .u64 %u0, %u1; - - mov.b128 %q0, {%u0, %u1}; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_scalar_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_scalar_allowed.ptx deleted file mode 100644 index 838da831f..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_scalar_allowed.ptx +++ /dev/null @@ -1,10 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_b128_scalar_allowed() { - .reg .b128 %q0, %q1; - - mov.b128 %q0, %q1; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_allowed.ptx deleted file mode 100644 index c192e3dbf..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_b128_unpack_allowed() { - .reg .b128 %q0; - .reg .b32 %r0, %r1, %r2, %r3; - - mov.b128 {%r0, %r1, %r2, %r3}, %q0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v2_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v2_allowed.ptx deleted file mode 100644 index ac73dfb6c..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v2_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_b128_unpack_v2_allowed() { - .reg .b128 %q0; - .reg .b64 %d0, %d1; - - mov.b128 {%d0, %d1}, %q0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_sink_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_sink_allowed.ptx deleted file mode 100644 index 1f72a20c5..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_sink_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_b128_unpack_v4_sink_allowed() { - .reg .b128 %q0; - .reg .b32 %r0, %r2; - - mov.b128 {%r0, _, %r2, _}, %q0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_u32_allowed.ptx b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_u32_allowed.ptx deleted file mode 100644 index 9e249ac68..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/ptx_datatype_cases/pass/instruction_mov_b128_unpack_v4_u32_allowed.ptx +++ /dev/null @@ -1,11 +0,0 @@ -.version 9.1 -.target sm_100 -.address_size 64 - -.visible .entry instruction_mov_b128_unpack_v4_u32_allowed() { - .reg .b128 %q0; - .reg .u32 %u0, %u1, %u2, %u3; - - mov.b128 {%u0, %u1, %u2, %u3}, %q0; - ret; -} diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/run_lowp_cvt_ptxas_crosscheck.sh b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/run_lowp_cvt_ptxas_crosscheck.sh deleted file mode 100755 index 240c93d4d..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/fp_lowp_cvt/run_lowp_cvt_ptxas_crosscheck.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -ne 1 ]]; then - echo "usage: $0 " >&2 - exit 2 -fi - -cases_root="$1" - -if ! command -v ptxas >/dev/null 2>&1; then - echo "ptxas not found; lowp cvt ptxas cross-check cannot run" >&2 - exit 1 -fi - -work_dir="$(mktemp -d)" -trap 'rm -rf "$work_dir"' EXIT - -failures=0 -ran=0 - -ptx_target() { - awk '$1 == ".target" {print $2; exit}' "$1" -} - -run_expect_pass() { - local rel="$1" - local file="$cases_root/$rel" - local arch - local out="$work_dir/$(basename "$file").out" - local cubin="$work_dir/$(basename "$file").cubin" - - ((ran += 1)) - arch="$(ptx_target "$file")" - if [[ -z "$arch" ]]; then - echo "[FAIL] missing .target in $file" - ((failures += 1)) - return - fi - - if ptxas -arch="$arch" "$file" -o "$cubin" >"$out" 2>&1; then - echo "[PASS] ptxas accepted: $rel" - else - echo "[FAIL] expected ptxas success: $rel" - cat "$out" - ((failures += 1)) - fi -} - -run_expect_fail() { - local rel="$1" - local pattern="$2" - local file="$cases_root/$rel" - local arch - local out="$work_dir/$(basename "$file").out" - local cubin="$work_dir/$(basename "$file").cubin" - - ((ran += 1)) - arch="$(ptx_target "$file")" - if [[ -z "$arch" ]]; then - echo "[FAIL] missing .target in $file" - ((failures += 1)) - return - fi - - if ptxas -arch="$arch" "$file" -o "$cubin" >"$out" 2>&1; then - echo "[FAIL] expected ptxas failure: $rel" - ((failures += 1)) - return - fi - - if [[ -n "$pattern" ]] && ! grep -Fq -- "$pattern" "$out"; then - echo "[FAIL] expected ptxas error text not found: $rel" - echo "[FAIL] missing pattern: $pattern" - cat "$out" - ((failures += 1)) - return - fi - - echo "[PASS] ptxas rejected as expected: $rel" -} - -ptxas_accepts_case() { - local rel="$1" - local file="$cases_root/$rel" - local arch - local out="$work_dir/probe_$(basename "$file").out" - local cubin="$work_dir/probe_$(basename "$file").cubin" - arch="$(ptx_target "$file")" - if [[ -z "$arch" ]]; then - return 1 - fi - if ptxas -arch="$arch" "$file" -o "$cubin" >"$out" 2>&1; then - return 0 - fi - return 1 -} - -spec_tracking_ue8m0x2_bf16x2_relu="${LOWP_CVT_SPEC_TRACK_UE8M0X2_BF16X2_RELU:-auto}" -spec_tracking_case_rel="pass/instruction_cvt_ue8m0x2_bf16x2_relu_allowed.ptx" -pass_case_count=0 -while IFS= read -r case_file; do - rel="${case_file#$cases_root/}" - if [[ "$rel" == "$spec_tracking_case_rel" ]]; then - continue - fi - run_expect_pass "$rel" - ((pass_case_count += 1)) -done < <( - find "$cases_root/pass" -maxdepth 1 -type f \ - -name "instruction_cvt_*_allowed.ptx" | sort -) -echo "[INFO] auto-covered parser-pass cvt forms: $pass_case_count" - -case "$spec_tracking_ue8m0x2_bf16x2_relu" in - auto) - if ptxas_accepts_case "$spec_tracking_case_rel"; then - echo "[SPEC-TRACKING:ue8m0x2_bf16x2_relu] ptxas currently accepts this PTX-legal form" - run_expect_pass "$spec_tracking_case_rel" - else - echo "[SPEC-TRACKING:ue8m0x2_bf16x2_relu] ptxas currently rejects this PTX-legal form (tracked divergence)" - run_expect_fail "$spec_tracking_case_rel" "" - fi - ;; - ptxas_rejects) - echo "[SPEC-TRACKING:ue8m0x2_bf16x2_relu] expecting ptxas rejection" - run_expect_fail "$spec_tracking_case_rel" "Illegal modifier '.relu'" - ;; - ptxas_accepts) - echo "[SPEC-TRACKING:ue8m0x2_bf16x2_relu] expecting ptxas acceptance" - run_expect_pass "$spec_tracking_case_rel" - ;; - spec_accepts) - echo "[SPEC-TRACKING:ue8m0x2_bf16x2_relu] enforcing spec expectation (accept)" - run_expect_pass "$spec_tracking_case_rel" - ;; - *) - echo "[FAIL] invalid LOWP_CVT_SPEC_TRACK_UE8M0X2_BF16X2_RELU value: $spec_tracking_ue8m0x2_bf16x2_relu (expected auto|ptxas_rejects|ptxas_accepts|spec_accepts)" >&2 - exit 2 - ;; -esac - -run_expect_fail "fail/instruction_cvt_ue8m0x2_f32_relu_not_allowed.ptx" "Illegal modifier '.relu'" -run_expect_fail "fail/instruction_cvt_ue8m0x2_f32_sm120_not_allowed.ptx" "not supported on .target 'sm_120'" -run_expect_fail "fail/instruction_cvt_e2m1x2_f32_sm120_not_allowed.ptx" "not supported on .target 'sm_120'" -run_expect_fail "fail/instruction_cvt_f32_bf16_ftz_sm80_not_allowed.ptx" "requires .target sm_90 or higher" -run_expect_fail "fail/instruction_cvt_bf16_f16_sm80_not_allowed.ptx" "requires .target sm_90 or higher" -run_expect_fail "fail/instruction_cvt_f16_bf16_sm80_not_allowed.ptx" "requires .target sm_90 or higher" -run_expect_fail "fail/instruction_cvt_bf16_s32_sm80_not_allowed.ptx" "requires .target sm_90 or higher" -run_expect_fail "fail/instruction_cvt_s32_bf16_sm80_not_allowed.ptx" "requires .target sm_90 or higher" -run_expect_fail "fail/instruction_cvt_rs_sm110a_not_allowed.ptx" "Feature '.rs' not supported" -run_expect_fail "fail/instruction_cvt_rs_sm100_not_allowed.ptx" "Feature '.rs' not supported" -run_expect_fail "fail/instruction_cvt_ue8m0x2_bf16x2_missing_rounding_not_allowed.ptx" "Rounding modifier required for instruction 'cvt'" -run_expect_fail "fail/instruction_cvt_s2f6x2_bf16x2_missing_satfinite_not_allowed.ptx" "'.satfinite' modifier required" -run_expect_fail "fail/instruction_cvt_s2f6x2_bf16x2_scaled_without_operand_not_allowed.ptx" "Illegal modifier '.scaled::n2::ue8m0'" -run_expect_fail "fail/instruction_cvt_s2f6x2_bf16x2_operand_without_scaled_not_allowed.ptx" "Arguments mismatch for instruction 'cvt'" -run_expect_fail "fail/instruction_cvt_s2f6x2_bf16x2_sm100f_not_allowed.ptx" "not supported on .target 'sm_100f'" -run_expect_fail "fail/instruction_cvt_s2f6x2_bf16x2_sm120f_not_allowed.ptx" "not supported on .target 'sm_120f'" -run_expect_fail "fail/instruction_cvt_tf32_f32_satfinite_rn_sm90_not_allowed.ptx" "requires .target sm_100 or higher" -run_expect_fail "fail/instruction_cvt_tf32_f32_rna_relu_not_allowed.ptx" "cannot be combined with modifier '.rna'" -run_expect_fail "fail/instruction_cvt_ue4m3_f32_not_allowed.ptx" "Unexpected instruction types specified for 'cvt'" -run_expect_fail "fail/instruction_cvt_ue8m0x2_scaled_not_allowed.ptx" "Illegal modifier '.scaled::n2::ue8m0'" -run_expect_fail "fail/instruction_cvt_rs_sm100f_not_allowed.ptx" "Feature '.rs' not supported" -run_expect_fail "fail/instruction_cvt_rs_sm120a_not_allowed.ptx" "Feature '.rs' not supported" -run_expect_fail "fail/instruction_cvt_rs_sm120f_not_allowed.ptx" "Feature '.rs' not supported" -run_expect_fail "fail/instruction_cvt_s32_bf16_sat_not_allowed.ptx" "Illegal modifier '.sat'" -run_expect_fail "fail/instruction_cvt_s32_s32_sat_not_allowed.ptx" "Illegal modifier '.sat'" - -if [[ $failures -ne 0 ]]; then - echo "lowp cvt ptxas cross-check failed: $failures / $ran cases failed" >&2 - exit 1 -fi - -echo "lowp cvt ptxas cross-check passed: $ran cases" diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/Makefile b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/Makefile deleted file mode 100644 index 0cdf5fee4..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -BASE_DIR := $(shell pwd) -BIN_DIR := $(BASE_DIR)/../../../bin - -CC := nvcc -EXE := gpgpusim_tma_tensor_benchmark -SRC := tma_tensor_benchmark.cu - -ifneq ($(filter 13 14 15 16,$(CUDA_VERSION_MAJOR)),) -ARCH ?= sm_90a sm_100a sm_110 sm_120 -else -ARCH ?= sm_90a sm_100a sm_101 sm_120 -endif - -CUDA_CPPFLAGS = -NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 - -.DEFAULT_GOAL := all - -all: $(BIN_DIR)/$(EXE) - -$(BIN_DIR): - mkdir -p $@ - -$(EXE).a: $(SRC) - $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ - -$(BIN_DIR)/$(EXE): $(EXE).a | $(BIN_DIR) - $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared - -run-fixtures: $(BIN_DIR)/$(EXE) - set -e; \ - tmp_dir="$$(mktemp -d)"; \ - trap 'rm -rf "$$tmp_dir"' EXIT; \ - for expected in expected/tma_tensor_test_*.txt; do \ - base="$$(basename "$$expected" .txt)"; \ - spec="$${base#tma_tensor_test_}"; \ - width="$${spec##*_}"; \ - spec="$${spec%_*}"; \ - height="$${spec##*_}"; \ - opcode="$${spec%_*}"; \ - ( cd "$$tmp_dir" && "$(BIN_DIR)/$(EXE)" -w "$$width" -h "$$height" -o "$$opcode" ); \ - actual="$$tmp_dir/$$(basename "$$expected")"; \ - cmp -s "$$expected" "$$actual" || diff -u "$$expected" "$$actual"; \ - done - -clean: - rm -f *.o *.a *.ptx *.ptxas *.sass $(EXE) - rm -f $(BIN_DIR)/$(EXE) - -.PHONY: all clean run-fixtures diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/README.md b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/README.md deleted file mode 100644 index ea41ab04f..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# TMA Tensor-Map Test - -This folder contains the standalone TMA tensor-map benchmark from the -GPGPU-Sim `tma` branch. Expected fixture outputs are kept under `expected/`. - -## Build - -```sh -make -``` - -The default build creates `gpgpusim_tma_tensor_benchmark` under -`GPU_Microbenchmark/bin`. - -## Fixture Check - -```sh -make run-fixtures -``` - -This target runs the imported fixture cases and compares generated output -against `expected/`. diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_REGULAR_LOAD_64_64.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_REGULAR_LOAD_64_64.txt deleted file mode 100644 index e3f4714b7..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_REGULAR_LOAD_64_64.txt +++ /dev/null @@ -1,8 +0,0 @@ -0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff 0x100 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x135 0x136 0x137 0x138 0x139 0x13a 0x13b 0x13c 0x13d 0x13e 0x13f 0x140 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x175 0x176 0x177 0x178 0x179 0x17a 0x17b 0x17c 0x17d 0x17e 0x17f 0x180 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x1b5 0x1b6 0x1b7 0x1b8 0x1b9 0x1ba 0x1bb 0x1bc 0x1bd 0x1be 0x1bf 0x1c0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x1f5 0x1f6 0x1f7 0x1f8 0x1f9 0x1fa 0x1fb 0x1fc 0x1fd 0x1fe 0x1ff 0x200 0x201 -0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x235 0x236 0x237 0x238 0x239 0x23a 0x23b 0x23c 0x23d 0x23e 0x23f 0x240 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x275 0x276 0x277 0x278 0x279 0x27a 0x27b 0x27c 0x27d 0x27e 0x27f 0x280 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x2b5 0x2b6 0x2b7 0x2b8 0x2b9 0x2ba 0x2bb 0x2bc 0x2bd 0x2be 0x2bf 0x2c0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x2f5 0x2f6 0x2f7 0x2f8 0x2f9 0x2fa 0x2fb 0x2fc 0x2fd 0x2fe 0x2ff 0x300 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x335 0x336 0x337 0x338 0x339 0x33a 0x33b 0x33c 0x33d 0x33e 0x33f 0x340 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x375 0x376 0x377 0x378 0x379 0x37a 0x37b 0x37c 0x37d 0x37e 0x37f 0x380 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x3b5 0x3b6 0x3b7 0x3b8 0x3b9 0x3ba 0x3bb 0x3bc 0x3bd 0x3be 0x3bf 0x3c0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x3f5 0x3f6 0x3f7 0x3f8 0x3f9 0x3fa 0x3fb 0x3fc 0x3fd 0x3fe 0x3ff 0x400 0x401 -0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x435 0x436 0x437 0x438 0x439 0x43a 0x43b 0x43c 0x43d 0x43e 0x43f 0x440 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x475 0x476 0x477 0x478 0x479 0x47a 0x47b 0x47c 0x47d 0x47e 0x47f 0x480 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x4b5 0x4b6 0x4b7 0x4b8 0x4b9 0x4ba 0x4bb 0x4bc 0x4bd 0x4be 0x4bf 0x4c0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x4f5 0x4f6 0x4f7 0x4f8 0x4f9 0x4fa 0x4fb 0x4fc 0x4fd 0x4fe 0x4ff 0x500 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x535 0x536 0x537 0x538 0x539 0x53a 0x53b 0x53c 0x53d 0x53e 0x53f 0x540 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x575 0x576 0x577 0x578 0x579 0x57a 0x57b 0x57c 0x57d 0x57e 0x57f 0x580 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x5b5 0x5b6 0x5b7 0x5b8 0x5b9 0x5ba 0x5bb 0x5bc 0x5bd 0x5be 0x5bf 0x5c0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x5f5 0x5f6 0x5f7 0x5f8 0x5f9 0x5fa 0x5fb 0x5fc 0x5fd 0x5fe 0x5ff 0x600 0x601 -0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x635 0x636 0x637 0x638 0x639 0x63a 0x63b 0x63c 0x63d 0x63e 0x63f 0x640 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x675 0x676 0x677 0x678 0x679 0x67a 0x67b 0x67c 0x67d 0x67e 0x67f 0x680 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x6b5 0x6b6 0x6b7 0x6b8 0x6b9 0x6ba 0x6bb 0x6bc 0x6bd 0x6be 0x6bf 0x6c0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x6f5 0x6f6 0x6f7 0x6f8 0x6f9 0x6fa 0x6fb 0x6fc 0x6fd 0x6fe 0x6ff 0x700 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x735 0x736 0x737 0x738 0x739 0x73a 0x73b 0x73c 0x73d 0x73e 0x73f 0x740 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x775 0x776 0x777 0x778 0x779 0x77a 0x77b 0x77c 0x77d 0x77e 0x77f 0x780 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x7b5 0x7b6 0x7b7 0x7b8 0x7b9 0x7ba 0x7bb 0x7bc 0x7bd 0x7be 0x7bf 0x7c0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x7f5 0x7f6 0x7f7 0x7f8 0x7f9 0x7fa 0x7fb 0x7fc 0x7fd 0x7fe 0x7ff 0x800 0x801 -0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x835 0x836 0x837 0x838 0x839 0x83a 0x83b 0x83c 0x83d 0x83e 0x83f 0x840 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x875 0x876 0x877 0x878 0x879 0x87a 0x87b 0x87c 0x87d 0x87e 0x87f 0x880 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x8b5 0x8b6 0x8b7 0x8b8 0x8b9 0x8ba 0x8bb 0x8bc 0x8bd 0x8be 0x8bf 0x8c0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x8f5 0x8f6 0x8f7 0x8f8 0x8f9 0x8fa 0x8fb 0x8fc 0x8fd 0x8fe 0x8ff 0x900 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x935 0x936 0x937 0x938 0x939 0x93a 0x93b 0x93c 0x93d 0x93e 0x93f 0x940 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x975 0x976 0x977 0x978 0x979 0x97a 0x97b 0x97c 0x97d 0x97e 0x97f 0x980 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x9b5 0x9b6 0x9b7 0x9b8 0x9b9 0x9ba 0x9bb 0x9bc 0x9bd 0x9be 0x9bf 0x9c0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x9f5 0x9f6 0x9f7 0x9f8 0x9f9 0x9fa 0x9fb 0x9fc 0x9fd 0x9fe 0x9ff 0xa00 0xa01 -0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0xa35 0xa36 0xa37 0xa38 0xa39 0xa3a 0xa3b 0xa3c 0xa3d 0xa3e 0xa3f 0xa40 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0xa75 0xa76 0xa77 0xa78 0xa79 0xa7a 0xa7b 0xa7c 0xa7d 0xa7e 0xa7f 0xa80 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0xab5 0xab6 0xab7 0xab8 0xab9 0xaba 0xabb 0xabc 0xabd 0xabe 0xabf 0xac0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0xaf5 0xaf6 0xaf7 0xaf8 0xaf9 0xafa 0xafb 0xafc 0xafd 0xafe 0xaff 0xb00 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0xb35 0xb36 0xb37 0xb38 0xb39 0xb3a 0xb3b 0xb3c 0xb3d 0xb3e 0xb3f 0xb40 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0xb75 0xb76 0xb77 0xb78 0xb79 0xb7a 0xb7b 0xb7c 0xb7d 0xb7e 0xb7f 0xb80 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0xbb5 0xbb6 0xbb7 0xbb8 0xbb9 0xbba 0xbbb 0xbbc 0xbbd 0xbbe 0xbbf 0xbc0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0xbf5 0xbf6 0xbf7 0xbf8 0xbf9 0xbfa 0xbfb 0xbfc 0xbfd 0xbfe 0xbff 0xc00 0xc01 -0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0xc35 0xc36 0xc37 0xc38 0xc39 0xc3a 0xc3b 0xc3c 0xc3d 0xc3e 0xc3f 0xc40 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0xc75 0xc76 0xc77 0xc78 0xc79 0xc7a 0xc7b 0xc7c 0xc7d 0xc7e 0xc7f 0xc80 0xc81 0xc82 0xc83 0xc84 0xc85 0xc86 0xc87 0xc88 0xc89 0xc8a 0xc8b 0xc8c 0xc8d 0xc8e 0xc8f 0xc90 0xc91 0xc92 0xc93 0xc94 0xc95 0xc96 0xc97 0xc98 0xc99 0xc9a 0xc9b 0xc9c 0xc9d 0xc9e 0xc9f 0xca0 0xca1 0xca2 0xca3 0xca4 0xca5 0xca6 0xca7 0xca8 0xca9 0xcaa 0xcab 0xcac 0xcad 0xcae 0xcaf 0xcb0 0xcb1 0xcb2 0xcb3 0xcb4 0xcb5 0xcb6 0xcb7 0xcb8 0xcb9 0xcba 0xcbb 0xcbc 0xcbd 0xcbe 0xcbf 0xcc0 0xcc1 0xcc2 0xcc3 0xcc4 0xcc5 0xcc6 0xcc7 0xcc8 0xcc9 0xcca 0xccb 0xccc 0xccd 0xcce 0xccf 0xcd0 0xcd1 0xcd2 0xcd3 0xcd4 0xcd5 0xcd6 0xcd7 0xcd8 0xcd9 0xcda 0xcdb 0xcdc 0xcdd 0xcde 0xcdf 0xce0 0xce1 0xce2 0xce3 0xce4 0xce5 0xce6 0xce7 0xce8 0xce9 0xcea 0xceb 0xcec 0xced 0xcee 0xcef 0xcf0 0xcf1 0xcf2 0xcf3 0xcf4 0xcf5 0xcf6 0xcf7 0xcf8 0xcf9 0xcfa 0xcfb 0xcfc 0xcfd 0xcfe 0xcff 0xd00 0xd01 0xd02 0xd03 0xd04 0xd05 0xd06 0xd07 0xd08 0xd09 0xd0a 0xd0b 0xd0c 0xd0d 0xd0e 0xd0f 0xd10 0xd11 0xd12 0xd13 0xd14 0xd15 0xd16 0xd17 0xd18 0xd19 0xd1a 0xd1b 0xd1c 0xd1d 0xd1e 0xd1f 0xd20 0xd21 0xd22 0xd23 0xd24 0xd25 0xd26 0xd27 0xd28 0xd29 0xd2a 0xd2b 0xd2c 0xd2d 0xd2e 0xd2f 0xd30 0xd31 0xd32 0xd33 0xd34 0xd35 0xd36 0xd37 0xd38 0xd39 0xd3a 0xd3b 0xd3c 0xd3d 0xd3e 0xd3f 0xd40 0xd41 0xd42 0xd43 0xd44 0xd45 0xd46 0xd47 0xd48 0xd49 0xd4a 0xd4b 0xd4c 0xd4d 0xd4e 0xd4f 0xd50 0xd51 0xd52 0xd53 0xd54 0xd55 0xd56 0xd57 0xd58 0xd59 0xd5a 0xd5b 0xd5c 0xd5d 0xd5e 0xd5f 0xd60 0xd61 0xd62 0xd63 0xd64 0xd65 0xd66 0xd67 0xd68 0xd69 0xd6a 0xd6b 0xd6c 0xd6d 0xd6e 0xd6f 0xd70 0xd71 0xd72 0xd73 0xd74 0xd75 0xd76 0xd77 0xd78 0xd79 0xd7a 0xd7b 0xd7c 0xd7d 0xd7e 0xd7f 0xd80 0xd81 0xd82 0xd83 0xd84 0xd85 0xd86 0xd87 0xd88 0xd89 0xd8a 0xd8b 0xd8c 0xd8d 0xd8e 0xd8f 0xd90 0xd91 0xd92 0xd93 0xd94 0xd95 0xd96 0xd97 0xd98 0xd99 0xd9a 0xd9b 0xd9c 0xd9d 0xd9e 0xd9f 0xda0 0xda1 0xda2 0xda3 0xda4 0xda5 0xda6 0xda7 0xda8 0xda9 0xdaa 0xdab 0xdac 0xdad 0xdae 0xdaf 0xdb0 0xdb1 0xdb2 0xdb3 0xdb4 0xdb5 0xdb6 0xdb7 0xdb8 0xdb9 0xdba 0xdbb 0xdbc 0xdbd 0xdbe 0xdbf 0xdc0 0xdc1 0xdc2 0xdc3 0xdc4 0xdc5 0xdc6 0xdc7 0xdc8 0xdc9 0xdca 0xdcb 0xdcc 0xdcd 0xdce 0xdcf 0xdd0 0xdd1 0xdd2 0xdd3 0xdd4 0xdd5 0xdd6 0xdd7 0xdd8 0xdd9 0xdda 0xddb 0xddc 0xddd 0xdde 0xddf 0xde0 0xde1 0xde2 0xde3 0xde4 0xde5 0xde6 0xde7 0xde8 0xde9 0xdea 0xdeb 0xdec 0xded 0xdee 0xdef 0xdf0 0xdf1 0xdf2 0xdf3 0xdf4 0xdf5 0xdf6 0xdf7 0xdf8 0xdf9 0xdfa 0xdfb 0xdfc 0xdfd 0xdfe 0xdff 0xe00 0xe01 -0xe02 0xe03 0xe04 0xe05 0xe06 0xe07 0xe08 0xe09 0xe0a 0xe0b 0xe0c 0xe0d 0xe0e 0xe0f 0xe10 0xe11 0xe12 0xe13 0xe14 0xe15 0xe16 0xe17 0xe18 0xe19 0xe1a 0xe1b 0xe1c 0xe1d 0xe1e 0xe1f 0xe20 0xe21 0xe22 0xe23 0xe24 0xe25 0xe26 0xe27 0xe28 0xe29 0xe2a 0xe2b 0xe2c 0xe2d 0xe2e 0xe2f 0xe30 0xe31 0xe32 0xe33 0xe34 0xe35 0xe36 0xe37 0xe38 0xe39 0xe3a 0xe3b 0xe3c 0xe3d 0xe3e 0xe3f 0xe40 0xe41 0xe42 0xe43 0xe44 0xe45 0xe46 0xe47 0xe48 0xe49 0xe4a 0xe4b 0xe4c 0xe4d 0xe4e 0xe4f 0xe50 0xe51 0xe52 0xe53 0xe54 0xe55 0xe56 0xe57 0xe58 0xe59 0xe5a 0xe5b 0xe5c 0xe5d 0xe5e 0xe5f 0xe60 0xe61 0xe62 0xe63 0xe64 0xe65 0xe66 0xe67 0xe68 0xe69 0xe6a 0xe6b 0xe6c 0xe6d 0xe6e 0xe6f 0xe70 0xe71 0xe72 0xe73 0xe74 0xe75 0xe76 0xe77 0xe78 0xe79 0xe7a 0xe7b 0xe7c 0xe7d 0xe7e 0xe7f 0xe80 0xe81 0xe82 0xe83 0xe84 0xe85 0xe86 0xe87 0xe88 0xe89 0xe8a 0xe8b 0xe8c 0xe8d 0xe8e 0xe8f 0xe90 0xe91 0xe92 0xe93 0xe94 0xe95 0xe96 0xe97 0xe98 0xe99 0xe9a 0xe9b 0xe9c 0xe9d 0xe9e 0xe9f 0xea0 0xea1 0xea2 0xea3 0xea4 0xea5 0xea6 0xea7 0xea8 0xea9 0xeaa 0xeab 0xeac 0xead 0xeae 0xeaf 0xeb0 0xeb1 0xeb2 0xeb3 0xeb4 0xeb5 0xeb6 0xeb7 0xeb8 0xeb9 0xeba 0xebb 0xebc 0xebd 0xebe 0xebf 0xec0 0xec1 0xec2 0xec3 0xec4 0xec5 0xec6 0xec7 0xec8 0xec9 0xeca 0xecb 0xecc 0xecd 0xece 0xecf 0xed0 0xed1 0xed2 0xed3 0xed4 0xed5 0xed6 0xed7 0xed8 0xed9 0xeda 0xedb 0xedc 0xedd 0xede 0xedf 0xee0 0xee1 0xee2 0xee3 0xee4 0xee5 0xee6 0xee7 0xee8 0xee9 0xeea 0xeeb 0xeec 0xeed 0xeee 0xeef 0xef0 0xef1 0xef2 0xef3 0xef4 0xef5 0xef6 0xef7 0xef8 0xef9 0xefa 0xefb 0xefc 0xefd 0xefe 0xeff 0xf00 0xf01 0xf02 0xf03 0xf04 0xf05 0xf06 0xf07 0xf08 0xf09 0xf0a 0xf0b 0xf0c 0xf0d 0xf0e 0xf0f 0xf10 0xf11 0xf12 0xf13 0xf14 0xf15 0xf16 0xf17 0xf18 0xf19 0xf1a 0xf1b 0xf1c 0xf1d 0xf1e 0xf1f 0xf20 0xf21 0xf22 0xf23 0xf24 0xf25 0xf26 0xf27 0xf28 0xf29 0xf2a 0xf2b 0xf2c 0xf2d 0xf2e 0xf2f 0xf30 0xf31 0xf32 0xf33 0xf34 0xf35 0xf36 0xf37 0xf38 0xf39 0xf3a 0xf3b 0xf3c 0xf3d 0xf3e 0xf3f 0xf40 0xf41 0xf42 0xf43 0xf44 0xf45 0xf46 0xf47 0xf48 0xf49 0xf4a 0xf4b 0xf4c 0xf4d 0xf4e 0xf4f 0xf50 0xf51 0xf52 0xf53 0xf54 0xf55 0xf56 0xf57 0xf58 0xf59 0xf5a 0xf5b 0xf5c 0xf5d 0xf5e 0xf5f 0xf60 0xf61 0xf62 0xf63 0xf64 0xf65 0xf66 0xf67 0xf68 0xf69 0xf6a 0xf6b 0xf6c 0xf6d 0xf6e 0xf6f 0xf70 0xf71 0xf72 0xf73 0xf74 0xf75 0xf76 0xf77 0xf78 0xf79 0xf7a 0xf7b 0xf7c 0xf7d 0xf7e 0xf7f 0xf80 0xf81 0xf82 0xf83 0xf84 0xf85 0xf86 0xf87 0xf88 0xf89 0xf8a 0xf8b 0xf8c 0xf8d 0xf8e 0xf8f 0xf90 0xf91 0xf92 0xf93 0xf94 0xf95 0xf96 0xf97 0xf98 0xf99 0xf9a 0xf9b 0xf9c 0xf9d 0xf9e 0xf9f 0xfa0 0xfa1 0xfa2 0xfa3 0xfa4 0xfa5 0xfa6 0xfa7 0xfa8 0xfa9 0xfaa 0xfab 0xfac 0xfad 0xfae 0xfaf 0xfb0 0xfb1 0xfb2 0xfb3 0xfb4 0xfb5 0xfb6 0xfb7 0xfb8 0xfb9 0xfba 0xfbb 0xfbc 0xfbd 0xfbe 0xfbf 0xfc0 0xfc1 0xfc2 0xfc3 0xfc4 0xfc5 0xfc6 0xfc7 0xfc8 0xfc9 0xfca 0xfcb 0xfcc 0xfcd 0xfce 0xfcf 0xfd0 0xfd1 0xfd2 0xfd3 0xfd4 0xfd5 0xfd6 0xfd7 0xfd8 0xfd9 0xfda 0xfdb 0xfdc 0xfdd 0xfde 0xfdf 0xfe0 0xfe1 0xfe2 0xfe3 0xfe4 0xfe5 0xfe6 0xfe7 0xfe8 0xfe9 0xfea 0xfeb 0xfec 0xfed 0xfee 0xfef 0xff0 0xff1 0xff2 0xff3 0xff4 0xff5 0xff6 0xff7 0xff8 0xff9 0xffa 0xffb 0xffc 0xffd 0xffe 0xfff 0x1000 0x1001 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMALDG_64_64.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMALDG_64_64.txt deleted file mode 100644 index 166808918..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMALDG_64_64.txt +++ /dev/null @@ -1,8 +0,0 @@ -0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff 0x100 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x135 0x136 0x137 0x138 0x139 0x13a 0x13b 0x13c 0x13d 0x13e 0x13f 0x140 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x175 0x176 0x177 0x178 0x179 0x17a 0x17b 0x17c 0x17d 0x17e 0x17f 0x180 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x1b5 0x1b6 0x1b7 0x1b8 0x1b9 0x1ba 0x1bb 0x1bc 0x1bd 0x1be 0x1bf 0x1c0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x1f5 0x1f6 0x1f7 0x1f8 0x1f9 0x1fa 0x1fb 0x1fc 0x1fd 0x1fe 0x1ff 0x200 -0x201 0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x235 0x236 0x237 0x238 0x239 0x23a 0x23b 0x23c 0x23d 0x23e 0x23f 0x240 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x275 0x276 0x277 0x278 0x279 0x27a 0x27b 0x27c 0x27d 0x27e 0x27f 0x280 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x2b5 0x2b6 0x2b7 0x2b8 0x2b9 0x2ba 0x2bb 0x2bc 0x2bd 0x2be 0x2bf 0x2c0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x2f5 0x2f6 0x2f7 0x2f8 0x2f9 0x2fa 0x2fb 0x2fc 0x2fd 0x2fe 0x2ff 0x300 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x335 0x336 0x337 0x338 0x339 0x33a 0x33b 0x33c 0x33d 0x33e 0x33f 0x340 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x375 0x376 0x377 0x378 0x379 0x37a 0x37b 0x37c 0x37d 0x37e 0x37f 0x380 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x3b5 0x3b6 0x3b7 0x3b8 0x3b9 0x3ba 0x3bb 0x3bc 0x3bd 0x3be 0x3bf 0x3c0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x3f5 0x3f6 0x3f7 0x3f8 0x3f9 0x3fa 0x3fb 0x3fc 0x3fd 0x3fe 0x3ff 0x400 -0x401 0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x435 0x436 0x437 0x438 0x439 0x43a 0x43b 0x43c 0x43d 0x43e 0x43f 0x440 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x475 0x476 0x477 0x478 0x479 0x47a 0x47b 0x47c 0x47d 0x47e 0x47f 0x480 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x4b5 0x4b6 0x4b7 0x4b8 0x4b9 0x4ba 0x4bb 0x4bc 0x4bd 0x4be 0x4bf 0x4c0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x4f5 0x4f6 0x4f7 0x4f8 0x4f9 0x4fa 0x4fb 0x4fc 0x4fd 0x4fe 0x4ff 0x500 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x535 0x536 0x537 0x538 0x539 0x53a 0x53b 0x53c 0x53d 0x53e 0x53f 0x540 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x575 0x576 0x577 0x578 0x579 0x57a 0x57b 0x57c 0x57d 0x57e 0x57f 0x580 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x5b5 0x5b6 0x5b7 0x5b8 0x5b9 0x5ba 0x5bb 0x5bc 0x5bd 0x5be 0x5bf 0x5c0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x5f5 0x5f6 0x5f7 0x5f8 0x5f9 0x5fa 0x5fb 0x5fc 0x5fd 0x5fe 0x5ff 0x600 -0x601 0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x635 0x636 0x637 0x638 0x639 0x63a 0x63b 0x63c 0x63d 0x63e 0x63f 0x640 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x675 0x676 0x677 0x678 0x679 0x67a 0x67b 0x67c 0x67d 0x67e 0x67f 0x680 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x6b5 0x6b6 0x6b7 0x6b8 0x6b9 0x6ba 0x6bb 0x6bc 0x6bd 0x6be 0x6bf 0x6c0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x6f5 0x6f6 0x6f7 0x6f8 0x6f9 0x6fa 0x6fb 0x6fc 0x6fd 0x6fe 0x6ff 0x700 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x735 0x736 0x737 0x738 0x739 0x73a 0x73b 0x73c 0x73d 0x73e 0x73f 0x740 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x775 0x776 0x777 0x778 0x779 0x77a 0x77b 0x77c 0x77d 0x77e 0x77f 0x780 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x7b5 0x7b6 0x7b7 0x7b8 0x7b9 0x7ba 0x7bb 0x7bc 0x7bd 0x7be 0x7bf 0x7c0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x7f5 0x7f6 0x7f7 0x7f8 0x7f9 0x7fa 0x7fb 0x7fc 0x7fd 0x7fe 0x7ff 0x800 -0x801 0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x835 0x836 0x837 0x838 0x839 0x83a 0x83b 0x83c 0x83d 0x83e 0x83f 0x840 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x875 0x876 0x877 0x878 0x879 0x87a 0x87b 0x87c 0x87d 0x87e 0x87f 0x880 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x8b5 0x8b6 0x8b7 0x8b8 0x8b9 0x8ba 0x8bb 0x8bc 0x8bd 0x8be 0x8bf 0x8c0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x8f5 0x8f6 0x8f7 0x8f8 0x8f9 0x8fa 0x8fb 0x8fc 0x8fd 0x8fe 0x8ff 0x900 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x935 0x936 0x937 0x938 0x939 0x93a 0x93b 0x93c 0x93d 0x93e 0x93f 0x940 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x975 0x976 0x977 0x978 0x979 0x97a 0x97b 0x97c 0x97d 0x97e 0x97f 0x980 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x9b5 0x9b6 0x9b7 0x9b8 0x9b9 0x9ba 0x9bb 0x9bc 0x9bd 0x9be 0x9bf 0x9c0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x9f5 0x9f6 0x9f7 0x9f8 0x9f9 0x9fa 0x9fb 0x9fc 0x9fd 0x9fe 0x9ff 0xa00 -0xa01 0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0xa35 0xa36 0xa37 0xa38 0xa39 0xa3a 0xa3b 0xa3c 0xa3d 0xa3e 0xa3f 0xa40 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0xa75 0xa76 0xa77 0xa78 0xa79 0xa7a 0xa7b 0xa7c 0xa7d 0xa7e 0xa7f 0xa80 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0xab5 0xab6 0xab7 0xab8 0xab9 0xaba 0xabb 0xabc 0xabd 0xabe 0xabf 0xac0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0xaf5 0xaf6 0xaf7 0xaf8 0xaf9 0xafa 0xafb 0xafc 0xafd 0xafe 0xaff 0xb00 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0xb35 0xb36 0xb37 0xb38 0xb39 0xb3a 0xb3b 0xb3c 0xb3d 0xb3e 0xb3f 0xb40 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0xb75 0xb76 0xb77 0xb78 0xb79 0xb7a 0xb7b 0xb7c 0xb7d 0xb7e 0xb7f 0xb80 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0xbb5 0xbb6 0xbb7 0xbb8 0xbb9 0xbba 0xbbb 0xbbc 0xbbd 0xbbe 0xbbf 0xbc0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0xbf5 0xbf6 0xbf7 0xbf8 0xbf9 0xbfa 0xbfb 0xbfc 0xbfd 0xbfe 0xbff 0xc00 -0xc01 0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0xc35 0xc36 0xc37 0xc38 0xc39 0xc3a 0xc3b 0xc3c 0xc3d 0xc3e 0xc3f 0xc40 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0xc75 0xc76 0xc77 0xc78 0xc79 0xc7a 0xc7b 0xc7c 0xc7d 0xc7e 0xc7f 0xc80 0xc81 0xc82 0xc83 0xc84 0xc85 0xc86 0xc87 0xc88 0xc89 0xc8a 0xc8b 0xc8c 0xc8d 0xc8e 0xc8f 0xc90 0xc91 0xc92 0xc93 0xc94 0xc95 0xc96 0xc97 0xc98 0xc99 0xc9a 0xc9b 0xc9c 0xc9d 0xc9e 0xc9f 0xca0 0xca1 0xca2 0xca3 0xca4 0xca5 0xca6 0xca7 0xca8 0xca9 0xcaa 0xcab 0xcac 0xcad 0xcae 0xcaf 0xcb0 0xcb1 0xcb2 0xcb3 0xcb4 0xcb5 0xcb6 0xcb7 0xcb8 0xcb9 0xcba 0xcbb 0xcbc 0xcbd 0xcbe 0xcbf 0xcc0 0xcc1 0xcc2 0xcc3 0xcc4 0xcc5 0xcc6 0xcc7 0xcc8 0xcc9 0xcca 0xccb 0xccc 0xccd 0xcce 0xccf 0xcd0 0xcd1 0xcd2 0xcd3 0xcd4 0xcd5 0xcd6 0xcd7 0xcd8 0xcd9 0xcda 0xcdb 0xcdc 0xcdd 0xcde 0xcdf 0xce0 0xce1 0xce2 0xce3 0xce4 0xce5 0xce6 0xce7 0xce8 0xce9 0xcea 0xceb 0xcec 0xced 0xcee 0xcef 0xcf0 0xcf1 0xcf2 0xcf3 0xcf4 0xcf5 0xcf6 0xcf7 0xcf8 0xcf9 0xcfa 0xcfb 0xcfc 0xcfd 0xcfe 0xcff 0xd00 0xd01 0xd02 0xd03 0xd04 0xd05 0xd06 0xd07 0xd08 0xd09 0xd0a 0xd0b 0xd0c 0xd0d 0xd0e 0xd0f 0xd10 0xd11 0xd12 0xd13 0xd14 0xd15 0xd16 0xd17 0xd18 0xd19 0xd1a 0xd1b 0xd1c 0xd1d 0xd1e 0xd1f 0xd20 0xd21 0xd22 0xd23 0xd24 0xd25 0xd26 0xd27 0xd28 0xd29 0xd2a 0xd2b 0xd2c 0xd2d 0xd2e 0xd2f 0xd30 0xd31 0xd32 0xd33 0xd34 0xd35 0xd36 0xd37 0xd38 0xd39 0xd3a 0xd3b 0xd3c 0xd3d 0xd3e 0xd3f 0xd40 0xd41 0xd42 0xd43 0xd44 0xd45 0xd46 0xd47 0xd48 0xd49 0xd4a 0xd4b 0xd4c 0xd4d 0xd4e 0xd4f 0xd50 0xd51 0xd52 0xd53 0xd54 0xd55 0xd56 0xd57 0xd58 0xd59 0xd5a 0xd5b 0xd5c 0xd5d 0xd5e 0xd5f 0xd60 0xd61 0xd62 0xd63 0xd64 0xd65 0xd66 0xd67 0xd68 0xd69 0xd6a 0xd6b 0xd6c 0xd6d 0xd6e 0xd6f 0xd70 0xd71 0xd72 0xd73 0xd74 0xd75 0xd76 0xd77 0xd78 0xd79 0xd7a 0xd7b 0xd7c 0xd7d 0xd7e 0xd7f 0xd80 0xd81 0xd82 0xd83 0xd84 0xd85 0xd86 0xd87 0xd88 0xd89 0xd8a 0xd8b 0xd8c 0xd8d 0xd8e 0xd8f 0xd90 0xd91 0xd92 0xd93 0xd94 0xd95 0xd96 0xd97 0xd98 0xd99 0xd9a 0xd9b 0xd9c 0xd9d 0xd9e 0xd9f 0xda0 0xda1 0xda2 0xda3 0xda4 0xda5 0xda6 0xda7 0xda8 0xda9 0xdaa 0xdab 0xdac 0xdad 0xdae 0xdaf 0xdb0 0xdb1 0xdb2 0xdb3 0xdb4 0xdb5 0xdb6 0xdb7 0xdb8 0xdb9 0xdba 0xdbb 0xdbc 0xdbd 0xdbe 0xdbf 0xdc0 0xdc1 0xdc2 0xdc3 0xdc4 0xdc5 0xdc6 0xdc7 0xdc8 0xdc9 0xdca 0xdcb 0xdcc 0xdcd 0xdce 0xdcf 0xdd0 0xdd1 0xdd2 0xdd3 0xdd4 0xdd5 0xdd6 0xdd7 0xdd8 0xdd9 0xdda 0xddb 0xddc 0xddd 0xdde 0xddf 0xde0 0xde1 0xde2 0xde3 0xde4 0xde5 0xde6 0xde7 0xde8 0xde9 0xdea 0xdeb 0xdec 0xded 0xdee 0xdef 0xdf0 0xdf1 0xdf2 0xdf3 0xdf4 0xdf5 0xdf6 0xdf7 0xdf8 0xdf9 0xdfa 0xdfb 0xdfc 0xdfd 0xdfe 0xdff 0xe00 -0xe01 0xe02 0xe03 0xe04 0xe05 0xe06 0xe07 0xe08 0xe09 0xe0a 0xe0b 0xe0c 0xe0d 0xe0e 0xe0f 0xe10 0xe11 0xe12 0xe13 0xe14 0xe15 0xe16 0xe17 0xe18 0xe19 0xe1a 0xe1b 0xe1c 0xe1d 0xe1e 0xe1f 0xe20 0xe21 0xe22 0xe23 0xe24 0xe25 0xe26 0xe27 0xe28 0xe29 0xe2a 0xe2b 0xe2c 0xe2d 0xe2e 0xe2f 0xe30 0xe31 0xe32 0xe33 0xe34 0xe35 0xe36 0xe37 0xe38 0xe39 0xe3a 0xe3b 0xe3c 0xe3d 0xe3e 0xe3f 0xe40 0xe41 0xe42 0xe43 0xe44 0xe45 0xe46 0xe47 0xe48 0xe49 0xe4a 0xe4b 0xe4c 0xe4d 0xe4e 0xe4f 0xe50 0xe51 0xe52 0xe53 0xe54 0xe55 0xe56 0xe57 0xe58 0xe59 0xe5a 0xe5b 0xe5c 0xe5d 0xe5e 0xe5f 0xe60 0xe61 0xe62 0xe63 0xe64 0xe65 0xe66 0xe67 0xe68 0xe69 0xe6a 0xe6b 0xe6c 0xe6d 0xe6e 0xe6f 0xe70 0xe71 0xe72 0xe73 0xe74 0xe75 0xe76 0xe77 0xe78 0xe79 0xe7a 0xe7b 0xe7c 0xe7d 0xe7e 0xe7f 0xe80 0xe81 0xe82 0xe83 0xe84 0xe85 0xe86 0xe87 0xe88 0xe89 0xe8a 0xe8b 0xe8c 0xe8d 0xe8e 0xe8f 0xe90 0xe91 0xe92 0xe93 0xe94 0xe95 0xe96 0xe97 0xe98 0xe99 0xe9a 0xe9b 0xe9c 0xe9d 0xe9e 0xe9f 0xea0 0xea1 0xea2 0xea3 0xea4 0xea5 0xea6 0xea7 0xea8 0xea9 0xeaa 0xeab 0xeac 0xead 0xeae 0xeaf 0xeb0 0xeb1 0xeb2 0xeb3 0xeb4 0xeb5 0xeb6 0xeb7 0xeb8 0xeb9 0xeba 0xebb 0xebc 0xebd 0xebe 0xebf 0xec0 0xec1 0xec2 0xec3 0xec4 0xec5 0xec6 0xec7 0xec8 0xec9 0xeca 0xecb 0xecc 0xecd 0xece 0xecf 0xed0 0xed1 0xed2 0xed3 0xed4 0xed5 0xed6 0xed7 0xed8 0xed9 0xeda 0xedb 0xedc 0xedd 0xede 0xedf 0xee0 0xee1 0xee2 0xee3 0xee4 0xee5 0xee6 0xee7 0xee8 0xee9 0xeea 0xeeb 0xeec 0xeed 0xeee 0xeef 0xef0 0xef1 0xef2 0xef3 0xef4 0xef5 0xef6 0xef7 0xef8 0xef9 0xefa 0xefb 0xefc 0xefd 0xefe 0xeff 0xf00 0xf01 0xf02 0xf03 0xf04 0xf05 0xf06 0xf07 0xf08 0xf09 0xf0a 0xf0b 0xf0c 0xf0d 0xf0e 0xf0f 0xf10 0xf11 0xf12 0xf13 0xf14 0xf15 0xf16 0xf17 0xf18 0xf19 0xf1a 0xf1b 0xf1c 0xf1d 0xf1e 0xf1f 0xf20 0xf21 0xf22 0xf23 0xf24 0xf25 0xf26 0xf27 0xf28 0xf29 0xf2a 0xf2b 0xf2c 0xf2d 0xf2e 0xf2f 0xf30 0xf31 0xf32 0xf33 0xf34 0xf35 0xf36 0xf37 0xf38 0xf39 0xf3a 0xf3b 0xf3c 0xf3d 0xf3e 0xf3f 0xf40 0xf41 0xf42 0xf43 0xf44 0xf45 0xf46 0xf47 0xf48 0xf49 0xf4a 0xf4b 0xf4c 0xf4d 0xf4e 0xf4f 0xf50 0xf51 0xf52 0xf53 0xf54 0xf55 0xf56 0xf57 0xf58 0xf59 0xf5a 0xf5b 0xf5c 0xf5d 0xf5e 0xf5f 0xf60 0xf61 0xf62 0xf63 0xf64 0xf65 0xf66 0xf67 0xf68 0xf69 0xf6a 0xf6b 0xf6c 0xf6d 0xf6e 0xf6f 0xf70 0xf71 0xf72 0xf73 0xf74 0xf75 0xf76 0xf77 0xf78 0xf79 0xf7a 0xf7b 0xf7c 0xf7d 0xf7e 0xf7f 0xf80 0xf81 0xf82 0xf83 0xf84 0xf85 0xf86 0xf87 0xf88 0xf89 0xf8a 0xf8b 0xf8c 0xf8d 0xf8e 0xf8f 0xf90 0xf91 0xf92 0xf93 0xf94 0xf95 0xf96 0xf97 0xf98 0xf99 0xf9a 0xf9b 0xf9c 0xf9d 0xf9e 0xf9f 0xfa0 0xfa1 0xfa2 0xfa3 0xfa4 0xfa5 0xfa6 0xfa7 0xfa8 0xfa9 0xfaa 0xfab 0xfac 0xfad 0xfae 0xfaf 0xfb0 0xfb1 0xfb2 0xfb3 0xfb4 0xfb5 0xfb6 0xfb7 0xfb8 0xfb9 0xfba 0xfbb 0xfbc 0xfbd 0xfbe 0xfbf 0xfc0 0xfc1 0xfc2 0xfc3 0xfc4 0xfc5 0xfc6 0xfc7 0xfc8 0xfc9 0xfca 0xfcb 0xfcc 0xfcd 0xfce 0xfcf 0xfd0 0xfd1 0xfd2 0xfd3 0xfd4 0xfd5 0xfd6 0xfd7 0xfd8 0xfd9 0xfda 0xfdb 0xfdc 0xfdd 0xfde 0xfdf 0xfe0 0xfe1 0xfe2 0xfe3 0xfe4 0xfe5 0xfe6 0xfe7 0xfe8 0xfe9 0xfea 0xfeb 0xfec 0xfed 0xfee 0xfef 0xff0 0xff1 0xff2 0xff3 0xff4 0xff5 0xff6 0xff7 0xff8 0xff9 0xffa 0xffb 0xffc 0xffd 0xffe 0xfff 0x1000 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAPF_64_64.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAPF_64_64.txt deleted file mode 100644 index 166808918..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAPF_64_64.txt +++ /dev/null @@ -1,8 +0,0 @@ -0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff 0x100 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x135 0x136 0x137 0x138 0x139 0x13a 0x13b 0x13c 0x13d 0x13e 0x13f 0x140 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x175 0x176 0x177 0x178 0x179 0x17a 0x17b 0x17c 0x17d 0x17e 0x17f 0x180 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x1b5 0x1b6 0x1b7 0x1b8 0x1b9 0x1ba 0x1bb 0x1bc 0x1bd 0x1be 0x1bf 0x1c0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x1f5 0x1f6 0x1f7 0x1f8 0x1f9 0x1fa 0x1fb 0x1fc 0x1fd 0x1fe 0x1ff 0x200 -0x201 0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x235 0x236 0x237 0x238 0x239 0x23a 0x23b 0x23c 0x23d 0x23e 0x23f 0x240 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x275 0x276 0x277 0x278 0x279 0x27a 0x27b 0x27c 0x27d 0x27e 0x27f 0x280 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x2b5 0x2b6 0x2b7 0x2b8 0x2b9 0x2ba 0x2bb 0x2bc 0x2bd 0x2be 0x2bf 0x2c0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x2f5 0x2f6 0x2f7 0x2f8 0x2f9 0x2fa 0x2fb 0x2fc 0x2fd 0x2fe 0x2ff 0x300 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x335 0x336 0x337 0x338 0x339 0x33a 0x33b 0x33c 0x33d 0x33e 0x33f 0x340 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x375 0x376 0x377 0x378 0x379 0x37a 0x37b 0x37c 0x37d 0x37e 0x37f 0x380 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x3b5 0x3b6 0x3b7 0x3b8 0x3b9 0x3ba 0x3bb 0x3bc 0x3bd 0x3be 0x3bf 0x3c0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x3f5 0x3f6 0x3f7 0x3f8 0x3f9 0x3fa 0x3fb 0x3fc 0x3fd 0x3fe 0x3ff 0x400 -0x401 0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x435 0x436 0x437 0x438 0x439 0x43a 0x43b 0x43c 0x43d 0x43e 0x43f 0x440 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x475 0x476 0x477 0x478 0x479 0x47a 0x47b 0x47c 0x47d 0x47e 0x47f 0x480 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x4b5 0x4b6 0x4b7 0x4b8 0x4b9 0x4ba 0x4bb 0x4bc 0x4bd 0x4be 0x4bf 0x4c0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x4f5 0x4f6 0x4f7 0x4f8 0x4f9 0x4fa 0x4fb 0x4fc 0x4fd 0x4fe 0x4ff 0x500 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x535 0x536 0x537 0x538 0x539 0x53a 0x53b 0x53c 0x53d 0x53e 0x53f 0x540 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x575 0x576 0x577 0x578 0x579 0x57a 0x57b 0x57c 0x57d 0x57e 0x57f 0x580 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x5b5 0x5b6 0x5b7 0x5b8 0x5b9 0x5ba 0x5bb 0x5bc 0x5bd 0x5be 0x5bf 0x5c0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x5f5 0x5f6 0x5f7 0x5f8 0x5f9 0x5fa 0x5fb 0x5fc 0x5fd 0x5fe 0x5ff 0x600 -0x601 0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x635 0x636 0x637 0x638 0x639 0x63a 0x63b 0x63c 0x63d 0x63e 0x63f 0x640 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x675 0x676 0x677 0x678 0x679 0x67a 0x67b 0x67c 0x67d 0x67e 0x67f 0x680 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x6b5 0x6b6 0x6b7 0x6b8 0x6b9 0x6ba 0x6bb 0x6bc 0x6bd 0x6be 0x6bf 0x6c0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x6f5 0x6f6 0x6f7 0x6f8 0x6f9 0x6fa 0x6fb 0x6fc 0x6fd 0x6fe 0x6ff 0x700 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x735 0x736 0x737 0x738 0x739 0x73a 0x73b 0x73c 0x73d 0x73e 0x73f 0x740 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x775 0x776 0x777 0x778 0x779 0x77a 0x77b 0x77c 0x77d 0x77e 0x77f 0x780 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x7b5 0x7b6 0x7b7 0x7b8 0x7b9 0x7ba 0x7bb 0x7bc 0x7bd 0x7be 0x7bf 0x7c0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x7f5 0x7f6 0x7f7 0x7f8 0x7f9 0x7fa 0x7fb 0x7fc 0x7fd 0x7fe 0x7ff 0x800 -0x801 0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x835 0x836 0x837 0x838 0x839 0x83a 0x83b 0x83c 0x83d 0x83e 0x83f 0x840 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x875 0x876 0x877 0x878 0x879 0x87a 0x87b 0x87c 0x87d 0x87e 0x87f 0x880 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x8b5 0x8b6 0x8b7 0x8b8 0x8b9 0x8ba 0x8bb 0x8bc 0x8bd 0x8be 0x8bf 0x8c0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x8f5 0x8f6 0x8f7 0x8f8 0x8f9 0x8fa 0x8fb 0x8fc 0x8fd 0x8fe 0x8ff 0x900 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x935 0x936 0x937 0x938 0x939 0x93a 0x93b 0x93c 0x93d 0x93e 0x93f 0x940 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x975 0x976 0x977 0x978 0x979 0x97a 0x97b 0x97c 0x97d 0x97e 0x97f 0x980 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x9b5 0x9b6 0x9b7 0x9b8 0x9b9 0x9ba 0x9bb 0x9bc 0x9bd 0x9be 0x9bf 0x9c0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x9f5 0x9f6 0x9f7 0x9f8 0x9f9 0x9fa 0x9fb 0x9fc 0x9fd 0x9fe 0x9ff 0xa00 -0xa01 0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0xa35 0xa36 0xa37 0xa38 0xa39 0xa3a 0xa3b 0xa3c 0xa3d 0xa3e 0xa3f 0xa40 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0xa75 0xa76 0xa77 0xa78 0xa79 0xa7a 0xa7b 0xa7c 0xa7d 0xa7e 0xa7f 0xa80 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0xab5 0xab6 0xab7 0xab8 0xab9 0xaba 0xabb 0xabc 0xabd 0xabe 0xabf 0xac0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0xaf5 0xaf6 0xaf7 0xaf8 0xaf9 0xafa 0xafb 0xafc 0xafd 0xafe 0xaff 0xb00 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0xb35 0xb36 0xb37 0xb38 0xb39 0xb3a 0xb3b 0xb3c 0xb3d 0xb3e 0xb3f 0xb40 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0xb75 0xb76 0xb77 0xb78 0xb79 0xb7a 0xb7b 0xb7c 0xb7d 0xb7e 0xb7f 0xb80 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0xbb5 0xbb6 0xbb7 0xbb8 0xbb9 0xbba 0xbbb 0xbbc 0xbbd 0xbbe 0xbbf 0xbc0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0xbf5 0xbf6 0xbf7 0xbf8 0xbf9 0xbfa 0xbfb 0xbfc 0xbfd 0xbfe 0xbff 0xc00 -0xc01 0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0xc35 0xc36 0xc37 0xc38 0xc39 0xc3a 0xc3b 0xc3c 0xc3d 0xc3e 0xc3f 0xc40 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0xc75 0xc76 0xc77 0xc78 0xc79 0xc7a 0xc7b 0xc7c 0xc7d 0xc7e 0xc7f 0xc80 0xc81 0xc82 0xc83 0xc84 0xc85 0xc86 0xc87 0xc88 0xc89 0xc8a 0xc8b 0xc8c 0xc8d 0xc8e 0xc8f 0xc90 0xc91 0xc92 0xc93 0xc94 0xc95 0xc96 0xc97 0xc98 0xc99 0xc9a 0xc9b 0xc9c 0xc9d 0xc9e 0xc9f 0xca0 0xca1 0xca2 0xca3 0xca4 0xca5 0xca6 0xca7 0xca8 0xca9 0xcaa 0xcab 0xcac 0xcad 0xcae 0xcaf 0xcb0 0xcb1 0xcb2 0xcb3 0xcb4 0xcb5 0xcb6 0xcb7 0xcb8 0xcb9 0xcba 0xcbb 0xcbc 0xcbd 0xcbe 0xcbf 0xcc0 0xcc1 0xcc2 0xcc3 0xcc4 0xcc5 0xcc6 0xcc7 0xcc8 0xcc9 0xcca 0xccb 0xccc 0xccd 0xcce 0xccf 0xcd0 0xcd1 0xcd2 0xcd3 0xcd4 0xcd5 0xcd6 0xcd7 0xcd8 0xcd9 0xcda 0xcdb 0xcdc 0xcdd 0xcde 0xcdf 0xce0 0xce1 0xce2 0xce3 0xce4 0xce5 0xce6 0xce7 0xce8 0xce9 0xcea 0xceb 0xcec 0xced 0xcee 0xcef 0xcf0 0xcf1 0xcf2 0xcf3 0xcf4 0xcf5 0xcf6 0xcf7 0xcf8 0xcf9 0xcfa 0xcfb 0xcfc 0xcfd 0xcfe 0xcff 0xd00 0xd01 0xd02 0xd03 0xd04 0xd05 0xd06 0xd07 0xd08 0xd09 0xd0a 0xd0b 0xd0c 0xd0d 0xd0e 0xd0f 0xd10 0xd11 0xd12 0xd13 0xd14 0xd15 0xd16 0xd17 0xd18 0xd19 0xd1a 0xd1b 0xd1c 0xd1d 0xd1e 0xd1f 0xd20 0xd21 0xd22 0xd23 0xd24 0xd25 0xd26 0xd27 0xd28 0xd29 0xd2a 0xd2b 0xd2c 0xd2d 0xd2e 0xd2f 0xd30 0xd31 0xd32 0xd33 0xd34 0xd35 0xd36 0xd37 0xd38 0xd39 0xd3a 0xd3b 0xd3c 0xd3d 0xd3e 0xd3f 0xd40 0xd41 0xd42 0xd43 0xd44 0xd45 0xd46 0xd47 0xd48 0xd49 0xd4a 0xd4b 0xd4c 0xd4d 0xd4e 0xd4f 0xd50 0xd51 0xd52 0xd53 0xd54 0xd55 0xd56 0xd57 0xd58 0xd59 0xd5a 0xd5b 0xd5c 0xd5d 0xd5e 0xd5f 0xd60 0xd61 0xd62 0xd63 0xd64 0xd65 0xd66 0xd67 0xd68 0xd69 0xd6a 0xd6b 0xd6c 0xd6d 0xd6e 0xd6f 0xd70 0xd71 0xd72 0xd73 0xd74 0xd75 0xd76 0xd77 0xd78 0xd79 0xd7a 0xd7b 0xd7c 0xd7d 0xd7e 0xd7f 0xd80 0xd81 0xd82 0xd83 0xd84 0xd85 0xd86 0xd87 0xd88 0xd89 0xd8a 0xd8b 0xd8c 0xd8d 0xd8e 0xd8f 0xd90 0xd91 0xd92 0xd93 0xd94 0xd95 0xd96 0xd97 0xd98 0xd99 0xd9a 0xd9b 0xd9c 0xd9d 0xd9e 0xd9f 0xda0 0xda1 0xda2 0xda3 0xda4 0xda5 0xda6 0xda7 0xda8 0xda9 0xdaa 0xdab 0xdac 0xdad 0xdae 0xdaf 0xdb0 0xdb1 0xdb2 0xdb3 0xdb4 0xdb5 0xdb6 0xdb7 0xdb8 0xdb9 0xdba 0xdbb 0xdbc 0xdbd 0xdbe 0xdbf 0xdc0 0xdc1 0xdc2 0xdc3 0xdc4 0xdc5 0xdc6 0xdc7 0xdc8 0xdc9 0xdca 0xdcb 0xdcc 0xdcd 0xdce 0xdcf 0xdd0 0xdd1 0xdd2 0xdd3 0xdd4 0xdd5 0xdd6 0xdd7 0xdd8 0xdd9 0xdda 0xddb 0xddc 0xddd 0xdde 0xddf 0xde0 0xde1 0xde2 0xde3 0xde4 0xde5 0xde6 0xde7 0xde8 0xde9 0xdea 0xdeb 0xdec 0xded 0xdee 0xdef 0xdf0 0xdf1 0xdf2 0xdf3 0xdf4 0xdf5 0xdf6 0xdf7 0xdf8 0xdf9 0xdfa 0xdfb 0xdfc 0xdfd 0xdfe 0xdff 0xe00 -0xe01 0xe02 0xe03 0xe04 0xe05 0xe06 0xe07 0xe08 0xe09 0xe0a 0xe0b 0xe0c 0xe0d 0xe0e 0xe0f 0xe10 0xe11 0xe12 0xe13 0xe14 0xe15 0xe16 0xe17 0xe18 0xe19 0xe1a 0xe1b 0xe1c 0xe1d 0xe1e 0xe1f 0xe20 0xe21 0xe22 0xe23 0xe24 0xe25 0xe26 0xe27 0xe28 0xe29 0xe2a 0xe2b 0xe2c 0xe2d 0xe2e 0xe2f 0xe30 0xe31 0xe32 0xe33 0xe34 0xe35 0xe36 0xe37 0xe38 0xe39 0xe3a 0xe3b 0xe3c 0xe3d 0xe3e 0xe3f 0xe40 0xe41 0xe42 0xe43 0xe44 0xe45 0xe46 0xe47 0xe48 0xe49 0xe4a 0xe4b 0xe4c 0xe4d 0xe4e 0xe4f 0xe50 0xe51 0xe52 0xe53 0xe54 0xe55 0xe56 0xe57 0xe58 0xe59 0xe5a 0xe5b 0xe5c 0xe5d 0xe5e 0xe5f 0xe60 0xe61 0xe62 0xe63 0xe64 0xe65 0xe66 0xe67 0xe68 0xe69 0xe6a 0xe6b 0xe6c 0xe6d 0xe6e 0xe6f 0xe70 0xe71 0xe72 0xe73 0xe74 0xe75 0xe76 0xe77 0xe78 0xe79 0xe7a 0xe7b 0xe7c 0xe7d 0xe7e 0xe7f 0xe80 0xe81 0xe82 0xe83 0xe84 0xe85 0xe86 0xe87 0xe88 0xe89 0xe8a 0xe8b 0xe8c 0xe8d 0xe8e 0xe8f 0xe90 0xe91 0xe92 0xe93 0xe94 0xe95 0xe96 0xe97 0xe98 0xe99 0xe9a 0xe9b 0xe9c 0xe9d 0xe9e 0xe9f 0xea0 0xea1 0xea2 0xea3 0xea4 0xea5 0xea6 0xea7 0xea8 0xea9 0xeaa 0xeab 0xeac 0xead 0xeae 0xeaf 0xeb0 0xeb1 0xeb2 0xeb3 0xeb4 0xeb5 0xeb6 0xeb7 0xeb8 0xeb9 0xeba 0xebb 0xebc 0xebd 0xebe 0xebf 0xec0 0xec1 0xec2 0xec3 0xec4 0xec5 0xec6 0xec7 0xec8 0xec9 0xeca 0xecb 0xecc 0xecd 0xece 0xecf 0xed0 0xed1 0xed2 0xed3 0xed4 0xed5 0xed6 0xed7 0xed8 0xed9 0xeda 0xedb 0xedc 0xedd 0xede 0xedf 0xee0 0xee1 0xee2 0xee3 0xee4 0xee5 0xee6 0xee7 0xee8 0xee9 0xeea 0xeeb 0xeec 0xeed 0xeee 0xeef 0xef0 0xef1 0xef2 0xef3 0xef4 0xef5 0xef6 0xef7 0xef8 0xef9 0xefa 0xefb 0xefc 0xefd 0xefe 0xeff 0xf00 0xf01 0xf02 0xf03 0xf04 0xf05 0xf06 0xf07 0xf08 0xf09 0xf0a 0xf0b 0xf0c 0xf0d 0xf0e 0xf0f 0xf10 0xf11 0xf12 0xf13 0xf14 0xf15 0xf16 0xf17 0xf18 0xf19 0xf1a 0xf1b 0xf1c 0xf1d 0xf1e 0xf1f 0xf20 0xf21 0xf22 0xf23 0xf24 0xf25 0xf26 0xf27 0xf28 0xf29 0xf2a 0xf2b 0xf2c 0xf2d 0xf2e 0xf2f 0xf30 0xf31 0xf32 0xf33 0xf34 0xf35 0xf36 0xf37 0xf38 0xf39 0xf3a 0xf3b 0xf3c 0xf3d 0xf3e 0xf3f 0xf40 0xf41 0xf42 0xf43 0xf44 0xf45 0xf46 0xf47 0xf48 0xf49 0xf4a 0xf4b 0xf4c 0xf4d 0xf4e 0xf4f 0xf50 0xf51 0xf52 0xf53 0xf54 0xf55 0xf56 0xf57 0xf58 0xf59 0xf5a 0xf5b 0xf5c 0xf5d 0xf5e 0xf5f 0xf60 0xf61 0xf62 0xf63 0xf64 0xf65 0xf66 0xf67 0xf68 0xf69 0xf6a 0xf6b 0xf6c 0xf6d 0xf6e 0xf6f 0xf70 0xf71 0xf72 0xf73 0xf74 0xf75 0xf76 0xf77 0xf78 0xf79 0xf7a 0xf7b 0xf7c 0xf7d 0xf7e 0xf7f 0xf80 0xf81 0xf82 0xf83 0xf84 0xf85 0xf86 0xf87 0xf88 0xf89 0xf8a 0xf8b 0xf8c 0xf8d 0xf8e 0xf8f 0xf90 0xf91 0xf92 0xf93 0xf94 0xf95 0xf96 0xf97 0xf98 0xf99 0xf9a 0xf9b 0xf9c 0xf9d 0xf9e 0xf9f 0xfa0 0xfa1 0xfa2 0xfa3 0xfa4 0xfa5 0xfa6 0xfa7 0xfa8 0xfa9 0xfaa 0xfab 0xfac 0xfad 0xfae 0xfaf 0xfb0 0xfb1 0xfb2 0xfb3 0xfb4 0xfb5 0xfb6 0xfb7 0xfb8 0xfb9 0xfba 0xfbb 0xfbc 0xfbd 0xfbe 0xfbf 0xfc0 0xfc1 0xfc2 0xfc3 0xfc4 0xfc5 0xfc6 0xfc7 0xfc8 0xfc9 0xfca 0xfcb 0xfcc 0xfcd 0xfce 0xfcf 0xfd0 0xfd1 0xfd2 0xfd3 0xfd4 0xfd5 0xfd6 0xfd7 0xfd8 0xfd9 0xfda 0xfdb 0xfdc 0xfdd 0xfde 0xfdf 0xfe0 0xfe1 0xfe2 0xfe3 0xfe4 0xfe5 0xfe6 0xfe7 0xfe8 0xfe9 0xfea 0xfeb 0xfec 0xfed 0xfee 0xfef 0xff0 0xff1 0xff2 0xff3 0xff4 0xff5 0xff6 0xff7 0xff8 0xff9 0xffa 0xffb 0xffc 0xffd 0xffe 0xfff 0x1000 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAREDG_64_64.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAREDG_64_64.txt deleted file mode 100644 index 166808918..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMAREDG_64_64.txt +++ /dev/null @@ -1,8 +0,0 @@ -0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff 0x100 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x135 0x136 0x137 0x138 0x139 0x13a 0x13b 0x13c 0x13d 0x13e 0x13f 0x140 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x175 0x176 0x177 0x178 0x179 0x17a 0x17b 0x17c 0x17d 0x17e 0x17f 0x180 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x1b5 0x1b6 0x1b7 0x1b8 0x1b9 0x1ba 0x1bb 0x1bc 0x1bd 0x1be 0x1bf 0x1c0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x1f5 0x1f6 0x1f7 0x1f8 0x1f9 0x1fa 0x1fb 0x1fc 0x1fd 0x1fe 0x1ff 0x200 -0x201 0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x235 0x236 0x237 0x238 0x239 0x23a 0x23b 0x23c 0x23d 0x23e 0x23f 0x240 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x275 0x276 0x277 0x278 0x279 0x27a 0x27b 0x27c 0x27d 0x27e 0x27f 0x280 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x2b5 0x2b6 0x2b7 0x2b8 0x2b9 0x2ba 0x2bb 0x2bc 0x2bd 0x2be 0x2bf 0x2c0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x2f5 0x2f6 0x2f7 0x2f8 0x2f9 0x2fa 0x2fb 0x2fc 0x2fd 0x2fe 0x2ff 0x300 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x335 0x336 0x337 0x338 0x339 0x33a 0x33b 0x33c 0x33d 0x33e 0x33f 0x340 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x375 0x376 0x377 0x378 0x379 0x37a 0x37b 0x37c 0x37d 0x37e 0x37f 0x380 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x3b5 0x3b6 0x3b7 0x3b8 0x3b9 0x3ba 0x3bb 0x3bc 0x3bd 0x3be 0x3bf 0x3c0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x3f5 0x3f6 0x3f7 0x3f8 0x3f9 0x3fa 0x3fb 0x3fc 0x3fd 0x3fe 0x3ff 0x400 -0x401 0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x435 0x436 0x437 0x438 0x439 0x43a 0x43b 0x43c 0x43d 0x43e 0x43f 0x440 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x475 0x476 0x477 0x478 0x479 0x47a 0x47b 0x47c 0x47d 0x47e 0x47f 0x480 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x4b5 0x4b6 0x4b7 0x4b8 0x4b9 0x4ba 0x4bb 0x4bc 0x4bd 0x4be 0x4bf 0x4c0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x4f5 0x4f6 0x4f7 0x4f8 0x4f9 0x4fa 0x4fb 0x4fc 0x4fd 0x4fe 0x4ff 0x500 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x535 0x536 0x537 0x538 0x539 0x53a 0x53b 0x53c 0x53d 0x53e 0x53f 0x540 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x575 0x576 0x577 0x578 0x579 0x57a 0x57b 0x57c 0x57d 0x57e 0x57f 0x580 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x5b5 0x5b6 0x5b7 0x5b8 0x5b9 0x5ba 0x5bb 0x5bc 0x5bd 0x5be 0x5bf 0x5c0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x5f5 0x5f6 0x5f7 0x5f8 0x5f9 0x5fa 0x5fb 0x5fc 0x5fd 0x5fe 0x5ff 0x600 -0x601 0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x635 0x636 0x637 0x638 0x639 0x63a 0x63b 0x63c 0x63d 0x63e 0x63f 0x640 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x675 0x676 0x677 0x678 0x679 0x67a 0x67b 0x67c 0x67d 0x67e 0x67f 0x680 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x6b5 0x6b6 0x6b7 0x6b8 0x6b9 0x6ba 0x6bb 0x6bc 0x6bd 0x6be 0x6bf 0x6c0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x6f5 0x6f6 0x6f7 0x6f8 0x6f9 0x6fa 0x6fb 0x6fc 0x6fd 0x6fe 0x6ff 0x700 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x735 0x736 0x737 0x738 0x739 0x73a 0x73b 0x73c 0x73d 0x73e 0x73f 0x740 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x775 0x776 0x777 0x778 0x779 0x77a 0x77b 0x77c 0x77d 0x77e 0x77f 0x780 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x7b5 0x7b6 0x7b7 0x7b8 0x7b9 0x7ba 0x7bb 0x7bc 0x7bd 0x7be 0x7bf 0x7c0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x7f5 0x7f6 0x7f7 0x7f8 0x7f9 0x7fa 0x7fb 0x7fc 0x7fd 0x7fe 0x7ff 0x800 -0x801 0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x835 0x836 0x837 0x838 0x839 0x83a 0x83b 0x83c 0x83d 0x83e 0x83f 0x840 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x875 0x876 0x877 0x878 0x879 0x87a 0x87b 0x87c 0x87d 0x87e 0x87f 0x880 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x8b5 0x8b6 0x8b7 0x8b8 0x8b9 0x8ba 0x8bb 0x8bc 0x8bd 0x8be 0x8bf 0x8c0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x8f5 0x8f6 0x8f7 0x8f8 0x8f9 0x8fa 0x8fb 0x8fc 0x8fd 0x8fe 0x8ff 0x900 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x935 0x936 0x937 0x938 0x939 0x93a 0x93b 0x93c 0x93d 0x93e 0x93f 0x940 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x975 0x976 0x977 0x978 0x979 0x97a 0x97b 0x97c 0x97d 0x97e 0x97f 0x980 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x9b5 0x9b6 0x9b7 0x9b8 0x9b9 0x9ba 0x9bb 0x9bc 0x9bd 0x9be 0x9bf 0x9c0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x9f5 0x9f6 0x9f7 0x9f8 0x9f9 0x9fa 0x9fb 0x9fc 0x9fd 0x9fe 0x9ff 0xa00 -0xa01 0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0xa35 0xa36 0xa37 0xa38 0xa39 0xa3a 0xa3b 0xa3c 0xa3d 0xa3e 0xa3f 0xa40 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0xa75 0xa76 0xa77 0xa78 0xa79 0xa7a 0xa7b 0xa7c 0xa7d 0xa7e 0xa7f 0xa80 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0xab5 0xab6 0xab7 0xab8 0xab9 0xaba 0xabb 0xabc 0xabd 0xabe 0xabf 0xac0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0xaf5 0xaf6 0xaf7 0xaf8 0xaf9 0xafa 0xafb 0xafc 0xafd 0xafe 0xaff 0xb00 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0xb35 0xb36 0xb37 0xb38 0xb39 0xb3a 0xb3b 0xb3c 0xb3d 0xb3e 0xb3f 0xb40 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0xb75 0xb76 0xb77 0xb78 0xb79 0xb7a 0xb7b 0xb7c 0xb7d 0xb7e 0xb7f 0xb80 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0xbb5 0xbb6 0xbb7 0xbb8 0xbb9 0xbba 0xbbb 0xbbc 0xbbd 0xbbe 0xbbf 0xbc0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0xbf5 0xbf6 0xbf7 0xbf8 0xbf9 0xbfa 0xbfb 0xbfc 0xbfd 0xbfe 0xbff 0xc00 -0xc01 0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0xc35 0xc36 0xc37 0xc38 0xc39 0xc3a 0xc3b 0xc3c 0xc3d 0xc3e 0xc3f 0xc40 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0xc75 0xc76 0xc77 0xc78 0xc79 0xc7a 0xc7b 0xc7c 0xc7d 0xc7e 0xc7f 0xc80 0xc81 0xc82 0xc83 0xc84 0xc85 0xc86 0xc87 0xc88 0xc89 0xc8a 0xc8b 0xc8c 0xc8d 0xc8e 0xc8f 0xc90 0xc91 0xc92 0xc93 0xc94 0xc95 0xc96 0xc97 0xc98 0xc99 0xc9a 0xc9b 0xc9c 0xc9d 0xc9e 0xc9f 0xca0 0xca1 0xca2 0xca3 0xca4 0xca5 0xca6 0xca7 0xca8 0xca9 0xcaa 0xcab 0xcac 0xcad 0xcae 0xcaf 0xcb0 0xcb1 0xcb2 0xcb3 0xcb4 0xcb5 0xcb6 0xcb7 0xcb8 0xcb9 0xcba 0xcbb 0xcbc 0xcbd 0xcbe 0xcbf 0xcc0 0xcc1 0xcc2 0xcc3 0xcc4 0xcc5 0xcc6 0xcc7 0xcc8 0xcc9 0xcca 0xccb 0xccc 0xccd 0xcce 0xccf 0xcd0 0xcd1 0xcd2 0xcd3 0xcd4 0xcd5 0xcd6 0xcd7 0xcd8 0xcd9 0xcda 0xcdb 0xcdc 0xcdd 0xcde 0xcdf 0xce0 0xce1 0xce2 0xce3 0xce4 0xce5 0xce6 0xce7 0xce8 0xce9 0xcea 0xceb 0xcec 0xced 0xcee 0xcef 0xcf0 0xcf1 0xcf2 0xcf3 0xcf4 0xcf5 0xcf6 0xcf7 0xcf8 0xcf9 0xcfa 0xcfb 0xcfc 0xcfd 0xcfe 0xcff 0xd00 0xd01 0xd02 0xd03 0xd04 0xd05 0xd06 0xd07 0xd08 0xd09 0xd0a 0xd0b 0xd0c 0xd0d 0xd0e 0xd0f 0xd10 0xd11 0xd12 0xd13 0xd14 0xd15 0xd16 0xd17 0xd18 0xd19 0xd1a 0xd1b 0xd1c 0xd1d 0xd1e 0xd1f 0xd20 0xd21 0xd22 0xd23 0xd24 0xd25 0xd26 0xd27 0xd28 0xd29 0xd2a 0xd2b 0xd2c 0xd2d 0xd2e 0xd2f 0xd30 0xd31 0xd32 0xd33 0xd34 0xd35 0xd36 0xd37 0xd38 0xd39 0xd3a 0xd3b 0xd3c 0xd3d 0xd3e 0xd3f 0xd40 0xd41 0xd42 0xd43 0xd44 0xd45 0xd46 0xd47 0xd48 0xd49 0xd4a 0xd4b 0xd4c 0xd4d 0xd4e 0xd4f 0xd50 0xd51 0xd52 0xd53 0xd54 0xd55 0xd56 0xd57 0xd58 0xd59 0xd5a 0xd5b 0xd5c 0xd5d 0xd5e 0xd5f 0xd60 0xd61 0xd62 0xd63 0xd64 0xd65 0xd66 0xd67 0xd68 0xd69 0xd6a 0xd6b 0xd6c 0xd6d 0xd6e 0xd6f 0xd70 0xd71 0xd72 0xd73 0xd74 0xd75 0xd76 0xd77 0xd78 0xd79 0xd7a 0xd7b 0xd7c 0xd7d 0xd7e 0xd7f 0xd80 0xd81 0xd82 0xd83 0xd84 0xd85 0xd86 0xd87 0xd88 0xd89 0xd8a 0xd8b 0xd8c 0xd8d 0xd8e 0xd8f 0xd90 0xd91 0xd92 0xd93 0xd94 0xd95 0xd96 0xd97 0xd98 0xd99 0xd9a 0xd9b 0xd9c 0xd9d 0xd9e 0xd9f 0xda0 0xda1 0xda2 0xda3 0xda4 0xda5 0xda6 0xda7 0xda8 0xda9 0xdaa 0xdab 0xdac 0xdad 0xdae 0xdaf 0xdb0 0xdb1 0xdb2 0xdb3 0xdb4 0xdb5 0xdb6 0xdb7 0xdb8 0xdb9 0xdba 0xdbb 0xdbc 0xdbd 0xdbe 0xdbf 0xdc0 0xdc1 0xdc2 0xdc3 0xdc4 0xdc5 0xdc6 0xdc7 0xdc8 0xdc9 0xdca 0xdcb 0xdcc 0xdcd 0xdce 0xdcf 0xdd0 0xdd1 0xdd2 0xdd3 0xdd4 0xdd5 0xdd6 0xdd7 0xdd8 0xdd9 0xdda 0xddb 0xddc 0xddd 0xdde 0xddf 0xde0 0xde1 0xde2 0xde3 0xde4 0xde5 0xde6 0xde7 0xde8 0xde9 0xdea 0xdeb 0xdec 0xded 0xdee 0xdef 0xdf0 0xdf1 0xdf2 0xdf3 0xdf4 0xdf5 0xdf6 0xdf7 0xdf8 0xdf9 0xdfa 0xdfb 0xdfc 0xdfd 0xdfe 0xdff 0xe00 -0xe01 0xe02 0xe03 0xe04 0xe05 0xe06 0xe07 0xe08 0xe09 0xe0a 0xe0b 0xe0c 0xe0d 0xe0e 0xe0f 0xe10 0xe11 0xe12 0xe13 0xe14 0xe15 0xe16 0xe17 0xe18 0xe19 0xe1a 0xe1b 0xe1c 0xe1d 0xe1e 0xe1f 0xe20 0xe21 0xe22 0xe23 0xe24 0xe25 0xe26 0xe27 0xe28 0xe29 0xe2a 0xe2b 0xe2c 0xe2d 0xe2e 0xe2f 0xe30 0xe31 0xe32 0xe33 0xe34 0xe35 0xe36 0xe37 0xe38 0xe39 0xe3a 0xe3b 0xe3c 0xe3d 0xe3e 0xe3f 0xe40 0xe41 0xe42 0xe43 0xe44 0xe45 0xe46 0xe47 0xe48 0xe49 0xe4a 0xe4b 0xe4c 0xe4d 0xe4e 0xe4f 0xe50 0xe51 0xe52 0xe53 0xe54 0xe55 0xe56 0xe57 0xe58 0xe59 0xe5a 0xe5b 0xe5c 0xe5d 0xe5e 0xe5f 0xe60 0xe61 0xe62 0xe63 0xe64 0xe65 0xe66 0xe67 0xe68 0xe69 0xe6a 0xe6b 0xe6c 0xe6d 0xe6e 0xe6f 0xe70 0xe71 0xe72 0xe73 0xe74 0xe75 0xe76 0xe77 0xe78 0xe79 0xe7a 0xe7b 0xe7c 0xe7d 0xe7e 0xe7f 0xe80 0xe81 0xe82 0xe83 0xe84 0xe85 0xe86 0xe87 0xe88 0xe89 0xe8a 0xe8b 0xe8c 0xe8d 0xe8e 0xe8f 0xe90 0xe91 0xe92 0xe93 0xe94 0xe95 0xe96 0xe97 0xe98 0xe99 0xe9a 0xe9b 0xe9c 0xe9d 0xe9e 0xe9f 0xea0 0xea1 0xea2 0xea3 0xea4 0xea5 0xea6 0xea7 0xea8 0xea9 0xeaa 0xeab 0xeac 0xead 0xeae 0xeaf 0xeb0 0xeb1 0xeb2 0xeb3 0xeb4 0xeb5 0xeb6 0xeb7 0xeb8 0xeb9 0xeba 0xebb 0xebc 0xebd 0xebe 0xebf 0xec0 0xec1 0xec2 0xec3 0xec4 0xec5 0xec6 0xec7 0xec8 0xec9 0xeca 0xecb 0xecc 0xecd 0xece 0xecf 0xed0 0xed1 0xed2 0xed3 0xed4 0xed5 0xed6 0xed7 0xed8 0xed9 0xeda 0xedb 0xedc 0xedd 0xede 0xedf 0xee0 0xee1 0xee2 0xee3 0xee4 0xee5 0xee6 0xee7 0xee8 0xee9 0xeea 0xeeb 0xeec 0xeed 0xeee 0xeef 0xef0 0xef1 0xef2 0xef3 0xef4 0xef5 0xef6 0xef7 0xef8 0xef9 0xefa 0xefb 0xefc 0xefd 0xefe 0xeff 0xf00 0xf01 0xf02 0xf03 0xf04 0xf05 0xf06 0xf07 0xf08 0xf09 0xf0a 0xf0b 0xf0c 0xf0d 0xf0e 0xf0f 0xf10 0xf11 0xf12 0xf13 0xf14 0xf15 0xf16 0xf17 0xf18 0xf19 0xf1a 0xf1b 0xf1c 0xf1d 0xf1e 0xf1f 0xf20 0xf21 0xf22 0xf23 0xf24 0xf25 0xf26 0xf27 0xf28 0xf29 0xf2a 0xf2b 0xf2c 0xf2d 0xf2e 0xf2f 0xf30 0xf31 0xf32 0xf33 0xf34 0xf35 0xf36 0xf37 0xf38 0xf39 0xf3a 0xf3b 0xf3c 0xf3d 0xf3e 0xf3f 0xf40 0xf41 0xf42 0xf43 0xf44 0xf45 0xf46 0xf47 0xf48 0xf49 0xf4a 0xf4b 0xf4c 0xf4d 0xf4e 0xf4f 0xf50 0xf51 0xf52 0xf53 0xf54 0xf55 0xf56 0xf57 0xf58 0xf59 0xf5a 0xf5b 0xf5c 0xf5d 0xf5e 0xf5f 0xf60 0xf61 0xf62 0xf63 0xf64 0xf65 0xf66 0xf67 0xf68 0xf69 0xf6a 0xf6b 0xf6c 0xf6d 0xf6e 0xf6f 0xf70 0xf71 0xf72 0xf73 0xf74 0xf75 0xf76 0xf77 0xf78 0xf79 0xf7a 0xf7b 0xf7c 0xf7d 0xf7e 0xf7f 0xf80 0xf81 0xf82 0xf83 0xf84 0xf85 0xf86 0xf87 0xf88 0xf89 0xf8a 0xf8b 0xf8c 0xf8d 0xf8e 0xf8f 0xf90 0xf91 0xf92 0xf93 0xf94 0xf95 0xf96 0xf97 0xf98 0xf99 0xf9a 0xf9b 0xf9c 0xf9d 0xf9e 0xf9f 0xfa0 0xfa1 0xfa2 0xfa3 0xfa4 0xfa5 0xfa6 0xfa7 0xfa8 0xfa9 0xfaa 0xfab 0xfac 0xfad 0xfae 0xfaf 0xfb0 0xfb1 0xfb2 0xfb3 0xfb4 0xfb5 0xfb6 0xfb7 0xfb8 0xfb9 0xfba 0xfbb 0xfbc 0xfbd 0xfbe 0xfbf 0xfc0 0xfc1 0xfc2 0xfc3 0xfc4 0xfc5 0xfc6 0xfc7 0xfc8 0xfc9 0xfca 0xfcb 0xfcc 0xfcd 0xfce 0xfcf 0xfd0 0xfd1 0xfd2 0xfd3 0xfd4 0xfd5 0xfd6 0xfd7 0xfd8 0xfd9 0xfda 0xfdb 0xfdc 0xfdd 0xfde 0xfdf 0xfe0 0xfe1 0xfe2 0xfe3 0xfe4 0xfe5 0xfe6 0xfe7 0xfe8 0xfe9 0xfea 0xfeb 0xfec 0xfed 0xfee 0xfef 0xff0 0xff1 0xff2 0xff3 0xff4 0xff5 0xff6 0xff7 0xff8 0xff9 0xffa 0xffb 0xffc 0xffd 0xffe 0xfff 0x1000 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_50_50.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_50_50.txt deleted file mode 100644 index f75acfe46..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_50_50.txt +++ /dev/null @@ -1,8 +0,0 @@ -0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 -0x201 0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 -0x401 0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 -0x601 0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 -0x801 0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 -0xa01 0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 -0xc01 0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 -0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_64_64.txt b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_64_64.txt deleted file mode 100644 index 166808918..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/expected/tma_tensor_test_UTMASTG_64_64.txt +++ /dev/null @@ -1,8 +0,0 @@ -0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff 0x100 0x101 0x102 0x103 0x104 0x105 0x106 0x107 0x108 0x109 0x10a 0x10b 0x10c 0x10d 0x10e 0x10f 0x110 0x111 0x112 0x113 0x114 0x115 0x116 0x117 0x118 0x119 0x11a 0x11b 0x11c 0x11d 0x11e 0x11f 0x120 0x121 0x122 0x123 0x124 0x125 0x126 0x127 0x128 0x129 0x12a 0x12b 0x12c 0x12d 0x12e 0x12f 0x130 0x131 0x132 0x133 0x134 0x135 0x136 0x137 0x138 0x139 0x13a 0x13b 0x13c 0x13d 0x13e 0x13f 0x140 0x141 0x142 0x143 0x144 0x145 0x146 0x147 0x148 0x149 0x14a 0x14b 0x14c 0x14d 0x14e 0x14f 0x150 0x151 0x152 0x153 0x154 0x155 0x156 0x157 0x158 0x159 0x15a 0x15b 0x15c 0x15d 0x15e 0x15f 0x160 0x161 0x162 0x163 0x164 0x165 0x166 0x167 0x168 0x169 0x16a 0x16b 0x16c 0x16d 0x16e 0x16f 0x170 0x171 0x172 0x173 0x174 0x175 0x176 0x177 0x178 0x179 0x17a 0x17b 0x17c 0x17d 0x17e 0x17f 0x180 0x181 0x182 0x183 0x184 0x185 0x186 0x187 0x188 0x189 0x18a 0x18b 0x18c 0x18d 0x18e 0x18f 0x190 0x191 0x192 0x193 0x194 0x195 0x196 0x197 0x198 0x199 0x19a 0x19b 0x19c 0x19d 0x19e 0x19f 0x1a0 0x1a1 0x1a2 0x1a3 0x1a4 0x1a5 0x1a6 0x1a7 0x1a8 0x1a9 0x1aa 0x1ab 0x1ac 0x1ad 0x1ae 0x1af 0x1b0 0x1b1 0x1b2 0x1b3 0x1b4 0x1b5 0x1b6 0x1b7 0x1b8 0x1b9 0x1ba 0x1bb 0x1bc 0x1bd 0x1be 0x1bf 0x1c0 0x1c1 0x1c2 0x1c3 0x1c4 0x1c5 0x1c6 0x1c7 0x1c8 0x1c9 0x1ca 0x1cb 0x1cc 0x1cd 0x1ce 0x1cf 0x1d0 0x1d1 0x1d2 0x1d3 0x1d4 0x1d5 0x1d6 0x1d7 0x1d8 0x1d9 0x1da 0x1db 0x1dc 0x1dd 0x1de 0x1df 0x1e0 0x1e1 0x1e2 0x1e3 0x1e4 0x1e5 0x1e6 0x1e7 0x1e8 0x1e9 0x1ea 0x1eb 0x1ec 0x1ed 0x1ee 0x1ef 0x1f0 0x1f1 0x1f2 0x1f3 0x1f4 0x1f5 0x1f6 0x1f7 0x1f8 0x1f9 0x1fa 0x1fb 0x1fc 0x1fd 0x1fe 0x1ff 0x200 -0x201 0x202 0x203 0x204 0x205 0x206 0x207 0x208 0x209 0x20a 0x20b 0x20c 0x20d 0x20e 0x20f 0x210 0x211 0x212 0x213 0x214 0x215 0x216 0x217 0x218 0x219 0x21a 0x21b 0x21c 0x21d 0x21e 0x21f 0x220 0x221 0x222 0x223 0x224 0x225 0x226 0x227 0x228 0x229 0x22a 0x22b 0x22c 0x22d 0x22e 0x22f 0x230 0x231 0x232 0x233 0x234 0x235 0x236 0x237 0x238 0x239 0x23a 0x23b 0x23c 0x23d 0x23e 0x23f 0x240 0x241 0x242 0x243 0x244 0x245 0x246 0x247 0x248 0x249 0x24a 0x24b 0x24c 0x24d 0x24e 0x24f 0x250 0x251 0x252 0x253 0x254 0x255 0x256 0x257 0x258 0x259 0x25a 0x25b 0x25c 0x25d 0x25e 0x25f 0x260 0x261 0x262 0x263 0x264 0x265 0x266 0x267 0x268 0x269 0x26a 0x26b 0x26c 0x26d 0x26e 0x26f 0x270 0x271 0x272 0x273 0x274 0x275 0x276 0x277 0x278 0x279 0x27a 0x27b 0x27c 0x27d 0x27e 0x27f 0x280 0x281 0x282 0x283 0x284 0x285 0x286 0x287 0x288 0x289 0x28a 0x28b 0x28c 0x28d 0x28e 0x28f 0x290 0x291 0x292 0x293 0x294 0x295 0x296 0x297 0x298 0x299 0x29a 0x29b 0x29c 0x29d 0x29e 0x29f 0x2a0 0x2a1 0x2a2 0x2a3 0x2a4 0x2a5 0x2a6 0x2a7 0x2a8 0x2a9 0x2aa 0x2ab 0x2ac 0x2ad 0x2ae 0x2af 0x2b0 0x2b1 0x2b2 0x2b3 0x2b4 0x2b5 0x2b6 0x2b7 0x2b8 0x2b9 0x2ba 0x2bb 0x2bc 0x2bd 0x2be 0x2bf 0x2c0 0x2c1 0x2c2 0x2c3 0x2c4 0x2c5 0x2c6 0x2c7 0x2c8 0x2c9 0x2ca 0x2cb 0x2cc 0x2cd 0x2ce 0x2cf 0x2d0 0x2d1 0x2d2 0x2d3 0x2d4 0x2d5 0x2d6 0x2d7 0x2d8 0x2d9 0x2da 0x2db 0x2dc 0x2dd 0x2de 0x2df 0x2e0 0x2e1 0x2e2 0x2e3 0x2e4 0x2e5 0x2e6 0x2e7 0x2e8 0x2e9 0x2ea 0x2eb 0x2ec 0x2ed 0x2ee 0x2ef 0x2f0 0x2f1 0x2f2 0x2f3 0x2f4 0x2f5 0x2f6 0x2f7 0x2f8 0x2f9 0x2fa 0x2fb 0x2fc 0x2fd 0x2fe 0x2ff 0x300 0x301 0x302 0x303 0x304 0x305 0x306 0x307 0x308 0x309 0x30a 0x30b 0x30c 0x30d 0x30e 0x30f 0x310 0x311 0x312 0x313 0x314 0x315 0x316 0x317 0x318 0x319 0x31a 0x31b 0x31c 0x31d 0x31e 0x31f 0x320 0x321 0x322 0x323 0x324 0x325 0x326 0x327 0x328 0x329 0x32a 0x32b 0x32c 0x32d 0x32e 0x32f 0x330 0x331 0x332 0x333 0x334 0x335 0x336 0x337 0x338 0x339 0x33a 0x33b 0x33c 0x33d 0x33e 0x33f 0x340 0x341 0x342 0x343 0x344 0x345 0x346 0x347 0x348 0x349 0x34a 0x34b 0x34c 0x34d 0x34e 0x34f 0x350 0x351 0x352 0x353 0x354 0x355 0x356 0x357 0x358 0x359 0x35a 0x35b 0x35c 0x35d 0x35e 0x35f 0x360 0x361 0x362 0x363 0x364 0x365 0x366 0x367 0x368 0x369 0x36a 0x36b 0x36c 0x36d 0x36e 0x36f 0x370 0x371 0x372 0x373 0x374 0x375 0x376 0x377 0x378 0x379 0x37a 0x37b 0x37c 0x37d 0x37e 0x37f 0x380 0x381 0x382 0x383 0x384 0x385 0x386 0x387 0x388 0x389 0x38a 0x38b 0x38c 0x38d 0x38e 0x38f 0x390 0x391 0x392 0x393 0x394 0x395 0x396 0x397 0x398 0x399 0x39a 0x39b 0x39c 0x39d 0x39e 0x39f 0x3a0 0x3a1 0x3a2 0x3a3 0x3a4 0x3a5 0x3a6 0x3a7 0x3a8 0x3a9 0x3aa 0x3ab 0x3ac 0x3ad 0x3ae 0x3af 0x3b0 0x3b1 0x3b2 0x3b3 0x3b4 0x3b5 0x3b6 0x3b7 0x3b8 0x3b9 0x3ba 0x3bb 0x3bc 0x3bd 0x3be 0x3bf 0x3c0 0x3c1 0x3c2 0x3c3 0x3c4 0x3c5 0x3c6 0x3c7 0x3c8 0x3c9 0x3ca 0x3cb 0x3cc 0x3cd 0x3ce 0x3cf 0x3d0 0x3d1 0x3d2 0x3d3 0x3d4 0x3d5 0x3d6 0x3d7 0x3d8 0x3d9 0x3da 0x3db 0x3dc 0x3dd 0x3de 0x3df 0x3e0 0x3e1 0x3e2 0x3e3 0x3e4 0x3e5 0x3e6 0x3e7 0x3e8 0x3e9 0x3ea 0x3eb 0x3ec 0x3ed 0x3ee 0x3ef 0x3f0 0x3f1 0x3f2 0x3f3 0x3f4 0x3f5 0x3f6 0x3f7 0x3f8 0x3f9 0x3fa 0x3fb 0x3fc 0x3fd 0x3fe 0x3ff 0x400 -0x401 0x402 0x403 0x404 0x405 0x406 0x407 0x408 0x409 0x40a 0x40b 0x40c 0x40d 0x40e 0x40f 0x410 0x411 0x412 0x413 0x414 0x415 0x416 0x417 0x418 0x419 0x41a 0x41b 0x41c 0x41d 0x41e 0x41f 0x420 0x421 0x422 0x423 0x424 0x425 0x426 0x427 0x428 0x429 0x42a 0x42b 0x42c 0x42d 0x42e 0x42f 0x430 0x431 0x432 0x433 0x434 0x435 0x436 0x437 0x438 0x439 0x43a 0x43b 0x43c 0x43d 0x43e 0x43f 0x440 0x441 0x442 0x443 0x444 0x445 0x446 0x447 0x448 0x449 0x44a 0x44b 0x44c 0x44d 0x44e 0x44f 0x450 0x451 0x452 0x453 0x454 0x455 0x456 0x457 0x458 0x459 0x45a 0x45b 0x45c 0x45d 0x45e 0x45f 0x460 0x461 0x462 0x463 0x464 0x465 0x466 0x467 0x468 0x469 0x46a 0x46b 0x46c 0x46d 0x46e 0x46f 0x470 0x471 0x472 0x473 0x474 0x475 0x476 0x477 0x478 0x479 0x47a 0x47b 0x47c 0x47d 0x47e 0x47f 0x480 0x481 0x482 0x483 0x484 0x485 0x486 0x487 0x488 0x489 0x48a 0x48b 0x48c 0x48d 0x48e 0x48f 0x490 0x491 0x492 0x493 0x494 0x495 0x496 0x497 0x498 0x499 0x49a 0x49b 0x49c 0x49d 0x49e 0x49f 0x4a0 0x4a1 0x4a2 0x4a3 0x4a4 0x4a5 0x4a6 0x4a7 0x4a8 0x4a9 0x4aa 0x4ab 0x4ac 0x4ad 0x4ae 0x4af 0x4b0 0x4b1 0x4b2 0x4b3 0x4b4 0x4b5 0x4b6 0x4b7 0x4b8 0x4b9 0x4ba 0x4bb 0x4bc 0x4bd 0x4be 0x4bf 0x4c0 0x4c1 0x4c2 0x4c3 0x4c4 0x4c5 0x4c6 0x4c7 0x4c8 0x4c9 0x4ca 0x4cb 0x4cc 0x4cd 0x4ce 0x4cf 0x4d0 0x4d1 0x4d2 0x4d3 0x4d4 0x4d5 0x4d6 0x4d7 0x4d8 0x4d9 0x4da 0x4db 0x4dc 0x4dd 0x4de 0x4df 0x4e0 0x4e1 0x4e2 0x4e3 0x4e4 0x4e5 0x4e6 0x4e7 0x4e8 0x4e9 0x4ea 0x4eb 0x4ec 0x4ed 0x4ee 0x4ef 0x4f0 0x4f1 0x4f2 0x4f3 0x4f4 0x4f5 0x4f6 0x4f7 0x4f8 0x4f9 0x4fa 0x4fb 0x4fc 0x4fd 0x4fe 0x4ff 0x500 0x501 0x502 0x503 0x504 0x505 0x506 0x507 0x508 0x509 0x50a 0x50b 0x50c 0x50d 0x50e 0x50f 0x510 0x511 0x512 0x513 0x514 0x515 0x516 0x517 0x518 0x519 0x51a 0x51b 0x51c 0x51d 0x51e 0x51f 0x520 0x521 0x522 0x523 0x524 0x525 0x526 0x527 0x528 0x529 0x52a 0x52b 0x52c 0x52d 0x52e 0x52f 0x530 0x531 0x532 0x533 0x534 0x535 0x536 0x537 0x538 0x539 0x53a 0x53b 0x53c 0x53d 0x53e 0x53f 0x540 0x541 0x542 0x543 0x544 0x545 0x546 0x547 0x548 0x549 0x54a 0x54b 0x54c 0x54d 0x54e 0x54f 0x550 0x551 0x552 0x553 0x554 0x555 0x556 0x557 0x558 0x559 0x55a 0x55b 0x55c 0x55d 0x55e 0x55f 0x560 0x561 0x562 0x563 0x564 0x565 0x566 0x567 0x568 0x569 0x56a 0x56b 0x56c 0x56d 0x56e 0x56f 0x570 0x571 0x572 0x573 0x574 0x575 0x576 0x577 0x578 0x579 0x57a 0x57b 0x57c 0x57d 0x57e 0x57f 0x580 0x581 0x582 0x583 0x584 0x585 0x586 0x587 0x588 0x589 0x58a 0x58b 0x58c 0x58d 0x58e 0x58f 0x590 0x591 0x592 0x593 0x594 0x595 0x596 0x597 0x598 0x599 0x59a 0x59b 0x59c 0x59d 0x59e 0x59f 0x5a0 0x5a1 0x5a2 0x5a3 0x5a4 0x5a5 0x5a6 0x5a7 0x5a8 0x5a9 0x5aa 0x5ab 0x5ac 0x5ad 0x5ae 0x5af 0x5b0 0x5b1 0x5b2 0x5b3 0x5b4 0x5b5 0x5b6 0x5b7 0x5b8 0x5b9 0x5ba 0x5bb 0x5bc 0x5bd 0x5be 0x5bf 0x5c0 0x5c1 0x5c2 0x5c3 0x5c4 0x5c5 0x5c6 0x5c7 0x5c8 0x5c9 0x5ca 0x5cb 0x5cc 0x5cd 0x5ce 0x5cf 0x5d0 0x5d1 0x5d2 0x5d3 0x5d4 0x5d5 0x5d6 0x5d7 0x5d8 0x5d9 0x5da 0x5db 0x5dc 0x5dd 0x5de 0x5df 0x5e0 0x5e1 0x5e2 0x5e3 0x5e4 0x5e5 0x5e6 0x5e7 0x5e8 0x5e9 0x5ea 0x5eb 0x5ec 0x5ed 0x5ee 0x5ef 0x5f0 0x5f1 0x5f2 0x5f3 0x5f4 0x5f5 0x5f6 0x5f7 0x5f8 0x5f9 0x5fa 0x5fb 0x5fc 0x5fd 0x5fe 0x5ff 0x600 -0x601 0x602 0x603 0x604 0x605 0x606 0x607 0x608 0x609 0x60a 0x60b 0x60c 0x60d 0x60e 0x60f 0x610 0x611 0x612 0x613 0x614 0x615 0x616 0x617 0x618 0x619 0x61a 0x61b 0x61c 0x61d 0x61e 0x61f 0x620 0x621 0x622 0x623 0x624 0x625 0x626 0x627 0x628 0x629 0x62a 0x62b 0x62c 0x62d 0x62e 0x62f 0x630 0x631 0x632 0x633 0x634 0x635 0x636 0x637 0x638 0x639 0x63a 0x63b 0x63c 0x63d 0x63e 0x63f 0x640 0x641 0x642 0x643 0x644 0x645 0x646 0x647 0x648 0x649 0x64a 0x64b 0x64c 0x64d 0x64e 0x64f 0x650 0x651 0x652 0x653 0x654 0x655 0x656 0x657 0x658 0x659 0x65a 0x65b 0x65c 0x65d 0x65e 0x65f 0x660 0x661 0x662 0x663 0x664 0x665 0x666 0x667 0x668 0x669 0x66a 0x66b 0x66c 0x66d 0x66e 0x66f 0x670 0x671 0x672 0x673 0x674 0x675 0x676 0x677 0x678 0x679 0x67a 0x67b 0x67c 0x67d 0x67e 0x67f 0x680 0x681 0x682 0x683 0x684 0x685 0x686 0x687 0x688 0x689 0x68a 0x68b 0x68c 0x68d 0x68e 0x68f 0x690 0x691 0x692 0x693 0x694 0x695 0x696 0x697 0x698 0x699 0x69a 0x69b 0x69c 0x69d 0x69e 0x69f 0x6a0 0x6a1 0x6a2 0x6a3 0x6a4 0x6a5 0x6a6 0x6a7 0x6a8 0x6a9 0x6aa 0x6ab 0x6ac 0x6ad 0x6ae 0x6af 0x6b0 0x6b1 0x6b2 0x6b3 0x6b4 0x6b5 0x6b6 0x6b7 0x6b8 0x6b9 0x6ba 0x6bb 0x6bc 0x6bd 0x6be 0x6bf 0x6c0 0x6c1 0x6c2 0x6c3 0x6c4 0x6c5 0x6c6 0x6c7 0x6c8 0x6c9 0x6ca 0x6cb 0x6cc 0x6cd 0x6ce 0x6cf 0x6d0 0x6d1 0x6d2 0x6d3 0x6d4 0x6d5 0x6d6 0x6d7 0x6d8 0x6d9 0x6da 0x6db 0x6dc 0x6dd 0x6de 0x6df 0x6e0 0x6e1 0x6e2 0x6e3 0x6e4 0x6e5 0x6e6 0x6e7 0x6e8 0x6e9 0x6ea 0x6eb 0x6ec 0x6ed 0x6ee 0x6ef 0x6f0 0x6f1 0x6f2 0x6f3 0x6f4 0x6f5 0x6f6 0x6f7 0x6f8 0x6f9 0x6fa 0x6fb 0x6fc 0x6fd 0x6fe 0x6ff 0x700 0x701 0x702 0x703 0x704 0x705 0x706 0x707 0x708 0x709 0x70a 0x70b 0x70c 0x70d 0x70e 0x70f 0x710 0x711 0x712 0x713 0x714 0x715 0x716 0x717 0x718 0x719 0x71a 0x71b 0x71c 0x71d 0x71e 0x71f 0x720 0x721 0x722 0x723 0x724 0x725 0x726 0x727 0x728 0x729 0x72a 0x72b 0x72c 0x72d 0x72e 0x72f 0x730 0x731 0x732 0x733 0x734 0x735 0x736 0x737 0x738 0x739 0x73a 0x73b 0x73c 0x73d 0x73e 0x73f 0x740 0x741 0x742 0x743 0x744 0x745 0x746 0x747 0x748 0x749 0x74a 0x74b 0x74c 0x74d 0x74e 0x74f 0x750 0x751 0x752 0x753 0x754 0x755 0x756 0x757 0x758 0x759 0x75a 0x75b 0x75c 0x75d 0x75e 0x75f 0x760 0x761 0x762 0x763 0x764 0x765 0x766 0x767 0x768 0x769 0x76a 0x76b 0x76c 0x76d 0x76e 0x76f 0x770 0x771 0x772 0x773 0x774 0x775 0x776 0x777 0x778 0x779 0x77a 0x77b 0x77c 0x77d 0x77e 0x77f 0x780 0x781 0x782 0x783 0x784 0x785 0x786 0x787 0x788 0x789 0x78a 0x78b 0x78c 0x78d 0x78e 0x78f 0x790 0x791 0x792 0x793 0x794 0x795 0x796 0x797 0x798 0x799 0x79a 0x79b 0x79c 0x79d 0x79e 0x79f 0x7a0 0x7a1 0x7a2 0x7a3 0x7a4 0x7a5 0x7a6 0x7a7 0x7a8 0x7a9 0x7aa 0x7ab 0x7ac 0x7ad 0x7ae 0x7af 0x7b0 0x7b1 0x7b2 0x7b3 0x7b4 0x7b5 0x7b6 0x7b7 0x7b8 0x7b9 0x7ba 0x7bb 0x7bc 0x7bd 0x7be 0x7bf 0x7c0 0x7c1 0x7c2 0x7c3 0x7c4 0x7c5 0x7c6 0x7c7 0x7c8 0x7c9 0x7ca 0x7cb 0x7cc 0x7cd 0x7ce 0x7cf 0x7d0 0x7d1 0x7d2 0x7d3 0x7d4 0x7d5 0x7d6 0x7d7 0x7d8 0x7d9 0x7da 0x7db 0x7dc 0x7dd 0x7de 0x7df 0x7e0 0x7e1 0x7e2 0x7e3 0x7e4 0x7e5 0x7e6 0x7e7 0x7e8 0x7e9 0x7ea 0x7eb 0x7ec 0x7ed 0x7ee 0x7ef 0x7f0 0x7f1 0x7f2 0x7f3 0x7f4 0x7f5 0x7f6 0x7f7 0x7f8 0x7f9 0x7fa 0x7fb 0x7fc 0x7fd 0x7fe 0x7ff 0x800 -0x801 0x802 0x803 0x804 0x805 0x806 0x807 0x808 0x809 0x80a 0x80b 0x80c 0x80d 0x80e 0x80f 0x810 0x811 0x812 0x813 0x814 0x815 0x816 0x817 0x818 0x819 0x81a 0x81b 0x81c 0x81d 0x81e 0x81f 0x820 0x821 0x822 0x823 0x824 0x825 0x826 0x827 0x828 0x829 0x82a 0x82b 0x82c 0x82d 0x82e 0x82f 0x830 0x831 0x832 0x833 0x834 0x835 0x836 0x837 0x838 0x839 0x83a 0x83b 0x83c 0x83d 0x83e 0x83f 0x840 0x841 0x842 0x843 0x844 0x845 0x846 0x847 0x848 0x849 0x84a 0x84b 0x84c 0x84d 0x84e 0x84f 0x850 0x851 0x852 0x853 0x854 0x855 0x856 0x857 0x858 0x859 0x85a 0x85b 0x85c 0x85d 0x85e 0x85f 0x860 0x861 0x862 0x863 0x864 0x865 0x866 0x867 0x868 0x869 0x86a 0x86b 0x86c 0x86d 0x86e 0x86f 0x870 0x871 0x872 0x873 0x874 0x875 0x876 0x877 0x878 0x879 0x87a 0x87b 0x87c 0x87d 0x87e 0x87f 0x880 0x881 0x882 0x883 0x884 0x885 0x886 0x887 0x888 0x889 0x88a 0x88b 0x88c 0x88d 0x88e 0x88f 0x890 0x891 0x892 0x893 0x894 0x895 0x896 0x897 0x898 0x899 0x89a 0x89b 0x89c 0x89d 0x89e 0x89f 0x8a0 0x8a1 0x8a2 0x8a3 0x8a4 0x8a5 0x8a6 0x8a7 0x8a8 0x8a9 0x8aa 0x8ab 0x8ac 0x8ad 0x8ae 0x8af 0x8b0 0x8b1 0x8b2 0x8b3 0x8b4 0x8b5 0x8b6 0x8b7 0x8b8 0x8b9 0x8ba 0x8bb 0x8bc 0x8bd 0x8be 0x8bf 0x8c0 0x8c1 0x8c2 0x8c3 0x8c4 0x8c5 0x8c6 0x8c7 0x8c8 0x8c9 0x8ca 0x8cb 0x8cc 0x8cd 0x8ce 0x8cf 0x8d0 0x8d1 0x8d2 0x8d3 0x8d4 0x8d5 0x8d6 0x8d7 0x8d8 0x8d9 0x8da 0x8db 0x8dc 0x8dd 0x8de 0x8df 0x8e0 0x8e1 0x8e2 0x8e3 0x8e4 0x8e5 0x8e6 0x8e7 0x8e8 0x8e9 0x8ea 0x8eb 0x8ec 0x8ed 0x8ee 0x8ef 0x8f0 0x8f1 0x8f2 0x8f3 0x8f4 0x8f5 0x8f6 0x8f7 0x8f8 0x8f9 0x8fa 0x8fb 0x8fc 0x8fd 0x8fe 0x8ff 0x900 0x901 0x902 0x903 0x904 0x905 0x906 0x907 0x908 0x909 0x90a 0x90b 0x90c 0x90d 0x90e 0x90f 0x910 0x911 0x912 0x913 0x914 0x915 0x916 0x917 0x918 0x919 0x91a 0x91b 0x91c 0x91d 0x91e 0x91f 0x920 0x921 0x922 0x923 0x924 0x925 0x926 0x927 0x928 0x929 0x92a 0x92b 0x92c 0x92d 0x92e 0x92f 0x930 0x931 0x932 0x933 0x934 0x935 0x936 0x937 0x938 0x939 0x93a 0x93b 0x93c 0x93d 0x93e 0x93f 0x940 0x941 0x942 0x943 0x944 0x945 0x946 0x947 0x948 0x949 0x94a 0x94b 0x94c 0x94d 0x94e 0x94f 0x950 0x951 0x952 0x953 0x954 0x955 0x956 0x957 0x958 0x959 0x95a 0x95b 0x95c 0x95d 0x95e 0x95f 0x960 0x961 0x962 0x963 0x964 0x965 0x966 0x967 0x968 0x969 0x96a 0x96b 0x96c 0x96d 0x96e 0x96f 0x970 0x971 0x972 0x973 0x974 0x975 0x976 0x977 0x978 0x979 0x97a 0x97b 0x97c 0x97d 0x97e 0x97f 0x980 0x981 0x982 0x983 0x984 0x985 0x986 0x987 0x988 0x989 0x98a 0x98b 0x98c 0x98d 0x98e 0x98f 0x990 0x991 0x992 0x993 0x994 0x995 0x996 0x997 0x998 0x999 0x99a 0x99b 0x99c 0x99d 0x99e 0x99f 0x9a0 0x9a1 0x9a2 0x9a3 0x9a4 0x9a5 0x9a6 0x9a7 0x9a8 0x9a9 0x9aa 0x9ab 0x9ac 0x9ad 0x9ae 0x9af 0x9b0 0x9b1 0x9b2 0x9b3 0x9b4 0x9b5 0x9b6 0x9b7 0x9b8 0x9b9 0x9ba 0x9bb 0x9bc 0x9bd 0x9be 0x9bf 0x9c0 0x9c1 0x9c2 0x9c3 0x9c4 0x9c5 0x9c6 0x9c7 0x9c8 0x9c9 0x9ca 0x9cb 0x9cc 0x9cd 0x9ce 0x9cf 0x9d0 0x9d1 0x9d2 0x9d3 0x9d4 0x9d5 0x9d6 0x9d7 0x9d8 0x9d9 0x9da 0x9db 0x9dc 0x9dd 0x9de 0x9df 0x9e0 0x9e1 0x9e2 0x9e3 0x9e4 0x9e5 0x9e6 0x9e7 0x9e8 0x9e9 0x9ea 0x9eb 0x9ec 0x9ed 0x9ee 0x9ef 0x9f0 0x9f1 0x9f2 0x9f3 0x9f4 0x9f5 0x9f6 0x9f7 0x9f8 0x9f9 0x9fa 0x9fb 0x9fc 0x9fd 0x9fe 0x9ff 0xa00 -0xa01 0xa02 0xa03 0xa04 0xa05 0xa06 0xa07 0xa08 0xa09 0xa0a 0xa0b 0xa0c 0xa0d 0xa0e 0xa0f 0xa10 0xa11 0xa12 0xa13 0xa14 0xa15 0xa16 0xa17 0xa18 0xa19 0xa1a 0xa1b 0xa1c 0xa1d 0xa1e 0xa1f 0xa20 0xa21 0xa22 0xa23 0xa24 0xa25 0xa26 0xa27 0xa28 0xa29 0xa2a 0xa2b 0xa2c 0xa2d 0xa2e 0xa2f 0xa30 0xa31 0xa32 0xa33 0xa34 0xa35 0xa36 0xa37 0xa38 0xa39 0xa3a 0xa3b 0xa3c 0xa3d 0xa3e 0xa3f 0xa40 0xa41 0xa42 0xa43 0xa44 0xa45 0xa46 0xa47 0xa48 0xa49 0xa4a 0xa4b 0xa4c 0xa4d 0xa4e 0xa4f 0xa50 0xa51 0xa52 0xa53 0xa54 0xa55 0xa56 0xa57 0xa58 0xa59 0xa5a 0xa5b 0xa5c 0xa5d 0xa5e 0xa5f 0xa60 0xa61 0xa62 0xa63 0xa64 0xa65 0xa66 0xa67 0xa68 0xa69 0xa6a 0xa6b 0xa6c 0xa6d 0xa6e 0xa6f 0xa70 0xa71 0xa72 0xa73 0xa74 0xa75 0xa76 0xa77 0xa78 0xa79 0xa7a 0xa7b 0xa7c 0xa7d 0xa7e 0xa7f 0xa80 0xa81 0xa82 0xa83 0xa84 0xa85 0xa86 0xa87 0xa88 0xa89 0xa8a 0xa8b 0xa8c 0xa8d 0xa8e 0xa8f 0xa90 0xa91 0xa92 0xa93 0xa94 0xa95 0xa96 0xa97 0xa98 0xa99 0xa9a 0xa9b 0xa9c 0xa9d 0xa9e 0xa9f 0xaa0 0xaa1 0xaa2 0xaa3 0xaa4 0xaa5 0xaa6 0xaa7 0xaa8 0xaa9 0xaaa 0xaab 0xaac 0xaad 0xaae 0xaaf 0xab0 0xab1 0xab2 0xab3 0xab4 0xab5 0xab6 0xab7 0xab8 0xab9 0xaba 0xabb 0xabc 0xabd 0xabe 0xabf 0xac0 0xac1 0xac2 0xac3 0xac4 0xac5 0xac6 0xac7 0xac8 0xac9 0xaca 0xacb 0xacc 0xacd 0xace 0xacf 0xad0 0xad1 0xad2 0xad3 0xad4 0xad5 0xad6 0xad7 0xad8 0xad9 0xada 0xadb 0xadc 0xadd 0xade 0xadf 0xae0 0xae1 0xae2 0xae3 0xae4 0xae5 0xae6 0xae7 0xae8 0xae9 0xaea 0xaeb 0xaec 0xaed 0xaee 0xaef 0xaf0 0xaf1 0xaf2 0xaf3 0xaf4 0xaf5 0xaf6 0xaf7 0xaf8 0xaf9 0xafa 0xafb 0xafc 0xafd 0xafe 0xaff 0xb00 0xb01 0xb02 0xb03 0xb04 0xb05 0xb06 0xb07 0xb08 0xb09 0xb0a 0xb0b 0xb0c 0xb0d 0xb0e 0xb0f 0xb10 0xb11 0xb12 0xb13 0xb14 0xb15 0xb16 0xb17 0xb18 0xb19 0xb1a 0xb1b 0xb1c 0xb1d 0xb1e 0xb1f 0xb20 0xb21 0xb22 0xb23 0xb24 0xb25 0xb26 0xb27 0xb28 0xb29 0xb2a 0xb2b 0xb2c 0xb2d 0xb2e 0xb2f 0xb30 0xb31 0xb32 0xb33 0xb34 0xb35 0xb36 0xb37 0xb38 0xb39 0xb3a 0xb3b 0xb3c 0xb3d 0xb3e 0xb3f 0xb40 0xb41 0xb42 0xb43 0xb44 0xb45 0xb46 0xb47 0xb48 0xb49 0xb4a 0xb4b 0xb4c 0xb4d 0xb4e 0xb4f 0xb50 0xb51 0xb52 0xb53 0xb54 0xb55 0xb56 0xb57 0xb58 0xb59 0xb5a 0xb5b 0xb5c 0xb5d 0xb5e 0xb5f 0xb60 0xb61 0xb62 0xb63 0xb64 0xb65 0xb66 0xb67 0xb68 0xb69 0xb6a 0xb6b 0xb6c 0xb6d 0xb6e 0xb6f 0xb70 0xb71 0xb72 0xb73 0xb74 0xb75 0xb76 0xb77 0xb78 0xb79 0xb7a 0xb7b 0xb7c 0xb7d 0xb7e 0xb7f 0xb80 0xb81 0xb82 0xb83 0xb84 0xb85 0xb86 0xb87 0xb88 0xb89 0xb8a 0xb8b 0xb8c 0xb8d 0xb8e 0xb8f 0xb90 0xb91 0xb92 0xb93 0xb94 0xb95 0xb96 0xb97 0xb98 0xb99 0xb9a 0xb9b 0xb9c 0xb9d 0xb9e 0xb9f 0xba0 0xba1 0xba2 0xba3 0xba4 0xba5 0xba6 0xba7 0xba8 0xba9 0xbaa 0xbab 0xbac 0xbad 0xbae 0xbaf 0xbb0 0xbb1 0xbb2 0xbb3 0xbb4 0xbb5 0xbb6 0xbb7 0xbb8 0xbb9 0xbba 0xbbb 0xbbc 0xbbd 0xbbe 0xbbf 0xbc0 0xbc1 0xbc2 0xbc3 0xbc4 0xbc5 0xbc6 0xbc7 0xbc8 0xbc9 0xbca 0xbcb 0xbcc 0xbcd 0xbce 0xbcf 0xbd0 0xbd1 0xbd2 0xbd3 0xbd4 0xbd5 0xbd6 0xbd7 0xbd8 0xbd9 0xbda 0xbdb 0xbdc 0xbdd 0xbde 0xbdf 0xbe0 0xbe1 0xbe2 0xbe3 0xbe4 0xbe5 0xbe6 0xbe7 0xbe8 0xbe9 0xbea 0xbeb 0xbec 0xbed 0xbee 0xbef 0xbf0 0xbf1 0xbf2 0xbf3 0xbf4 0xbf5 0xbf6 0xbf7 0xbf8 0xbf9 0xbfa 0xbfb 0xbfc 0xbfd 0xbfe 0xbff 0xc00 -0xc01 0xc02 0xc03 0xc04 0xc05 0xc06 0xc07 0xc08 0xc09 0xc0a 0xc0b 0xc0c 0xc0d 0xc0e 0xc0f 0xc10 0xc11 0xc12 0xc13 0xc14 0xc15 0xc16 0xc17 0xc18 0xc19 0xc1a 0xc1b 0xc1c 0xc1d 0xc1e 0xc1f 0xc20 0xc21 0xc22 0xc23 0xc24 0xc25 0xc26 0xc27 0xc28 0xc29 0xc2a 0xc2b 0xc2c 0xc2d 0xc2e 0xc2f 0xc30 0xc31 0xc32 0xc33 0xc34 0xc35 0xc36 0xc37 0xc38 0xc39 0xc3a 0xc3b 0xc3c 0xc3d 0xc3e 0xc3f 0xc40 0xc41 0xc42 0xc43 0xc44 0xc45 0xc46 0xc47 0xc48 0xc49 0xc4a 0xc4b 0xc4c 0xc4d 0xc4e 0xc4f 0xc50 0xc51 0xc52 0xc53 0xc54 0xc55 0xc56 0xc57 0xc58 0xc59 0xc5a 0xc5b 0xc5c 0xc5d 0xc5e 0xc5f 0xc60 0xc61 0xc62 0xc63 0xc64 0xc65 0xc66 0xc67 0xc68 0xc69 0xc6a 0xc6b 0xc6c 0xc6d 0xc6e 0xc6f 0xc70 0xc71 0xc72 0xc73 0xc74 0xc75 0xc76 0xc77 0xc78 0xc79 0xc7a 0xc7b 0xc7c 0xc7d 0xc7e 0xc7f 0xc80 0xc81 0xc82 0xc83 0xc84 0xc85 0xc86 0xc87 0xc88 0xc89 0xc8a 0xc8b 0xc8c 0xc8d 0xc8e 0xc8f 0xc90 0xc91 0xc92 0xc93 0xc94 0xc95 0xc96 0xc97 0xc98 0xc99 0xc9a 0xc9b 0xc9c 0xc9d 0xc9e 0xc9f 0xca0 0xca1 0xca2 0xca3 0xca4 0xca5 0xca6 0xca7 0xca8 0xca9 0xcaa 0xcab 0xcac 0xcad 0xcae 0xcaf 0xcb0 0xcb1 0xcb2 0xcb3 0xcb4 0xcb5 0xcb6 0xcb7 0xcb8 0xcb9 0xcba 0xcbb 0xcbc 0xcbd 0xcbe 0xcbf 0xcc0 0xcc1 0xcc2 0xcc3 0xcc4 0xcc5 0xcc6 0xcc7 0xcc8 0xcc9 0xcca 0xccb 0xccc 0xccd 0xcce 0xccf 0xcd0 0xcd1 0xcd2 0xcd3 0xcd4 0xcd5 0xcd6 0xcd7 0xcd8 0xcd9 0xcda 0xcdb 0xcdc 0xcdd 0xcde 0xcdf 0xce0 0xce1 0xce2 0xce3 0xce4 0xce5 0xce6 0xce7 0xce8 0xce9 0xcea 0xceb 0xcec 0xced 0xcee 0xcef 0xcf0 0xcf1 0xcf2 0xcf3 0xcf4 0xcf5 0xcf6 0xcf7 0xcf8 0xcf9 0xcfa 0xcfb 0xcfc 0xcfd 0xcfe 0xcff 0xd00 0xd01 0xd02 0xd03 0xd04 0xd05 0xd06 0xd07 0xd08 0xd09 0xd0a 0xd0b 0xd0c 0xd0d 0xd0e 0xd0f 0xd10 0xd11 0xd12 0xd13 0xd14 0xd15 0xd16 0xd17 0xd18 0xd19 0xd1a 0xd1b 0xd1c 0xd1d 0xd1e 0xd1f 0xd20 0xd21 0xd22 0xd23 0xd24 0xd25 0xd26 0xd27 0xd28 0xd29 0xd2a 0xd2b 0xd2c 0xd2d 0xd2e 0xd2f 0xd30 0xd31 0xd32 0xd33 0xd34 0xd35 0xd36 0xd37 0xd38 0xd39 0xd3a 0xd3b 0xd3c 0xd3d 0xd3e 0xd3f 0xd40 0xd41 0xd42 0xd43 0xd44 0xd45 0xd46 0xd47 0xd48 0xd49 0xd4a 0xd4b 0xd4c 0xd4d 0xd4e 0xd4f 0xd50 0xd51 0xd52 0xd53 0xd54 0xd55 0xd56 0xd57 0xd58 0xd59 0xd5a 0xd5b 0xd5c 0xd5d 0xd5e 0xd5f 0xd60 0xd61 0xd62 0xd63 0xd64 0xd65 0xd66 0xd67 0xd68 0xd69 0xd6a 0xd6b 0xd6c 0xd6d 0xd6e 0xd6f 0xd70 0xd71 0xd72 0xd73 0xd74 0xd75 0xd76 0xd77 0xd78 0xd79 0xd7a 0xd7b 0xd7c 0xd7d 0xd7e 0xd7f 0xd80 0xd81 0xd82 0xd83 0xd84 0xd85 0xd86 0xd87 0xd88 0xd89 0xd8a 0xd8b 0xd8c 0xd8d 0xd8e 0xd8f 0xd90 0xd91 0xd92 0xd93 0xd94 0xd95 0xd96 0xd97 0xd98 0xd99 0xd9a 0xd9b 0xd9c 0xd9d 0xd9e 0xd9f 0xda0 0xda1 0xda2 0xda3 0xda4 0xda5 0xda6 0xda7 0xda8 0xda9 0xdaa 0xdab 0xdac 0xdad 0xdae 0xdaf 0xdb0 0xdb1 0xdb2 0xdb3 0xdb4 0xdb5 0xdb6 0xdb7 0xdb8 0xdb9 0xdba 0xdbb 0xdbc 0xdbd 0xdbe 0xdbf 0xdc0 0xdc1 0xdc2 0xdc3 0xdc4 0xdc5 0xdc6 0xdc7 0xdc8 0xdc9 0xdca 0xdcb 0xdcc 0xdcd 0xdce 0xdcf 0xdd0 0xdd1 0xdd2 0xdd3 0xdd4 0xdd5 0xdd6 0xdd7 0xdd8 0xdd9 0xdda 0xddb 0xddc 0xddd 0xdde 0xddf 0xde0 0xde1 0xde2 0xde3 0xde4 0xde5 0xde6 0xde7 0xde8 0xde9 0xdea 0xdeb 0xdec 0xded 0xdee 0xdef 0xdf0 0xdf1 0xdf2 0xdf3 0xdf4 0xdf5 0xdf6 0xdf7 0xdf8 0xdf9 0xdfa 0xdfb 0xdfc 0xdfd 0xdfe 0xdff 0xe00 -0xe01 0xe02 0xe03 0xe04 0xe05 0xe06 0xe07 0xe08 0xe09 0xe0a 0xe0b 0xe0c 0xe0d 0xe0e 0xe0f 0xe10 0xe11 0xe12 0xe13 0xe14 0xe15 0xe16 0xe17 0xe18 0xe19 0xe1a 0xe1b 0xe1c 0xe1d 0xe1e 0xe1f 0xe20 0xe21 0xe22 0xe23 0xe24 0xe25 0xe26 0xe27 0xe28 0xe29 0xe2a 0xe2b 0xe2c 0xe2d 0xe2e 0xe2f 0xe30 0xe31 0xe32 0xe33 0xe34 0xe35 0xe36 0xe37 0xe38 0xe39 0xe3a 0xe3b 0xe3c 0xe3d 0xe3e 0xe3f 0xe40 0xe41 0xe42 0xe43 0xe44 0xe45 0xe46 0xe47 0xe48 0xe49 0xe4a 0xe4b 0xe4c 0xe4d 0xe4e 0xe4f 0xe50 0xe51 0xe52 0xe53 0xe54 0xe55 0xe56 0xe57 0xe58 0xe59 0xe5a 0xe5b 0xe5c 0xe5d 0xe5e 0xe5f 0xe60 0xe61 0xe62 0xe63 0xe64 0xe65 0xe66 0xe67 0xe68 0xe69 0xe6a 0xe6b 0xe6c 0xe6d 0xe6e 0xe6f 0xe70 0xe71 0xe72 0xe73 0xe74 0xe75 0xe76 0xe77 0xe78 0xe79 0xe7a 0xe7b 0xe7c 0xe7d 0xe7e 0xe7f 0xe80 0xe81 0xe82 0xe83 0xe84 0xe85 0xe86 0xe87 0xe88 0xe89 0xe8a 0xe8b 0xe8c 0xe8d 0xe8e 0xe8f 0xe90 0xe91 0xe92 0xe93 0xe94 0xe95 0xe96 0xe97 0xe98 0xe99 0xe9a 0xe9b 0xe9c 0xe9d 0xe9e 0xe9f 0xea0 0xea1 0xea2 0xea3 0xea4 0xea5 0xea6 0xea7 0xea8 0xea9 0xeaa 0xeab 0xeac 0xead 0xeae 0xeaf 0xeb0 0xeb1 0xeb2 0xeb3 0xeb4 0xeb5 0xeb6 0xeb7 0xeb8 0xeb9 0xeba 0xebb 0xebc 0xebd 0xebe 0xebf 0xec0 0xec1 0xec2 0xec3 0xec4 0xec5 0xec6 0xec7 0xec8 0xec9 0xeca 0xecb 0xecc 0xecd 0xece 0xecf 0xed0 0xed1 0xed2 0xed3 0xed4 0xed5 0xed6 0xed7 0xed8 0xed9 0xeda 0xedb 0xedc 0xedd 0xede 0xedf 0xee0 0xee1 0xee2 0xee3 0xee4 0xee5 0xee6 0xee7 0xee8 0xee9 0xeea 0xeeb 0xeec 0xeed 0xeee 0xeef 0xef0 0xef1 0xef2 0xef3 0xef4 0xef5 0xef6 0xef7 0xef8 0xef9 0xefa 0xefb 0xefc 0xefd 0xefe 0xeff 0xf00 0xf01 0xf02 0xf03 0xf04 0xf05 0xf06 0xf07 0xf08 0xf09 0xf0a 0xf0b 0xf0c 0xf0d 0xf0e 0xf0f 0xf10 0xf11 0xf12 0xf13 0xf14 0xf15 0xf16 0xf17 0xf18 0xf19 0xf1a 0xf1b 0xf1c 0xf1d 0xf1e 0xf1f 0xf20 0xf21 0xf22 0xf23 0xf24 0xf25 0xf26 0xf27 0xf28 0xf29 0xf2a 0xf2b 0xf2c 0xf2d 0xf2e 0xf2f 0xf30 0xf31 0xf32 0xf33 0xf34 0xf35 0xf36 0xf37 0xf38 0xf39 0xf3a 0xf3b 0xf3c 0xf3d 0xf3e 0xf3f 0xf40 0xf41 0xf42 0xf43 0xf44 0xf45 0xf46 0xf47 0xf48 0xf49 0xf4a 0xf4b 0xf4c 0xf4d 0xf4e 0xf4f 0xf50 0xf51 0xf52 0xf53 0xf54 0xf55 0xf56 0xf57 0xf58 0xf59 0xf5a 0xf5b 0xf5c 0xf5d 0xf5e 0xf5f 0xf60 0xf61 0xf62 0xf63 0xf64 0xf65 0xf66 0xf67 0xf68 0xf69 0xf6a 0xf6b 0xf6c 0xf6d 0xf6e 0xf6f 0xf70 0xf71 0xf72 0xf73 0xf74 0xf75 0xf76 0xf77 0xf78 0xf79 0xf7a 0xf7b 0xf7c 0xf7d 0xf7e 0xf7f 0xf80 0xf81 0xf82 0xf83 0xf84 0xf85 0xf86 0xf87 0xf88 0xf89 0xf8a 0xf8b 0xf8c 0xf8d 0xf8e 0xf8f 0xf90 0xf91 0xf92 0xf93 0xf94 0xf95 0xf96 0xf97 0xf98 0xf99 0xf9a 0xf9b 0xf9c 0xf9d 0xf9e 0xf9f 0xfa0 0xfa1 0xfa2 0xfa3 0xfa4 0xfa5 0xfa6 0xfa7 0xfa8 0xfa9 0xfaa 0xfab 0xfac 0xfad 0xfae 0xfaf 0xfb0 0xfb1 0xfb2 0xfb3 0xfb4 0xfb5 0xfb6 0xfb7 0xfb8 0xfb9 0xfba 0xfbb 0xfbc 0xfbd 0xfbe 0xfbf 0xfc0 0xfc1 0xfc2 0xfc3 0xfc4 0xfc5 0xfc6 0xfc7 0xfc8 0xfc9 0xfca 0xfcb 0xfcc 0xfcd 0xfce 0xfcf 0xfd0 0xfd1 0xfd2 0xfd3 0xfd4 0xfd5 0xfd6 0xfd7 0xfd8 0xfd9 0xfda 0xfdb 0xfdc 0xfdd 0xfde 0xfdf 0xfe0 0xfe1 0xfe2 0xfe3 0xfe4 0xfe5 0xfe6 0xfe7 0xfe8 0xfe9 0xfea 0xfeb 0xfec 0xfed 0xfee 0xfef 0xff0 0xff1 0xff2 0xff3 0xff4 0xff5 0xff6 0xff7 0xff8 0xff9 0xffa 0xffb 0xffc 0xffd 0xffe 0xfff 0x1000 diff --git a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/tma_tensor_benchmark.cu b/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/tma_tensor_benchmark.cu deleted file mode 100644 index a33ac4b13..000000000 --- a/src/cuda/GPU_Microbenchmark/ubench/gpgpusim_tests/tma_tensormap/tma_tensor_benchmark.cu +++ /dev/null @@ -1,415 +0,0 @@ -// Adapt from https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html?highlight=tma#using-tma-to-transfer-multi-dimensional-arrays -#include // PFN_cuTensorMapEncodeTiled, CUtensorMap -#include // CUtensormap -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using barrier = cuda::barrier; -namespace ptx = cuda::ptx; - -/* - * Test application for TMA tensor operations. - * - * Usage: ./tma_tensor -w -h -o - * - */ - -#define CUDA_SAFECALL(call) \ - { \ - call; \ - cudaError err = cudaGetLastError(); \ - if (cudaSuccess != err) \ - { \ - fprintf( \ - stderr, \ - "Cuda error in function '%s' file '%s' in line %i : %s.\n", \ - #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ - fflush(stderr); \ - exit(EXIT_FAILURE); \ - } \ - } - -#define GMEM_WIDTH 1024 -#define GMEM_HEIGHT 1024 -#define SMEM_WIDTH 32 -#define SMEM_HEIGHT 32 -#define DEFAULT_RUN_ITERS 128 - -enum class TestType { - UTMAPF, - UTMALDG, - UTMALDG_L2Hint, - UTMASTG, - UTMAREDG, - REGULAR_LOAD -}; - -static const std::unordered_map opcode_map = { - {"UTMAPF", TestType::UTMAPF}, - {"UTMALDG", TestType::UTMALDG}, - {"UTMASTG", TestType::UTMASTG}, - {"UTMAREDG", TestType::UTMAREDG}, - {"REGULAR_LOAD", TestType::REGULAR_LOAD} -}; - -__global__ void test_kernel(const CUtensorMap *tensor_map, int *mat, TestType test_type, int width_stride, int run_iters); -__device__ void test_UTMAPF_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters); -__device__ void test_UTMALDG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters); -__device__ void test_UTMASTG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters); -__device__ void test_UTMAREDG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters); -__device__ void test_REGULAR_LOAD_kernel(int *mat, int x, int y, int width_stride, int run_iters); - -__global__ void test_kernel(const CUtensorMap *tensor_map, int *mat, TestType test_type, int width_stride, int run_iters) { - int x = blockDim.x * blockIdx.x; - int y = blockDim.y * blockIdx.y; - if (blockIdx.x == 0 && blockIdx.y == 0 && - threadIdx.x == 0 && threadIdx.y == 0) { - printf("TensorMap address: %p\n", tensor_map); - } - switch (test_type) { - case TestType::UTMAPF: - test_UTMAPF_kernel(tensor_map, x, y, run_iters); - break; - case TestType::UTMALDG: - test_UTMALDG_kernel(tensor_map, x, y, run_iters); - break; - case TestType::UTMASTG: - test_UTMASTG_kernel(tensor_map, x, y, run_iters); - break; - case TestType::UTMAREDG: - test_UTMAREDG_kernel(tensor_map, x, y, run_iters); - break; - case TestType::REGULAR_LOAD: - test_REGULAR_LOAD_kernel(mat, x, y, width_stride, run_iters); - break; - default: - test_UTMAPF_kernel(tensor_map, x, y, run_iters); - break; - } -} - -__device__ void test_UTMAPF_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters) { - // TensorMap prefetch at tensor_map with tensor coord {x, y} - if (threadIdx.x == 0 && threadIdx.y == 0) { - asm volatile ( - "cp.async.bulk.prefetch.tensor.2d.L2.global.tile" - " [%0, {%1, %2}];" - : - : "l"(tensor_map), - "r"(x), - "r"(y) - : "memory"); - } -} - -__device__ void test_UTMALDG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters) { - // The destination shared memory buffer of a bulk tensor operation should be - // 128 byte aligned. - __shared__ alignas(128) int smem_buffer[SMEM_HEIGHT][SMEM_WIDTH]; - -// Initialize shared memory barrier with the number of threads participating in the barrier. -#pragma nv_diag_suppress static_var_with_dynamic_init - __shared__ barrier bar; - - if (threadIdx.x == 0 && threadIdx.y == 0) - { - // Initialize barrier. All threads in block participate. - init(&bar, blockDim.x * blockDim.y); - // Make initialized barrier visible in async proxy. - ptx::fence_proxy_async(ptx::space_shared); - } - // Syncthreads so initialized barrier is visible to all threads. - __syncthreads(); - - for (int i = 0; i < run_iters; i++) { - barrier::arrival_token token; - if (threadIdx.x == 0 && threadIdx.y == 0) { - // Initiate bulk tensor copy. - ptx::cp_async_bulk_tensor( - ptx::space_cluster, - ptx::space_global, - &smem_buffer, - tensor_map, - {x, y}, - cuda::device::barrier_native_handle(bar) - ); - // Arrive on the barrier and tell how many bytes are expected to come in. - token = cuda::device::barrier_arrive_tx(bar, 1, sizeof(smem_buffer)); - } - else - { - // Other threads just arrive. - token = bar.arrive(); - } - // Wait for the data to have arrived. - bar.wait(std::move(token)); - } -} - -__device__ void test_UTMASTG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters) { - __shared__ alignas(128) int smem_buffer[SMEM_HEIGHT][SMEM_WIDTH]; - - // Compute a unique value for the thread - int thread_x = threadIdx.x + x; - int thread_y = threadIdx.y + y; - smem_buffer[threadIdx.y][threadIdx.x] = 1 + thread_x + thread_y * blockDim.x * gridDim.x; - - // Wait for shared memory writes to be visible to TMA engine. - ptx::fence_proxy_async(ptx::space_shared); - __syncthreads(); - - // Initiate TMA transfer to copy shared memory to global memory - for (int i = 0; i < run_iters; i++) { - if (threadIdx.x == 0 && threadIdx.y == 0) { - ptx::cp_async_bulk_tensor( - ptx::space_global, - ptx::space_shared, - tensor_map, - {x, y}, - &smem_buffer - ); - ptx::cp_async_bulk_commit_group(); - ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); - } - } -} - -__device__ void test_UTMAREDG_kernel(const CUtensorMap *tensor_map, int x, int y, int run_iters) { - __shared__ alignas(128) int smem_buffer[SMEM_HEIGHT][SMEM_WIDTH]; - - // Compute a unique value for the thread - int thread_x = threadIdx.x + x; - int thread_y = threadIdx.y + y; - // Add 1 so the max op will not be effective - smem_buffer[threadIdx.y][threadIdx.x] = 1 + thread_x + thread_y * blockDim.x * gridDim.x; - - // Wait for shared memory writes to be visible to TMA engine. - ptx::fence_proxy_async(ptx::space_shared); - __syncthreads(); - - // Initiate TMA transfer to copy shared memory to global memory - for (int i = 0; i < run_iters; i++) { - if (threadIdx.x == 0 && threadIdx.y == 0) { - ptx::cp_reduce_async_bulk_tensor( - ptx::space_global, - ptx::space_shared, - ptx::op_max, - tensor_map, - {x, y}, - &smem_buffer - ); - ptx::cp_async_bulk_commit_group(); - ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); - } - } -} - -__device__ void test_REGULAR_LOAD_kernel(int *mat, int x, int y, int width_stride, int run_iters) { - __shared__ alignas(128) int smem_buffer[SMEM_HEIGHT][SMEM_WIDTH]; - - // Compute a unique value for the thread - int thread_x = threadIdx.x + x; - int thread_y = threadIdx.y + y; - // Mimic a TMA load pattern here - for (int i = 0; i < run_iters; i++) { - if (threadIdx.x == 0 && threadIdx.y == 0) { - for (int row = 0; row < SMEM_HEIGHT; row++) { - for (int col = 0; col < SMEM_WIDTH; col++) { - smem_buffer[row][col] = mat[(y + row) * width_stride + (x + col)] + 1; - } - } - } - __syncthreads(); - // Mimic a TMA store pattern here to make compiler happy - if (threadIdx.x == 0 && threadIdx.y == 0) { - for (int row = 0; row < SMEM_HEIGHT; row++) { - for (int col = 0; col < SMEM_WIDTH; col++) { - mat[(y + row) * width_stride + (x + col)] = smem_buffer[row][col]; - } - } - } - __syncthreads(); - } -} - -PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() -{ - // Get pointer to cuTensorMapEncodeTiled - cudaDriverEntryPointQueryResult driver_status; - void *cuTensorMapEncodeTiled_ptr = nullptr; - CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", &cuTensorMapEncodeTiled_ptr, 12000, cudaEnableDefault, &driver_status)); - assert(driver_status == cudaDriverEntryPointSuccess); - - return reinterpret_cast(cuTensorMapEncodeTiled_ptr); -} - - -int main(int argc, char *argv[]) { - uint64_t width = GMEM_WIDTH; - uint64_t height = GMEM_HEIGHT; - std::string opcode = "UTMAPF"; - TestType test_type = TestType::UTMAPF; - int run_iters = DEFAULT_RUN_ITERS; - int opt; - while ((opt = getopt(argc, argv, "w:h:o:i:")) != -1) { - switch (opt) { - case 'w': - width = uint64_t(atoi(optarg)); - break; - case 'h': - height = uint64_t(atoi(optarg)); - break; - case 'o': - opcode = std::string(optarg); - break; - case 'i': - run_iters = atoi(optarg); - break; - default: - fprintf(stderr, "Usage: %s -w -h -o \n", argv[0]); - fprintf(stderr, " Block size: %d x %d\n", SMEM_WIDTH, SMEM_HEIGHT); - fprintf(stderr, " -w : width of the matrix\n"); - fprintf(stderr, " -h : height of the matrix\n"); - fprintf(stderr, " -o : opcode\n"); - fprintf(stderr, " -o UTMAPF: tensor prefetch\n"); - fprintf(stderr, " -o UTMALDG: tensor load async\n"); - fprintf(stderr, " -o UTMASTG: tensor store async\n"); - fprintf(stderr, " -o UTMAREDG: tensor reduce async\n"); - fprintf(stderr, " -i : number of iterations\n"); - return 1; - } - } - - // Check if opcode is valid - if (opcode_map.find(opcode) == opcode_map.end()) { - fprintf(stderr, "Invalid opcode\n"); - return 1; - } - - test_type = opcode_map.at(opcode); - - // Initialize data matrix - int *mat, *out_mat, *d_mat; - CUtensorMap *d_tensor_map; - // height|width_stride must be a multiple of 16 and must be greater than height|width - // Here we make it multiple of SMEM_HEIGHT and SMEM_WIDTH to fit our shmem size - uint64_t height_stride = ((height + (SMEM_HEIGHT - 1)) / SMEM_HEIGHT) * SMEM_HEIGHT; - uint64_t width_stride = ((width + (SMEM_WIDTH - 1)) / SMEM_WIDTH) * SMEM_WIDTH; - printf("height: %lu, width: %lu\n", height, width); - printf("height_stride: %lu, width_stride: %lu\n", height_stride, width_stride); - size_t byte_count = height_stride * width_stride * sizeof(int); - mat = (int *)malloc(byte_count); - out_mat = (int *)malloc(byte_count); - cudaMalloc(&d_mat, byte_count); - cudaMalloc(&d_tensor_map, sizeof(CUtensorMap)); - - // Initialize the matrix, set the values to 0 for out of bounds - uint64_t i = 1; - for (uint64_t r = 0; r < height_stride; r++) { - for (uint64_t c = 0; c < width_stride; c++) { - // If within the tensor, set some unique values - if (r < height && c < width) { - mat[r * width_stride + c] = i; - i++; - } - // Otherwise, set the value to 0 - else { - mat[r * width_stride + c] = 0; - } - } - } - - // TMA tensor map object - CUtensorMap tensor_map{}; - // rank is the number of dimensions of the array. - constexpr uint32_t rank = 2; - // The tensor size - uint64_t size[rank] = {width, height}; - // The stride is the number of bytes to traverse from the first element of one row to the next. - // It must be a multiple of 16. - uint64_t stride[rank - 1] = {width_stride * sizeof(int)}; - // The box_size is the size of the shared memory buffer that is used as the - // destination of a TMA transfer. - uint32_t box_size[rank] = {SMEM_WIDTH, SMEM_HEIGHT}; - // The distance between elements in units of sizeof(element). A stride of 2 - // can be used to load only the real component of a complex-valued tensor, for instance. - uint32_t elem_stride[rank] = {1, 1}; - - // Get a function pointer to the cuTensorMapEncodeTiled driver API. - auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); - - // Create the tensor descriptor. - CUresult res = cuTensorMapEncodeTiled( - &tensor_map, // CUtensorMap *tensorMap, - CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, - rank, // cuuint32_t tensorRank, - d_mat, // void *globalAddress, - size, // const cuuint64_t *globalDim, - stride, // const cuuint64_t *globalStrides, - box_size, // const cuuint32_t *boxDim, - elem_stride, // const cuuint32_t *elementStrides, - // Interleave patterns can be used to accelerate loading of values that - // are less than 4 bytes long. - CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, - // Swizzling can be used to avoid shared memory bank conflicts. - CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, - // L2 Promotion can be used to widen the effect of a cache-policy to a wider - // set of L2 cache lines. - CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, - // Any element that is outside of bounds will be set to zero by the TMA transfer. - CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); - if (res != CUDA_SUCCESS) { - fprintf(stderr, "cuTensorMapEncodeTiled failed with CUresult=%d\n", (int)res); - return 1; - } - cudaMemcpy(d_tensor_map, &tensor_map, sizeof(CUtensorMap), cudaMemcpyHostToDevice); - - // Kernel launch - dim3 grid_dim(width_stride / SMEM_WIDTH, height_stride / SMEM_HEIGHT); - dim3 block_dim(SMEM_WIDTH, SMEM_HEIGHT); - printf("grid_dim: x: %d, y: %d\n", grid_dim.x, grid_dim.y); - printf("block_dim: x: %d, y: %d\n", block_dim.x, block_dim.y); - cudaMemcpy(d_mat, mat, byte_count, cudaMemcpyHostToDevice); - CUDA_SAFECALL((test_kernel<<>>(d_tensor_map, d_mat, test_type, width_stride, run_iters))); - CUDA_SAFECALL(cudaDeviceSynchronize()); - CUDA_SAFECALL(cudaMemcpy(out_mat, d_mat, byte_count, cudaMemcpyDeviceToHost)); - - // Print the matrix to output file - // Dump the values to a file in hex format - // For load operations, use host source array - int *ptr = mat; - - // For store operations, use destination array - if (test_type == TestType::UTMASTG || test_type == TestType::UTMAREDG) { - ptr = out_mat; - } - - char filename[100]; - sprintf(filename, "tma_tensor_test_%s_%lu_%lu.txt", opcode.c_str(), height, width); - FILE *f = fopen(filename, "w"); - for (int i = 0; i < height_stride * width_stride; i++) { - if (i != 0) { - fprintf(f, (i % 512 == 0) ? "\n" : " "); - } - fprintf(f, "0x%x", ptr[i]); - } - fprintf(f, "\n"); - fclose(f); - printf("Values dumped to %s\n", filename); - - // Release device memory - cudaFree(d_tensor_map); - cudaFree(d_mat); - - // Release host memory - free(out_mat); - free(mat); - return 0; -} From d2ef816816c108f6100026eab85613ef4c171468 Mon Sep 17 00:00:00 2001 From: Raunak Dani Date: Tue, 9 Jun 2026 16:39:01 -0400 Subject: [PATCH 15/35] added elect_sync kernel to src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync --- .../ubench/tma/elect_sync/Makefile | 14 ++++++++++ .../ubench/tma/elect_sync/elect_sync.cu | 27 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile new file mode 100644 index 000000000..e2bca61ae --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile @@ -0,0 +1,14 @@ +SRC = elect_sync.cu + +EXE = elect_sync + +# TMA is supported on SM_90a and above +ARCH?=sm_90a sm_100a sm_101 sm_120 +# Unset the CUDA_CPPFLAGS which is set based on CUDA version +# but TMA is only supported on SM_90a and above +CUDA_CPPFLAGS= + +# Generate code for both sm_XXX and compute_XXX (SASS and PTX) +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +include ../../../common/common.mk diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu new file mode 100644 index 000000000..a777a2f05 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu @@ -0,0 +1,27 @@ +#include + +__global__ void elect_sync_kernel() { + unsigned int is_leader; + unsigned int dynamic_mask = __activemask(); //this is current mask -- better for configurability + + asm volatile( + "{\n\t" + " .reg .b32 rx;\n\t" // 1. Declare a dummy 32-bit register 'rx' for threadID + " .reg .pred p;\n\t" // 2. Declare a temporary predicate register 'p' + " elect.sync rx|p, %1;\n\t" // 3. Syntax requires BOTH separated by a '|' + " selp.u32 %0, 1, 0, p;\n\t" // 4. If 'p' is true, set output to 1. Else, 0. (this controls the is_leader variable) + "}" + : "=r"(is_leader) // Output %0 + : "r"(dynamic_mask) // Input %1 + ); + + // Print the result to satisfy the compiler and verify execution (commented out for now since GPGPUsim doesn't support printf) + printf("Thread %02d | Leader Status: %u\n", threadIdx.x, is_leader); +} + + +int main() { + elect_sync_kernel<<<1, 32>>>(); + cudaDeviceSynchronize(); + return 0; +} \ No newline at end of file From 4a0efce86e016123020aefadead3dd4cd272740d Mon Sep 17 00:00:00 2001 From: Raunak Dani Date: Wed, 10 Jun 2026 17:51:12 -0400 Subject: [PATCH 16/35] fix kernel --- src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu index a777a2f05..6b3d775ca 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu @@ -16,7 +16,7 @@ __global__ void elect_sync_kernel() { ); // Print the result to satisfy the compiler and verify execution (commented out for now since GPGPUsim doesn't support printf) - printf("Thread %02d | Leader Status: %u\n", threadIdx.x, is_leader); + // printf("Thread %02d | Leader Status: %u\n", threadIdx.x, is_leader); } From f6340fdd52f2ff0b7059f2e5a0762a42935a2b6f Mon Sep 17 00:00:00 2001 From: Raunak Dani Date: Tue, 16 Jun 2026 23:31:47 -0400 Subject: [PATCH 17/35] added elect.sync kernel, fully updated + tested --- .../ubench/tma/elect_sync/Makefile | 33 +++- .../ubench/tma/elect_sync/elect_sync.cu | 175 +++++++++++++++--- 2 files changed, 179 insertions(+), 29 deletions(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile index e2bca61ae..a9eb5283f 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/Makefile @@ -1,14 +1,31 @@ -SRC = elect_sync.cu +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ -EXE = elect_sync +# elect.sync microbenchmark. +ALL_BENCHES := elect_sync -# TMA is supported on SM_90a and above -ARCH?=sm_90a sm_100a sm_101 sm_120 -# Unset the CUDA_CPPFLAGS which is set based on CUDA version -# but TMA is only supported on SM_90a and above +# elect.sync is a Hopper+ PTX instruction (SM_90a and above). +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. CUDA_CPPFLAGS= -# Generate code for both sm_XXX and compute_XXX (SASS and PTX) +# Generate both SASS and PTX. NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 -include ../../../common/common.mk +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu index 6b3d775ca..1b46b1aae 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/elect_sync/elect_sync.cu @@ -1,27 +1,160 @@ #include +#include -__global__ void elect_sync_kernel() { - unsigned int is_leader; - unsigned int dynamic_mask = __activemask(); //this is current mask -- better for configurability - - asm volatile( - "{\n\t" - " .reg .b32 rx;\n\t" // 1. Declare a dummy 32-bit register 'rx' for threadID - " .reg .pred p;\n\t" // 2. Declare a temporary predicate register 'p' - " elect.sync rx|p, %1;\n\t" // 3. Syntax requires BOTH separated by a '|' - " selp.u32 %0, 1, 0, p;\n\t" // 4. If 'p' is true, set output to 1. Else, 0. (this controls the is_leader variable) - "}" - : "=r"(is_leader) // Output %0 - : "r"(dynamic_mask) // Input %1 - ); - - // Print the result to satisfy the compiler and verify execution (commented out for now since GPGPUsim doesn't support printf) - // printf("Thread %02d | Leader Status: %u\n", threadIdx.x, is_leader); -} +// Verify elect.sync by forcing a *specific* leader lane in each warp. The +// instruction has TWO results we want to check: +// +// 1. The predicate p -> true only for the elected leader lane. +// 2. The register rx -> the laneid of the elected leader, returned to +// EVERY participating thread in the membermask +// (not just the leader). +// +// We test both with one slot per thread in two contiguous buffers: +// - d_leader[tid] : leader writes 500, every other (participating) thread 0. +// - d_rx[tid] : each participating thread writes the laneid it read from +// rx; non-participating threads write the sentinel -1. +// +// elect.sync elects the lowest active lane in the membermask. So to make lane w +// the leader of warp w, we pass a membermask with the low w bits cleared +// (0xFFFFFFFF << w); the lowest remaining active lane is then exactly w: +// warp 0 -> lane 0 leader, warp 1 -> lane 1 leader, ... warp w -> lane w. +// +// After the kernel runs, the host checks each warp's 32-slot region: +// - d_leader: a single 500 at lane == warp id, 0 at the other participants. +// - d_rx : every participating lane (l >= warp id) reads rx == warp id. + +#define WARP_SIZE 32 + +__global__ void elect_sync_verify_kernel(int *d_leader, int *d_rx) { + unsigned int tid = blockIdx.x * blockDim.x + threadIdx.x; + unsigned int warp_id = tid / WARP_SIZE; + unsigned int lane = threadIdx.x % WARP_SIZE; // lane index within the warp + + // We want lane == warp_id to be the elected leader of warp w. + // + // elect.sync requires that EVERY thread executing it is named in the + // membermask, and that all named threads are converged/active. The mask + // 0xFFFFFFFF << warp_id names lanes [warp_id .. 31], so we must guard the + // instruction so that only those lanes actually execute it. Inside the + // branch the converged active set is exactly the masked set, which is the + // well-defined contract for elect.sync. It then elects the lowest active + // lane — which is exactly lane == warp_id. + unsigned int member_mask = 0xFFFFFFFFu << warp_id; + + int value = 0; + int elected_lane = -1; // sentinel for threads that don't participate + if (lane >= warp_id) { + unsigned int is_leader; + unsigned int rx; // laneid of the elected leader + asm volatile( + "{\n\t" + " .reg .pred p;\n\t" // predicate: true only for the leader + " elect.sync %0|p, %2;\n\t" // rx = elected laneid, p = is-leader + " selp.u32 %1, 1, 0, p;\n\t" // is_leader = p ? 1 : 0 + "}" + : "=r"(rx), // %0 + "=r"(is_leader) // %1 + : "r"(member_mask) // %2 + ); + elected_lane = (int)rx; + if (is_leader) value = 500; + } + // Leader writes 500, everyone else writes 0 — contiguous, one slot per thread. + d_leader[tid] = value; + // Each participating thread records the laneid it got back from rx. + d_rx[tid] = elected_lane; +} int main() { - elect_sync_kernel<<<1, 32>>>(); + const int total_warps = 5; // need warp 4 to exist + const int total_threads = total_warps * WARP_SIZE; + + // ── Allocations ────────────────────────────────────────────────────────── + int *h_leader = (int *)malloc(total_threads * sizeof(int)); + int *h_rx = (int *)malloc(total_threads * sizeof(int)); + int *d_leader = NULL; + int *d_rx = NULL; + cudaMalloc(&d_leader, total_threads * sizeof(int)); + cudaMalloc(&d_rx, total_threads * sizeof(int)); + + // Sentinel-fill so any unwritten slot is obvious (0xFF bytes -> -1). + cudaMemset(d_leader, 0xFF, total_threads * sizeof(int)); + cudaMemset(d_rx, 0xFF, total_threads * sizeof(int)); + + // ── Launch (single block, one warp per intended leader lane) ───────────── + elect_sync_verify_kernel<<<1, total_threads>>>(d_leader, d_rx); cudaDeviceSynchronize(); - return 0; -} \ No newline at end of file + + // ── Copy results back ──────────────────────────────────────────────────── + cudaMemcpy(h_leader, d_leader, total_threads * sizeof(int), cudaMemcpyDeviceToHost); + cudaMemcpy(h_rx, d_rx, total_threads * sizeof(int), cudaMemcpyDeviceToHost); + + // ── Dump the raw buffers ───────────────────────────────────────────────── + printf("Leader buffer (one slot per thread, 500 == elected leader):\n"); + for (int w = 0; w < total_warps; ++w) { + printf("warp %d:", w); + for (int l = 0; l < WARP_SIZE; ++l) { + printf(" %d", h_leader[w * WARP_SIZE + l]); + } + printf("\n"); + } + printf("\n"); + + printf("rx buffer (one slot per thread, elected laneid; -1 == did not participate):\n"); + for (int w = 0; w < total_warps; ++w) { + printf("warp %d:", w); + for (int l = 0; l < WARP_SIZE; ++l) { + printf(" %d", h_rx[w * WARP_SIZE + l]); + } + printf("\n"); + } + printf("\n"); + + // ── Host-side verification ─────────────────────────────────────────────── + // Expected leader lane in warp w is lane w. Every participating lane + // (l >= w) must read rx == w; non-participating lanes (l < w) stay -1. + int failures = 0; + for (int w = 0; w < total_warps; ++w) { + int expected_leader = w; + int warp_ok = 1; + for (int l = 0; l < WARP_SIZE; ++l) { + int idx = w * WARP_SIZE + l; + + // Check 1: predicate result via the leader marker. + int v = h_leader[idx]; + int expected_v = (l == expected_leader) ? 500 : 0; + if (v != expected_v) { + printf(" [FAIL] warp %d lane %d: leader marker got %d, expected %d\n", + w, l, v, expected_v); + warp_ok = 0; + failures++; + } + + // Check 2: rx register holds the elected laneid for participants. + int rx = h_rx[idx]; + int expected_rx = (l >= expected_leader) ? expected_leader : -1; + if (rx != expected_rx) { + printf(" [FAIL] warp %d lane %d: rx got %d, expected %d\n", + w, l, rx, expected_rx); + warp_ok = 0; + failures++; + } + } + if (warp_ok) { + printf(" [PASS] warp %d: leader is lane %d (value 500), " + "all participants read rx == %d\n", + w, expected_leader, expected_leader); + } + } + + printf("\n==> %s\n", failures == 0 ? "ALL CHECKS PASSED" : "SOME CHECKS FAILED"); + + // ── Cleanup ────────────────────────────────────────────────────────────── + cudaFree(d_leader); + cudaFree(d_rx); + free(h_leader); + free(h_rx); + + return failures == 0 ? 0 : 1; +} From 688129dbeb775d358b09b88b97b17e8771b1e991 Mon Sep 17 00:00:00 2001 From: jyasuumi <144069912+jyasuumi@users.noreply.github.com> Date: Mon, 22 Jun 2026 01:47:45 -0400 Subject: [PATCH 18/35] Implement TMA tensor microbenchmarks with multiple test cases for tensor loads, prefetching, and reductions. Add new files for UTMALDG, UTMAPF, and UTMAREDG tests, enhancing coverage for TMA operations on rank-2 and rank-3 tensors. Update Makefile to support new benchmarks and streamline build process. --- .../ubench/tma/tma_tensor/Makefile | 39 ++- .../ubench/tma/tma_tensor/utmaldg.cu | 209 ++++++++++++++++ .../ubench/tma/tma_tensor/utmaldg_3d.cu | 229 ++++++++++++++++++ .../ubench/tma/tma_tensor/utmaldg_3d_tx.cu | 220 +++++++++++++++++ .../ubench/tma/tma_tensor/utmapf.cu | 200 +++++++++++++++ .../ubench/tma/tma_tensor/utmapf_3d.cu | 212 ++++++++++++++++ .../ubench/tma/tma_tensor/utmaredg.cu | 194 +++++++++++++++ .../ubench/tma/tma_tensor/utmaredg_3d.cu | 201 +++++++++++++++ .../ubench/tma/tma_tensor/utmastg.cu | 191 +++++++++++++++ .../ubench/tma/tma_tensor/utmastg_3d.cu | 205 ++++++++++++++++ 10 files changed, 1891 insertions(+), 9 deletions(-) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/Makefile index b24b4dac9..9fe44a16c 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/Makefile @@ -1,14 +1,35 @@ -SRC = tma_tensor.cu +# TMA Tensor microbenchmarks +# Builds the monolithic tma_tensor (manual -o flag) plus per-variant standalone +# binaries (utmapf, utmaldg, utmastg, utmaredg) that the CI runner discovers +# and gates automatically. -EXE = tma_tensor +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ -# TMA is supported on SM_90a and above -ARCH?=sm_90a sm_100a sm_101 sm_120 -# Unset the CUDA_CPPFLAGS which is set based on CUDA version -# but TMA is only supported on SM_90a and above -CUDA_CPPFLAGS= +ALL_BENCHES := tma_tensor utmapf utmaldg utmastg utmaredg \ + utmapf_3d utmaldg_3d utmastg_3d utmaredg_3d \ + utmaldg_3d_tx -# Generate code for both sm_XXX and compute_XXX (SASS and PTX) +# TMA tensor requires Hopper-class (sm_90a) or newer. +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Generate both SASS and PTX for each target architecture. NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 -include ../../../common/common.mk +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +# Two-stage build: .cu -> .a (preserves PTX in binary) -> executable +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu new file mode 100644 index 000000000..9b4ff2d2b --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu @@ -0,0 +1,209 @@ +/** + * UTMALDG Unit Test: TMA Tensor Load (Global -> Shared) + * ====================================================== + * Tests that cp.async.bulk.tensor correctly loads a 2D tile from global + * memory into shared memory using a CUtensorMap descriptor. + * + * Strategy: + * 1. Global source matrix initialized with unique values (1, 2, 3, ...) + * 2. Destination buffer (d_dst) poisoned with 0xDEADBEEF + * 3. Kernel: thread (0,0) issues TMA tensor load via mbarrier; all threads wait + * 4. Trusted scalar copy: each thread writes smem_buffer[ty][tx] -> d_dst + * PASS: d_dst matches original source for in-bounds elements + * FAIL_NOP: d_dst still contains poison (TMA load did nothing) + * FAIL_MISMATCH: d_dst has wrong values (descriptor or load produced bad data) + * + * Usage: ./utmaldg [-w ] [-h ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; +using barrier = cuda::barrier; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t SMEM_W = 32; +static constexpr uint32_t SMEM_H = 32; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMALDG(const __grid_constant__ CUtensorMap tensor_map, + int32_t *d_dst, uint32_t width_stride, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[SMEM_H][SMEM_W]; +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + + // bar is intentionally left uninitialized. The functional simulator executes + // TMA loads synchronously so the mbarrier completion path is never taken. + // Calling init() would register the mbarrier in the sim's tracking table + // and trigger a duplicate-registration assertion across multiple CTAs. + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + ptx::cp_async_bulk_tensor( + ptx::space_cluster, ptx::space_global, + &smem_buffer, &tensor_map, {x, y}, + cuda::device::barrier_native_handle(bar)); + } + __syncthreads(); + } + + // Trusted scalar copy: verifies TMA populated smem correctly + uint32_t gx = (uint32_t)x + threadIdx.x; + uint32_t gy = (uint32_t)y + threadIdx.y; + d_dst[gy * width_stride + gx] = smem_buffer[threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_dst, const int32_t *h_src, + uint64_t height, uint64_t width, + uint64_t height_stride, uint64_t width_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t r = 0; r < height; r++) { + for (uint64_t c = 0; c < width; c++) { + int32_t got = h_dst[r * width_stride + c]; + int32_t exp = h_src[r * width_stride + c]; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu]: got 0x%08X, expected %d\n", + r, c, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu]: got %d, expected %d\n", + r, c, got, exp); + mismatch_count++; + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t width = 64, height = 64; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "w:h:i:")) != -1) { + switch (opt) { + case 'w': width = (uint64_t)atoi(optarg); break; + case 'h': height = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-w ] [-h ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t width_stride = ((width + SMEM_W - 1) / SMEM_W) * SMEM_W; + uint64_t height_stride = ((height + SMEM_H - 1) / SMEM_H) * SMEM_H; + size_t bytes = height_stride * width_stride * sizeof(int32_t); + + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + // Source: unique values in-bounds, 0 for padding + int32_t val = 1; + for (uint64_t r = 0; r < height_stride; r++) + for (uint64_t c = 0; c < width_stride; c++) + h_src[r * width_stride + c] = (r < height && c < width) ? val++ : 0; + + // Destination: poisoned + for (size_t i = 0; i < height_stride * width_stride; i++) + h_dst[i] = POISON; + + int32_t *d_mat, *d_dst; + cudaMalloc(&d_mat, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_mat, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Build tensor descriptor + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 2; + uint64_t size[rank] = {width, height}; + uint64_t stride[1] = {width_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {SMEM_W, SMEM_H}; + uint32_t elem_stride[rank] = {1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(width_stride / SMEM_W, height_stride / SMEM_H); + dim3 block(SMEM_W, SMEM_H); + CUDA_SAFECALL((test_UTMALDG<<>>( + tensor_map, d_dst, (uint32_t)width_stride, run_iters))); + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMALDG (TMA Tensor Load: Global -> Shared) ===\n"); + bool pass = verify(h_dst, h_src, height, width, height_stride, width_stride); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_src); free(h_dst); + cudaFree(d_mat); cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu new file mode 100644 index 000000000..6a51d7897 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu @@ -0,0 +1,229 @@ +/** + * UTMALDG_3D Unit Test: TMA Tensor Load, rank-3 (Global -> Shared) + * ================================================================= + * Tests cp.async.bulk.tensor.3d.shared::cluster.global with a rank-3 + * CUtensorMap descriptor, exercising the 3D coordinate path through the + * simulator's cp.async.bulk.tensor handler and tensormap decoder. + * + * Tensor layout: W x H x D (dim0=innermost, dim2=outermost) + * globalDim = {W, H, D} + * boxDim = {TILE_W, TILE_H, TILE_D} + * globalStrides[0] = W_stride * sizeof(elem) (bytes per H-row) + * globalStrides[1] = W_stride * H_stride * sizeof(elem) (bytes per D-plane) + * + * Strategy: + * 1. Global source tensor initialized with unique values (1, 2, 3, ...) + * 2. Destination buffer (d_dst) poisoned with 0xDEADBEEF + * 3. Kernel: thread (0,0,0) issues 3D TMA load; __syncthreads() as fence + * 4. Trusted scalar copy: each thread writes its smem element to d_dst + * PASS: d_dst matches original source for in-bounds elements + * FAIL_NOP: d_dst still contains poison + * FAIL_MISMATCH: d_dst has wrong values + * + * Usage: ./utmaldg_3d [-W ] [-H ] [-D ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; +using barrier = cuda::barrier; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t TILE_W = 32; +static constexpr uint32_t TILE_H = 4; +static constexpr uint32_t TILE_D = 2; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMALDG_3D(const __grid_constant__ CUtensorMap tensor_map, + int32_t *d_dst, + uint32_t w_stride, uint32_t h_stride, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[TILE_D][TILE_H][TILE_W]; +#pragma nv_diag_suppress static_var_with_dynamic_init + __shared__ barrier bar; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + int z = (int)(blockDim.z * blockIdx.z); + + // bar is left uninitialized: functional sim executes TMA synchronously; + // calling init() triggers a multi-CTA mbarrier duplicate-registration assertion. + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + ptx::cp_async_bulk_tensor( + ptx::space_cluster, ptx::space_global, + &smem_buffer, &tensor_map, {x, y, z}, + cuda::device::barrier_native_handle(bar)); + } + __syncthreads(); + } + + // Trusted scalar copy: verify TMA load populated smem correctly + uint32_t gx = (uint32_t)x + threadIdx.x; + uint32_t gy = (uint32_t)y + threadIdx.y; + uint32_t gz = (uint32_t)z + threadIdx.z; + d_dst[gz * h_stride * w_stride + gy * w_stride + gx] = + smem_buffer[threadIdx.z][threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_dst, const int32_t *h_src, + uint64_t depth, uint64_t height, uint64_t width, + uint64_t h_stride, uint64_t w_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t z = 0; z < depth; z++) { + for (uint64_t y = 0; y < height; y++) { + for (uint64_t x = 0; x < width; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + int32_t got = h_dst[idx]; + int32_t exp = h_src[idx]; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu][%lu]: got 0x%08X, expected %d\n", + z, y, x, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu][%lu]: got %d, expected %d\n", + z, y, x, got, exp); + mismatch_count++; + } + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, depth * height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, depth * height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t W = 64, H = 8, D = 4; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "W:H:D:i:")) != -1) { + switch (opt) { + case 'W': W = (uint64_t)atoi(optarg); break; + case 'H': H = (uint64_t)atoi(optarg); break; + case 'D': D = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, + "Usage: %s [-W ] [-H ] [-D ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t w_stride = ((W + TILE_W - 1) / TILE_W) * TILE_W; + uint64_t h_stride = ((H + TILE_H - 1) / TILE_H) * TILE_H; + uint64_t d_stride = ((D + TILE_D - 1) / TILE_D) * TILE_D; + size_t bytes = d_stride * h_stride * w_stride * sizeof(int32_t); + + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + // Source: unique values for in-bounds, 0 for padding + int32_t val = 1; + for (uint64_t z = 0; z < d_stride; z++) + for (uint64_t y = 0; y < h_stride; y++) + for (uint64_t x = 0; x < w_stride; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + h_src[idx] = (z < D && y < H && x < W) ? val++ : 0; + } + + for (size_t i = 0; i < d_stride * h_stride * w_stride; i++) + h_dst[i] = POISON; + + int32_t *d_mat, *d_dst; + cudaMalloc(&d_mat, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_mat, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Build rank-3 tensor descriptor + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 3; + uint64_t size[rank] = {W, H, D}; + uint64_t stride[rank - 1] = {w_stride * sizeof(int32_t), + w_stride * h_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {TILE_W, TILE_H, TILE_D}; + uint32_t elem_stride[rank] = {1, 1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(w_stride / TILE_W, h_stride / TILE_H, d_stride / TILE_D); + dim3 block(TILE_W, TILE_H, TILE_D); + CUDA_SAFECALL((test_UTMALDG_3D<<>>( + tensor_map, d_dst, + (uint32_t)w_stride, (uint32_t)h_stride, run_iters))); + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMALDG_3D (TMA Tensor Load rank-3: Global -> Shared) ===\n"); + bool pass = verify(h_dst, h_src, D, H, W, h_stride, w_stride); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_src); free(h_dst); + cudaFree(d_mat); cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu new file mode 100644 index 000000000..4b48953a9 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu @@ -0,0 +1,220 @@ +/** + * UTMALDG_3D_TX: TMA Tensor Load rank-3 with mbarrier::complete_tx + L2::cache_hint + * ==================================================================================== + * Exercises: + * cp.async.bulk.tensor.3d.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint + * + * This is the exact instruction emitted by CUTLASS's SM90_TMA_LOAD_3D::copy() + * (cute/arch/copy_sm90_tma.hpp) and used by example 48_hopper_warp_specialized_gemm. + * + * Unlike utmaldg_3d.cu (which avoids mbarrier by using __syncthreads), this ubench + * exercises the full mbarrier completion path: + * 1. mbarrier.init.shared::cta.b64 — initialize mbarrier, thread_count=1 + * 2. mbarrier.arrive.expect_tx.shared::cta.b64 — thread 0 arrives + declares expected bytes + * 3. TMA load with .mbarrier::complete_tx::bytes.L2::cache_hint — hw signals mbar on done + * 4. mbarrier.try_wait.parity.shared::cta.b64 — spin until mbarrier complete + * + * Single-CTA launch: avoids the GPGPU-Sim assertion + * "A mbarrier should only exist once in local/remote shmem" + * which fires when multiple CTAs each call mbarrier.init at the same relative smem offset. + * + * PASS: smem_buffer was correctly populated by TMA; scalar readback matches source. + * FAIL: smem_buffer contains poison or wrong values. + * TIMEOUT: mbarrier wait never returns (TMA completion signaling not implemented in sim). + * + * Usage: ./utmaldg_3d_tx + */ + +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t TILE_W = 32; +static constexpr uint32_t TILE_H = 4; +static constexpr uint32_t TILE_D = 2; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMALDG_3D_TX(const __grid_constant__ CUtensorMap tensor_map, + int32_t *d_dst) +{ + __shared__ alignas(128) int32_t smem_buffer[TILE_D][TILE_H][TILE_W]; + // Raw uint64_t mbarrier — same as CUTLASS SM90_TMA_LOAD_3D uses. + // Not cuda::barrier: we replicate the exact CUTLASS pattern with raw PTX. + __shared__ uint64_t mbar; + + const int x = 0, y = 0, z = 0; // single tile at origin (single-CTA launch) + + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + uint32_t smem_buf_addr = static_cast(__cvta_generic_to_shared(&smem_buffer)); + uint32_t smem_mbar_addr = static_cast(__cvta_generic_to_shared(&mbar)); + + // 1. Init mbarrier: 1 expected arrival (thread 0 only, producer-warp pattern) + asm volatile( + "mbarrier.init.shared::cta.b64 [%0], %1;" + :: "r"(smem_mbar_addr), "r"(1) + : "memory"); + + // 2. arrive.expect_tx: thread 0 provides its 1 arrival + declares tx byte count. + // mbarrier is now complete when TMA delivers sizeof(smem_buffer) bytes. + asm volatile( + "mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" + :: "r"(smem_mbar_addr), "r"((uint32_t)sizeof(smem_buffer)) + : "memory"); + + // 3. TMA load: 3D, shared::cluster dst, global src, mbarrier::complete_tx, L2::cache_hint. + // cache_hint=1 is a valid evict-first hint; treated as NOP by GPGPU-Sim. + // Argument order matches CUTLASS SM90_TMA_LOAD_3D (copy_sm90_tma.hpp): + // [smem_dst], [tmap, {crd0, crd1, crd2}], [mbar], cache_hint + uint64_t cache_hint = 1ULL; + asm volatile( + "cp.async.bulk.tensor.3d.shared::cluster.global" + ".mbarrier::complete_tx::bytes.L2::cache_hint" + " [%0], [%1, {%2, %3, %4}], [%5], %6;" + :: "r"(smem_buf_addr), + "l"((uint64_t)&tensor_map), + "r"(x), "r"(y), "r"(z), + "r"(smem_mbar_addr), + "l"(cache_hint) + : "memory"); + + // 4. Spin-wait on mbarrier phase 0 (matches CUTLASS wait_barrier pattern). + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra DONE;\n" + "bra LAB_WAIT;\n" + "DONE:\n" + "}\n" + :: "r"(static_cast(__cvta_generic_to_shared(&mbar))), + "r"(0) // phase 0 (initial phase) + : "memory"); + } + __syncthreads(); + + // Scalar readback: verify TMA populated smem_buffer correctly + d_dst[threadIdx.z * TILE_H * TILE_W + threadIdx.y * TILE_W + threadIdx.x] = + smem_buffer[threadIdx.z][threadIdx.y][threadIdx.x]; +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_dst, const int32_t *h_src, uint32_t n) +{ + int poison_count = 0, mismatch_count = 0; + for (uint32_t i = 0; i < n; i++) { + int32_t got = h_dst[i]; + int32_t exp = h_src[i]; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%u]: got 0x%08X, expected %d\n", + i, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%u]: got %d, expected %d\n", i, got, exp); + mismatch_count++; + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%u elements contain poison (TMA NOP or mbarrier wait skipped)\n", + poison_count, n); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%u elements have wrong values\n", mismatch_count, n); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int, char **) +{ + constexpr uint32_t N = TILE_W * TILE_H * TILE_D; // 256 elements, one tile + size_t bytes = N * sizeof(int32_t); + + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + for (uint32_t i = 0; i < N; i++) h_src[i] = (int32_t)(i + 1); + for (uint32_t i = 0; i < N; i++) h_dst[i] = POISON; + + int32_t *d_src, *d_dst; + cudaMalloc(&d_src, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_src, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Rank-3 descriptor: tensor exactly one tile in size. + // globalDim = {TILE_W, TILE_H, TILE_D} + // globalStrides (bytes between rows): stride[0]=TILE_W*4, stride[1]=TILE_W*TILE_H*4 + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 3; + uint64_t size[rank] = {TILE_W, TILE_H, TILE_D}; + uint64_t stride[rank - 1] = {TILE_W * sizeof(int32_t), + TILE_W * TILE_H * sizeof(int32_t)}; + uint32_t box_size[rank] = {TILE_W, TILE_H, TILE_D}; + uint32_t elem_stride[rank] = {1, 1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_src, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + // Single CTA: grid=(1,1,1), block=(TILE_W, TILE_H, TILE_D) = 256 threads. + // This avoids the GPGPU-Sim mbarrier duplicate-registration assertion + // that fires when multiple CTAs each init a mbarrier at the same relative smem offset. + dim3 grid(1, 1, 1); + dim3 block(TILE_W, TILE_H, TILE_D); + CUDA_SAFECALL((test_UTMALDG_3D_TX<<>>(tensor_map, d_dst))); + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMALDG_3D_TX (TMA rank-3 load: mbarrier::complete_tx + L2::cache_hint) ===\n"); + bool pass = verify(h_dst, h_src, N); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_src); free(h_dst); + cudaFree(d_src); cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu new file mode 100644 index 000000000..73942905f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu @@ -0,0 +1,200 @@ +/** + * UTMAPF Unit Test: TMA Tensor Prefetch (L2 hint) + * ================================================ + * Tests that cp.async.bulk.prefetch.tensor issues without error and does not + * corrupt data. The prefetch is a performance hint that warms the L2 cache; + * correctness is verified by a subsequent scalar readback. + * + * Strategy: + * 1. Source matrix (d_mat) initialized with unique values (1, 2, 3, ...) + * 2. Destination buffer (d_dst) poisoned with 0xDEADBEEF + * 3. Kernel: thread (0,0) issues TMA tensor prefetch for this block's tile + * 4. Scalar readback: each thread copies d_mat[gy*stride+gx] -> d_dst + * PASS: d_dst matches original source (prefetch did not corrupt data) + * FAIL_NOP: d_dst still contains poison (readback did not execute) + * FAIL_MISMATCH: d_dst has wrong values (unexpected corruption) + * + * Usage: ./utmapf [-w ] [-h ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t SMEM_W = 32; +static constexpr uint32_t SMEM_H = 32; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMAPF(const __grid_constant__ CUtensorMap tensor_map, + const int32_t *d_mat, int32_t *d_dst, + uint32_t width_stride, int run_iters) +{ + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + + if (threadIdx.x == 0 && threadIdx.y == 0) { + for (int i = 0; i < run_iters; i++) { + asm volatile( + "cp.async.bulk.prefetch.tensor.2d.L2.global.tile" + " [%0, {%1, %2}];" + : : "l"(&tensor_map), "r"(x), "r"(y) : "memory"); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } + __syncthreads(); + + // Scalar readback: verifies prefetch didn't corrupt the source + uint32_t gx = (uint32_t)x + threadIdx.x; + uint32_t gy = (uint32_t)y + threadIdx.y; + d_dst[gy * width_stride + gx] = d_mat[gy * width_stride + gx]; +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_dst, const int32_t *h_src, + uint64_t height, uint64_t width, + uint64_t height_stride, uint64_t width_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t r = 0; r < height; r++) { + for (uint64_t c = 0; c < width; c++) { + int32_t got = h_dst[r * width_stride + c]; + int32_t exp = h_src[r * width_stride + c]; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu]: got 0x%08X, expected %d\n", + r, c, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu]: got %d, expected %d\n", + r, c, got, exp); + mismatch_count++; + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t width = 64, height = 64; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "w:h:i:")) != -1) { + switch (opt) { + case 'w': width = (uint64_t)atoi(optarg); break; + case 'h': height = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-w ] [-h ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t width_stride = ((width + SMEM_W - 1) / SMEM_W) * SMEM_W; + uint64_t height_stride = ((height + SMEM_H - 1) / SMEM_H) * SMEM_H; + size_t bytes = height_stride * width_stride * sizeof(int32_t); + + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + // Source: unique values in-bounds, 0 for padding + int32_t val = 1; + for (uint64_t r = 0; r < height_stride; r++) + for (uint64_t c = 0; c < width_stride; c++) + h_src[r * width_stride + c] = (r < height && c < width) ? val++ : 0; + + // Destination: poisoned + for (size_t i = 0; i < height_stride * width_stride; i++) + h_dst[i] = POISON; + + int32_t *d_mat, *d_dst; + cudaMalloc(&d_mat, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_mat, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + // Build tensor descriptor (needed to issue prefetch; descriptor encodes the tensor layout) + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 2; + uint64_t size[rank] = {width, height}; + uint64_t stride[1] = {width_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {SMEM_W, SMEM_H}; + uint32_t elem_stride[rank] = {1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(width_stride / SMEM_W, height_stride / SMEM_H); + dim3 block(SMEM_W, SMEM_H); + CUDA_SAFECALL((test_UTMAPF<<>>( + tensor_map, d_mat, d_dst, (uint32_t)width_stride, run_iters))); + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMAPF (TMA Tensor Prefetch + Scalar Readback) ===\n"); + bool pass = verify(h_dst, h_src, height, width, height_stride, width_stride); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_src); free(h_dst); + cudaFree(d_mat); cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu new file mode 100644 index 000000000..8360f1571 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu @@ -0,0 +1,212 @@ +/** + * UTMAPF_3D Unit Test: TMA Tensor Prefetch, rank-3 (L2 hint) + * ============================================================ + * Tests cp.async.bulk.prefetch.tensor.3d.L2.global.tile with a rank-3 + * CUtensorMap descriptor. Prefetch is a performance hint; correctness is + * verified by a subsequent scalar readback. + * + * Strategy: + * 1. Source tensor (d_mat) initialized with unique values (1, 2, 3, ...) + * 2. Destination buffer (d_dst) poisoned with 0xDEADBEEF + * 3. Thread (0,0,0) issues 3D TMA prefetch for this block's tile + * 4. Scalar readback: each thread copies d_mat -> d_dst + * PASS: d_dst matches original source (no corruption) + * FAIL_NOP: d_dst still contains poison (readback did not execute) + * + * Usage: ./utmapf_3d [-W ] [-H ] [-D ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t TILE_W = 32; +static constexpr uint32_t TILE_H = 4; +static constexpr uint32_t TILE_D = 2; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMAPF_3D(const __grid_constant__ CUtensorMap tensor_map, + const int32_t *d_mat, int32_t *d_dst, + uint32_t w_stride, uint32_t h_stride, + int run_iters) +{ + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + int z = (int)(blockDim.z * blockIdx.z); + + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + for (int i = 0; i < run_iters; i++) { + asm volatile( + "cp.async.bulk.prefetch.tensor.3d.L2.global.tile" + " [%0, {%1, %2, %3}];" + : : "l"(&tensor_map), "r"(x), "r"(y), "r"(z) : "memory"); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } + __syncthreads(); + + // Scalar readback: verify prefetch didn't corrupt the source + uint32_t gx = (uint32_t)x + threadIdx.x; + uint32_t gy = (uint32_t)y + threadIdx.y; + uint32_t gz = (uint32_t)z + threadIdx.z; + uint32_t idx = gz * h_stride * w_stride + gy * w_stride + gx; + d_dst[idx] = d_mat[idx]; +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_dst, const int32_t *h_src, + uint64_t depth, uint64_t height, uint64_t width, + uint64_t h_stride, uint64_t w_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t z = 0; z < depth; z++) { + for (uint64_t y = 0; y < height; y++) { + for (uint64_t x = 0; x < width; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + int32_t got = h_dst[idx]; + int32_t exp = h_src[idx]; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu][%lu]: got 0x%08X, expected %d\n", + z, y, x, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu][%lu]: got %d, expected %d\n", + z, y, x, got, exp); + mismatch_count++; + } + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, depth * height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, depth * height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t W = 64, H = 8, D = 4; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "W:H:D:i:")) != -1) { + switch (opt) { + case 'W': W = (uint64_t)atoi(optarg); break; + case 'H': H = (uint64_t)atoi(optarg); break; + case 'D': D = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, + "Usage: %s [-W ] [-H ] [-D ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t w_stride = ((W + TILE_W - 1) / TILE_W) * TILE_W; + uint64_t h_stride = ((H + TILE_H - 1) / TILE_H) * TILE_H; + uint64_t d_stride = ((D + TILE_D - 1) / TILE_D) * TILE_D; + size_t bytes = d_stride * h_stride * w_stride * sizeof(int32_t); + + int32_t *h_src = (int32_t *)malloc(bytes); + int32_t *h_dst = (int32_t *)malloc(bytes); + + int32_t val = 1; + for (uint64_t z = 0; z < d_stride; z++) + for (uint64_t y = 0; y < h_stride; y++) + for (uint64_t x = 0; x < w_stride; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + h_src[idx] = (z < D && y < H && x < W) ? val++ : 0; + } + + for (size_t i = 0; i < d_stride * h_stride * w_stride; i++) + h_dst[i] = POISON; + + int32_t *d_mat, *d_dst; + cudaMalloc(&d_mat, bytes); + cudaMalloc(&d_dst, bytes); + cudaMemcpy(d_mat, h_src, bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_dst, h_dst, bytes, cudaMemcpyHostToDevice); + + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 3; + uint64_t size[rank] = {W, H, D}; + uint64_t stride[rank - 1] = {w_stride * sizeof(int32_t), + w_stride * h_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {TILE_W, TILE_H, TILE_D}; + uint32_t elem_stride[rank] = {1, 1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(w_stride / TILE_W, h_stride / TILE_H, d_stride / TILE_D); + dim3 block(TILE_W, TILE_H, TILE_D); + CUDA_SAFECALL((test_UTMAPF_3D<<>>( + tensor_map, d_mat, d_dst, + (uint32_t)w_stride, (uint32_t)h_stride, run_iters))); + cudaMemcpy(h_dst, d_dst, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMAPF_3D (TMA Tensor Prefetch rank-3 + Scalar Readback) ===\n"); + bool pass = verify(h_dst, h_src, D, H, W, h_stride, w_stride); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_src); free(h_dst); + cudaFree(d_mat); cudaFree(d_dst); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu new file mode 100644 index 000000000..66f574c52 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu @@ -0,0 +1,194 @@ +/** + * UTMAREDG Unit Test: TMA Tensor Reduce (Shared -> Global, MAX) + * ============================================================== + * Tests that cp.reduce.async.bulk.tensor correctly applies an atomic MAX + * reduction from shared memory into global memory via a CUtensorMap descriptor. + * + * Strategy: + * 1. Global matrix (d_mat) initialized to BASE_VALUE=100 + * 2. Kernel: each thread writes a pattern into shared memory. + * Pattern at global (gx, gy): 1 + gx + gy * width_stride + * 3. Thread (0,0) issues TMA tensor reduce MAX: shared -> global + * PASS: d_mat[r][c] == max(BASE_VALUE, 1 + c + r*width_stride) + * FAIL_NOP: d_mat still contains BASE_VALUE for all elements that should + * have been updated (TMA reduce did nothing) + * FAIL_MISMATCH: d_mat has values that differ from the expected max + * + * Usage: ./utmaredg [-w ] [-h ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t SMEM_W = 32; +static constexpr uint32_t SMEM_H = 32; +static constexpr int32_t BASE_VALUE = 100; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMAREDG(const __grid_constant__ CUtensorMap tensor_map, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[SMEM_H][SMEM_W]; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + + // Each thread fills smem with a globally-unique pattern + int gx = x + (int)threadIdx.x; + int gy = y + (int)threadIdx.y; + smem_buffer[threadIdx.y][threadIdx.x] = + 1 + gx + gy * (int)(blockDim.x * gridDim.x); + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + ptx::cp_reduce_async_bulk_tensor( + ptx::space_global, ptx::space_shared, + ptx::op_max, &tensor_map, {x, y}, &smem_buffer); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_out, + uint64_t height, uint64_t width, uint64_t width_stride) +{ + int nop_count = 0, mismatch_count = 0; + for (uint64_t r = 0; r < height; r++) { + for (uint64_t c = 0; c < width; c++) { + int32_t got = h_out[r * width_stride + c]; + int32_t pattern = 1 + (int32_t)c + (int32_t)r * (int32_t)width_stride; + int32_t exp = (pattern > BASE_VALUE) ? pattern : BASE_VALUE; + // NOP: element should have been raised above BASE_VALUE but wasn't + bool should_update = (pattern > BASE_VALUE); + if (should_update && got == BASE_VALUE) { + if (nop_count == 0) + printf(" NOP at [%lu][%lu]: got %d, expected %d (pattern=%d)\n", + r, c, got, exp, pattern); + nop_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu]: got %d, expected %d\n", + r, c, got, exp); + mismatch_count++; + } + } + } + if (nop_count > 0) { + printf(" FAIL: %d/%lu elements not updated (TMA reduce was NOP)\n", + nop_count, height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t width = 64, height = 64; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "w:h:i:")) != -1) { + switch (opt) { + case 'w': width = (uint64_t)atoi(optarg); break; + case 'h': height = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-w ] [-h ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t width_stride = ((width + SMEM_W - 1) / SMEM_W) * SMEM_W; + uint64_t height_stride = ((height + SMEM_H - 1) / SMEM_H) * SMEM_H; + size_t bytes = height_stride * width_stride * sizeof(int32_t); + + // Host buffer: all BASE_VALUE; TMA reduce MAX will raise values where pattern > BASE_VALUE + int32_t *h_buf = (int32_t *)malloc(bytes); + for (size_t i = 0; i < height_stride * width_stride; i++) + h_buf[i] = BASE_VALUE; + + int32_t *d_mat; + cudaMalloc(&d_mat, bytes); + cudaMemcpy(d_mat, h_buf, bytes, cudaMemcpyHostToDevice); + + // Build tensor descriptor + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 2; + uint64_t size[rank] = {width, height}; + uint64_t stride[1] = {width_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {SMEM_W, SMEM_H}; + uint32_t elem_stride[rank] = {1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(width_stride / SMEM_W, height_stride / SMEM_H); + dim3 block(SMEM_W, SMEM_H); + CUDA_SAFECALL((test_UTMAREDG<<>>(tensor_map, run_iters))); + cudaMemcpy(h_buf, d_mat, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMAREDG (TMA Tensor Reduce MAX: Shared -> Global) ===\n"); + bool pass = verify(h_buf, height, width, width_stride); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_buf); + cudaFree(d_mat); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu new file mode 100644 index 000000000..f8d28dbe0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu @@ -0,0 +1,201 @@ +/** + * UTMAREDG_3D Unit Test: TMA Tensor Reduce MAX, rank-3 (Shared -> Global) + * ========================================================================= + * Tests cp.reduce.async.bulk.tensor.3d.global.shared::cta.op::max with a + * rank-3 CUtensorMap descriptor. + * + * Strategy: + * 1. d_mat initialized to BASE_VALUE=100 for all elements + * 2. Each thread writes pattern (1 + linear_index) into smem + * 3. Thread (0,0,0) issues 3D TMA reduce MAX: global = max(global, shared) + * PASS: d_mat[idx] == max(100, 1 + idx) for in-bounds elements + * FAIL_NOP: elements that should increase above 100 still equal 100 + * FAIL_MISMATCH: wrong values + * + * Usage: ./utmaredg_3d [-W ] [-H ] [-D ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t TILE_W = 32; +static constexpr uint32_t TILE_H = 4; +static constexpr uint32_t TILE_D = 2; +static constexpr int32_t BASE_VALUE = 100; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMAREDG_3D(const __grid_constant__ CUtensorMap tensor_map, + uint32_t w_stride, uint32_t h_stride, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[TILE_D][TILE_H][TILE_W]; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + int z = (int)(blockDim.z * blockIdx.z); + + int gx = x + (int)threadIdx.x; + int gy = y + (int)threadIdx.y; + int gz = z + (int)threadIdx.z; + smem_buffer[threadIdx.z][threadIdx.y][threadIdx.x] = + 1 + gx + gy * (int)w_stride + gz * (int)(h_stride * w_stride); + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + ptx::cp_reduce_async_bulk_tensor( + ptx::space_global, ptx::space_shared, + ptx::op_max, &tensor_map, {x, y, z}, &smem_buffer); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_out, + uint64_t depth, uint64_t height, uint64_t width, + uint64_t h_stride, uint64_t w_stride) +{ + int nop_count = 0, mismatch_count = 0; + for (uint64_t z = 0; z < depth; z++) { + for (uint64_t y = 0; y < height; y++) { + for (uint64_t x = 0; x < width; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + int32_t got = h_out[idx]; + int32_t pattern = 1 + (int32_t)idx; + int32_t exp = (pattern > BASE_VALUE) ? pattern : BASE_VALUE; + bool should_update = (pattern > BASE_VALUE); + if (should_update && got == BASE_VALUE) { + if (nop_count == 0) + printf(" NOP at [%lu][%lu][%lu]: got %d, expected %d\n", + z, y, x, got, exp); + nop_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu][%lu]: got %d, expected %d\n", + z, y, x, got, exp); + mismatch_count++; + } + } + } + } + if (nop_count > 0) { + printf(" FAIL: %d/%lu elements not updated (TMA reduce was NOP)\n", + nop_count, depth * height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, depth * height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t W = 64, H = 8, D = 4; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "W:H:D:i:")) != -1) { + switch (opt) { + case 'W': W = (uint64_t)atoi(optarg); break; + case 'H': H = (uint64_t)atoi(optarg); break; + case 'D': D = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, + "Usage: %s [-W ] [-H ] [-D ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t w_stride = ((W + TILE_W - 1) / TILE_W) * TILE_W; + uint64_t h_stride = ((H + TILE_H - 1) / TILE_H) * TILE_H; + uint64_t d_stride = ((D + TILE_D - 1) / TILE_D) * TILE_D; + size_t bytes = d_stride * h_stride * w_stride * sizeof(int32_t); + + int32_t *h_buf = (int32_t *)malloc(bytes); + for (size_t i = 0; i < d_stride * h_stride * w_stride; i++) + h_buf[i] = BASE_VALUE; + + int32_t *d_mat; + cudaMalloc(&d_mat, bytes); + cudaMemcpy(d_mat, h_buf, bytes, cudaMemcpyHostToDevice); + + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 3; + uint64_t size[rank] = {W, H, D}; + uint64_t stride[rank - 1] = {w_stride * sizeof(int32_t), + w_stride * h_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {TILE_W, TILE_H, TILE_D}; + uint32_t elem_stride[rank] = {1, 1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(w_stride / TILE_W, h_stride / TILE_H, d_stride / TILE_D); + dim3 block(TILE_W, TILE_H, TILE_D); + CUDA_SAFECALL((test_UTMAREDG_3D<<>>( + tensor_map, (uint32_t)w_stride, (uint32_t)h_stride, run_iters))); + cudaMemcpy(h_buf, d_mat, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMAREDG_3D (TMA Tensor Reduce MAX rank-3: Shared -> Global) ===\n"); + bool pass = verify(h_buf, D, H, W, h_stride, w_stride); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_buf); + cudaFree(d_mat); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu new file mode 100644 index 000000000..fdd0bf9d3 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu @@ -0,0 +1,191 @@ +/** + * UTMASTG Unit Test: TMA Tensor Store (Shared -> Global) + * ======================================================= + * Tests that cp.async.bulk.tensor correctly stores a 2D tile from shared + * memory into global memory using a CUtensorMap descriptor. + * + * Strategy: + * 1. Global destination matrix (d_mat) poisoned with 0xDEADBEEF + * 2. Kernel: each thread writes a unique pattern into shared memory + * 3. Thread (0,0) issues TMA tensor store: shared -> global + * PASS: d_mat contains expected pattern for in-bounds elements + * FAIL_NOP: d_mat still contains poison (TMA store did nothing) + * FAIL_MISMATCH: d_mat has wrong values (descriptor or store produced bad data) + * + * Expected pattern at global position (gx, gy): 1 + gx + gy * width_stride + * + * Usage: ./utmastg [-w ] [-h ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t SMEM_W = 32; +static constexpr uint32_t SMEM_H = 32; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMASTG(const __grid_constant__ CUtensorMap tensor_map, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[SMEM_H][SMEM_W]; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + + // Each thread fills smem with a globally-unique pattern + int gx = x + (int)threadIdx.x; + int gy = y + (int)threadIdx.y; + smem_buffer[threadIdx.y][threadIdx.x] = + 1 + gx + gy * (int)(blockDim.x * gridDim.x); + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0) { + ptx::cp_async_bulk_tensor( + ptx::space_global, ptx::space_shared, + &tensor_map, {x, y}, &smem_buffer); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_out, + uint64_t height, uint64_t width, uint64_t width_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t r = 0; r < height; r++) { + for (uint64_t c = 0; c < width; c++) { + int32_t got = h_out[r * width_stride + c]; + int32_t exp = 1 + (int32_t)c + (int32_t)r * (int32_t)width_stride; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu]: got 0x%08X, expected %d\n", + r, c, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu]: got %d, expected %d\n", + r, c, got, exp); + mismatch_count++; + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t width = 64, height = 64; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "w:h:i:")) != -1) { + switch (opt) { + case 'w': width = (uint64_t)atoi(optarg); break; + case 'h': height = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, "Usage: %s [-w ] [-h ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t width_stride = ((width + SMEM_W - 1) / SMEM_W) * SMEM_W; + uint64_t height_stride = ((height + SMEM_H - 1) / SMEM_H) * SMEM_H; + size_t bytes = height_stride * width_stride * sizeof(int32_t); + + // Host buffer: poisoned (TMA store must overwrite) + int32_t *h_buf = (int32_t *)malloc(bytes); + for (size_t i = 0; i < height_stride * width_stride; i++) + h_buf[i] = POISON; + + int32_t *d_mat; + cudaMalloc(&d_mat, bytes); + cudaMemcpy(d_mat, h_buf, bytes, cudaMemcpyHostToDevice); + + // Build tensor descriptor + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 2; + uint64_t size[rank] = {width, height}; + uint64_t stride[1] = {width_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {SMEM_W, SMEM_H}; + uint32_t elem_stride[rank] = {1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(width_stride / SMEM_W, height_stride / SMEM_H); + dim3 block(SMEM_W, SMEM_H); + CUDA_SAFECALL((test_UTMASTG<<>>(tensor_map, run_iters))); + cudaMemcpy(h_buf, d_mat, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMASTG (TMA Tensor Store: Shared -> Global) ===\n"); + bool pass = verify(h_buf, height, width, width_stride); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_buf); + cudaFree(d_mat); + return pass ? 0 : 1; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu new file mode 100644 index 000000000..107d77f64 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu @@ -0,0 +1,205 @@ +/** + * UTMASTG_3D Unit Test: TMA Tensor Store, rank-3 (Shared -> Global) + * ================================================================== + * Tests cp.async.bulk.tensor.3d.global.shared::cta with a rank-3 + * CUtensorMap descriptor, exercising the 3D coordinate store path. + * + * Tensor layout: W x H x D (dim0=innermost) + * globalStrides[0] = W_stride * sizeof(elem) + * globalStrides[1] = W_stride * H_stride * sizeof(elem) + * + * Strategy: + * 1. Global destination (d_mat) poisoned with 0xDEADBEEF + * 2. Each thread writes a globally-unique pattern into smem + * pattern at (gx, gy, gz): 1 + gx + gy*W_stride + gz*H_stride*W_stride + * (equals 1 + linear_index for this layout) + * 3. Thread (0,0,0) issues 3D TMA store: shared -> global + * PASS: d_mat[idx] == 1 + idx for in-bounds elements + * FAIL_NOP: d_mat still contains poison + * FAIL_MISMATCH: wrong values + * + * Usage: ./utmastg_3d [-W ] [-H ] [-D ] [-i ] + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CUDA_SAFECALL(call) \ + { \ + call; \ + cudaError err = cudaGetLastError(); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in '%s' at %s:%d : %s.\n", \ + #call, __FILE__, __LINE__, cudaGetErrorString(err)); \ + fflush(stderr); exit(EXIT_FAILURE); \ + } \ + } + +namespace ptx = cuda::ptx; + +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900 +static_assert(false, "Requires sm_90a or newer for TMA tensor operations."); +#endif + +static constexpr uint32_t TILE_W = 32; +static constexpr uint32_t TILE_H = 4; +static constexpr uint32_t TILE_D = 2; +static constexpr int32_t POISON = (int32_t)0xDEADBEEF; + +// ============================================================================ +// Kernel +// ============================================================================ +__global__ void test_UTMASTG_3D(const __grid_constant__ CUtensorMap tensor_map, + uint32_t w_stride, uint32_t h_stride, + int run_iters) +{ + __shared__ alignas(128) int32_t smem_buffer[TILE_D][TILE_H][TILE_W]; + + int x = (int)(blockDim.x * blockIdx.x); + int y = (int)(blockDim.y * blockIdx.y); + int z = (int)(blockDim.z * blockIdx.z); + + int gx = x + (int)threadIdx.x; + int gy = y + (int)threadIdx.y; + int gz = z + (int)threadIdx.z; + smem_buffer[threadIdx.z][threadIdx.y][threadIdx.x] = + 1 + gx + gy * (int)w_stride + gz * (int)(h_stride * w_stride); + + ptx::fence_proxy_async(ptx::space_shared); + __syncthreads(); + + for (int i = 0; i < run_iters; i++) { + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + ptx::cp_async_bulk_tensor( + ptx::space_global, ptx::space_shared, + &tensor_map, {x, y, z}, &smem_buffer); + ptx::cp_async_bulk_commit_group(); + ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>()); + } + } +} + +// ============================================================================ +// Helpers +// ============================================================================ +static PFN_cuTensorMapEncodeTiled_v12000 get_cuTensorMapEncodeTiled() +{ + cudaDriverEntryPointQueryResult status; + void *fn = nullptr; + CUDA_SAFECALL(cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", + &fn, 12000, cudaEnableDefault, &status)); + assert(status == cudaDriverEntryPointSuccess); + return reinterpret_cast(fn); +} + +// ============================================================================ +// Verification +// ============================================================================ +static bool verify(const int32_t *h_out, + uint64_t depth, uint64_t height, uint64_t width, + uint64_t h_stride, uint64_t w_stride) +{ + int poison_count = 0, mismatch_count = 0; + for (uint64_t z = 0; z < depth; z++) { + for (uint64_t y = 0; y < height; y++) { + for (uint64_t x = 0; x < width; x++) { + uint64_t idx = z * h_stride * w_stride + y * w_stride + x; + int32_t got = h_out[idx]; + int32_t exp = 1 + (int32_t)idx; + if (got == POISON) { + if (poison_count == 0) + printf(" NOP at [%lu][%lu][%lu]: got 0x%08X, expected %d\n", + z, y, x, (unsigned)got, exp); + poison_count++; + } else if (got != exp) { + if (mismatch_count == 0) + printf(" Mismatch at [%lu][%lu][%lu]: got %d, expected %d\n", + z, y, x, got, exp); + mismatch_count++; + } + } + } + } + if (poison_count > 0) { + printf(" FAIL: %d/%lu elements still contain poison\n", + poison_count, depth * height * width); + return false; + } + if (mismatch_count > 0) { + printf(" FAIL: %d/%lu elements have wrong values\n", + mismatch_count, depth * height * width); + return false; + } + return true; +} + +// ============================================================================ +// Main +// ============================================================================ +int main(int argc, char *argv[]) +{ + uint64_t W = 64, H = 8, D = 4; + int run_iters = 1; + int opt; + while ((opt = getopt(argc, argv, "W:H:D:i:")) != -1) { + switch (opt) { + case 'W': W = (uint64_t)atoi(optarg); break; + case 'H': H = (uint64_t)atoi(optarg); break; + case 'D': D = (uint64_t)atoi(optarg); break; + case 'i': run_iters = atoi(optarg); break; + default: + fprintf(stderr, + "Usage: %s [-W ] [-H ] [-D ] [-i ]\n", + argv[0]); + return 1; + } + } + + uint64_t w_stride = ((W + TILE_W - 1) / TILE_W) * TILE_W; + uint64_t h_stride = ((H + TILE_H - 1) / TILE_H) * TILE_H; + uint64_t d_stride = ((D + TILE_D - 1) / TILE_D) * TILE_D; + size_t bytes = d_stride * h_stride * w_stride * sizeof(int32_t); + + int32_t *h_buf = (int32_t *)malloc(bytes); + for (size_t i = 0; i < d_stride * h_stride * w_stride; i++) + h_buf[i] = POISON; + + int32_t *d_mat; + cudaMalloc(&d_mat, bytes); + cudaMemcpy(d_mat, h_buf, bytes, cudaMemcpyHostToDevice); + + CUtensorMap tensor_map{}; + constexpr uint32_t rank = 3; + uint64_t size[rank] = {W, H, D}; + uint64_t stride[rank - 1] = {w_stride * sizeof(int32_t), + w_stride * h_stride * sizeof(int32_t)}; + uint32_t box_size[rank] = {TILE_W, TILE_H, TILE_D}; + uint32_t elem_stride[rank] = {1, 1, 1}; + auto cuTensorMapEncodeTiled = get_cuTensorMapEncodeTiled(); + cuTensorMapEncodeTiled( + &tensor_map, CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_INT32, + rank, d_mat, size, stride, box_size, elem_stride, + CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_NONE, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE, + CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + + dim3 grid(w_stride / TILE_W, h_stride / TILE_H, d_stride / TILE_D); + dim3 block(TILE_W, TILE_H, TILE_D); + CUDA_SAFECALL((test_UTMASTG_3D<<>>( + tensor_map, (uint32_t)w_stride, (uint32_t)h_stride, run_iters))); + cudaMemcpy(h_buf, d_mat, bytes, cudaMemcpyDeviceToHost); + + printf("=== UTMASTG_3D (TMA Tensor Store rank-3: Shared -> Global) ===\n"); + bool pass = verify(h_buf, D, H, W, h_stride, w_stride); + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + + free(h_buf); + cudaFree(d_mat); + return pass ? 0 : 1; +} From a8b21b8f92a43d439e26d9ef57a938f6fb94c2c7 Mon Sep 17 00:00:00 2001 From: jyasuumi <144069912+jyasuumi@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:45:19 -0400 Subject: [PATCH 19/35] Update result output formatting in microbenchmark files to use "PASSED" and "FAILED" so CI respects output --- .../ubench/tma/mbarrier/mbarrier_reduction.cu | 1 + .../ubench/tma/mbarrier/mbarrier_reduction2.cu | 1 + src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_g_s.cu | 2 +- src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_s_g.cu | 2 +- src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu | 2 +- src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu | 2 +- src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu | 2 +- src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu | 2 +- .../GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu | 2 +- src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu | 2 +- src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu | 2 +- src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu | 2 +- .../GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu | 2 +- src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu | 2 +- src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu | 2 +- 15 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu index 4c40a89f0..5ee7702d7 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction.cu @@ -112,6 +112,7 @@ int main() { printf("FAILURE: Results mismatch.\n"); printf("Likely Cause: Threads raced ahead without waiting.\n"); } + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); // Cleanup free(h_in); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu index 9357de736..7dbe47617 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_reduction2.cu @@ -134,6 +134,7 @@ int main() { printf("FAILURE: Results mismatch.\n"); printf("Likely Cause: Threads raced ahead without waiting.\n"); } + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); // Cleanup free(h_in); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_g_s.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_g_s.cu index 60c000980..f4086c4ab 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_g_s.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_g_s.cu @@ -150,7 +150,7 @@ int main(int argc, char *argv[]) // Verify printf("=== UBLKCP_G_S (TMA Bulk Store: Shared -> Global) ===\n"); bool pass = verify(h_result, n); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_result); cudaFree(d_a); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_s_g.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_s_g.cu index 143415d4f..1dbd13d07 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_s_g.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkcp_s_g.cu @@ -165,7 +165,7 @@ int main(int argc, char *argv[]) // Verify printf("=== UBLKCP_S_G (TMA Bulk Load: Global -> Shared) ===\n"); bool pass = verify(h_src, h_dst, n); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_src); free(h_dst); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu index acd5ce4aa..002bcb4f0 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkpf.cu @@ -212,7 +212,7 @@ int main(int argc, char *argv[]) printf(" Iterations: %d\n", run_iters); bool pass = verify(h_src, h_dst, (int)verified_elems); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); // Dump results to file for easy expected-vs-actual comparison char filename[128]; diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu index 4ba1c2bbf..2a721d195 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_bulk/ublkred_g_s.cu @@ -273,7 +273,7 @@ static bool run_single_reduction(ReductionKind kind, int n, int run_iters) // Verify printf("=== UBLKRED_G_S (TMA Bulk Reduction: %s) ===\n", reduction_kind_name(kind)); bool pass = verify_reduction(h_data, n, kind, run_iters); - printf("RESULT: %s\n\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n\n", pass ? "PASSED" : "FAILED"); free(h_data); cudaFree(d_a); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu index 9b4ff2d2b..55dce1054 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg.cu @@ -201,7 +201,7 @@ int main(int argc, char *argv[]) printf("=== UTMALDG (TMA Tensor Load: Global -> Shared) ===\n"); bool pass = verify(h_dst, h_src, height, width, height_stride, width_stride); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_src); free(h_dst); cudaFree(d_mat); cudaFree(d_dst); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu index 6a51d7897..62c0972f2 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d.cu @@ -221,7 +221,7 @@ int main(int argc, char *argv[]) printf("=== UTMALDG_3D (TMA Tensor Load rank-3: Global -> Shared) ===\n"); bool pass = verify(h_dst, h_src, D, H, W, h_stride, w_stride); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_src); free(h_dst); cudaFree(d_mat); cudaFree(d_dst); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu index 4b48953a9..966379266 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaldg_3d_tx.cu @@ -212,7 +212,7 @@ int main(int, char **) printf("=== UTMALDG_3D_TX (TMA rank-3 load: mbarrier::complete_tx + L2::cache_hint) ===\n"); bool pass = verify(h_dst, h_src, N); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_src); free(h_dst); cudaFree(d_src); cudaFree(d_dst); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu index 73942905f..4bd35aa78 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf.cu @@ -192,7 +192,7 @@ int main(int argc, char *argv[]) printf("=== UTMAPF (TMA Tensor Prefetch + Scalar Readback) ===\n"); bool pass = verify(h_dst, h_src, height, width, height_stride, width_stride); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_src); free(h_dst); cudaFree(d_mat); cudaFree(d_dst); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu index 8360f1571..20b1302dc 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmapf_3d.cu @@ -204,7 +204,7 @@ int main(int argc, char *argv[]) printf("=== UTMAPF_3D (TMA Tensor Prefetch rank-3 + Scalar Readback) ===\n"); bool pass = verify(h_dst, h_src, D, H, W, h_stride, w_stride); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_src); free(h_dst); cudaFree(d_mat); cudaFree(d_dst); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu index 66f574c52..1dc3ed50e 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg.cu @@ -186,7 +186,7 @@ int main(int argc, char *argv[]) printf("=== UTMAREDG (TMA Tensor Reduce MAX: Shared -> Global) ===\n"); bool pass = verify(h_buf, height, width, width_stride); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_buf); cudaFree(d_mat); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu index f8d28dbe0..c459b6129 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmaredg_3d.cu @@ -193,7 +193,7 @@ int main(int argc, char *argv[]) printf("=== UTMAREDG_3D (TMA Tensor Reduce MAX rank-3: Shared -> Global) ===\n"); bool pass = verify(h_buf, D, H, W, h_stride, w_stride); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_buf); cudaFree(d_mat); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu index fdd0bf9d3..7253b1ef6 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg.cu @@ -183,7 +183,7 @@ int main(int argc, char *argv[]) printf("=== UTMASTG (TMA Tensor Store: Shared -> Global) ===\n"); bool pass = verify(h_buf, height, width, width_stride); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_buf); cudaFree(d_mat); diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu index 107d77f64..784a2bf2c 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/tma_tensor/utmastg_3d.cu @@ -197,7 +197,7 @@ int main(int argc, char *argv[]) printf("=== UTMASTG_3D (TMA Tensor Store rank-3: Shared -> Global) ===\n"); bool pass = verify(h_buf, D, H, W, h_stride, w_stride); - printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + printf("RESULT: %s\n", pass ? "PASSED" : "FAILED"); free(h_buf); cudaFree(d_mat); From a269690b09600c209aea2c52e893c77d25949c38 Mon Sep 17 00:00:00 2001 From: Ivorchu <41135347+Ivorchu@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:12:22 -0400 Subject: [PATCH 20/35] cutlass48 specific testbench --- .../ubench/wgmma/wgmma/Makefile | 4 +- .../ubench/wgmma/wgmma/wgmma_tf32_128.cu | 258 ++++++++++++++++++ 2 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_tf32_128.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile index f43d1c4a5..f9bbbf0f5 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile @@ -3,8 +3,8 @@ BIN_DIR := $(BASE_DIR)/../../../bin/ CUTLASS_ROOT ?= $(BASE_DIR)/../../../../cutlass-bench # WGMMA microbenchmarks. -# wgmma_no_tma uses CuTe (CUTLASS) headers; wgmma_verify is standalone. -ALL_BENCHES := wgmma_verify wgmma_no_tma +# wgmma_no_tma / wgmma_tf32_128 use CuTe (CUTLASS) headers; wgmma_verify is standalone. +ALL_BENCHES := wgmma_verify wgmma_no_tma wgmma_tf32_128 # WGMMA requires Hopper (SM90a). ARCH ?= sm_90a diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_tf32_128.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_tf32_128.cu new file mode 100644 index 000000000..6501babda --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_tf32_128.cu @@ -0,0 +1,258 @@ +// ============================================================================= +// wgmma_tf32_128.cu +// +// Functional-correctness microbenchmark for the Hopper WGMMA F32+=TF32*TF32 +// atom at its native N=128 tile size — the shape/dtype combo CUTLASS example +// 48 (Hopper warp-specialized GEMM) uses for its TF32 kernel: +// +// wgmma.mma_async.sync.aligned.m64n128k8.f32.tf32.tf32 +// wgmma.fence.sync.aligned +// wgmma.commit_group.sync.aligned +// wgmma.wait_group.sync.aligned +// +// wgmma_verify.cu hand-writes wgmma PTX with an explicit output-operand list, +// which is only practical up to N=16 (8 D-registers/thread). The m64n128k8 +// atom has 64 D-registers/thread, so this benchmark instead goes through +// CUTLASS/CuTe's MMA_64x128x8_F32TF32TF32_SS_TN atom +// (cute/arch/mma_sm90_gmma.hpp) — the compiler emits the real instruction +// with all 64 accumulator registers wired up, and CuTe's partition_C() +// handles the fragment-to-global-(m,n) mapping so no hand-derived layout +// formula is needed (unlike wgmma_verify.cu's d_frag_pos()). +// +// GEMM shape: D[M×N] = A[M×K] * B^T[N×K] (TN / K-major smem, single tile, +// exactly the atom's native shape — one wgmma call, no K-loop) +// M=64 N=128 K=8 +// +// Smem fill is single-threaded (thread 0 only), same workaround as +// wgmma_no_tma.cu: GPGPU-Sim's sequential execution resolves intra-warp smem +// write ordering differently than real H100 concurrent execution, so a +// TiledCopy-based fill can produce spurious conflicts. +// +// Numeric note: GPGPU-Sim's tf32 functional model currently reads operands +// as full-precision float32 (no mantissa truncation on read — see +// quantize_to_type()/element_to_double() in instructions.cc). Real hardware +// truncates to 10 mantissa bits inside the tensor core. To stay correct +// against both, this test follows wgmma_verify.cu's convention: store raw +// (untruncated) float32 bits in the tf32 smem buffers, but compute the host +// reference from tf32-truncated inputs, and use the same loose tolerance +// (1e-3 absolute / 1% relative) that already absorbs the difference. +// +// Usage: +// ./wgmma_tf32_128 – real H100 +// PTX_SIM_USE_PTX_FILE=1 ./wgmma_tf32_128 – GPGPU-Sim functional sim +// ============================================================================= + +#include +#include +#include +#include +#include + +#include + +#include "cutlass/util/print_error.hpp" +#include "cutlass/util/helper_cuda.hpp" + +using namespace cute; + +static constexpr int M = 64, N = 128, K = 8; + +// --------------------------------------------------------------------------- +// Round float to the nearest tf32-representable value (10 mantissa bits). +// Matches wgmma_verify.cu's quant_tf32(); used only for the host reference. +// --------------------------------------------------------------------------- +static double quant_tf32(double f) { + float fp = (float)f; + uint32_t b; memcpy(&b, &fp, 4); + b &= 0xFFFFE000u; + memcpy(&fp, &b, 4); + return (double)fp; +} + +// ============================================================================ +// Shared memory (K-major, no swizzle — GMMA::Layout_K_INTER_Atom) +// ============================================================================ +template +struct SharedStorage { + alignas(128) cute::ArrayEngine> A; + alignas(128) cute::ArrayEngine> B; +}; + +// ============================================================================ +// Kernel: single-tile GEMM — exactly one wgmma.m64n128k8 call. +// ============================================================================ +template +__global__ static +__launch_bounds__(decltype(size(TiledMma{}))::value) +void wgmma_tf32_128_kernel(tfloat32_t const* A_g, tfloat32_t const* B_g, + float* C_g, uint32_t* clk_g, + TiledMma mma, ASmemLayout sA_layout, BSmemLayout sB_layout) +{ + extern __shared__ char shared_memory[]; + using SS = SharedStorage; + SS& smem = *reinterpret_cast(shared_memory); + Tensor sA = make_tensor(make_smem_ptr(smem.A.begin()), sA_layout); // (M,K) + Tensor sB = make_tensor(make_smem_ptr(smem.B.begin()), sB_layout); // (N,K) + + Tensor gA = make_tensor(make_gmem_ptr(A_g), make_shape(Int{}, Int{}), + make_stride(Int{}, Int<1>{})); // row-major (M,K) + Tensor gB = make_tensor(make_gmem_ptr(B_g), make_shape(Int{}, Int{}), + make_stride(Int{}, Int<1>{})); // row-major (N,K) == B^T + Tensor gC = make_tensor(make_gmem_ptr(C_g), make_shape(Int{}, Int{}), + make_stride(Int{}, Int<1>{})); // row-major (M,N) + + const int tid = threadIdx.x; + if (tid == 0) { uint32_t c; asm volatile("mov.u32 %0,%%clock;\n" : "=r"(c)); clk_g[0] = c; } + + // Single-threaded smem fill (see file header for why). + if (tid == 0) { + for (int m = 0; m < M; ++m) for (int k = 0; k < K; ++k) sA(m, k) = gA(m, k); + for (int n = 0; n < N; ++n) for (int k = 0; k < K; ++k) sB(n, k) = gB(n, k); + } + __syncthreads(); + if (tid == 0) { uint32_t c; asm volatile("mov.u32 %0,%%clock;\n" : "=r"(c)); clk_g[1] = c; } + + ThrMMA thr_mma = mma.get_slice(tid); + Tensor tCsA = thr_mma.partition_A(sA); + Tensor tCsB = thr_mma.partition_B(sB); + Tensor tCgC = thr_mma.partition_C(gC); + + Tensor tCrA = thr_mma.make_fragment_A(tCsA); + Tensor tCrB = thr_mma.make_fragment_B(tCsB); + Tensor tCrC = thr_mma.make_fragment_C(tCgC); + clear(tCrC); + + warpgroup_fence_operand(tCrC); + warpgroup_arrive(); // wgmma.fence.sync.aligned + cute::gemm(mma, tCrA, tCrB, tCrC); // wgmma.mma_async.sync.aligned.m64n128k8.f32.tf32.tf32 + warpgroup_commit_batch(); // wgmma.commit_group.sync.aligned + warpgroup_wait<0>(); // wgmma.wait_group.sync.aligned 0 + warpgroup_fence_operand(tCrC); + if (tid == 0) { uint32_t c; asm volatile("mov.u32 %0,%%clock;\n" : "=r"(c)); clk_g[2] = c; } + + axpby(1.0f, tCrC, 0.0f, tCgC); +} + +// ============================================================================ +// Host launcher +// ============================================================================ +static void run_kernel(tfloat32_t const* A, tfloat32_t const* B, float* C, uint32_t* clk, + cudaStream_t stream = 0) { + auto sA_layout = tile_to_shape(GMMA::Layout_K_INTER_Atom{}, make_shape(Int{}, Int{})); + auto sB_layout = tile_to_shape(GMMA::Layout_K_INTER_Atom{}, make_shape(Int{}, Int{})); + + TiledMMA tiled_mma = make_tiled_mma(SM90_64x128x8_F32TF32TF32_SS_TN<>{}); + + using SS = SharedStorage; + int smem_size = int(sizeof(SS)); + + dim3 block(size(tiled_mma)); // 128 threads = 1 warpgroup + dim3 grid(1, 1, 1); + + auto kernel_ptr = &wgmma_tf32_128_kernel; + CUTE_CHECK_ERROR(cudaFuncSetAttribute( + kernel_ptr, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + + kernel_ptr<<>>(A, B, C, clk, tiled_mma, sA_layout, sB_layout); + CUTE_CHECK_LAST(); +} + +// ============================================================================ +// Test runner: fill A/B, launch, compare against a tf32-truncated CPU +// reference, print PASS/FAIL like wgmma_verify.cu. +// ============================================================================ +static bool run_test(const char* name, const float* A_host, const float* B_host) { + std::vector A_d(M * K), B_d(N * K); + std::vector A_ref(M * K), B_ref(N * K); + for (int i = 0; i < M * K; ++i) { + uint32_t bits; memcpy(&bits, &A_host[i], 4); + A_d[i] = tfloat32_t::bitcast(bits); // raw float32 bits (no truncation) + A_ref[i] = quant_tf32(A_host[i]); // tf32-truncated reference input + } + for (int i = 0; i < N * K; ++i) { + uint32_t bits; memcpy(&bits, &B_host[i], 4); + B_d[i] = tfloat32_t::bitcast(bits); + B_ref[i] = quant_tf32(B_host[i]); + } + + tfloat32_t *d_A, *d_B; + float *d_C; + uint32_t *d_clk; + cudaMalloc(&d_A, M * K * sizeof(tfloat32_t)); + cudaMalloc(&d_B, N * K * sizeof(tfloat32_t)); + cudaMalloc(&d_C, M * N * sizeof(float)); + cudaMalloc(&d_clk, 3 * sizeof(uint32_t)); + cudaMemcpy(d_A, A_d.data(), M * K * sizeof(tfloat32_t), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, B_d.data(), N * K * sizeof(tfloat32_t), cudaMemcpyHostToDevice); + + run_kernel(d_A, d_B, d_C, d_clk); + cudaError_t err = cudaDeviceSynchronize(); + if (err != cudaSuccess) { + printf("[%s] CUDA error: %s\n", name, cudaGetErrorString(err)); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_C); cudaFree(d_clk); + return false; + } + + std::vector D_sim(M * N); + uint32_t clk[3]; + cudaMemcpy(D_sim.data(), d_C, M * N * sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk, d_clk, 3 * sizeof(uint32_t), cudaMemcpyDeviceToHost); + cudaFree(d_A); cudaFree(d_B); cudaFree(d_C); cudaFree(d_clk); + + // CPU reference: D[m][n] = sum_k A_ref[m][k] * B_ref[n][k] + double max_err = 0; int max_m = 0, max_n = 0; + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + double s = 0.0; + for (int k = 0; k < K; ++k) s += A_ref[m * K + k] * B_ref[n * K + k]; + double e = fabs((double)D_sim[m * N + n] - s); + if (e > max_err) { max_err = e; max_m = m; max_n = n; } + } + } + double ref_check = 0.0; + for (int k = 0; k < K; ++k) ref_check += A_ref[max_m * K + k] * B_ref[max_n * K + k]; + double ref_mag = fabs(ref_check); + double tol = (ref_mag < 1.0) ? 1e-3 : ref_mag * 1e-2; + bool pass = (max_err <= tol); + + printf("[%-24s] %s max_err=%.4e fill=%u wgmma=%u cycles\n", + name, pass ? "PASS" : "FAIL", max_err, clk[1] - clk[0], clk[2] - clk[1]); + if (!pass) + printf(" Worst: D[%d][%d] sim=%.6f ref=%.6f\n", + max_m, max_n, D_sim[max_m * N + max_n], (float)ref_check); + return pass; +} + +// ============================================================================ +// main +// ============================================================================ +int main() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_tf32_128 — %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("Tests wgmma.mma_async.sync.aligned.m64n128k8.f32.tf32.tf32 (CUTLASS SM90_64x128x8_F32TF32TF32_SS_TN)\n"); + printf("plus wgmma.fence/commit_group/wait_group.sync.aligned\n"); + printf("================================================================\n"); + + bool all_pass = true; + + // all-ones: D[m][n] = K = 8 (trivially verifiable) + { + static float A[M][K], B[N][K]; + for (int m = 0; m < M; ++m) for (int k = 0; k < K; ++k) A[m][k] = 1.f; + for (int n = 0; n < N; ++n) for (int k = 0; k < K; ++k) B[n][k] = 1.f; + all_pass &= run_test("tf32 m64n128k8 all-ones", (float*)A, (float*)B); + } + + // sequential: small deterministic values vs. tf32-truncated double reference + { + static float A[M][K], B[N][K]; + for (int m = 0; m < M; ++m) for (int k = 0; k < K; ++k) A[m][k] = (m * K + k + 1) * 0.1f; + for (int n = 0; n < N; ++n) for (int k = 0; k < K; ++k) B[n][k] = (k * N + n + 1) * 0.1f; + all_pass &= run_test("tf32 m64n128k8 sequential", (float*)A, (float*)B); + } + + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + return all_pass ? 0 : 1; +} From d27771bc57b5eb7f538d9f1d3163c096af44cfbc Mon Sep 17 00:00:00 2001 From: Raunak Dani Date: Fri, 10 Jul 2026 01:29:52 -0400 Subject: [PATCH 21/35] Add bar.sync CTA barrier microbenchmark New ubench under tma/bar.sync that exercises the CTA-wide barrier (PTX `bar.sync 0;`, i.e. __syncthreads()) via explicit inline PTX and verifies the result on the host. The kernel runs 4 blocks x 256 threads (8 warps/block) and builds two cross-warp data dependencies that are only correct if the barrier synchronized every warp in the CTA: - Phase 1: each thread writes global_tid+1 into its shared-memory slot, bar.sync, then reads the mirrored slot smem[blockDim-1-tid] (owned by a different warp) out to global memory. - Phase 2: block-wide tree reduction with a bar.sync between every step. A per-warp clock64 skew forces the warps out of lockstep so the barrier is genuinely required; a USE_BARRIER compile toggle strips the barriers for a control run. Results are copied back and checked on the host, and printed as PASSED/FAILED so CI can parse the outcome. Adds a dedicated Makefile (built by `make gpu_microbenchmark`, auto- discovered via the top-level ubench/*/*/ wildcard) that compiles bar_sync.cu into the bin/ directory. ARCH defaults to the tma/ suite set (sm_80 sm_90a sm_100a sm_101 sm_120) since bar.sync is universal. Co-authored-by: Cursor --- .../ubench/tma/bar.sync/Makefile | 36 +++ .../ubench/tma/bar.sync/bar_sync.cu | 242 ++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/bar_sync.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/Makefile new file mode 100644 index 000000000..3f01a50f7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/Makefile @@ -0,0 +1,36 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# bar.sync microbenchmark: exercises the CTA-wide barrier (PTX `bar.sync 0;`, +# i.e. __syncthreads()) via explicit inline PTX and verifies the result on the +# host. The single source file bar_sync.cu builds a single executable. +ALL_BENCHES := bar_sync + +# bar.sync is a universal barrier instruction (present on every SM generation), +# so we target a broad range while matching the architectures used by the rest +# of the tma/ suite. Override on the command line, e.g. `make ARCH=sm_90a`, to +# build for a single GPU. +ARCH ?= sm_80 sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS (code=sm_XX) and PTX (code=compute_XX) for each arch. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/bar_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/bar_sync.cu new file mode 100644 index 000000000..6219893aa --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/bar_sync.cu @@ -0,0 +1,242 @@ +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// bar_sync.cu — exercise the CTA-wide barrier (PTX `bar.sync`, i.e. the C-level +// __syncthreads()) and verify from the CPU that it actually synchronized every +// warp in the block. +// +// WHY THIS IS A REAL TEST +// ----------------------- +// In GPGPU-Sim each warp (32 lanes) executes in lockstep, but *different warps +// in the same CTA are scheduled independently* — there is no ordering between +// warp 0 and warp 7 unless you force one. bar.sync is that force: it is a gate +// that no warp may pass until EVERY warp in the block has arrived. +// +// A barrier test only proves something if the result is correct *only when the +// barrier works*. So every thread here consumes data produced by a thread in a +// DIFFERENT warp: +// +// Phase 1 (cross-warp exchange): +// shared[tid] = value(tid) // each warp produces its slice +// __syncthreads() // <-- all warps must finish +// exchange_out[tid] = shared[BS-1-tid] // read the *reversed* slot, +// // always owned by another warp +// +// Phase 2 (tree reduction — many barriers across warps): +// shared[tid] = value(tid) +// for (stride = BS/2; stride > 0; stride >>= 1) { +// if (tid < stride) shared[tid] += shared[tid + stride]; +// __syncthreads() // <-- between every step +// } +// block_sum = shared[0] +// +// Without the barriers, the reversed reads and the reduction steps race on +// shared memory produced by other warps, and the CPU checks below fail. +// +// CONTROL EXPERIMENT +// ------------------ +// Compile normally -> barriers ON -> ALL CHECKS PASSED. +// Compile with -DUSE_BARRIER=0 -> barriers stripped -> checks are expected to +// FAIL (or read garbage), demonstrating that bar.sync is doing real work in the +// simulator rather than being a no-op. +// +// make KERNEL=bar_sync +// ./run.sh bar_sync +// # control: +// nvcc -DUSE_BARRIER=0 ... bar_sync.cu -o bar_sync_nobar +// ───────────────────────────────────────────────────────────────────────────── + +#define WARP_SIZE 32 + +// Barriers are on by default; -DUSE_BARRIER=0 strips them for the control run. +#ifndef USE_BARRIER +#define USE_BARRIER 1 +#endif + +// Emit the CTA-wide barrier as EXPLICIT inline PTX so there is no ambiguity +// about which instruction is under test. `bar.sync 0;` is barrier number 0 with +// the implicit "all threads in the CTA" count — exactly what __syncthreads() +// lowers to. In GPGPU-Sim's PTX front-end (src/cuda-sim/ptx.l) the mnemonic +// `bar` lexes to BAR_OP and the `.sync` suffix to SYNC_OPTION, which the decoder +// (cuda-sim.cc set_bar_type) turns into bar_type = SYNC and the shader executes +// via barrier_set_t::warp_reaches_barrier. The "memory" clobber prevents the +// compiler from reordering shared-memory accesses across the barrier, so the +// cross-warp dependency below stays a genuine test. +#if USE_BARRIER +#define BARRIER() asm volatile("bar.sync 0;" ::: "memory") +#else +#define BARRIER() ((void)0) +#endif + +// The per-thread "work" result. Kept simple and deterministic so the CPU can +// predict the exact expected value for every slot. +__device__ __forceinline__ int produce(int global_tid) { + return global_tid + 1; // value(tid) = tid + 1 (nonzero, so 0 == "unwritten") +} + +// Deliberately skew when each warp arrives, so the warps do NOT march in +// lockstep. Later warps (higher warp id) start their shared-memory writes much +// later. This matters because Phase 1 has warp 0 consume the slot produced by +// the *last* warp: if that producer is delayed and there is no barrier, the +// consumer races ahead and reads stale data. With the barrier, everyone waits +// regardless of skew, so the result is still correct. The spin reads the cycle +// counter each iteration, so it cannot be optimized away and it advances real +// simulated time. +__device__ __forceinline__ void skew_by_warp(int warp_id) { + long long start = clock64(); + long long budget = (long long)warp_id * 4000; + while (clock64() - start < budget) { + // busy-wait; the loop condition has a side effect (%clock64 read) + } +} + +__global__ void bar_sync_kernel(int *exchange_out, long long *block_sums, + int block_size) { + extern __shared__ int smem[]; + + const int local_tid = threadIdx.x; + const int global_tid = blockIdx.x * blockDim.x + threadIdx.x; + const int warp_id = local_tid / WARP_SIZE; + + // Force the warps out of lockstep so the cross-warp dependency below is a + // genuine race when the barrier is absent. + skew_by_warp(warp_id); + + // ── Phase 1: every warp produces its slice of shared memory ────────────── + smem[local_tid] = produce(global_tid); + + // Barrier #1: no warp may read another warp's slot until all warps wrote. + BARRIER(); + + // Cross-warp exchange: read the mirrored slot. For a 256-thread block, + // lane 0 (warp 0) reads slot 255 (warp 7), and vice-versa — every thread + // reads a slot owned by a *different* warp. This is only correct if the + // barrier above completed for the whole CTA. + int mirror = block_size - 1 - local_tid; + exchange_out[global_tid] = smem[mirror]; + + // Barrier #2: make sure all mirror-reads are done before Phase 2 overwrites + // shared memory. + BARRIER(); + + // ── Phase 2: block-wide tree reduction (a barrier between each step) ────── + smem[local_tid] = produce(global_tid); + BARRIER(); + + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (local_tid < stride) { + smem[local_tid] += smem[local_tid + stride]; + } + // Each step consumes partial sums written by threads that may live in + // other warps, so a barrier is required after every step. + BARRIER(); + } + + if (local_tid == 0) { + block_sums[blockIdx.x] = (long long)smem[0]; + } +} + +int main() { + const int block_size = 256; // 8 warps per block + const int num_blocks = 4; // 4 CTAs -> tests per-CTA scoping + const int total_threads = num_blocks * block_size; + const size_t smem_bytes = block_size * sizeof(int); + + // ── Allocations ────────────────────────────────────────────────────────── + int *h_exchange = (int *)malloc(total_threads * sizeof(int)); + long long *h_sums = (long long *)malloc(num_blocks * sizeof(long long)); + int *d_exchange = NULL; + long long *d_sums = NULL; + cudaMalloc(&d_exchange, total_threads * sizeof(int)); + cudaMalloc(&d_sums, num_blocks * sizeof(long long)); + + // Sentinel-fill so any unwritten slot is obvious (0xFF bytes -> -1). + cudaMemset(d_exchange, 0xFF, total_threads * sizeof(int)); + cudaMemset(d_sums, 0xFF, num_blocks * sizeof(long long)); + + // ── Launch ─────────────────────────────────────────────────────────────── + printf("Launching bar_sync_kernel: %d blocks x %d threads (%d warps/block), " + "barriers %s\n\n", + num_blocks, block_size, block_size / WARP_SIZE, + USE_BARRIER ? "ON" : "OFF (control experiment)"); + + bar_sync_kernel<<>>(d_exchange, d_sums, + block_size); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("Kernel launch error: %s\n", cudaGetErrorString(err)); + return 1; + } + cudaDeviceSynchronize(); + + // ── GPU -> CPU transfer ──────────────────────────────────────────────────── + cudaMemcpy(h_exchange, d_exchange, total_threads * sizeof(int), + cudaMemcpyDeviceToHost); + cudaMemcpy(h_sums, d_sums, num_blocks * sizeof(long long), + cudaMemcpyDeviceToHost); + + // ── Host-side verification ───────────────────────────────────────────────── + int failures = 0; + + // Check 1: cross-warp exchange. For block b, local thread L must have read + // the mirrored producer value = produce(global index of (block_size-1-L)). + printf("Phase 1 — cross-warp shared-memory exchange (reversed index):\n"); + for (int b = 0; b < num_blocks; ++b) { + int block_ok = 1; + for (int l = 0; l < block_size; ++l) { + int global_tid = b * block_size + l; + int mirror_local = block_size - 1 - l; + int mirror_global = b * block_size + mirror_local; + int expected = mirror_global + 1; // produce(mirror_global) + int got = h_exchange[global_tid]; + if (got != expected) { + if (failures < 12) { // cap the noise + printf(" [FAIL] block %d lane %d: got %d, expected %d " + "(reversed slot owned by warp %d)\n", + b, l, got, expected, mirror_local / WARP_SIZE); + } + block_ok = 0; + failures++; + } + } + if (block_ok) { + printf(" [PASS] block %d: all %d threads read the correct " + "cross-warp mirrored value\n", b, block_size); + } + } + printf("\n"); + + // Check 2: block-wide reduction. Sum of produce(tid) over the block. + // produce(tid) = tid + 1, tids in block b are [b*BS .. b*BS+BS-1]. + printf("Phase 2 — block-wide tree reduction (barrier between each step):\n"); + for (int b = 0; b < num_blocks; ++b) { + long long expected = 0; + for (int l = 0; l < block_size; ++l) { + expected += (b * block_size + l) + 1; // produce(global_tid) + } + long long got = h_sums[b]; + if (got != expected) { + printf(" [FAIL] block %d: sum got %lld, expected %lld\n", + b, got, expected); + failures++; + } else { + printf(" [PASS] block %d: reduced sum = %lld\n", b, got); + } + } + + printf("\n==> %s\n", failures == 0 ? "ALL CHECKS PASSED" : "SOME CHECKS FAILED"); +#if !USE_BARRIER + printf(" (barriers were OFF; failures/garbage here confirm bar.sync is " + "doing real work)\n"); +#endif + + // ── Cleanup ──────────────────────────────────────────────────────────────── + cudaFree(d_exchange); + cudaFree(d_sums); + free(h_exchange); + free(h_sums); + + return failures == 0 ? 0 : 1; +} From b1d98db1eb1af7b971aef7147a9ac700fec2d0cd Mon Sep 17 00:00:00 2001 From: Raunak Dani Date: Fri, 10 Jul 2026 02:17:55 -0400 Subject: [PATCH 22/35] Add bar.warp.sync (__syncwarp) microbenchmark New ubench under tma/bar_warp_sync that exercises the warp-level barrier (PTX `bar.warp.sync`, i.e. __syncwarp()) via explicit inline PTX and verifies an intra-warp exchange on the host. What the kernel does: - Each lane writes produce(global_tid) = global_tid+1 into its slot of shared memory, issues `bar.warp.sync 0xffffffff;` (full membermask), then reads the MIRRORED lane WITHIN THE SAME WARP (lane l reads lane 31-l) and writes it to global memory. Runs 2 blocks x 256 threads (16 warps). The host copies the buffer back and checks every lane got its within-warp mirror value, printing PASSED/FAILED. How it is NOP'd (and why that is correct): - bar.warp.sync only synchronizes the 32 lanes within one warp. GPGPU-Sim already executes each warp in lockstep and reconverges divergent lanes with the SIMT/PDOM stack, so that intra-warp synchronization is already guaranteed by the model. The instruction therefore has no work to do and is NOP'd in the PTX front-end (src/cuda-sim/ptx.l maps the `bar.warp.*` family to NOP_OP). Its real purpose on silicon is Independent Thread Scheduling (Volta+), which the SIMT-stack model does not implement. - A passing run validates both that the simulator parses and drops bar.warp.sync cleanly (a missing lexer rule would abort PTX parsing) and that intra-warp lockstep ordering holds without the barrier doing anything. Note there is deliberately NO cross-warp dependency and NO warp skew (unlike the bar.sync test), because intra-warp ordering is implicit in the model. Adds a dedicated Makefile (built by `make gpu_microbenchmark`, auto- discovered via the top-level ubench/*/*/ wildcard) that compiles bar_warp_sync.cu into the bin/ directory. ARCH defaults to the tma/ suite set (sm_80 sm_90a sm_100a sm_101 sm_120) since bar.warp.sync is universal. Co-authored-by: Cursor --- .../tma/{bar.sync => bar_sync}/Makefile | 0 .../tma/{bar.sync => bar_sync}/bar_sync.cu | 0 .../ubench/tma/bar_warp_sync/Makefile | 35 +++++ .../ubench/tma/bar_warp_sync/bar_warp_sync.cu | 135 ++++++++++++++++++ 4 files changed, 170 insertions(+) rename src/cuda/GPU_Microbenchmark/ubench/tma/{bar.sync => bar_sync}/Makefile (100%) rename src/cuda/GPU_Microbenchmark/ubench/tma/{bar.sync => bar_sync}/bar_sync.cu (100%) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/bar_warp_sync.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/bar_sync/Makefile similarity index 100% rename from src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/Makefile rename to src/cuda/GPU_Microbenchmark/ubench/tma/bar_sync/Makefile diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/bar_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/bar_sync/bar_sync.cu similarity index 100% rename from src/cuda/GPU_Microbenchmark/ubench/tma/bar.sync/bar_sync.cu rename to src/cuda/GPU_Microbenchmark/ubench/tma/bar_sync/bar_sync.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/Makefile new file mode 100644 index 000000000..2f23c4b14 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/Makefile @@ -0,0 +1,35 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# bar.warp.sync microbenchmark: issues the warp-level barrier (PTX +# `bar.warp.sync`, i.e. __syncwarp()) via explicit inline PTX and verifies an +# intra-warp exchange on the host. Single source -> single executable. +ALL_BENCHES := bar_warp_sync + +# bar.warp.sync is a universal instruction, so target a broad range while +# matching the architectures used by the rest of the tma/ suite. Override on the +# command line, e.g. `make ARCH=sm_90a`, to build for a single GPU. +ARCH ?= sm_80 sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS (code=sm_XX) and PTX (code=compute_XX) for each arch. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/bar_warp_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/bar_warp_sync.cu new file mode 100644 index 000000000..bb9057f30 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/bar_warp_sync.cu @@ -0,0 +1,135 @@ +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// bar_warp_sync.cu — exercise the WARP-level barrier (PTX `bar.warp.sync`, i.e. +// the C-level __syncwarp()) and verify from the CPU that the program is still +// correct. +// +// WHY THIS IS DIFFERENT FROM bar.sync +// ----------------------------------- +// bar.sync is a CTA-wide barrier: it synchronizes *all warps* in the block, and +// GPGPU-Sim must implement it because warps are scheduled independently. +// +// bar.warp.sync only synchronizes the 32 lanes *within a single warp*. GPGPU-Sim +// already executes every warp in lockstep (all 32 lanes step together) and +// reconverges divergent lanes with the SIMT/PDOM stack, so the synchronization +// bar.warp.sync would provide is ALREADY guaranteed by the model. The +// instruction therefore has no work to do and is NOP'd in the PTX front-end +// (src/cuda-sim/ptx.l: `bar\.warp[...]* -> NOP_OP`). Its real purpose on silicon +// is Independent Thread Scheduling (Volta+), which the SIMT-stack model does not +// implement. +// +// WHAT THIS TEST CHECKS +// --------------------- +// 1. The simulator PARSES and NOPs `bar.warp.sync` without error (a missing +// lexer rule would abort PTX parsing). +// 2. A warp-internal data exchange guarded by __syncwarp() produces the correct +// result. Each lane writes its value to shared memory, __syncwarp(), then +// reads the MIRRORED lane *within the same warp* (lane l reads lane 31-l). +// Because the exchange stays inside one warp, lockstep execution makes it +// correct — which is exactly why the barrier can be a NOP here. +// +// Note: unlike the bar.sync test, there is no cross-warp dependency and no need +// to skew warps — the whole point is that intra-warp ordering is implicit. +// ───────────────────────────────────────────────────────────────────────────── + +#define WARP_SIZE 32 +#define FULL_MASK 0xffffffffu + +// Explicit inline PTX for the warp barrier. `bar.warp.sync ;` is +// what __syncwarp() lowers to. In GPGPU-Sim this lexes to NOP_OP (see ptx.l), +// so it is dropped; the "memory" clobber still prevents the compiler from +// reordering the shared-memory accesses around it. +__device__ __forceinline__ void warp_sync(unsigned mask) { + asm volatile("bar.warp.sync %0;" ::"r"(mask) : "memory"); +} + +__device__ __forceinline__ int produce(int global_tid) { + return global_tid + 1; // nonzero, so 0 means "unwritten" +} + +__global__ void bar_warp_sync_kernel(int *exchange_out) { + extern __shared__ int smem[]; + + const int local_tid = threadIdx.x; + const int global_tid = blockIdx.x * blockDim.x + threadIdx.x; + const int lane = local_tid % WARP_SIZE; + const int warp_base = local_tid - lane; // first local tid of this warp + + // Each lane publishes its value into this warp's slice of shared memory. + smem[local_tid] = produce(global_tid); + + // Warp-level barrier: all 32 lanes have published before anyone reads. + warp_sync(FULL_MASK); + + // Intra-warp mirrored read: lane l reads lane (31 - l) of the SAME warp. + int partner = warp_base + (WARP_SIZE - 1 - lane); + exchange_out[global_tid] = smem[partner]; +} + +int main() { + const int block_size = 256; // 8 warps per block + const int num_blocks = 2; + const int total_threads = num_blocks * block_size; + const size_t smem_bytes = block_size * sizeof(int); + + int *h_exchange = (int *)malloc(total_threads * sizeof(int)); + int *d_exchange = NULL; + cudaMalloc(&d_exchange, total_threads * sizeof(int)); + cudaMemset(d_exchange, 0xFF, total_threads * sizeof(int)); // -1 sentinel + + printf("Launching bar_warp_sync_kernel: %d blocks x %d threads " + "(%d warps/block)\n\n", + num_blocks, block_size, block_size / WARP_SIZE); + + bar_warp_sync_kernel<<>>(d_exchange); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("Kernel launch error: %s\n", cudaGetErrorString(err)); + return 1; + } + cudaDeviceSynchronize(); + + cudaMemcpy(h_exchange, d_exchange, total_threads * sizeof(int), + cudaMemcpyDeviceToHost); + + // Verify: each lane l must have read the value produced by lane (31 - l) + // of the SAME warp. + int failures = 0; + printf("Intra-warp mirrored exchange (lane l reads lane 31-l of its warp):\n"); + for (int b = 0; b < num_blocks; ++b) { + for (int w = 0; w < block_size / WARP_SIZE; ++w) { + int warp_ok = 1; + for (int lane = 0; lane < WARP_SIZE; ++lane) { + int local_tid = w * WARP_SIZE + lane; + int global_tid = b * block_size + local_tid; + int partner_lane = WARP_SIZE - 1 - lane; + int partner_global = b * block_size + w * WARP_SIZE + partner_lane; + int expected = partner_global + 1; // produce(partner) + int got = h_exchange[global_tid]; + if (got != expected) { + if (failures < 12) { + printf(" [FAIL] block %d warp %d lane %d: got %d, " + "expected %d\n", b, w, lane, got, expected); + } + warp_ok = 0; + failures++; + } + } + if (warp_ok) { + printf(" [PASS] block %d warp %d: all 32 lanes read the " + "correct mirrored lane\n", b, w); + } + } + } + + printf("\n==> %s\n", failures == 0 ? "ALL CHECKS PASSED" : "SOME CHECKS FAILED"); + printf(" (a passing run confirms bar.warp.sync was parsed and NOP'd " + "correctly, and intra-warp lockstep ordering holds)\n"); + + cudaFree(d_exchange); + free(h_exchange); + + return failures == 0 ? 0 : 1; +} From e9387269bfc08fccf4c63e10b57ff6560d20c547 Mon Sep 17 00:00:00 2001 From: Raunak Dani Date: Fri, 10 Jul 2026 11:08:36 -0400 Subject: [PATCH 23/35] Move bar_sync and bar_warp_sync ubenches from tma/ to system/ Relocate the bar.sync and bar.warp.sync microbenchmarks to the system/ category. Folder move only; no source or Makefile changes. Co-authored-by: Cursor --- .../GPU_Microbenchmark/ubench/{tma => system}/bar_sync/Makefile | 0 .../ubench/{tma => system}/bar_sync/bar_sync.cu | 0 .../ubench/{tma => system}/bar_warp_sync/Makefile | 0 .../ubench/{tma => system}/bar_warp_sync/bar_warp_sync.cu | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename src/cuda/GPU_Microbenchmark/ubench/{tma => system}/bar_sync/Makefile (100%) rename src/cuda/GPU_Microbenchmark/ubench/{tma => system}/bar_sync/bar_sync.cu (100%) rename src/cuda/GPU_Microbenchmark/ubench/{tma => system}/bar_warp_sync/Makefile (100%) rename src/cuda/GPU_Microbenchmark/ubench/{tma => system}/bar_warp_sync/bar_warp_sync.cu (100%) diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/bar_sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/system/bar_sync/Makefile similarity index 100% rename from src/cuda/GPU_Microbenchmark/ubench/tma/bar_sync/Makefile rename to src/cuda/GPU_Microbenchmark/ubench/system/bar_sync/Makefile diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/bar_sync/bar_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/system/bar_sync/bar_sync.cu similarity index 100% rename from src/cuda/GPU_Microbenchmark/ubench/tma/bar_sync/bar_sync.cu rename to src/cuda/GPU_Microbenchmark/ubench/system/bar_sync/bar_sync.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/Makefile b/src/cuda/GPU_Microbenchmark/ubench/system/bar_warp_sync/Makefile similarity index 100% rename from src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/Makefile rename to src/cuda/GPU_Microbenchmark/ubench/system/bar_warp_sync/Makefile diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/bar_warp_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/system/bar_warp_sync/bar_warp_sync.cu similarity index 100% rename from src/cuda/GPU_Microbenchmark/ubench/tma/bar_warp_sync/bar_warp_sync.cu rename to src/cuda/GPU_Microbenchmark/ubench/system/bar_warp_sync/bar_warp_sync.cu From 3ae7c330eb1f2cf3103719b4a7df0445ae0f15ad Mon Sep 17 00:00:00 2001 From: Ivorchu <41135347+Ivorchu@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:09:39 -0400 Subject: [PATCH 24/35] wgmma ubenches print PASSED or FAILED for CI --- src/cuda/GPU_Microbenchmark/Makefile | 3 + .../ubench/wgmma/wgmma/Makefile | 9 +- .../ubench/wgmma/wgmma/wgmma_verify.cu | 2 +- .../ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu | 2 +- .../ubench/wgmma/wgmma_multik/wgmma_multik.cu | 2 +- .../ubench/wgmma/wgmma_rs_verify/Makefile | 9 + .../ubench/wgmma/wgmma_scale/wgmma_scale.cu | 2 +- .../ubench/wgmma/wgmma_shapes/wgmma_shapes.cu | 2 +- .../ubench/wgmma/wgmma_sweep/Makefile | 9 + .../ubench/wgmma/wgmma_sync/wgmma_fence.cu | 2 +- .../ubench/wgmma/wgmma_sync/wgmma_sync.cu | 2 +- .../ubench/wgmma/wgmma_timing_edge/Makefile | 28 + .../wgmma_timing_edge/wgmma_timing_edge.cu | 687 ++++++++++++++++++ 13 files changed, 750 insertions(+), 9 deletions(-) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu diff --git a/src/cuda/GPU_Microbenchmark/Makefile b/src/cuda/GPU_Microbenchmark/Makefile index b3ad4de64..91eb0ed7b 100644 --- a/src/cuda/GPU_Microbenchmark/Makefile +++ b/src/cuda/GPU_Microbenchmark/Makefile @@ -2,6 +2,9 @@ BASE_DIR := $(shell pwd) BIN_DIR := $(BASE_DIR)/bin ALL_SUB_DIRS = $(wildcard ubench/*/*/) GMMA_DIRS = $(filter %gmma/,$(ALL_SUB_DIRS)) +# ubench/wgmma/wgmma builds wgmma_verify (standalone, no CUTLASS); its CuTe +# benches are guarded inside its own Makefile, so keep it in the default build. +GMMA_DIRS := $(filter-out ubench/wgmma/wgmma/,$(GMMA_DIRS)) SUB_DIRS = $(filter-out $(GMMA_DIRS),$(ALL_SUB_DIRS)) SUB_DIRS_ALL = $(SUB_DIRS:%=all-%) SUB_DIRS_CLEAN = $(SUB_DIRS:%=clean-%) diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile index f9bbbf0f5..09798c3d7 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/Makefile @@ -3,8 +3,13 @@ BIN_DIR := $(BASE_DIR)/../../../bin/ CUTLASS_ROOT ?= $(BASE_DIR)/../../../../cutlass-bench # WGMMA microbenchmarks. -# wgmma_no_tma / wgmma_tf32_128 use CuTe (CUTLASS) headers; wgmma_verify is standalone. -ALL_BENCHES := wgmma_verify wgmma_no_tma wgmma_tf32_128 +# wgmma_no_tma / wgmma_tf32_128 use CuTe (CUTLASS) headers; wgmma_verify is +# standalone. Only build the CuTe benches when the headers are present +# (cutlass-bench submodule initialized), so CI can still build wgmma_verify. +ALL_BENCHES := wgmma_verify +ifneq ($(wildcard $(CUTLASS_ROOT)/include/cute/tensor.hpp),) +ALL_BENCHES += wgmma_no_tma wgmma_tf32_128 +endif # WGMMA requires Hopper (SM90a). ARCH ?= sm_90a diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu index ec86afe1e..c9db9b67f 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu @@ -975,6 +975,6 @@ int main() { } printf("\n================================================================\n"); - printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); return all_pass ? 0 : 1; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu index c76e51d3e..6cce61abe 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu @@ -185,6 +185,6 @@ int main() { all_pass &= run("f16←f16×f16 m64n16k16 A=2 B=3", 2.f, 3.f, (float)(2*3*K)); printf("\n================================================================\n"); - printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); return all_pass ? 0 : 1; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu index c95b6102f..0940fc352 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu @@ -665,6 +665,6 @@ int main() { fclose(outfile); printf("\n================================================================\n"); - printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); return all_pass ? 0 : 1; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile index 3867f9231..a17977b56 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_rs_verify/Makefile @@ -9,4 +9,13 @@ HOPPER_CUDA_CPPFLAGS=$(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$ NVCC_FLAGS := $(HOPPER_CUDA_CPPFLAGS) -std=c++17 +# Needs CuTe/CUTLASS headers (bundled lat_gmma/include or the cutlass-bench +# submodule). Skip gracefully when absent (e.g. CI containers). +CUTE_HDR := $(or $(wildcard ../lat_gmma/include/cute/tensor.hpp), \ + $(wildcard $(GPUAPPS_ROOT)/src/cuda/cutlass-bench/include/cute/tensor.hpp)) +ifeq ($(CUTE_HDR),) +release clean: + @echo "$(EXE): SKIPPED (CuTe/CUTLASS headers not found)" +else include ../../../common/common.mk +endif diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu index 070395bc0..12a9be3ae 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu @@ -295,6 +295,6 @@ int main() { } printf("\n================================================================\n"); - printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); return all_pass ? 0 : 1; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu index bfbf2837f..3915b1190 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu @@ -243,6 +243,6 @@ int main() { for (auto& t : tests) all_pass &= run(t); printf("\n================================================================\n"); - printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); return all_pass ? 0 : 1; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile index 30010cd5f..5e54f2c2b 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sweep/Makefile @@ -9,4 +9,13 @@ HOPPER_CUDA_CPPFLAGS=$(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$ NVCC_FLAGS := $(HOPPER_CUDA_CPPFLAGS) -std=c++17 +# Needs CuTe/CUTLASS headers (bundled lat_gmma/include or the cutlass-bench +# submodule). Skip gracefully when absent (e.g. CI containers). +CUTE_HDR := $(or $(wildcard ../lat_gmma/include/cute/tensor.hpp), \ + $(wildcard $(GPUAPPS_ROOT)/src/cuda/cutlass-bench/include/cute/tensor.hpp)) +ifeq ($(CUTE_HDR),) +release clean: + @echo "$(EXE): SKIPPED (CuTe/CUTLASS headers not found)" +else include ../../../common/common.mk +endif diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu index 6f3504b3c..fb9fdfcda 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu @@ -326,6 +326,6 @@ int main() { } printf("\n================================================================\n"); - printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); return all_pass ? 0 : 1; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu index 6cadbd674..2163c9d07 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu @@ -285,6 +285,6 @@ int main() { } printf("\n================================================================\n"); - printf("Overall: %s\n", all_pass ? "PASS" : "FAIL"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); return all_pass ? 0 : 1; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/Makefile b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/Makefile new file mode 100644 index 000000000..c7f70f9d7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/Makefile @@ -0,0 +1,28 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := wgmma_timing_edge + +ARCH ?= sm_90a +CC := nvcc + +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu new file mode 100644 index 000000000..c3b56eae4 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu @@ -0,0 +1,687 @@ +// ============================================================================= +// wgmma_timing_edge.cu +// +// Silicon-validation microbenchmarks for WGMMA *timing-model edge cases* that +// lat_gmma / wgmma_multik do not cover. Every test self-checks functional +// correctness (exit 0 = all PASS) and reports %clock-based cycle counts to +// stdout + wgmma_timing_edge_results.csv for sim-vs-silicon comparison. +// +// Tests (one named kernel each, so Nsight Compute can profile them separately): +// +// 1. kernel_group_size - S MMAs in ONE commit group, S = 1..32. +// total(S) ~= LAT + (S-1)*II -> linear fit +// separates true completion latency (intercept) +// from initiation interval (slope). Directly +// tests commit_wgmma_group_with_ops()'s formula. +// 2. kernel_wait_partial - 4 MMAs in 4 separate groups; timed staged +// wait_group 3 -> 2 -> 1 -> 0. Tests the +// "all but latest N" drain semantics + timing. +// 3. kernel_serialize - 2 groups of 1 MMA vs 1 group of 2 MMAs. +// Tests inter-group serialization (model forces +// group completion >= previous group completion). +// 4. kernel_overlap - MMA + commit, then a dependent FMA chain of +// length X, then wait. Sweeping X shows how much +// independent ALU work hides under the async MMA +// (the knee = true async latency; cross-checks #1). +// 5. kernel_inflight - 16x (MMA;COMMIT) with a %clock stamp after each +// pair, no wait until the end. A knee in the +// per-iteration deltas reveals the hardware +// in-flight group limit (model: wgmma_max_pending). +// 6. kernel_swizzle - 32-MMA chain with swizzle mode 0/3/2/1 +// (none/32B/64B/128B). Model predicts identical +// timing; functional check validates the swizzled +// smem layout on silicon. +// 7. kernel_multiwg - 1 vs 2 warpgroups (128 vs 256 threads) each +// running a 64-MMA chain on private smem tiles. +// Tests tensor-core contention across warpgroups +// (model tracks warpgroups independently). +// +// All tests: m64n16k16 f32 <- f16 x f16, A = B = 1 so D element = K * #MMAs. +// +// Run on silicon: +// ./wgmma_timing_edge # all tests + CSV + exit code +// ncu --kernel-name 'regex:kernel_.*' \ +// --metrics gpu__time_duration.sum,sm__cycles_elapsed.avg,\ +// sm__pipe_tensor_op_hmma_cycles_active.avg \ +// ./wgmma_timing_edge # per-kernel timing cross-check +// nsys profile --stats=true ./wgmma_timing_edge # coarse launch timeline +// +// Run in GPGPU-Sim: +// functional: PTX_SIM_USE_PTX_FILE=1 ./wgmma_timing_edge (mode 1 config) +// timing: PTX_SIM_MODE_FUNC=0 ./wgmma_timing_edge +// +// Exit status: 0 = all functional checks PASS, 1 = any FAIL. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr int M = 64; +static constexpr int N = 16; +static constexpr int K = 16; +static constexpr int WGSIZE = 128; +static constexpr int D_ELEMS = N / 2; // 8 f32 registers per thread +static constexpr int E = 2; // bytes per f16 +static constexpr int SBO = 128; +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int SMEM_A = M * K * E; // 2048 +static constexpr int SMEM_B = K * N * E; // 512 + +// --------------------------------------------------------------------------- +// Device helpers (same conventions as wgmma_multik.cu / sim wgmma_layout.h) +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smem_addr(const void* ptr) { + uint32_t a; + asm volatile( + "{ .reg .u64 _p; cvta.to.shared.u64 _p, %1; cvt.u32.u64 %0, _p; }" + : "=r"(a) : "l"(ptr)); + return a; +} + +__device__ __forceinline__ uint64_t make_gmma_desc( + uint32_t base, uint32_t lbo, uint32_t sbo, uint32_t sw = 0) { + uint64_t d = 0; + d |= (uint64_t)(base >> 4) & 0x3FFFu; + d |= ((uint64_t)((lbo >> 4) & 0x3FFFu)) << 16; + d |= ((uint64_t)((sbo >> 4) & 0x3FFFu)) << 32; + d |= ((uint64_t)(sw & 0x3u)) << 62; + return d; +} + +// Byte offset of logical element (leading, stride) - matches sim wgmma_smem_offset. +__host__ __device__ __forceinline__ +int smem_off(int leading, int stride, int e, int LBO) { + int T = 16 / e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +// XOR swizzle - matches sim wgmma_apply_swizzle (mode 0=none,1=128B,2=64B,3=32B). +__host__ __device__ __forceinline__ +int apply_swizzle(int off, int sw) { + if (sw == 0) return off; + int stride = (sw == 1) ? 128 : (sw == 2) ? 64 : 32; + int block = off / stride; + int in_block = off % stride; + int xm = (block & 1) ? (stride >> 1) : 0; + return block * stride + (in_block ^ xm); +} + +#define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") +#define WGMMA_COMMIT asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory") +#define WGMMA_WAIT_N(n) asm volatile("wgmma.wait_group.sync.aligned " #n ";\n" ::: "memory") + +#define CLOCK(v) asm volatile("mov.u32 %0, %%clock;" : "=r"(v) :: "memory") + +__device__ __forceinline__ void mma_f16( + uint64_t da, uint64_t db, + float& d0, float& d1, float& d2, float& d3, + float& d4, float& d5, float& d6, float& d7) { + asm volatile( + "{ .reg .pred p; setp.ne.b32 p,1,0;\n" + "wgmma.mma_async.sync.aligned.m64n16k16.f32.f16.f16 " + "{%0,%1,%2,%3,%4,%5,%6,%7},%8,%9,p,1,1,0,0; }\n" + : "+f"(d0),"+f"(d1),"+f"(d2),"+f"(d3), + "+f"(d4),"+f"(d5),"+f"(d6),"+f"(d7) + : "l"(da),"l"(db) : "memory"); +} + +// Fill smem with the canonical (optionally swizzled) layout; A/B value = 1.0. +// lane_stride lets a 256-thread block fill two private tiles. +__device__ __forceinline__ void fill_tiles( + char* smA, char* smB, const half* A_g, const half* B_g, + int tid, int nthreads, int sw) +{ + for (int i = tid; i < M*K; i += nthreads) { + int m = i/K, k = i%K; + *(half*)(smA + apply_swizzle(smem_off(k, m, E, LBO_A), sw)) = A_g[i]; + } + for (int i = tid; i < K*N; i += nthreads) { + int k = i/N, n = i%N; + *(half*)(smB + apply_swizzle(smem_off(k, n, E, LBO_B), sw)) = B_g[i]; + } +} + +__device__ __forceinline__ void store_d( + float* D_g, int tid, + float d0, float d1, float d2, float d3, + float d4, float d5, float d6, float d7) { + const int b = tid * D_ELEMS; + D_g[b]=d0; D_g[b+1]=d1; D_g[b+2]=d2; D_g[b+3]=d3; + D_g[b+4]=d4; D_g[b+5]=d5; D_g[b+6]=d6; D_g[b+7]=d7; +} + +// --------------------------------------------------------------------------- +// 1. kernel_group_size: S MMAs in one commit group; time fence..wait0. +// total(S) = LAT + (S-1)*II -> host does the linear fit. +// --------------------------------------------------------------------------- +__global__ void kernel_group_size( + const half* A_g, const half* B_g, float* D_g, + uint32_t* clk_out, int S) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) CLOCK(t0); + __syncthreads(); + + WGMMA_FENCE; + for (int i = 0; i < S; i++) + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT_N(0); + WGMMA_FENCE; + + __syncthreads(); + if (tid == 0) CLOCK(t1); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 2. kernel_wait_partial: 4 single-MMA groups pending, then staged waits. +// clk stamps: [0]=start, [1]=after wait 3, [2]=after wait 2, +// [3]=after wait 1, [4]=after wait 0. +// --------------------------------------------------------------------------- +__global__ void kernel_wait_partial( + const half* A_g, const half* B_g, float* D_g, uint32_t* clk_out) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + uint32_t c0=0,c1=0,c2=0,c3=0,c4=0; + + __syncthreads(); + WGMMA_FENCE; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; // group 0 + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; // group 1 + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; // group 2 + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; // group 3 + if (tid == 0) CLOCK(c0); + WGMMA_WAIT_N(3); // oldest group (0) must be done + if (tid == 0) CLOCK(c1); + WGMMA_WAIT_N(2); // groups 0,1 done + if (tid == 0) CLOCK(c2); + WGMMA_WAIT_N(1); // groups 0,1,2 done + if (tid == 0) CLOCK(c3); + WGMMA_WAIT_N(0); // all done + WGMMA_FENCE; + if (tid == 0) CLOCK(c4); + + if (tid == 0) { + clk_out[0]=c0; clk_out[1]=c1; clk_out[2]=c2; clk_out[3]=c3; clk_out[4]=c4; + } + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 3. kernel_serialize: mode 0 = 1 group of 2 MMAs; mode 1 = 2 groups of 1 MMA. +// Model serializes groups: mode1 total >= mode0 total expected on sim. +// --------------------------------------------------------------------------- +__global__ void kernel_serialize( + const half* A_g, const half* B_g, float* D_g, + uint32_t* clk_out, int mode) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) CLOCK(t0); + __syncthreads(); + + WGMMA_FENCE; + if (mode == 0) { + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + } else { + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); WGMMA_COMMIT; + } + WGMMA_WAIT_N(0); + WGMMA_FENCE; + + __syncthreads(); + if (tid == 0) CLOCK(t1); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 4. kernel_overlap: MMA+commit, dependent FMA chain of length X, then wait. +// do_mma / X let the host measure mma-only, fma-only and combined runs. +// acc result is checked so the chain can't be optimized away. +// --------------------------------------------------------------------------- +__global__ void kernel_overlap( + const half* A_g, const half* B_g, float* D_g, + float* acc_g, uint32_t* clk_out, int X, int do_mma) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + float acc = acc_g[tid]; // opaque start value (1.0f) + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) CLOCK(t0); + __syncthreads(); + + WGMMA_FENCE; + if (do_mma) { + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + } + // Dependent FMA chain, pinned between commit and wait by volatile asm. + asm volatile("" : "+f"(acc)); + #pragma unroll 1 + for (int i = 0; i < X; i++) + acc = __fmaf_rn(acc, 1.0f, 1.0f); // acc += 1 each step (dependent) + asm volatile("" : "+f"(acc)); + if (do_mma) { + WGMMA_WAIT_N(0); + WGMMA_FENCE; + } + + __syncthreads(); + if (tid == 0) CLOCK(t1); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + acc_g[tid] = acc; + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 5. kernel_inflight: NGRP x (MMA;COMMIT) with a clock stamp after each pair; +// wait 0 only at the end. Stall knee in the deltas = HW in-flight limit. +// --------------------------------------------------------------------------- +static constexpr int NGRP = 16; + +__global__ void kernel_inflight( + const half* A_g, const half* B_g, float* D_g, uint32_t* clk_out) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + uint32_t stamps[NGRP + 2]; + + __syncthreads(); + if (tid == 0) CLOCK(stamps[0]); + __syncthreads(); + + WGMMA_FENCE; + #pragma unroll + for (int g = 0; g < NGRP; g++) { + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + if (tid == 0) CLOCK(stamps[g + 1]); + } + WGMMA_WAIT_N(0); + WGMMA_FENCE; + __syncthreads(); + if (tid == 0) CLOCK(stamps[NGRP + 1]); + + if (tid == 0) + for (int i = 0; i < NGRP + 2; i++) clk_out[i] = stamps[i]; + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 6. kernel_swizzle: 32-MMA chain with swizzle mode sw (0/1/2/3). +// Functional check validates the swizzled fill; timing compares modes. +// --------------------------------------------------------------------------- +static constexpr int SW_MMAS = 32; + +__global__ void kernel_swizzle( + const half* A_g, const half* B_g, float* D_g, + uint32_t* clk_out, int sw) +{ + __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; + const int tid = threadIdx.x; + fill_tiles(smA, smB, A_g, B_g, tid, WGSIZE, sw); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, (uint32_t)sw); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, (uint32_t)sw); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); + if (tid == 0) CLOCK(t0); + __syncthreads(); + + WGMMA_FENCE; + for (int i = 0; i < SW_MMAS; i++) + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT_N(0); + WGMMA_FENCE; + + __syncthreads(); + if (tid == 0) CLOCK(t1); + if (tid == 0) { clk_out[0] = t0; clk_out[1] = t1; } + store_d(D_g, tid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// --------------------------------------------------------------------------- +// 7. kernel_multiwg: blockDim = 128 or 256 (1 or 2 warpgroups). Each +// warpgroup runs a WG_MMAS chain on its OWN smem tiles. Per-warpgroup +// start/stop clocks -> contention shows as higher cyc/mma with 2 wgs. +// --------------------------------------------------------------------------- +static constexpr int WG_MMAS = 64; +static constexpr int MAX_WG = 2; + +__global__ void kernel_multiwg( + const half* A_g, const half* B_g, float* D_g, uint32_t* clk_out) +{ + __shared__ __align__(128) char smA[MAX_WG][SMEM_A], smB[MAX_WG][SMEM_B]; + const int tid = threadIdx.x; + const int wg = tid / WGSIZE; // warpgroup index in block + const int wtid = tid % WGSIZE; // thread index within warpgroup + + fill_tiles(smA[wg], smB[wg], A_g, B_g, wtid, WGSIZE, 0); + __syncthreads(); + + uint64_t da = make_gmma_desc(smem_addr(smA[wg]), LBO_A, SBO); + uint64_t db = make_gmma_desc(smem_addr(smB[wg]), LBO_B, SBO); + float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; + + uint32_t t0 = 0, t1 = 0; + __syncthreads(); // common start line for all wgs + if (wtid == 0) CLOCK(t0); + + WGMMA_FENCE; + for (int i = 0; i < WG_MMAS; i++) + mma_f16(da, db, d0,d1,d2,d3,d4,d5,d6,d7); + WGMMA_COMMIT; + WGMMA_WAIT_N(0); + WGMMA_FENCE; + + if (wtid == 0) CLOCK(t1); + if (wtid == 0) { clk_out[2*wg] = t0; clk_out[2*wg + 1] = t1; } + store_d(D_g + wg * WGSIZE * D_ELEMS, wtid, d0,d1,d2,d3,d4,d5,d6,d7); +} + +// =========================================================================== +// Host side +// =========================================================================== +static void d_frag_pos(int T, int e, int* row, int* col) { + int warp = T/32, lane = T%32, s = e/4, k = e%4; + *row = (lane/4)*2 + k/2 + warp*16; + *col = (lane%4)*2 + k%2 + s*8; +} + +static bool check_uniform_flat(const char* name, const float* flat, + float expected, FILE* csv, + const char* param, float cycles, + float cyc_per_mma) { + float max_err = 0.f; + for (int T = 0; T < WGSIZE; T++) + for (int e = 0; e < D_ELEMS; e++) { + int row, col; + d_frag_pos(T, e, &row, &col); + if (row >= M || col >= N) continue; + float err = fabsf(flat[T*D_ELEMS + e] - expected); + if (err > max_err) max_err = err; + } + float tol = fabsf(expected) * 1e-2f + 1e-3f; + bool pass = (max_err <= tol); + printf("[%-28s %-12s] %s expected=%.1f max_err=%.3e cycles=%.0f", + name, param, pass ? "PASS" : "FAIL", expected, max_err, cycles); + if (cyc_per_mma > 0) printf(" cyc/mma=%.2f", cyc_per_mma); + printf("\n"); + if (csv) + fprintf(csv, "%s,%s,%s,%.1f,%.4e,%.0f,%.2f\n", + name, param, pass ? "PASS" : "FAIL", expected, max_err, + cycles, cyc_per_mma); + return pass; +} + +struct DevBufs { + half *A, *B; + float *D; + uint32_t *clk; + float *acc; +}; + +static DevBufs alloc_bufs(int d_mult = 1) { + DevBufs b; + std::vector h_A(M*K, __float2half(1.f)); + std::vector h_B(K*N, __float2half(1.f)); + cudaMalloc(&b.A, M*K*sizeof(half)); + cudaMalloc(&b.B, K*N*sizeof(half)); + cudaMalloc(&b.D, d_mult*WGSIZE*D_ELEMS*sizeof(float)); + cudaMalloc(&b.clk, 32*sizeof(uint32_t)); + cudaMalloc(&b.acc, WGSIZE*sizeof(float)); + cudaMemcpy(b.A, h_A.data(), M*K*sizeof(half), cudaMemcpyHostToDevice); + cudaMemcpy(b.B, h_B.data(), K*N*sizeof(half), cudaMemcpyHostToDevice); + return b; +} + +static void free_bufs(DevBufs& b) { + cudaFree(b.A); cudaFree(b.B); cudaFree(b.D); + cudaFree(b.clk); cudaFree(b.acc); +} + +int main(int argc, char** argv) { + const char* filter = (argc > 1) ? argv[1] : NULL; // substring test filter + auto run = [&](const char* name) { + return !filter || strstr(name, filter); + }; + + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + printf("wgmma_timing_edge - %s (SM %d.%d)\n", prop.name, prop.major, prop.minor); + printf("================================================================\n"); + + FILE* csv = fopen("wgmma_timing_edge_results.csv", "w"); + if (csv) fprintf(csv, "test,param,status,expected,max_err,cycles,cycles_per_mma\n"); + + bool all_pass = true; + std::vector flat(2*WGSIZE*D_ELEMS); + std::vector clk(32); + + // --- 1. group-size sweep: LAT + (S-1)*II fit ------------------------- + if (run("group_size")) { + printf("\n--- group_size: S MMAs in one group (fit: total = LAT + (S-1)*II) ---\n"); + const int Ss[] = {1, 2, 4, 8, 16, 32}; + double sx=0, sy=0, sxx=0, sxy=0; int np = 0; + DevBufs b = alloc_bufs(); + for (int S : Ss) { + kernel_group_size<<<1, WGSIZE>>>(b.A, b.B, b.D, b.clk, S); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + float cyc = (float)(clk[1] - clk[0]); + char param[32]; snprintf(param, 32, "S=%d", S); + all_pass &= check_uniform_flat("group_size", flat.data(), + (float)(S*K), csv, param, cyc, cyc/S); + sx += S; sy += cyc; sxx += (double)S*S; sxy += (double)S*cyc; np++; + } + free_bufs(b); + double ii = (np*sxy - sx*sy) / (np*sxx - sx*sx); + double lat = (sy - ii*sx) / np + ii; // intercept at S=1... total(1)=LAT + printf(" linear fit: II (slope) = %.2f cyc/mma, LAT (total at S=1) = %.2f cyc\n", ii, lat); + if (csv) fprintf(csv, "group_size_fit,II=%.2f;LAT=%.2f,TIMING,,,,\n", ii, lat); + } + + // --- 2. staged partial waits ----------------------------------------- + if (run("wait_partial")) { + printf("\n--- wait_partial: 4 groups pending; wait 3 -> 2 -> 1 -> 0 ---\n"); + DevBufs b = alloc_bufs(); + kernel_wait_partial<<<1, WGSIZE>>>(b.A, b.B, b.D, b.clk); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 5*sizeof(uint32_t), cudaMemcpyDeviceToHost); + free_bufs(b); + float total = (float)(clk[4] - clk[0]); + all_pass &= check_uniform_flat("wait_partial", flat.data(), + (float)(4*K), csv, "4grp", total, 0); + printf(" wait3=%u wait2=+%u wait1=+%u wait0=+%u cycles\n", + clk[1]-clk[0], clk[2]-clk[1], clk[3]-clk[2], clk[4]-clk[3]); + if (csv) fprintf(csv, "wait_partial_stages,w3=%u;w2=%u;w1=%u;w0=%u,TIMING,,,%'.0f,\n", + clk[1]-clk[0], clk[2]-clk[1], clk[3]-clk[2], clk[4]-clk[3], total); + } + + // --- 3. group serialization ------------------------------------------ + if (run("serialize")) { + printf("\n--- serialize: 1 group x 2 MMAs vs 2 groups x 1 MMA ---\n"); + DevBufs b = alloc_bufs(); + float cyc_mode[2]; + for (int mode = 0; mode < 2; mode++) { + kernel_serialize<<<1, WGSIZE>>>(b.A, b.B, b.D, b.clk, mode); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + cyc_mode[mode] = (float)(clk[1] - clk[0]); + all_pass &= check_uniform_flat("serialize", flat.data(), (float)(2*K), + csv, mode == 0 ? "1grpx2mma" : "2grpx1mma", + cyc_mode[mode], cyc_mode[mode]/2); + } + free_bufs(b); + printf(" delta (2grp - 1grp) = %.0f cycles\n", cyc_mode[1] - cyc_mode[0]); + } + + // --- 4. async overlap sweep ------------------------------------------ + if (run("overlap")) { + printf("\n--- overlap: MMA || dependent FMA chain of length X ---\n"); + DevBufs b = alloc_bufs(); + const int Xs[] = {0, 8, 16, 32, 64, 96, 128, 192, 256}; + std::vector h_acc(WGSIZE, 1.f), r_acc(WGSIZE); + for (int X : Xs) { + for (int do_mma = (X == 0 ? 1 : 0); do_mma < 2; do_mma++) { + cudaMemcpy(b.acc, h_acc.data(), WGSIZE*sizeof(float), cudaMemcpyHostToDevice); + kernel_overlap<<<1, WGSIZE>>>(b.A, b.B, b.D, b.acc, b.clk, X, do_mma); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(r_acc.data(), b.acc, WGSIZE*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + float cyc = (float)(clk[1] - clk[0]); + // functional: acc = 1 + X for every thread; D = K if do_mma + bool acc_ok = true; + for (int t = 0; t < WGSIZE; t++) + if (fabsf(r_acc[t] - (1.f + X)) > 1e-3f) acc_ok = false; + char param[40]; + snprintf(param, 40, "X=%d%s", X, do_mma ? "+mma" : ",fma_only"); + if (do_mma) { + all_pass &= check_uniform_flat("overlap", flat.data(), (float)K, + csv, param, cyc, 0) && acc_ok; + } else { + printf("[%-28s %-12s] %s cycles=%.0f (fma chain alone)\n", + "overlap", param, acc_ok ? "PASS" : "FAIL", cyc); + if (csv) fprintf(csv, "overlap,%s,%s,,,%0.f,\n", + param, acc_ok ? "PASS" : "FAIL", cyc); + all_pass &= acc_ok; + } + if (!acc_ok) printf(" FMA chain result wrong (X=%d)\n", X); + } + } + free_bufs(b); + printf(" (flat combined-total until X exceeds the MMA latency = async overlap works)\n"); + } + + // --- 5. in-flight group limit ----------------------------------------- + if (run("inflight")) { + printf("\n--- inflight: %d x (MMA;COMMIT) with per-commit clock stamps ---\n", NGRP); + DevBufs b = alloc_bufs(); + kernel_inflight<<<1, WGSIZE>>>(b.A, b.B, b.D, b.clk); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, (NGRP+2)*sizeof(uint32_t), cudaMemcpyDeviceToHost); + free_bufs(b); + float total = (float)(clk[NGRP+1] - clk[0]); + all_pass &= check_uniform_flat("inflight", flat.data(), (float)(NGRP*K), + csv, "16grp", total, total/NGRP); + printf(" per-(mma;commit) deltas: "); + for (int g = 1; g <= NGRP; g++) printf("%u ", clk[g] - clk[g-1]); + printf("\n drain after last commit: %u cycles\n", clk[NGRP+1] - clk[NGRP]); + if (csv) { + fprintf(csv, "inflight_deltas,"); + for (int g = 1; g <= NGRP; g++) + fprintf(csv, "%u%s", clk[g] - clk[g-1], g == NGRP ? "" : ";"); + fprintf(csv, ",TIMING,,,%0.f,\n", total); + } + } + + // --- 6. swizzle modes -------------------------------------------------- + if (run("swizzle")) { + printf("\n--- swizzle: %d-MMA chain, modes none/32B/64B/128B ---\n", SW_MMAS); + static const char* swname[] = {"none", "128B", "64B", "32B"}; + DevBufs b = alloc_bufs(); + for (int sw = 0; sw < 4; sw++) { + kernel_swizzle<<<1, WGSIZE>>>(b.A, b.B, b.D, b.clk, sw); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 2*sizeof(uint32_t), cudaMemcpyDeviceToHost); + float cyc = (float)(clk[1] - clk[0]); + char param[24]; snprintf(param, 24, "sw=%s", swname[sw]); + all_pass &= check_uniform_flat("swizzle", flat.data(), (float)(SW_MMAS*K), + csv, param, cyc, cyc/SW_MMAS); + } + free_bufs(b); + } + + // --- 7. multi-warpgroup contention -------------------------------------- + if (run("multiwg")) { + printf("\n--- multiwg: %d-MMA chain per warpgroup, 1 vs 2 warpgroups ---\n", WG_MMAS); + DevBufs b = alloc_bufs(MAX_WG); + for (int nwg = 1; nwg <= MAX_WG; nwg++) { + cudaMemset(b.D, 0, MAX_WG*WGSIZE*D_ELEMS*sizeof(float)); + kernel_multiwg<<<1, nwg*WGSIZE>>>(b.A, b.B, b.D, b.clk); + cudaDeviceSynchronize(); + cudaMemcpy(flat.data(), b.D, nwg*WGSIZE*D_ELEMS*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(clk.data(), b.clk, 2*nwg*sizeof(uint32_t), cudaMemcpyDeviceToHost); + for (int wg = 0; wg < nwg; wg++) { + float cyc = (float)(clk[2*wg+1] - clk[2*wg]); + char param[24]; snprintf(param, 24, "nwg=%d;wg=%d", nwg, wg); + all_pass &= check_uniform_flat("multiwg", + flat.data() + wg*WGSIZE*D_ELEMS, + (float)(WG_MMAS*K), csv, param, + cyc, cyc/WG_MMAS); + } + } + free_bufs(b); + printf(" (2-wg cyc/mma vs 1-wg cyc/mma = tensor-core contention factor)\n"); + } + + if (csv) fclose(csv); + printf("\n================================================================\n"); + printf("Overall: %s\n", all_pass ? "PASSED" : "FAILED"); + return all_pass ? 0 : 1; +} From d445d336a5597b152c8a587c7db82a3966bebaeb Mon Sep 17 00:00:00 2001 From: jyasuumi <144069912+jyasuumi@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:12:24 -0400 Subject: [PATCH 25/35] fence.mbarrier_init.release.cluster ubench verification for single CTA pure functional mode --- .../ubench/tma/ufence_mbarrier_init/Makefile | 29 +++++ .../ufence_mbarrier_init.cu | 115 ++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/Makefile b/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/Makefile new file mode 100644 index 000000000..ebdcfbbd0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/Makefile @@ -0,0 +1,29 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# fence.mbarrier_init.release.cluster status-probe microbenchmark. +ALL_BENCHES := ufence_mbarrier_init + +# fence.mbarrier_init and mbarrier.init.shared::cta are Hopper+ PTX (SM_90a and above). +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS and PTX. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu new file mode 100644 index 000000000..d7f417b21 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu @@ -0,0 +1,115 @@ +#include +#include +#include + +// Status probe for `fence.mbarrier_init.release.cluster` in the GPGPU-Sim +// functional model. +// +// This Hopper (sm_90+) fence is emitted by CUTLASS example 48 right after +// `mbarrier.init`, to make the barrier initialization visible cluster-wide +// (release, .cluster scope) before any CTA arrives on it. Example 48 launches +// with ClusterShape = 1x1x1 — a SINGLE CTA per cluster. In pure functional +// simulation, shared-memory/mbarrier effects are instantaneous and there is +// only one CTA, so the cluster-scope release fence has nothing to order across +// CTAs: it is functionally a no-op. The simulator already treats it as one +// (lexer maps `fence.mbarrier_init...` -> NOP_OP), and this ubench verifies that +// a single-CTA kernel containing the exact instruction both parses and produces +// correct results under pure functional mode. +// +// The barrier is made semantically load-bearing so a broken fence/barrier would +// be caught: every thread publishes payload[tid], then (after the barrier) reads +// its NEIGHBOR's slot. A correct neighbor read requires every producer to have +// executed past the fence. Expected: out[t] == ((t+1) % N) * 2. +// +// The surrounding mbarrier arrive/try_wait path mirrors how CUTLASS uses the +// barrier the fence is initializing; its result is informational (kept live via +// the `info` buffer) and is not part of the pass criterion, which relies on the +// pure-functional-exact __syncthreads() ordering. + +#define N 32 // one warp / one CTA + +__global__ void ufence_mbarrier_init_kernel(unsigned int *out, unsigned int *info) { + __shared__ unsigned long long bar; + __shared__ unsigned int payload[N]; + const unsigned int t = threadIdx.x; + + // 1. Initialize the mbarrier (thread 0): expect all N threads to arrive. + if (t == 0) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" + : : "l"(&bar), "r"(N) : "memory"); + } + + // 2. INSTRUCTION UNDER TEST — exact CUTLASS-48 spelling and position: + // release, cluster-scope fence ordering the mbarrier init before arrives. + asm volatile("fence.mbarrier_init.release.cluster;" : : : "memory"); + + __syncthreads(); + + // 3. Producer: publish this thread's slot. + payload[t] = t * 2u; + + // 4. Exercise the mbarrier arrive/wait path (as CUTLASS does around the + // barrier this fence initializes). Informational only. + unsigned long long state; + asm volatile("mbarrier.arrive.b64 %0, [%1], %2;" + : "=l"(state) : "l"(&bar), "n"(1) : "memory"); + + unsigned int phase = 0u; + unsigned int done = 0u; + asm volatile( + "{\n\t" + ".reg .pred p;\n\t" + "mbarrier.try_wait.parity.b64 p, [%1], %2;\n\t" + "selp.u32 %0, 1, 0, p;\n\t" + "}" + : "=r"(done) : "l"(&bar), "r"(phase) : "memory"); + + // 5. Barrier-ordered consumer: neighbor read requires every producer to have + // run past the fence. __syncthreads() is the functional ordering the + // pure-functional simulator models exactly. + __syncthreads(); + out[t] = payload[(t + 1u) % N]; + info[t] = done; // keep try_wait live; not part of the pass criterion. +} + +int main() { + // Unbuffered stdout so the verdict survives even if a later stage aborts + // hard inside the simulator. + setvbuf(stdout, NULL, _IONBF, 0); + + const int n = N; + unsigned int *d_out = NULL, *d_info = NULL; + unsigned int h_out[N], h_info[N]; + cudaMalloc(&d_out, n * sizeof(unsigned int)); + cudaMalloc(&d_info, n * sizeof(unsigned int)); + cudaMemset(d_out, 0xFF, n * sizeof(unsigned int)); + cudaMemset(d_info, 0xFF, n * sizeof(unsigned int)); + + // Single-CTA launch — matches CUTLASS-48's per-cluster CTA count (1). + ufence_mbarrier_init_kernel<<<1, N>>>(d_out, d_info); + cudaError_t err = cudaDeviceSynchronize(); + + int fails = 0; + if (err != cudaSuccess) { + printf("launch/sync error: %s\n", cudaGetErrorString(err)); + fails++; + } else { + cudaMemcpy(h_out, d_out, n * sizeof(unsigned int), cudaMemcpyDeviceToHost); + cudaMemcpy(h_info, d_info, n * sizeof(unsigned int), cudaMemcpyDeviceToHost); + + printf("neighbor-exchange after fence.mbarrier_init.release.cluster:\n"); + for (int i = 0; i < n; ++i) { + unsigned int expected = (unsigned int)(((i + 1) % N) * 2); + const char *mark = (h_out[i] == expected) ? "ok" : "MISMATCH"; + if (h_out[i] != expected) fails++; + printf(" t %2d: out = %3u (expected %3u) [%s] try_wait=%u\n", + i, h_out[i], expected, mark, h_info[i]); + } + } + + cudaFree(d_out); + cudaFree(d_info); + + printf("RESULT: %s\n", fails == 0 ? "PASSED" : "FAILED"); + return fails == 0 ? 0 : 1; +} From 176cad1efb3a03caa8f735934f2eba5f455df6af Mon Sep 17 00:00:00 2001 From: jyasuumi <144069912+jyasuumi@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:40:42 -0400 Subject: [PATCH 26/35] using cvta generic to shared for mbarrier init fix --- .../ufence_mbarrier_init.cu | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu index d7f417b21..87134b1de 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/ufence_mbarrier_init/ufence_mbarrier_init.cu @@ -33,10 +33,16 @@ __global__ void ufence_mbarrier_init_kernel(unsigned int *out, unsigned int *inf __shared__ unsigned int payload[N]; const unsigned int t = threadIdx.x; + // CUTLASS-48 addresses the mbarrier with a 32-bit .shared-window offset + // (e.g. `mbarrier.init.shared::cta.b64 [%r141], ...`), not a 64-bit generic + // pointer. Mirror that: the simulator's mbarrier tracking keys barriers by + // this shared offset, so init/arrive/try_wait must all use the same form. + const unsigned int bar_s = (unsigned int)__cvta_generic_to_shared(&bar); + // 1. Initialize the mbarrier (thread 0): expect all N threads to arrive. if (t == 0) { asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" - : : "l"(&bar), "r"(N) : "memory"); + : : "r"(bar_s), "r"(N) : "memory"); } // 2. INSTRUCTION UNDER TEST — exact CUTLASS-48 spelling and position: @@ -49,20 +55,21 @@ __global__ void ufence_mbarrier_init_kernel(unsigned int *out, unsigned int *inf payload[t] = t * 2u; // 4. Exercise the mbarrier arrive/wait path (as CUTLASS does around the - // barrier this fence initializes). Informational only. - unsigned long long state; - asm volatile("mbarrier.arrive.b64 %0, [%1], %2;" - : "=l"(state) : "l"(&bar), "n"(1) : "memory"); + // barrier this fence initializes), using CUTLASS-48's exact forms + // (`mbarrier.arrive.shared::cta.b64 _, [%r];` — token sunk to `_`). + // Informational only. + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];" + : : "r"(bar_s) : "memory"); unsigned int phase = 0u; unsigned int done = 0u; asm volatile( "{\n\t" ".reg .pred p;\n\t" - "mbarrier.try_wait.parity.b64 p, [%1], %2;\n\t" + "mbarrier.try_wait.parity.shared::cta.b64 p, [%1], %2;\n\t" "selp.u32 %0, 1, 0, p;\n\t" "}" - : "=r"(done) : "l"(&bar), "r"(phase) : "memory"); + : "=r"(done) : "r"(bar_s), "r"(phase) : "memory"); // 5. Barrier-ordered consumer: neighbor read requires every producer to have // run past the fence. __syncthreads() is the functional ordering the From cb0559ffe03f7675d6a6066ef85e1a21cf58a559 Mon Sep 17 00:00:00 2001 From: Raunak Dani Date: Thu, 16 Jul 2026 23:44:51 -0400 Subject: [PATCH 27/35] Add ld.global.L2::128B.u32 functional-verification microbenchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New ubench under l2_cache/ld_global_l2_128b that exercises the CUTLASS- style global load `ld.global.L2::128B.u32` via explicit inline PTX and verifies from the host that the load still returns correct data — proving the .L2::128B prefetch hint is functionally inert while the underlying global load performs a real, CPU-observable memory transaction. Instruction under test: - Base opcode: ordinary ld.global.u32 (reads 4 bytes from global memory into a register). Already implemented in GPGPU-Sim. - Qualifier: .L2::128B is a PTX .level::prefetch_size hint (PTX ISA 7.4+, SM_75+). It asks the hardware to also prefetch a 128-byte region into L2 alongside the load. It does NOT change the value returned by the load — only (maybe) timing on silicon. How the hint is NOP'd in GPGPU-Sim (no simulator patch required): - ptx.l already lexes `.L2::64B` / `.L2::128B` / `.L2::256B` / `.L2::cache_hint` / `.L2` as L2_OPTION, so PTX carrying the qualifier parses cleanly instead of aborting. - ptx_ir.cc already accepts L2_OPTION in the instruction-option switch and takes the "recognized but no special action needed" path — i.e. the hint is ignored for functional correctness while ld.global still executes through the normal load path. - This matches the team's reading of the CUTLASS / Accel-Sim coverage table: ld.global is in; L2::128B prefetching can be ignored for now; confirming that unblocks CUTLASS-48 functional runs. What the kernel does: - Issues EXPLICIT inline PTX for both paths so the exercised instruction is unambiguous: path A (under test): ld.global.L2::128B.u32 %0, [%1]; path B (control): ld.global.u32 %0, [%1]; - Host fills a global input buffer with unique nonzero values (tid*3+7). Each of 4 blocks x 256 threads loads in[tid] via both paths, then stores (loaded+1) into two separate output buffers. Depending the store on the loaded value prevents DCE and guarantees a real GPU->CPU-visible memory transaction seeded by the load. - Host copies both buffers back and checks: (1) every hinted load returned input+1 (functional load worked); (2) hinted == plain for every tid (L2::128B did not change results). Prints PASSED/FAILED so CI respects the result. Why a passing run validates the NOP'd hint: 1. Parsing: a missing L2_OPTION lexer/option rule would abort PTX parsing on `.L2::128B`. A completed run proves the qualifier is recognized and ignored cleanly. 2. Functional load: if the whole instruction were wrongly NOP'd (not just the hint), outputs would keep their 0xFF sentinel and Check 1 would fail. Passing Check 1 proves a real global load occurred. 3. Hint inertness: Check 2 shows the hinted path is bit-identical to plain ld.global.u32 — the functional-mode contract for a prefetch- size hint. Adds a dedicated Makefile (built by `make GPU_Microbenchmark`, auto- discovered via the top-level ubench/*/*/ wildcard) that compiles ld_global_l2_128b.cu into bin/ with shared cudart for silicon runs. ARCH defaults to sm_80 sm_90a sm_100a sm_101 sm_120 (Ampere+; L2::128B is SM_75+). Override with e.g. `make ARCH=sm_90a` for a single GPU. Verified under GPGPU-Sim (1024 threads): both checks PASS; cuobjdump confirms the binary embeds `ld.global.L2::128B.u32` alongside the plain control load. Co-authored-by: Cursor --- .../l2_cache/ld_global_l2_128b/Makefile | 37 ++++ .../ld_global_l2_128b/ld_global_l2_128b.cu | 182 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/ld_global_l2_128b.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/Makefile b/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/Makefile new file mode 100644 index 000000000..38058ccf7 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/Makefile @@ -0,0 +1,37 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# ld.global.L2::128B microbenchmark: issues `ld.global.L2::128B.u32` via +# explicit inline PTX (plus a plain `ld.global.u32` control path) and verifies +# from the host that the load still returns correct data — proving the +# .L2::128B prefetch hint is functionally inert while the underlying global +# load performs a real memory transaction. Single source -> single executable. +ALL_BENCHES := ld_global_l2_128b + +# .L2::128B is a PTX .level::prefetch_size hint (PTX ISA 7.4+, SM_75+). +# Target Ampere and newer to match the rest of the modern ubench suite. +# Override on the command line, e.g. `make ARCH=sm_90a`, for a single GPU. +ARCH ?= sm_80 sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS (code=sm_XX) and PTX (code=compute_XX) for each arch. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/ld_global_l2_128b.cu b/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/ld_global_l2_128b.cu new file mode 100644 index 000000000..57b008819 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/l2_cache/ld_global_l2_128b/ld_global_l2_128b.cu @@ -0,0 +1,182 @@ +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// ld_global_l2_128b.cu — exercise `ld.global.L2::128B.u32` and verify from the +// CPU that the *load* still works functionally while the L2::128B qualifier is +// treated as a no-op performance hint. +// +// CONTEXT (CUTLASS / GPGPU-Sim) +// ----------------------------- +// CUTLASS emits loads of the form: +// +// ld.global.L2::128B.u32 %r, [%addr]; +// +// `.L2::128B` is a PTX `.level::prefetch_size` hint (PTX ISA 7.4+): it asks the +// hardware to prefetch 128B into L2 alongside the load. It does NOT change the +// data returned by the load — only (maybe) timing on silicon. +// +// In GPGPU-Sim the qualifier is already recognized: +// ptx.l : `.L2::128B` -> L2_OPTION +// ptx_ir : L2_OPTION falls into the "recognized, no special action" path +// so the hint is effectively NOP'd for functional mode, while `ld.global.u32` +// still performs a real global memory read. +// +// WHAT THIS TEST CHECKS +// --------------------- +// 1. The simulator PARSES `ld.global.L2::128B.u32` without error (a missing +// lexer/option rule would abort PTX parsing). +// 2. The underlying global load still happens: each thread reads a known value +// from a host-initialized global buffer via the hinted load, transforms it, +// and writes to an output buffer. The host checks every slot — proving a +// real memory transaction occurred, not a silent no-op of the whole insn. +// 3. A plain `ld.global.u32` control path on the same input produces identical +// results, confirming L2::128B does not affect functional correctness. +// +// make KERNEL=ld_global_l2_128b +// ./run.sh ld_global_l2_128b +// ───────────────────────────────────────────────────────────────────────────── + +// The instruction under test: load with the L2::128B prefetch hint. +__device__ __forceinline__ unsigned ld_global_l2_128b(const unsigned *ptr) { + unsigned val; + asm volatile("ld.global.L2::128B.u32 %0, [%1];" + : "=r"(val) + : "l"(ptr)); + return val; +} + +// Control: same load without the prefetch hint. +__device__ __forceinline__ unsigned ld_global_plain(const unsigned *ptr) { + unsigned val; + asm volatile("ld.global.u32 %0, [%1];" + : "=r"(val) + : "l"(ptr)); + return val; +} + +// Each thread: +// 1. loads in[tid] via ld.global.L2::128B.u32 -> out_hinted[tid] +// 2. loads in[tid] via ld.global.u32 -> out_plain[tid] +// 3. stores (loaded + 1) so the host sees a write that depends on the load +__global__ void ld_global_l2_128b_kernel(const unsigned *in, + unsigned *out_hinted, + unsigned *out_plain) { + unsigned tid = blockIdx.x * blockDim.x + threadIdx.x; + + unsigned hinted = ld_global_l2_128b(&in[tid]); + unsigned plain = ld_global_plain(&in[tid]); + + // Depend on the loaded values so neither load can be DCE'd, and so the + // host-visible stores are genuine memory transactions seeded by the loads. + out_hinted[tid] = hinted + 1u; + out_plain[tid] = plain + 1u; +} + +int main() { + const int block_size = 256; + const int num_blocks = 4; + const int total_threads = num_blocks * block_size; + + unsigned *h_in = (unsigned *)malloc(total_threads * sizeof(unsigned)); + unsigned *h_hinted = (unsigned *)malloc(total_threads * sizeof(unsigned)); + unsigned *h_plain = (unsigned *)malloc(total_threads * sizeof(unsigned)); + unsigned *d_in = NULL; + unsigned *d_hinted = NULL; + unsigned *d_plain = NULL; + + for (int i = 0; i < total_threads; ++i) { + h_in[i] = (unsigned)(i * 3 + 7); // matches produce(tid) + } + + cudaMalloc(&d_in, total_threads * sizeof(unsigned)); + cudaMalloc(&d_hinted, total_threads * sizeof(unsigned)); + cudaMalloc(&d_plain, total_threads * sizeof(unsigned)); + + // Sentinel-fill outputs so unwritten slots are obvious (0xFF -> all bits 1). + cudaMemset(d_hinted, 0xFF, total_threads * sizeof(unsigned)); + cudaMemset(d_plain, 0xFF, total_threads * sizeof(unsigned)); + + cudaMemcpy(d_in, h_in, total_threads * sizeof(unsigned), + cudaMemcpyHostToDevice); + + printf("Launching ld_global_l2_128b_kernel: %d blocks x %d threads\n" + " path A: ld.global.L2::128B.u32 (hint under test)\n" + " path B: ld.global.u32 (plain control)\n\n", + num_blocks, block_size); + + ld_global_l2_128b_kernel<<>>(d_in, d_hinted, d_plain); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("Kernel launch error: %s\n", cudaGetErrorString(err)); + return 1; + } + cudaDeviceSynchronize(); + + cudaMemcpy(h_hinted, d_hinted, total_threads * sizeof(unsigned), + cudaMemcpyDeviceToHost); + cudaMemcpy(h_plain, d_plain, total_threads * sizeof(unsigned), + cudaMemcpyDeviceToHost); + + int failures = 0; + + // Check 1: hinted load produced the correct functional value for every tid. + printf("Check 1 — ld.global.L2::128B.u32 functional correctness:\n"); + { + int ok = 1; + for (int i = 0; i < total_threads; ++i) { + unsigned expected = h_in[i] + 1u; + unsigned got = h_hinted[i]; + if (got != expected) { + if (failures < 12) { + printf(" [FAIL] tid %d: got %u, expected %u " + "(input was %u)\n", + i, got, expected, h_in[i]); + } + ok = 0; + failures++; + } + } + if (ok) { + printf(" [PASS] all %d threads: hinted load returned input+1\n", + total_threads); + } + } + printf("\n"); + + // Check 2: plain load matches the hinted path exactly. + printf("Check 2 — L2::128B hint is functionally equivalent to plain ld:\n"); + { + int ok = 1; + for (int i = 0; i < total_threads; ++i) { + if (h_hinted[i] != h_plain[i]) { + if (failures < 12) { + printf(" [FAIL] tid %d: hinted=%u plain=%u\n", + i, h_hinted[i], h_plain[i]); + } + ok = 0; + failures++; + } + } + if (ok) { + printf(" [PASS] all %d threads: hinted == plain " + "(prefetch hint did not change results)\n", + total_threads); + } + } + + printf("\n==> %s\n", + failures == 0 ? "ALL CHECKS PASSED" : "SOME CHECKS FAILED"); + printf(" (a passing run confirms L2::128B was parsed/ignored as a\n" + " prefetch hint, while ld.global still performed a real load\n" + " that the host can observe through the output buffer)\n"); + + cudaFree(d_in); + cudaFree(d_hinted); + cudaFree(d_plain); + free(h_in); + free(h_hinted); + free(h_plain); + + return failures == 0 ? 0 : 1; +} From b3d3f3e882a003147daa18f04b9c50255b9639c4 Mon Sep 17 00:00:00 2001 From: Kai Ze Ee <136288731+definitelyka1ze@users.noreply.github.com> Date: Sat, 18 Jul 2026 10:58:15 -0700 Subject: [PATCH 28/35] mbarrier cluster scope test --- .../ubench/tma/mbarrier/mbarrier_cluster.cu | 657 ++++++++++++++++++ 1 file changed, 657 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_cluster.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_cluster.cu b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_cluster.cu new file mode 100644 index 000000000..3d57292cd --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/tma/mbarrier/mbarrier_cluster.cu @@ -0,0 +1,657 @@ +// mbarrier_cluster.cu +// +// Aggressive microbenchmark for mbarrier on sm_90 (Hopper). +// Covers every PTX form used by CUTLASS 4.8 and the known simulator +// correctness gaps, with one dedicated test per issue. +// +// PTX forms exercised (matching CUTLASS cute/arch/copy_sm90_desc.hpp and +// cutlass/arch/barrier.h exactly): +// +// mbarrier.init.shared::cta.b64 [bar], count +// mbarrier.arrive.expect_tx.shared::cta.b64 _, [bar], bytes (no token) +// mbarrier.arrive.shared::cta.b64 _, [bar] (no token, CUTLASS arrive_barrier) +// mbarrier.arrive.shared::cta.b64 tok, [bar] (token form) +// mbarrier.arrive.shared::cta.b64 tok, [bar], count (token + count) +// mbarrier.arrive.shared::cluster.b64 _, [remote_bar] (cross-CTA, no token) +// mbarrier.try_wait.parity.shared::cta.b64 P, [bar], phase_bit (CUTLASS wait form) +// mbarrier.try_wait.shared.b64 P, [bar], token (token wait form) +// mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 [bar], tx +// mbarrier.inval.shared.b64 [bar] +// +// Test index → output slot: +// h[0] TEST 1: mbarrier.init.shared::cta / arrive(token) / try_wait(token) — baseline +// h[1] TEST 2: mbarrier.try_wait.parity — sim crash test (SYNCS_MAX_ENUM assert) +// h[2] TEST 3: arrive.expect_tx + try_wait.parity (CUTLASS TMA wait pattern) +// h[3] TEST 4: arrive(no-token) _, [bar] — CUTLASS arrive_barrier form +// h[4] TEST 5: arrive.shared::cluster (remote) + try_wait.parity — CUTLASS cluster arrive +// h[5] TEST 6: Multi-phase ping-pong using try_wait.parity (looping parity bit) +// h[6] TEST 7: count=0 observe-only arrive must not decrement pending +// h[7] TEST 8: mbarrier.inval then re-init on same address +// h[8] TEST 9: expect_tx + complete_tx + all-thread arrive (combined barrier) +// +// All outputs: 1 = PASS, -1 = FAIL, 0 = did not run (from cudaMemset). +// +// Compile: +// nvcc -arch=sm_90 -std=c++17 mbarrier_cluster.cu -o mbarrier_cluster +// +// PTX audit: +// cuobjdump -ptx mbarrier_cluster | grep mbarrier + +#include +#include +#include + +namespace cg = cooperative_groups; + +// --------------------------------------------------------------------------- +// Inline PTX helpers — each one maps to exactly one CUTLASS or PTX form +// --------------------------------------------------------------------------- + +// 32-bit shared-memory address from a generic pointer (cvta.to.shared) +__device__ static inline uint32_t smem_u32(const void* ptr) { + uint32_t a; + asm("{ .reg .u64 tmp; cvta.to.shared.u64 tmp, %1; cvt.u32.u64 %0, tmp; }" + : "=r"(a) : "l"((uint64_t)ptr)); + return a; +} + +// mbarrier.init.shared::cta.b64 [bar], count +// CUTLASS: initialize_barrier() (copy_sm90_desc.hpp:69) +__device__ static inline void mbar_init(uint64_t* bar, uint32_t count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" + :: "r"(smem_u32(bar)), "r"(count) : "memory"); +} + +// mbarrier.inval.shared.b64 [bar] +__device__ static inline void mbar_inval(uint64_t* bar) { + asm volatile("mbarrier.inval.shared.b64 [%0];" :: "r"(smem_u32(bar)) : "memory"); +} + +// mbarrier.arrive.shared::cta.b64 tok, [bar], count — token form with count +// PTX ISA §9.7.12.4: decrements pending by count, returns prior-phase token +__device__ static inline uint64_t mbar_arrive_token(uint64_t* bar, uint32_t count = 1) { + uint64_t tok; + asm volatile("mbarrier.arrive.shared::cta.b64 %0, [%1], %2;" + : "=l"(tok) : "r"(smem_u32(bar)), "r"(count) : "memory"); + return tok; +} + +// mbarrier.arrive.shared::cta.b64 _, [bar] +// CUTLASS: arrive_barrier() (copy_sm90_desc.hpp:122) — no token returned +// Matches CUTLASS exactly: no volatile, no memory clobber (token discarded) +__device__ static inline void mbar_arrive_no_token(uint64_t* bar) { + asm("{ .reg .b64 state; mbarrier.arrive.shared::cta.b64 state, [%0]; }" + :: "r"(smem_u32(bar))); +} + +// mbarrier.arrive.expect_tx.shared::cta.b64 _, [bar], bytes +// CUTLASS: set_barrier_transaction_bytes() (copy_sm90_desc.hpp:83) +// Adds tx_bytes to tx_count; decrements pending by 1; no token returned. +__device__ static inline void mbar_arrive_expect_tx(uint64_t* bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" + :: "r"(smem_u32(bar)), "r"(bytes) : "memory"); +} + +// mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 [bar], tx +// Called after simulated TMA data arrives; decrements tx_count +__device__ static inline void mbar_complete_tx(uint64_t* bar, uint32_t tx) { + asm volatile("mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 [%0], %1;" + :: "r"(smem_u32(bar)), "r"(tx) : "memory"); +} + +// mbarrier.try_wait.parity.shared::cta.b64 P, [bar], phase_bit, ticks +// Matches CUTLASS ClusterBarrier::wait() in barrier.h exactly: +// 3-operand form with timeout ticks (0x989680); loops on bra until P=true. +// PTX ISA §9.7.12.6: returns P=true when phase%2 == phase_bit. +// Unique-label helper: two levels of macro indirection so __COUNTER__ expands +// before stringification, producing labels like LAB_WAIT_42 / DONE_42. +#define MBAR_WAIT_STR2(x) #x +#define MBAR_WAIT_STR(x) MBAR_WAIT_STR2(x) +#define MBAR_WAIT_PARITY_IMPL(sa, pb, tk, id) \ + asm volatile( \ + "{\n\t" \ + ".reg .pred P1;\n\t" \ + "LAB_WAIT_" MBAR_WAIT_STR(id) ":\n\t" \ + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1, %2;\n\t" \ + "@P1 bra DONE_" MBAR_WAIT_STR(id) ";\n\t" \ + "bra LAB_WAIT_" MBAR_WAIT_STR(id) ";\n\t" \ + "DONE_" MBAR_WAIT_STR(id) ":\n\t" \ + "}" \ + :: "r"(sa), "r"(pb), "r"(tk) : "memory") +// __LINE__ is unique per call site in the source file and survives inlining. +#define MBAR_WAIT_PARITY(sa, pb, tk) MBAR_WAIT_PARITY_IMPL(sa, pb, tk, __LINE__) + +// mbar_wait_parity intentionally removed — use mbar_spin_wait_parity macro instead, +// which captures __LINE__ at the call site to guarantee unique PTX labels per use. + +// Non-blocking one-shot parity test — returns true if barrier is already done +__device__ static inline bool mbar_test_wait_parity(uint64_t* bar, uint32_t phase_bit) { + uint32_t done; + asm volatile( + "{ .reg .pred P;\n\t" + "mbarrier.try_wait.parity.shared::cta.b64 P, [%0], %1;\n\t" + "selp.u32 %2, 1, 0, P; }" + : "=r"(done) : "r"(smem_u32(bar)), "r"(phase_bit) : "memory"); + return (bool)done; +} + +// mbarrier.try_wait.shared.b64 P, [bar], token — token form (used in tests 1, 7) +__device__ static inline bool mbar_try_wait_token(uint64_t* bar, uint64_t tok) { + uint32_t done; + asm volatile( + "{ .reg .pred P;\n\t" + "mbarrier.try_wait.shared.b64 P, [%1], %2;\n\t" + "selp.u32 %0, 1, 0, P; }" + : "=r"(done) : "r"(smem_u32(bar)), "l"(tok) : "memory"); + return (bool)done; +} + +// mbarrier.arrive.shared::cluster.b64 _, [remote_bar] +// CUTLASS: ClusterBarrier::arrive(smem_ptr, cta_id) (barrier.h:489) +// remote_bar is a 32-bit shared-memory address in another CTA's space +// obtained via mapa.shared::cluster.u32. +__device__ static inline void mbar_arrive_cluster_remote32(uint32_t remote_smem_addr) { + asm volatile( + "mbarrier.arrive.shared::cluster.b64 _, [%0];" + :: "r"(remote_smem_addr) : "memory"); +} + +// mapa.shared::cluster.u32 dst, src, cta_rank +// Maps a 32-bit shared-memory address in this CTA's space to the same +// offset in another CTA's shared memory, within the cluster. +// This is the CUTLASS barrier.h form (u32), distinct from mapa.u64. +__device__ static inline uint32_t mapa32(const void* local_ptr, uint32_t cta_rank) { + uint32_t dst; + asm("mapa.shared::cluster.u32 %0, %1, %2;" + : "=r"(dst) : "r"(smem_u32(local_ptr)), "r"(cta_rank)); + return dst; +} + +// Spin-wait helpers +__device__ static inline void mbar_spin_wait_token(uint64_t* bar, uint64_t tok) { + while (!mbar_try_wait_token(bar, tok)) __nanosleep(20); +} +// mbar_spin_wait_parity must be a macro so __LINE__ captures the call site, +// giving each inline PTX block a unique label. Two calls in the same kernel +// with the same function would otherwise generate duplicate LAB_WAIT labels +// that the gpgpu-sim PTX parser aliases to the first occurrence. +#define mbar_spin_wait_parity(bar, phase_bit) do { \ + uint32_t _sa = smem_u32(bar); \ + uint32_t _pb = (phase_bit); \ + uint32_t _tk = 0x989680; \ + MBAR_WAIT_PARITY(_sa, _pb, _tk); \ +} while(0) + +// ============================================================ +// TEST 1: mbarrier.init.shared::cta + arrive(token) + try_wait.token +// PTX: mbarrier.init.shared::cta.b64 / .arrive.shared::cta.b64 tok / .try_wait.shared.b64 +// +// Baseline: all 32 threads arrive with count=1, wait on 64-bit token. +// Verifies the fundamental init→arrive→wait cycle works. +// h[0] = 1 if sum of smem[0..31] == 496 after barrier completes. +// ============================================================ +__global__ void test1_baseline_token_wait(int* out) { + __shared__ uint64_t bar; + __shared__ int smem[32]; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + smem[threadIdx.x] = threadIdx.x; + __threadfence_block(); + + uint64_t tok = mbar_arrive_token(&bar); + mbar_spin_wait_token(&bar, tok); + + if (threadIdx.x == 0) { + int sum = 0; + for (int i = 0; i < 32; i++) sum += smem[i]; + out[0] = (sum == 496) ? 1 : -1; + } +} + +// ============================================================ +// TEST 2: mbarrier.try_wait.parity — the CUTLASS wait form +// PTX: mbarrier.try_wait.parity.shared::cta.b64 P, [bar], phase_bit +// +// CUTLASS uses parity-form exclusively (copy_sm90_desc.hpp wait_barrier and +// barrier.h ClusterBarrier::try_wait). This takes a plain u32 phase_bit (0 or 1) +// instead of a 64-bit token. The simulator must route PARITY_OPTION to +// SYNCS_TRY_WAIT, not to SYNCS_MAX_ENUM_NO_USED. +// +// SIMULATOR ISSUE: ptx_ir.cc PARITY_OPTION falls to default → SYNCS_MAX_ENUM_NO_USED +// → shader.cc assert(false) crash. Fix: add PARITY_OPTION → SYNCS_TRY_WAIT. +// PTX ISA §9.7.12.6: try_wait.parity completes when phase%2 == phase_bit. +// h[1] = 1 if barrier completes and smem sum is correct. +// ============================================================ +__global__ void test2_try_wait_parity(int* out) { + __shared__ uint64_t bar; + __shared__ int smem[32]; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + smem[threadIdx.x] = threadIdx.x; + __threadfence_block(); + + // All threads arrive without token — CUTLASS arrive_barrier form + mbar_arrive_no_token(&bar); + + // Wait using parity form: current phase=0, parity=0 → phase_bit=0 (CUTLASS convention) + mbar_spin_wait_parity(&bar, 0); + + if (threadIdx.x == 0) { + int sum = 0; + for (int i = 0; i < 32; i++) sum += smem[i]; + out[1] = (sum == 496) ? 1 : -1; + } +} + +// ============================================================ +// TEST 3: arrive.expect_tx + complete_tx + try_wait.parity +// PTX: mbarrier.arrive.expect_tx.shared::cta.b64 _, [bar], bytes +// mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 [bar], bytes +// mbarrier.try_wait.parity.shared::cta.b64 P, [bar], phase_bit +// +// This is the CUTLASS TMA producer/consumer pattern (copy_sm90_desc.hpp): +// 1. Consumer thread calls arrive.expect_tx (announces incoming async bytes, +// decrements pending by 1, adds bytes to tx_count) +// 2. Simulated TMA data arrives, producer calls complete_tx (decrements tx_count) +// 3. All threads wait with parity form +// Barrier completes only when BOTH pending==0 AND tx_count==0. +// PTX ISA §9.7.12.5: expect_tx adds to the expected transaction count. +// h[2] = 1 if data is correct after wait. +// ============================================================ +__global__ void test3_expect_tx_parity_wait(int* out) { + __shared__ uint64_t bar; + __shared__ int data[32]; + + // Init: 1 arrive expected (the expect_tx thread), plus tx bytes + if (threadIdx.x == 0) { + mbar_init(&bar, blockDim.x); + } + __syncthreads(); + + if (threadIdx.x == 0) { + // Announce 32*4=128 bytes of incoming async data and arrive (count=1) + mbar_arrive_expect_tx(&bar, 32 * sizeof(int)); + + // Simulate the async data arriving (in real use this is TMA) + for (int i = 0; i < 32; i++) data[i] = i + 100; + __threadfence_block(); + + // Signal tx completion + mbar_complete_tx(&bar, 32 * sizeof(int)); + } else { + // All other threads arrive without token + mbar_arrive_no_token(&bar); + } + + // All threads wait: current phase=0, parity=0 → phase_bit=0 (CUTLASS convention) + mbar_spin_wait_parity(&bar, 0); + + if (threadIdx.x == 0) { + int ok = 1; + for (int i = 0; i < 32; i++) if (data[i] != i + 100) { ok = 0; break; } + out[2] = ok ? 1 : -1; + } +} + +// ============================================================ +// TEST 4: arrive(no-token) _, [bar] — CUTLASS arrive_barrier form +// PTX: mbarrier.arrive.shared::cta.b64 tok, [bar] (token discarded) +// +// CUTLASS arrive_barrier() (copy_sm90_desc.hpp:122) issues this form with +// a .reg .b64 tok that is read but never stored — the destination is +// architecturally a register but the C++ wrapper ignores it. +// This verifies that arrive with implicit dst (vs explicit "_") is handled +// correctly by the simulator's dst.is_non_arch_reg() check. +// PTX ISA §9.7.12.4: the token is valid but the caller may discard it. +// h[3] = 1 if barrier completes (all 32 threads arrived, waited with parity). +// ============================================================ +__global__ void test4_arrive_no_token_form(int* out) { + __shared__ uint64_t bar; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + // All threads call arrive_no_token — token captured in reg but discarded + mbar_arrive_no_token(&bar); + + // current phase=0, parity=0 → phase_bit=0 + mbar_spin_wait_parity(&bar, 0); + + if (threadIdx.x == 0) out[3] = 1; +} + +// ============================================================ +// TEST 5: mbarrier.arrive.shared::cluster.b64 via mapa.shared::cluster.u32 +// PTX: mapa.shared::cluster.u32 remAddr, localAddr, cta_rank +// mbarrier.arrive.shared::cluster.b64 _, [remAddr] +// mbarrier.try_wait.parity.shared::cta.b64 P, [bar], phase_bit +// +// This is the CUTLASS ClusterBarrier::arrive(smem_ptr, cta_id) form (barrier.h:489). +// Uses 32-bit mapa (not 64-bit mapa.u64) to map the local shmem address to a +// remote CTA's shmem space. Thread 0 of each CTA signals the partner CTA's barrier. +// Each barrier is init'd for 1 arrival (from the partner's thread 0). +// Each CTA then waits with parity form for its own barrier to complete. +// +// SIMULATOR ISSUE: mapa.shared::cluster.u32 address encoding may differ from +// mapa.u64. The address passed to arrive.shared::cluster.b64 must be decoded +// from a 32-bit generic shared address, not a 64-bit one. +// h[4] = 1 (both CTAs write 1 to same slot) +// ============================================================ +__global__ void __cluster_dims__(2, 1, 1) +test5_cluster_arrive_mapa32(int* out) { + auto cluster = cg::this_cluster(); + unsigned my_rank = cluster.block_rank(); + unsigned partner = 1 - my_rank; + + __shared__ uint64_t bar; + + if (threadIdx.x == 0) mbar_init(&bar, 1); + cluster.sync(); + + if (threadIdx.x == 0) { + // Map partner's bar address using 32-bit mapa (CUTLASS barrier.h form) + uint32_t remote_addr = mapa32(&bar, partner); + mbar_arrive_cluster_remote32(remote_addr); + } + + // current phase=0, parity=0 → phase_bit=0 + if (threadIdx.x == 0) { + mbar_spin_wait_parity(&bar, 0); + out[4] = 1; // both CTAs write to same slot; that's fine, both write 1 + } +} + +// ============================================================ +// TEST 6: Two-phase parity alternation — arrive+try_wait.parity, phase 0 then phase 1 +// PTX: mbarrier.arrive.shared::cta.b64 / mbarrier.try_wait.parity.shared::cta.b64 +// +// Tests that parity form correctly handles two consecutive phases with alternating +// phase_bit (0 then 1), matching CUTLASS PipelineState::phase_ toggle (^= 1). +// +// Two separate barrier cycles in one kernel: +// Cycle A: init(count=32), all arrive, wait(phase_bit=0) — phase 0 in-progress +// Cycle B: re-use same bar (now at phase=1), all arrive, wait(phase_bit=1) — phase 1 +// +// Simulator note: each cycle is a single arrive→wait, so two-pass mode advances +// the barrier by exactly 1 per cycle. No parity ambiguity (phase 2 never reached +// within a single kernel execution, only phase 0→1 and 1→2). +// h[5] = 1 if both cycles complete with correct data. +// ============================================================ +__global__ void test6_looping_parity(int* out) { + __shared__ uint64_t bar; + __shared__ int smemA[32]; + __shared__ int smemB[32]; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + // Cycle A: phase 0 → phase_bit=0 + smemA[threadIdx.x] = threadIdx.x; + __threadfence_block(); + mbar_arrive_no_token(&bar); + mbar_spin_wait_parity(&bar, 0); + + // Cycle B: phase 1 → phase_bit=1 + smemB[threadIdx.x] = threadIdx.x + 100; + __threadfence_block(); + mbar_arrive_no_token(&bar); + mbar_spin_wait_parity(&bar, 1); + + if (threadIdx.x == 0) { + int sumA = 0, sumB = 0; + for (int i = 0; i < 32; i++) { sumA += smemA[i]; sumB += smemB[i]; } + int ok = (sumA == 496) && (sumB == 496 + 32*100); + out[5] = ok ? 1 : -1; + } +} + +// ============================================================ +// TEST 7: count=0 observe-only arrive must not decrement pending +// PTX: mbarrier.arrive.shared::cta.b64 tok, [bar], 0 (count=0) +// +// PTX ISA §9.7.12.4: "If count is 0, the operation acts as an observe and +// does not modify the pending count of the mbarrier object." +// Thread 0 does the real arrive (count=1); all others call count=0 to get +// a token. A simulator that subtracts 0 from a uint32 pending count and then +// calls try_phase_transition() could spuriously fire the transition or +// underflow the counter (wrapping below 0 as unsigned). +// h[6] = 1 if all threads see the barrier complete correctly. +// ============================================================ +__global__ void test7_count0_observe(int* out) { + __shared__ uint64_t bar; + __shared__ int flag; + + if (threadIdx.x == 0) { + mbar_init(&bar, 1); // only 1 real arrival expected + flag = 0; + } + __syncthreads(); + + uint64_t tok; + if (threadIdx.x == 0) { + flag = 42; + __threadfence_block(); + tok = mbar_arrive_token(&bar, 1); // real arrive: pending 1→0 → transition + } else { + tok = mbar_arrive_token(&bar, 0); // observe-only: no pending change + } + + mbar_spin_wait_token(&bar, tok); + + if (threadIdx.x == 0) + out[6] = (flag == 42) ? 1 : -1; +} + +// ============================================================ +// TEST 8: mbarrier.inval then re-init on same address +// PTX: mbarrier.inval.shared.b64 / mbarrier.init.shared::cta.b64 (re-init) +// +// PTX ISA §9.7.12.9: after inval, the mbarrier object is in an undefined +// state. A subsequent init on the same address must reinitialize it cleanly. +// SIMULATOR ISSUE: the mbarrier_clear_for_cta mechanism clears on CTA exit, +// but inval within a kernel should also work. If the simulator doesn't remove +// the mbarrier on inval and a second init hits the "already exists" assert, +// this test exposes it. +// h[7] = 1 if re-init and second barrier cycle complete successfully. +// ============================================================ +__global__ void test8_inval_reinit(int* out) { + __shared__ uint64_t bar; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + // First cycle + uint64_t tok = mbar_arrive_token(&bar); + mbar_spin_wait_token(&bar, tok); + + if (threadIdx.x == 0) mbar_inval(&bar); + __syncthreads(); + + // Re-init same address + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + __syncthreads(); + + // Second cycle — parity form (phase resets to 0 after re-init, so phase_bit=0) + mbar_arrive_no_token(&bar); + mbar_spin_wait_parity(&bar, 0); + + if (threadIdx.x == 0) out[7] = 1; +} + +// ============================================================ +// TEST 9: expect_tx + complete_tx + all-thread cluster arrive + parity wait +// PTX: mbarrier.arrive.expect_tx.shared::cta.b64 +// mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 +// mbarrier.arrive.shared::cta.b64 (all threads, no token) +// mbarrier.try_wait.parity.shared::cta.b64 +// +// This is the full CUTLASS TMA warp-specialized producer pattern: +// - Thread 0 calls expect_tx (arrives + announces TX bytes) +// - Thread 0 simulates data arriving then calls complete_tx +// - All 32 threads call arrive (so the barrier needs 32 arrives total, +// but thread 0's arrive was already done inside expect_tx) +// - All wait with parity form +// +// Note: expect_tx decrements pending by 1 AND adds tx_count. So init must +// count the total thread arrivals including the implicit one from expect_tx. +// Init with blockDim.x (32): expect_tx contributes 1 arrive (from thread 0), +// remaining 31 threads contribute 1 each = 32 total → transition fires when +// tx_count also reaches 0. +// h[8] = 1 if data is correct after wait. +// ============================================================ +__global__ void __cluster_dims__(2, 1, 1) +test9_full_tma_pattern(int* out) { + auto cluster = cg::this_cluster(); + unsigned my_rank = cluster.block_rank(); + + __shared__ uint64_t bar; + __shared__ int data[32]; + + if (threadIdx.x == 0) mbar_init(&bar, blockDim.x); + cluster.sync(); + + if (my_rank == 0) { + if (threadIdx.x == 0) { + // arrive.expect_tx: arrives (pending 32→31) + announces 128 bytes tx + mbar_arrive_expect_tx(&bar, 32 * sizeof(int)); + + // Simulate data arriving asynchronously + for (int i = 0; i < 32; i++) data[i] = i + 300; + __threadfence_block(); + + // complete_tx: tx_count 128→0 + mbar_complete_tx(&bar, 32 * sizeof(int)); + } else { + // Remaining 31 threads arrive (no token) + mbar_arrive_no_token(&bar); + } + } else { + // CTA1: all 32 threads just arrive normally + mbar_arrive_no_token(&bar); + } + + // current phase=0, parity=0 → phase_bit=0 + mbar_spin_wait_parity(&bar, 0); + + if (my_rank == 0 && threadIdx.x == 0) { + int ok = 1; + for (int i = 0; i < 32; i++) if (data[i] != i + 300) { ok = 0; break; } + out[8] = ok ? 1 : -1; + } +} + +// ============================================================ +// main +// ============================================================ +int main() { + const int N = 9; + int h[N] = {}; + int* d; + cudaMalloc(&d, N * sizeof(int)); + + // Zero all slots once upfront — each test writes only its own slot(s) + cudaMemset(d, 0, N * sizeof(int)); + + // 2-CTA cluster config + cudaLaunchConfig_t cfg2 = {}; + cudaLaunchAttribute attr2[1]; + attr2[0].id = cudaLaunchAttributeClusterDimension; + attr2[0].val.clusterDim = {2, 1, 1}; + cfg2.gridDim = {2, 1, 1}; + cfg2.blockDim = {32, 1, 1}; + cfg2.attrs = attr2; + cfg2.numAttrs = 1; + + auto sync = [&]() { + cudaDeviceSynchronize(); + cudaMemcpy(h, d, N * sizeof(int), cudaMemcpyDeviceToHost); + }; + auto pf = [](int v) { return v == 1 ? "PASS" : (v == -1 ? "FAIL" : "SKIP"); }; + + printf("=== mbarrier Cluster Microbenchmark (sm_90 / Hopper) ===\n"); + printf("PTX forms covered:\n"); + printf(" mbarrier.init.shared::cta.b64\n"); + printf(" mbarrier.arrive.shared::cta.b64 tok/_ (token and no-token forms)\n"); + printf(" mbarrier.arrive.expect_tx.shared::cta.b64 _\n"); + printf(" mbarrier.arrive.shared::cluster.b64 _ (via mapa.shared::cluster.u32)\n"); + printf(" mbarrier.try_wait.parity.shared::cta.b64 (CUTLASS wait form)\n"); + printf(" mbarrier.try_wait.shared.b64 (token wait form)\n"); + printf(" mbarrier.complete_tx.relaxed.cluster.shared::cta.b64\n"); + printf(" mbarrier.inval.shared.b64\n\n"); + + // TEST 1 — h[0] + printf("TEST 1 [init/arrive-token/try_wait-token baseline]... "); + fflush(stdout); + test1_baseline_token_wait<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[0])); + + // TEST 2 — h[1] + printf("TEST 2 [try_wait.parity - CUTLASS wait form]... "); + fflush(stdout); + test2_try_wait_parity<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[1])); + + // TEST 3 — h[2] + printf("TEST 3 [arrive.expect_tx + complete_tx + try_wait.parity]... "); + fflush(stdout); + test3_expect_tx_parity_wait<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[2])); + + // TEST 4 — h[3] + printf("TEST 4 [arrive(no-token/_, [bar]) - CUTLASS arrive_barrier form]... "); + fflush(stdout); + test4_arrive_no_token_form<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[3])); + + // TEST 5 — h[4] (2-CTA cluster) + printf("TEST 5 [arrive.shared::cluster via mapa.shared::cluster.u32]... "); + fflush(stdout); + cudaLaunchKernelEx(&cfg2, test5_cluster_arrive_mapa32, d); + sync(); + printf("%s\n", pf(h[4])); + + // TEST 6 — h[5] + printf("TEST 6 [looping parity: 4 phase transitions, alternating phase_bit]... "); + fflush(stdout); + test6_looping_parity<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[5])); + + // TEST 7 — h[6] + printf("TEST 7 [count=0 observe-only arrive must not decrement pending]... "); + fflush(stdout); + test7_count0_observe<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[6])); + + // TEST 8 — h[7] + printf("TEST 8 [mbarrier.inval then re-init on same address]... "); + fflush(stdout); + test8_inval_reinit<<<1, 32>>>(d); + sync(); + printf("%s\n", pf(h[7])); + + // TEST 9 — h[8] (2-CTA cluster) + printf("TEST 9 [full CUTLASS TMA pattern: expect_tx+complete_tx+arrive+parity-wait]... "); + fflush(stdout); + cudaLaunchKernelEx(&cfg2, test9_full_tma_pattern, d); + sync(); + printf("%s\n", pf(h[8])); + + printf("\n=== Results: [%d,%d,%d,%d,%d,%d,%d,%d,%d] ===\n", + h[0],h[1],h[2],h[3],h[4],h[5],h[6],h[7],h[8]); + bool all = true; + for (int i = 0; i < N; i++) all = all && (h[i] == 1); + printf("Overall: %s\n", all ? "PASS" : "PARTIAL/FAIL"); + + cudaFree(d); + return 0; +} From d4d153c09aaa9fb661a9bdf75fb37ea09570243a Mon Sep 17 00:00:00 2001 From: Raunak Dani Date: Sun, 19 Jul 2026 23:42:54 -0400 Subject: [PATCH 29/35] Add mapa.shared::cluster.u32 careful functional-verification microbenchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New ubench under mem/mapa_shared_cluster that exercises the Hopper+ PTX instruction `mapa.shared::cluster.u32` (PTX ISA 7.8+, sm_90+) via explicit inline PTX and verifies end-to-end cluster shared-memory remapping from the host. Built through the existing GPU_Microbenchmark recursion (`make GPU_Microbenchmark`); installs mapa_shared_cluster into bin/ with shared cudart for silicon and simulator runs. Instruction under test: mapa.shared::cluster.u32 d, a, b; Remaps shared-window address `a` onto cluster CTA rank `b`. Does NOT move data; a later ld/st.shared::cluster through `d` touches CTA b's copy of that offset. Cluster launch uses __cluster_dims__(2,1,1) with barrier.cluster arrive/wait around remapped stores. Why the gates are strict (a stub/identity mapa must fail): Gate A — Rank sensitivity (address-only): from the same local address, mapa(., 0) and mapa(., 1) MUST differ. A stub that ignores operand b and copies a→a fails this check hard. Gate B — Remote store / local load: each CTA writes its magic into the *peer* slot via st.shared::cluster [mapa(local, peer)], then reads its *own* slot with an ordinary shared load (no mapa). Expected local value is the peer's magic; identity mapa writes self and fails. Gate C — Self-map: mapa(local, own_rank) must still name this CTA's slot (load through that address returns the peer magic after Gate B). Host D2H checks all gates; device buffers are poison-filled so a silent no-op is obvious. Makefile targets Hopper+ arches (sm_90a / sm_100a / sm_101 / sm_120) with SASS+PTX gencode, matching the elect_sync-style per-instruction ubench pattern. Co-authored-by: Cursor --- .../ubench/mem/mapa_shared_cluster/Makefile | 34 +++ .../mapa_shared_cluster.cu | 262 ++++++++++++++++++ 2 files changed, 296 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile new file mode 100644 index 000000000..a162019de --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile @@ -0,0 +1,34 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +# mapa.shared::cluster.u32 careful functional microbenchmark. +# Remaps a shared-window address onto a cluster CTA rank; Gates A/B/C +# verify rank sensitivity and remote store / local load semantics. +ALL_BENCHES := mapa_shared_cluster + +# mapa.shared::cluster is Hopper+ PTX (PTX ISA 7.8+, SM_90 and above). +# Override on the command line, e.g. `make ARCH=sm_90a`, for a single GPU. +ARCH ?= sm_90a sm_100a sm_101 sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +# Generate both SASS (code=sm_XX) and PTX (code=compute_XX) for each arch. +NVCC_FLAGS := $(foreach arch,$(ARCH),-gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster.cu b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster.cu new file mode 100644 index 000000000..13c08eb98 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster.cu @@ -0,0 +1,262 @@ +#include +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// mapa_shared_cluster.cu — careful functional gate for `mapa.shared::cluster.u32` +// +// WHAT mapa DOES (PTX ISA 7.8+, sm_90+) +// -------------------------------------- +// mapa.shared::cluster.u32 d, a, b; +// +// Remap shared-window address `a` onto cluster CTA rank `b`. Does NOT move data; +// a later ld/st.shared::cluster uses `d` to touch CTA b's copy of that offset. +// +// WHY THE PREVIOUS UBENCH WAS TOO WEAK +// ------------------------------------ +// It wrote each CTA's magic into its *own* slot, then remote-loaded via mapa. +// A stub mapa that returns `a` unchanged (ignoring rank) can still look plausible +// when combined with incidental cluster-address encoding elsewhere. This version +// closes those gaps: +// +// Gate A — RANK SENSITIVITY (address-only) +// From the same local address, mapa(., 0) and mapa(., 1) MUST differ. +// The current GPGPU-Sim stub ignores operand b and copies a → a, so +// mapped0 == mapped1 and this gate fails hard. +// +// Gate B — REMOTE STORE, LOCAL LOAD (end-to-end) +// Each CTA writes its magic into the *peer* slot via +// st.shared::cluster [mapa(local, peer)] +// then reads its *own* slot with an ordinary shared load (no mapa). +// Expected local value == peer's magic. Identity mapa writes own slot +// instead, so the local load sees own magic and fails. +// +// Gate C — SELF MAP IS IDENTITY-ISH +// mapa(local, own_rank) must still address this CTA's slot (local load +// through that mapped addr returns what we wrote remotely into ourselves, +// i.e. the peer's magic after Gate B). +// +// Host checks all three after D2H. Poison-fill so a silent no-op is obvious. +// +// make KERNEL=mapa_shared_cluster +// ./run.sh mapa_shared_cluster +// ./run.sh mapa_shared_cluster silicon +// ───────────────────────────────────────────────────────────────────────────── + +#define CLUSTER_SIZE 2 +#define BLOCK_SIZE 32 +#define MAGIC_BASE 0xA000u +#define POISON 0xFFFFFFFFu + +__device__ __forceinline__ unsigned cluster_ctarank() { + unsigned rank; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(rank)); + return rank; +} + +// Instruction under test: remap shared-window address `addr` onto CTA `rank`. +__device__ __forceinline__ unsigned mapa_shared_cluster_u32(unsigned addr, + unsigned rank) { + unsigned mapped; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" + : "=r"(mapped) + : "r"(addr), "r"(rank)); + return mapped; +} + +__device__ __forceinline__ void st_shared_cluster_u32(unsigned addr, + unsigned val) { + asm volatile("st.shared::cluster.u32 [%0], %1;" + : + : "r"(addr), "r"(val) + : "memory"); +} + +__device__ __forceinline__ unsigned ld_shared_cluster_u32(unsigned addr) { + unsigned val; + asm volatile("ld.shared::cluster.u32 %0, [%1];" + : "=r"(val) + : "r"(addr) + : "memory"); + return val; +} + +__global__ void __cluster_dims__(CLUSTER_SIZE, 1, 1) +mapa_shared_cluster_kernel(unsigned *d_local_after, // Gate B result + unsigned *d_self_via_mapa, // Gate C result + unsigned *d_mapped_rank0, // Gate A + unsigned *d_mapped_rank1, // Gate A + unsigned *d_local_addr, + unsigned *d_mapped_peer) { + __shared__ unsigned slot; + + const unsigned tid = threadIdx.x; + const unsigned rank = cluster_ctarank(); + const unsigned peer = rank ^ 1u; + + // Shared-window address of our local slot (CUTLASS / PTX mapa input form). + const unsigned local_addr = + static_cast(__cvta_generic_to_shared(&slot)); + + // Start from a known empty slot so Gate B cannot see a self-written magic. + if (tid == 0) { + slot = 0u; + } + __syncthreads(); + + // Cluster barrier so every CTA's zeroing is visible before remote stores. + // (GPGPU-Sim currently lexes these to NOP_OP; silicon is a real barrier.) + asm volatile("barrier.cluster.arrive;" ::: "memory"); + asm volatile("barrier.cluster.wait;" ::: "memory"); + + if (tid == 0) { + // ── Gate B: write *into the peer CTA* via mapa ───────────────────── + // (Gate A address checks are recorded after the barrier below.) + const unsigned mapped_peer = mapa_shared_cluster_u32(local_addr, peer); + st_shared_cluster_u32(mapped_peer, MAGIC_BASE + rank); + } + + // Make remote stores visible before anyone reads their local slot. + asm volatile("barrier.cluster.arrive;" ::: "memory"); + asm volatile("barrier.cluster.wait;" ::: "memory"); + __syncthreads(); + + if (tid == 0) { + // Ordinary .shared::cta load of *our* slot — peer should have written + // MAGIC_BASE + peer into it through Gate B's remapped store. + const unsigned local_after = slot; + + // ── Gate C: self-map still names our own slot ────────────────────── + const unsigned mapped_self = + mapa_shared_cluster_u32(local_addr, rank); + const unsigned self_via_mapa = ld_shared_cluster_u32(mapped_self); + + // Recompute peer map for the host-visible address dump (same as Gate B). + const unsigned mapped_peer = mapa_shared_cluster_u32(local_addr, peer); + const unsigned mapped0 = mapa_shared_cluster_u32(local_addr, 0u); + const unsigned mapped1 = mapa_shared_cluster_u32(local_addr, 1u); + + d_local_after[rank] = local_after; + d_self_via_mapa[rank] = self_via_mapa; + d_mapped_rank0[rank] = mapped0; + d_mapped_rank1[rank] = mapped1; + d_local_addr[rank] = local_addr; + d_mapped_peer[rank] = mapped_peer; + } +} + +int main() { + setvbuf(stdout, NULL, _IONBF, 0); + + printf("=== mapa.shared::cluster.u32 careful functional ubench ===\n"); + printf("cluster=%d CTAs, block=%d threads, magic_base=0x%X\n\n", + CLUSTER_SIZE, BLOCK_SIZE, MAGIC_BASE); + + const int n = CLUSTER_SIZE; + const size_t bytes = n * sizeof(unsigned); + + unsigned *h_local_after = (unsigned *)malloc(bytes); + unsigned *h_self_via_mapa = (unsigned *)malloc(bytes); + unsigned *h_mapped0 = (unsigned *)malloc(bytes); + unsigned *h_mapped1 = (unsigned *)malloc(bytes); + unsigned *h_local_addr = (unsigned *)malloc(bytes); + unsigned *h_mapped_peer = (unsigned *)malloc(bytes); + + unsigned *d_local_after = NULL, *d_self_via_mapa = NULL; + unsigned *d_mapped0 = NULL, *d_mapped1 = NULL; + unsigned *d_local_addr = NULL, *d_mapped_peer = NULL; + + cudaMalloc(&d_local_after, bytes); + cudaMalloc(&d_self_via_mapa, bytes); + cudaMalloc(&d_mapped0, bytes); + cudaMalloc(&d_mapped1, bytes); + cudaMalloc(&d_local_addr, bytes); + cudaMalloc(&d_mapped_peer, bytes); + + cudaMemset(d_local_after, 0xFF, bytes); + cudaMemset(d_self_via_mapa, 0xFF, bytes); + cudaMemset(d_mapped0, 0xFF, bytes); + cudaMemset(d_mapped1, 0xFF, bytes); + cudaMemset(d_local_addr, 0xFF, bytes); + cudaMemset(d_mapped_peer, 0xFF, bytes); + + mapa_shared_cluster_kernel<<>>( + d_local_after, d_self_via_mapa, d_mapped0, d_mapped1, + d_local_addr, d_mapped_peer); + cudaError_t err = cudaDeviceSynchronize(); + + int fails = 0; + if (err != cudaSuccess) { + printf("launch/sync error: %s\n", cudaGetErrorString(err)); + fails++; + } else { + cudaMemcpy(h_local_after, d_local_after, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_self_via_mapa, d_self_via_mapa, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_mapped0, d_mapped0, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_mapped1, d_mapped1, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_local_addr, d_local_addr, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_mapped_peer, d_mapped_peer, bytes, cudaMemcpyDeviceToHost); + + printf("per-CTA results (after D2H):\n"); + for (int r = 0; r < n; ++r) { + // After remote stores, my slot holds the peer CTA's magic. + const unsigned expect_local = MAGIC_BASE + (unsigned)(r ^ 1); + const unsigned expect_self = expect_local; // self-map → same slot + + const int wrote = + (h_local_after[r] != POISON) && (h_self_via_mapa[r] != POISON) && + (h_mapped0[r] != POISON) && (h_mapped1[r] != POISON); + + // Gate A: mapa(.,0) and mapa(.,1) must disagree. + const int rank_sensitive = (h_mapped0[r] != h_mapped1[r]); + + // Gate A': peer map must differ from self map when peer != self. + const unsigned mapped_self_expect = + (r == 0) ? h_mapped0[r] : h_mapped1[r]; + const int peer_addr_differs = + (h_mapped_peer[r] != mapped_self_expect); + + // Gate B / C value checks. + const int local_ok = (h_local_after[r] == expect_local); + const int self_ok = (h_self_via_mapa[r] == expect_self); + + if (!wrote) fails++; + if (!rank_sensitive) fails++; + if (!peer_addr_differs) fails++; + if (!local_ok) fails++; + if (!self_ok) fails++; + + printf(" CTA rank %d:\n", r); + printf(" local_addr = 0x%08X\n", h_local_addr[r]); + printf(" mapa(local, 0) = 0x%08X\n", h_mapped0[r]); + printf(" mapa(local, 1) = 0x%08X [%s]\n", + h_mapped1[r], + rank_sensitive ? "rank-sensitive ok" : "RANK IGNORED"); + printf(" mapped_peer_addr = 0x%08X [%s]\n", + h_mapped_peer[r], + peer_addr_differs ? "differs from self ok" : "SAME AS SELF"); + printf(" local_after = 0x%08X (expected 0x%08X) [%s]\n", + h_local_after[r], expect_local, + local_ok ? "ok" : "MISMATCH"); + printf(" self_via_mapa = 0x%08X (expected 0x%08X) [%s]\n", + h_self_via_mapa[r], expect_self, + self_ok ? "ok" : "MISMATCH"); + } + } + + cudaFree(d_local_after); + cudaFree(d_self_via_mapa); + cudaFree(d_mapped0); + cudaFree(d_mapped1); + cudaFree(d_local_addr); + cudaFree(d_mapped_peer); + free(h_local_after); + free(h_self_via_mapa); + free(h_mapped0); + free(h_mapped1); + free(h_local_addr); + free(h_mapped_peer); + + printf("\nRESULT: %s\n", fails == 0 ? "PASSED" : "FAILED"); + return fails == 0 ? 0 : 1; +} From 1c78e94d3366eee7df1dbafaca597059dfdc1010 Mon Sep 17 00:00:00 2001 From: lara kawasme Date: Tue, 21 Jul 2026 14:43:34 -0700 Subject: [PATCH 30/35] ubenchmark for cta_rank complete --- .../cluster_ctarank/Makefile | 33 +++ .../cluster_ctarank/README.md | 38 ++++ .../cluster_ctarank/cluster_ctarank.cu | 191 ++++++++++++++++++ .../cluster_ctarank/cluster_ctarank_3d.cu | 187 +++++++++++++++++ 4 files changed, 449 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/README.md create mode 100644 src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank.cu create mode 100644 src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/Makefile b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/Makefile new file mode 100644 index 000000000..7fe7b8680 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/Makefile @@ -0,0 +1,33 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +ALL_BENCHES := cluster_ctarank cluster_ctarank_3d + +# Thread-block clusters and %cluster_ctarank require SM_90 or newer. +ARCH ?= sm_120 +CC := nvcc + +# Unset CUDA_CPPFLAGS from setup_environment; use explicit codegen here. +CUDA_CPPFLAGS= + +NVCC_FLAGS := $(foreach arch,$(ARCH),\ + -gencode=arch=compute_$(subst sm_,,$(arch)),code=$(arch) \ + -gencode=arch=compute_$(subst sm_,,$(arch)),code=compute_$(subst sm_,,$(arch))) \ + -std=c++17 + +.DEFAULT_GOAL := all + +all: $(ALL_BENCHES) + +%.a: %.cu + $(CC) $(NVCC_FLAGS) $(CUDA_CPPFLAGS) --lib $< -o $@ + +$(ALL_BENCHES): %: %.a + $(CC) $(NVCC_FLAGS) $< -o $@ -lcudart --cudart shared + mkdir -p $(BIN_DIR) + mv $@ $(BIN_DIR) + +clean: + rm -f *.o *.a *.ptx *.ptxas *.sass $(ALL_BENCHES) + +.PHONY: all clean diff --git a/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/README.md b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/README.md new file mode 100644 index 000000000..2098d418a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/README.md @@ -0,0 +1,38 @@ +# `%cluster_ctarank` microbenchmark + +This benchmark launches multiple one-dimensional thread-block clusters and +records the PTX `%cluster_ctarank` special register once per CTA. For a cluster +containing `N` CTAs, the expected ranks are `0` through `N - 1`; the sequence +starts again at zero in the next cluster. It also reads `%ctaid.x` directly and +checks that it equals CUDA's `blockIdx.x`, the CTA's grid-wide x coordinate. + +Build for an RTX 5070/5070 Ti (`sm_120`) and run the default case (four clusters, +four CTAs per cluster): + +```sh +make +../../../bin/cluster_ctarank +``` + +Choose another number of clusters and CTAs per cluster at runtime: + +```sh +../../../bin/cluster_ctarank 3 8 +``` + +The grid always contains `clusters * ctas-per-cluster` CTAs. The second argument +is limited to the portable maximum of eight CTAs per cluster. + +## Three-dimensional test + +`cluster_ctarank_3d` uses a `2 x 2 x 2` cluster shape and a `4 x 4 x 4` grid, +giving eight clusters with eight CTAs each. It reads `%ctaid.x`, `%ctaid.y`, and +`%ctaid.z` directly and compares them with the corresponding `blockIdx` +components. It also checks the flattened cluster-local rank: + +```text +rank = local_x + 2 * (local_y + 2 * local_z) +``` + +Run it with: + diff --git a/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank.cu b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank.cu new file mode 100644 index 000000000..5c12a383a --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank.cu @@ -0,0 +1,191 @@ +#include + +#include +#include +#include +#include + +namespace { +/* + modify these values using cli args to test different numbers of clusters and + ctas per clusters. + example: 4 clusters, 8 ctas per cluster + 'cluster_ctarank 4 8' +*/ +constexpr unsigned kDefaultClusters = 4; +constexpr unsigned kDefaultCTAsPerCluster = 4; +constexpr unsigned kThreadsPerCTA = 32; + +// struct used to report results +struct CTAResult { + unsigned block_idx; + unsigned ctaid_x; + unsigned cluster_ctarank; +}; + +//error checking macro +#define CHECK_CUDA(call) \ + do { \ + const cudaError_t error = (call); \ + if (error != cudaSuccess) { \ + std::fprintf(stderr, "%s:%d: CUDA error: %s\n", __FILE__, \ + __LINE__, cudaGetErrorString(error)); \ + std::exit(EXIT_FAILURE); \ + } \ + } while (0) + +// gpu fn to read the %cluster_ctarank register +__device__ __forceinline__ unsigned read_cluster_ctarank() { + unsigned rank; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(rank)); + return rank; +} +// gpu fn to read the %ctaid register in x dimension only +__device__ __forceinline__ unsigned read_ctaid_x() { + unsigned ctaid_x; + asm volatile("mov.u32 %0, %%ctaid.x;" : "=r"(ctaid_x)); + return ctaid_x; +} + +__global__ void cluster_ctarank_kernel(CTAResult *results) { + // The special register is the same for every thread in the CTA, so only read if + // thread 0 to get single print and avoid interleaving of prints + // blockIdx.x == ctaid.x + if (threadIdx.x == 0) { + results[blockIdx.x] = + {blockIdx.x, read_ctaid_x(), read_cluster_ctarank()}; + } +} + +//parse cli args and ensure positive +unsigned parse_positive(const char *text, const char *name) { + errno = 0; + char *end = nullptr; + const unsigned long value = std::strtoul(text, &end, 10); + if (errno != 0 || end == text || *end != '\0' || value == 0 || + value > UINT_MAX) { + std::fprintf(stderr, "%s must be a positive integer (got '%s')\n", + name, text); + std::exit(EXIT_FAILURE); + } + return static_cast(value); +} + +} // namespace + +int main(int argc, char **argv) { + if (argc > 3) { + std::fprintf(stderr, "usage: %s [clusters] [ctas-per-cluster]\n", + argv[0]); + return EXIT_FAILURE; + } + + const unsigned num_clusters = + argc >= 2 ? parse_positive(argv[1], "clusters") : kDefaultClusters; + const unsigned ctas_per_cluster = + argc >= 3 ? parse_positive(argv[2], "ctas-per-cluster") + : kDefaultCTAsPerCluster; + + int device = 0; + CHECK_CUDA(cudaGetDevice(&device)); + + // check clusters are supported + cudaDeviceProp properties{}; + CHECK_CUDA(cudaGetDeviceProperties(&properties, device)); + + int cluster_launch_supported = 0; + CHECK_CUDA(cudaDeviceGetAttribute(&cluster_launch_supported, + cudaDevAttrClusterLaunch, device)); + if (!cluster_launch_supported) { + std::fprintf(stderr, "%s does not support thread-block clusters\n", + properties.name); + return EXIT_FAILURE; + } + const unsigned total_ctas = num_clusters * ctas_per_cluster; + + //descirbe requested cluster shape + cudaLaunchAttribute attribute{}; + attribute.id = cudaLaunchAttributeClusterDimension; + attribute.val.clusterDim.x = ctas_per_cluster; + attribute.val.clusterDim.y = 1; + attribute.val.clusterDim.z = 1; + + // build kernel launch config + cudaLaunchConfig_t config{}; + config.gridDim = dim3(total_ctas, 1, 1); + config.blockDim = dim3(kThreadsPerCTA, 1, 1); + config.attrs = &attribute; + config.numAttrs = 1; + + //check supported cluster size + int max_cluster_size = 0; + CHECK_CUDA(cudaOccupancyMaxPotentialClusterSize( + &max_cluster_size, + (void*)cluster_ctarank_kernel, + &config)); + + printf("Maximum supported cluster size: %d\n", max_cluster_size); + + // verify ctas/cluster requested is valid + if (ctas_per_cluster > static_cast(max_cluster_size)) { + std::fprintf(stderr, + "Requested %u CTAs/cluster, but this GPU supports at most %d\n", + ctas_per_cluster, max_cluster_size); + return EXIT_FAILURE; + } + + // calculate total number of CTAS and malloc enough space for device results + const size_t bytes = total_ctas * sizeof(CTAResult); + CTAResult *device_results = nullptr; + CHECK_CUDA(cudaMalloc(&device_results, bytes)); + + //launch kernel + CHECK_CUDA(cudaLaunchKernelEx( + &config, + cluster_ctarank_kernel, + device_results)); + CHECK_CUDA(cudaDeviceSynchronize()); + + CTAResult *host_results = + static_cast(std::malloc(bytes)); + if (host_results == nullptr) { + std::fprintf(stderr, "host allocation failed\n"); + CHECK_CUDA(cudaFree(device_results)); + return EXIT_FAILURE; + } + CHECK_CUDA(cudaMemcpy(host_results, device_results, bytes, + cudaMemcpyDeviceToHost)); + + std::printf("GPU: %s (compute capability %d.%d)\n", properties.name, + properties.major, properties.minor); + std::printf("launch: %u clusters x %u CTAs/cluster = %u CTAs\n\n", + num_clusters, ctas_per_cluster, total_ctas); + std::printf("logical_cluster blockIdx.x %%ctaid.x " + "%%cluster_ctarank expected_rank\n"); + + unsigned failures = 0; + for (unsigned block = 0; block < total_ctas; ++block) { + const unsigned logical_cluster = block / ctas_per_cluster; + const unsigned expected_rank = block % ctas_per_cluster; + const CTAResult result = host_results[block]; + const bool pass = result.block_idx == block && + result.ctaid_x == result.block_idx && + result.cluster_ctarank == expected_rank; + std::printf("%15u %10u %8u %18u %13u %s\n", logical_cluster, + result.block_idx, result.ctaid_x, result.cluster_ctarank, + expected_rank, + pass ? "PASS" : "FAIL"); + failures += !pass; + } + + std::free(host_results); + CHECK_CUDA(cudaFree(device_results)); + + if (failures != 0) { + std::printf("\nFAIL: %u CTA result(s) did not match\n", failures); + return EXIT_FAILURE; + } + std::printf("\nPASS: %%ctaid.x matched blockIdx.x and %%cluster_ctarank " + "restarted at zero in every cluster\n"); + return EXIT_SUCCESS; +} diff --git a/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu new file mode 100644 index 000000000..6d4c266a0 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu @@ -0,0 +1,187 @@ +#include + +#include +#include + +namespace { + +constexpr unsigned kClusterX = 2; +constexpr unsigned kClusterY = 2; +constexpr unsigned kClusterZ = 2; +constexpr unsigned kGridX = 4; +constexpr unsigned kGridY = 4; +constexpr unsigned kGridZ = 4; +constexpr unsigned kThreadsPerCta = 32; +constexpr unsigned kTotalCtas = kGridX * kGridY * kGridZ; +constexpr unsigned kCtasPerCluster = kClusterX * kClusterY * kClusterZ; +// struct used to report results +struct CtaResult { + unsigned block_x; + unsigned block_y; + unsigned block_z; + unsigned ctaid_x; + unsigned ctaid_y; + unsigned ctaid_z; + unsigned cluster_ctarank; +}; +//error checking macro +#define CHECK_CUDA(call) \ + do { \ + const cudaError_t error = (call); \ + if (error != cudaSuccess) { \ + std::fprintf(stderr, "%s:%d: CUDA error: %s\n", __FILE__, \ + __LINE__, cudaGetErrorString(error)); \ + std::exit(EXIT_FAILURE); \ + } \ + } while (0) + +// gpu fns to read the %ctaid register in each dimension +__device__ __forceinline__ unsigned read_ctaid_x() { + unsigned value; + asm volatile("mov.u32 %0, %%ctaid.x;" : "=r"(value)); + return value; +} + +__device__ __forceinline__ unsigned read_ctaid_y() { + unsigned value; + asm volatile("mov.u32 %0, %%ctaid.y;" : "=r"(value)); + return value; +} + +__device__ __forceinline__ unsigned read_ctaid_z() { + unsigned value; + asm volatile("mov.u32 %0, %%ctaid.z;" : "=r"(value)); + return value; +} + +// gpu fn to read the %cluster_ctarank register +__device__ __forceinline__ unsigned read_cluster_ctarank() { + unsigned value; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(value)); + return value; +} + +__global__ void cluster_ctarank_3d_kernel(CtaResult *results) { + // The special register is the same for every thread in the CTA, so only read if + // thread 0 to get single print and avoid interleaving of prints + if (threadIdx.x != 0) { + return; + } + + const unsigned linearized_block_idx = + blockIdx.x + gridDim.x * (blockIdx.y + gridDim.y * blockIdx.z); + results[linearized_block_idx] = {blockIdx.x, + blockIdx.y, + blockIdx.z, + read_ctaid_x(), + read_ctaid_y(), + read_ctaid_z(), + read_cluster_ctarank()}; +} + +} // namespace + +int main() { + int device = 0; + CHECK_CUDA(cudaGetDevice(&device)); + + cudaDeviceProp properties{}; + CHECK_CUDA(cudaGetDeviceProperties(&properties, device)); + + int cluster_launch_supported = 0; + CHECK_CUDA(cudaDeviceGetAttribute(&cluster_launch_supported, + cudaDevAttrClusterLaunch, device)); + if (!cluster_launch_supported) { + std::fprintf(stderr, "%s does not support thread-block clusters\n", + properties.name); + return EXIT_FAILURE; + } + + CtaResult *device_results = nullptr; + CHECK_CUDA(cudaMalloc(&device_results, kTotalCtas * sizeof(CtaResult))); + + cudaLaunchAttribute attribute{}; + attribute.id = cudaLaunchAttributeClusterDimension; + attribute.val.clusterDim.x = kClusterX; + attribute.val.clusterDim.y = kClusterY; + attribute.val.clusterDim.z = kClusterZ; + + cudaLaunchConfig_t config{}; + config.gridDim = dim3(kGridX, kGridY, kGridZ); + config.blockDim = dim3(kThreadsPerCta, 1, 1); + config.attrs = &attribute; + config.numAttrs = 1; + + //check supported cluster size + int max_cluster_size = 0; + CHECK_CUDA(cudaOccupancyMaxPotentialClusterSize( + &max_cluster_size, + (void*)cluster_ctarank_3d_kernel, + &config)); + + printf("Maximum supported cluster size: %d\n", max_cluster_size); + + // verify ctas/cluster requested is valid + if (kCtasPerCluster > static_cast(max_cluster_size)) { + std::fprintf(stderr, + "Requested %u CTAs/cluster, but this GPU supports at most %d\n", + kTotalCtas, max_cluster_size); + return EXIT_FAILURE; + } + + CHECK_CUDA(cudaLaunchKernelEx(&config, cluster_ctarank_3d_kernel, + device_results)); + CHECK_CUDA(cudaDeviceSynchronize()); + + CtaResult host_results[kTotalCtas]; + CHECK_CUDA(cudaMemcpy(host_results, device_results, sizeof(host_results), + cudaMemcpyDeviceToHost)); + CHECK_CUDA(cudaFree(device_results)); + + std::printf("GPU: %s (compute capability %d.%d)\n", properties.name, + properties.major, properties.minor); + std::printf("grid=(%u,%u,%u), cluster=(%u,%u,%u), total=%u CTAs\n\n", + kGridX, kGridY, kGridZ, kClusterX, kClusterY, kClusterZ, + kTotalCtas); + std::printf("cluster(x,y,z) blockIdx(x,y,z) %%ctaid(x,y,z) " + "%%cluster_ctarank expected\n"); + + unsigned failures = 0; + for (unsigned z = 0; z < kGridZ; ++z) { + for (unsigned y = 0; y < kGridY; ++y) { + for (unsigned x = 0; x < kGridX; ++x) { + const unsigned index = x + kGridX * (y + kGridY * z); + const CtaResult result = host_results[index]; + const unsigned cluster_x = x / kClusterX; + const unsigned cluster_y = y / kClusterY; + const unsigned cluster_z = z / kClusterZ; + const unsigned local_x = x % kClusterX; + const unsigned local_y = y % kClusterY; + const unsigned local_z = z % kClusterZ; + const unsigned expected_rank = + local_x + kClusterX * (local_y + kClusterY * local_z); + + const bool pass = result.block_x == x && result.block_y == y && + result.block_z == z && result.ctaid_x == x && + result.ctaid_y == y && result.ctaid_z == z && + result.cluster_ctarank == expected_rank; + std::printf("(%u,%u,%u) (%u,%u,%u) " + "(%u,%u,%u) %2u %2u %s\n", + cluster_x, cluster_y, cluster_z, result.block_x, + result.block_y, result.block_z, result.ctaid_x, + result.ctaid_y, result.ctaid_z, + result.cluster_ctarank, expected_rank, + pass ? "PASS" : "FAIL"); + failures += !pass; + } + } + } + + if (failures != 0) { + std::printf("\nFAIL: %u CTA result(s) did not match\n", failures); + return EXIT_FAILURE; + } + std::printf("\nPASS: 3D %%ctaid matched blockIdx and all cluster-local " + "ranks matched\n"); + return EXIT_SUCCESS; +} From ac8951449e6225060599a40ae3e6d031bd82e54e Mon Sep 17 00:00:00 2001 From: lara kawasme Date: Tue, 21 Jul 2026 14:51:02 -0700 Subject: [PATCH 31/35] added few more comments for ctarank 3d benchmark --- .../cluster_ctarank/cluster_ctarank_3d.cu | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu index 6d4c266a0..296f56025 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/special_registers/cluster_ctarank/cluster_ctarank_3d.cu @@ -147,24 +147,28 @@ int main() { "%%cluster_ctarank expected\n"); unsigned failures = 0; + //walk through every cta in grid for (unsigned z = 0; z < kGridZ; ++z) { for (unsigned y = 0; y < kGridY; ++y) { for (unsigned x = 0; x < kGridX; ++x) { - const unsigned index = x + kGridX * (y + kGridY * z); + const unsigned index = x + kGridX * (y + kGridY * z); //convert 3d corrd to 1d index const CtaResult result = host_results[index]; + //figure out which cluster has the cta we care abt const unsigned cluster_x = x / kClusterX; const unsigned cluster_y = y / kClusterY; const unsigned cluster_z = z / kClusterZ; + //figure out the local coordinate of the cta we care abt within ints cluster const unsigned local_x = x % kClusterX; const unsigned local_y = y % kClusterY; const unsigned local_z = z % kClusterZ; + //from this, we have the expected rank const unsigned expected_rank = local_x + kClusterX * (local_y + kClusterY * local_z); const bool pass = result.block_x == x && result.block_y == y && result.block_z == z && result.ctaid_x == x && - result.ctaid_y == y && result.ctaid_z == z && - result.cluster_ctarank == expected_rank; + result.ctaid_y == y && result.ctaid_z == z && //ctaid is same as blockidx + result.cluster_ctarank == expected_rank; //does our expected rank match cluster ctarank? std::printf("(%u,%u,%u) (%u,%u,%u) " "(%u,%u,%u) %2u %2u %s\n", cluster_x, cluster_y, cluster_z, result.block_x, From 350a7671ea75a83dc5a50febd2ba9fdd011caed8 Mon Sep 17 00:00:00 2001 From: jyasuumi <144069912+jyasuumi@users.noreply.github.com> Date: Sun, 2 Aug 2026 17:40:52 -0400 Subject: [PATCH 32/35] wgmma ubench: match CuTe GMMA canonical smem layout The wgmma microbenchmarks write their A/B operand tiles into shared memory with a device-side smem_off() helper and then run wgmma.mma_async, which the simulator interprets. The simulator's wgmma_smem_offset() was corrected to CuTe's canonical GMMA Major-K layout, so these helpers -- which encoded the old (non-hardware) formula -- must be updated to stay consistent with the reader. smem_off() now computes, in uint128_t units, u128 = (leading/T)*(LBO/16) + (stride%8)*Wsw + (stride/8)*(SBO/16) off = u128*16 + (leading%T)*E matching the simulator. For the SW=0 tiles used by most benches the leading byte offset becomes LBO = MN*16 (dense); wgmma_verify additionally exercises SW=1/2/3 with the sparse swizzled layout (SBO=128*Wsw, LBO=16) and larger shared-memory buffers. The Swizzle XOR is omitted to match the reader. All built wgmma benches (verify, dtypes, multik, shapes, scale, sync, fence, timing_edge) pass under functional simulation. Co-Authored-By: Claude Opus 4.8 --- .../ubench/wgmma/wgmma/wgmma_verify.cu | 84 +++++++++++-------- .../ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu | 13 +-- .../ubench/wgmma/wgmma_multik/wgmma_multik.cu | 13 +-- .../ubench/wgmma/wgmma_scale/wgmma_scale.cu | 17 ++-- .../ubench/wgmma/wgmma_shapes/wgmma_shapes.cu | 15 ++-- .../ubench/wgmma/wgmma_sync/wgmma_fence.cu | 13 +-- .../ubench/wgmma/wgmma_sync/wgmma_sync.cu | 13 +-- .../wgmma_timing_edge/wgmma_timing_edge.cu | 13 +-- 8 files changed, 109 insertions(+), 72 deletions(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu index c9db9b67f..1f65cc5b2 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu @@ -14,11 +14,14 @@ // b1 k256 D=s32 (GPGPU-Sim functional sim only; // real H100 needs bit-packed smem, not 1 byte/bit) // -// Smem layout (K-major, no swizzle unless SW!=0): -// T = 16 / E (elements per 128-bit column) -// off = (stride%T + leading%8 * T)*E + (stride/T)*SBO + (leading/8)*LBO -// SBO = 128 always. LBO_A = M*E*8, LBO_B = N*E*8. -// All shapes satisfy M*K*E = 2048 and K*N*E = 512. +// Smem layout: canonical CuTe GMMA Major-K tiling, identical to the simulator's +// wgmma_smem_offset() (gpgpu-sim src/cuda-sim/wgmma_layout.h). In uint128_t units: +// u128 = (leading/T)*(LBO/16) + (stride%8)*Wsw + (stride/8)*(SBO/16) +// off = u128*16 + (leading%T)*E (T = 16/E; leading=k, stride=mn) +// Wsw is the swizzle width in u128 (128B=8, 64B=4, 32B=2, none=1). SW=0 uses a +// dense layout (SBO=128, LBO=MN*16); SW>0 uses the sparse swizzled layout +// (SBO=128*Wsw, LBO=16). The Swizzle XOR is a bank-conflict optimization +// and is intentionally omitted on both producer and consumer. // // Two test cases per type: // all-ones – A=B=1 → D[m][n] = K (trivially verifiable) @@ -52,10 +55,11 @@ static constexpr int N = 16; static constexpr int WGSIZE = 128; static constexpr int D_ELEMS = N / 2; // 8 D-registers per thread -// For all standard types: M*K*E = 2048, K*N*E = 512 (fixed by WGMMA design) -static constexpr int SMEM_A = 2048; -static constexpr int SMEM_B = 512; -static constexpr int SBO = 128; // stride byte offset (always 128 for K-major) +// SW=0 (dense) needs M*K*E / K*N*E bytes; the swizzled f16 layout (SW>0) is +// sparse and needs up to ~8 KB for A and ~2 KB for B, so size for the worst case. +static constexpr int SMEM_A = 8192; +static constexpr int SMEM_B = 2048; +static constexpr int SBO = 128; // stride byte offset for SW=0 (K-major) // b1 (sim convention): 1 byte per bit, K=256 "elements" static constexpr int B1_K = 256; @@ -83,28 +87,38 @@ __device__ __forceinline__ uint64_t make_gmma_desc( return d; } -// K-major smem byte offset (before swizzle). +// Canonical CuTe GMMA Major-K smem byte offset, matching the simulator's +// wgmma_smem_offset() (src/cuda-sim/wgmma_layout.h). In units of uint128_t: +// u128 = (leading/T)*(LBO/16) + (stride%8)*Wsw + (stride/8)*(SBO/16) +// off = u128*16 + (leading%T)*E // For A (M×K): leading=k, stride=m. For B (K×N): leading=k, stride=n. -__host__ __device__ __forceinline__ -int smem_off(int leading, int stride, int E, int LBO) { - int T = 16 / E; - return (stride % T + (leading % 8) * T) * E - + (stride / T) * SBO - + (leading / 8) * LBO; -} +// Wsw (MN inner stride, in u128) is the swizzle width: 128B=8,64B=4,32B=2,none=1. +// The Swizzle XOR is a bank-conflict optimization only and is omitted on +// both producer and consumer (it does not change which element is multiplied). +__host__ __device__ __forceinline__ constexpr +int gmma_wsw(int sw) { return sw == 1 ? 8 : sw == 2 ? 4 : sw == 3 ? 2 : 1; } + +// Stride byte offset (SBO) and leading byte offset (LBO) for a canonical +// K-major operand tile whose MN extent is MN, matching the sim reader. +__host__ __device__ __forceinline__ constexpr +int gmma_sbo(int sw) { return 128 * gmma_wsw(sw); } +__host__ __device__ __forceinline__ constexpr +int gmma_lbo(int MN, int sw) { return sw ? 16 : MN * 16; } -// XOR-swizzle (self-inverse) applied to a byte offset. __host__ __device__ __forceinline__ -int apply_sw(int off, int sw) { - if (!sw) return off; - int s = (sw == 1) ? 128 : (sw == 2) ? 64 : 32; - int b = off / s, i = off % s; - return b * s + (i ^ ((b & 1) ? (s >> 1) : 0)); +int smem_off(int leading, int stride, int E, int LBO, int sw = 0) { + int T = 16 / E; + int Wsw = gmma_wsw(sw); + int SBOb = gmma_sbo(sw); + int u128 = (leading / T) * (LBO / 16) + + (stride % 8) * Wsw + + (stride / 8) * (SBOb / 16); + return u128 * 16 + (leading % T) * E; } __host__ __device__ __forceinline__ int smem_off_sw(int leading, int stride, int E, int LBO, int sw) { - return apply_sw(smem_off(leading, stride, E, LBO), sw); + return smem_off(leading, stride, E, LBO, sw); } // --------------------------------------------------------------------------- @@ -127,7 +141,9 @@ __global__ void kernel_f16(const half* A_g, const half* B_g, float* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 16, E = 2, LBO_A = M*E*8, LBO_B = N*E*8; // 1024, 256 + constexpr int K = 16, E = 2; + constexpr int LBO_A = gmma_lbo(M, SW), LBO_B = gmma_lbo(N, SW); + constexpr int SBOe = gmma_sbo(SW); if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -141,8 +157,8 @@ __global__ void kernel_f16(const half* A_g, const half* B_g, __syncthreads(); if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } - uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, SW); - uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, SW); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBOe, SW); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBOe, SW); float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; WGMMA_FENCE; asm volatile( @@ -166,7 +182,7 @@ __global__ void kernel_bf16(const __nv_bfloat16* A_g, const __nv_bfloat16* B_g, float* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 16, E = 2, LBO_A = M*E*8, LBO_B = N*E*8; + constexpr int K = 16, E = 2, LBO_A = M*16, LBO_B = N*16; if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -205,7 +221,7 @@ __global__ void kernel_tf32(const float* A_g, const float* B_g, float* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 8, E = 4, LBO_A = M*E*8, LBO_B = N*E*8; // 2048, 512 + constexpr int K = 8, E = 4, LBO_A = M*16, LBO_B = N*16; // 2048, 512 if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -245,7 +261,7 @@ __global__ void kernel_e4m3(const uint8_t* A_g, const uint8_t* B_g, float* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; // 512, 128 + constexpr int K = 32, E = 1, LBO_A = M*16, LBO_B = N*16; // 512, 128 if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -284,7 +300,7 @@ __global__ void kernel_e5m2(const uint8_t* A_g, const uint8_t* B_g, float* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; + constexpr int K = 32, E = 1, LBO_A = M*16, LBO_B = N*16; if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -323,7 +339,7 @@ __global__ void kernel_s8(const int8_t* A_g, const int8_t* B_g, int32_t* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; + constexpr int K = 32, E = 1, LBO_A = M*16, LBO_B = N*16; if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -362,7 +378,7 @@ __global__ void kernel_u8(const uint8_t* A_g, const uint8_t* B_g, int32_t* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; + constexpr int K = 32, E = 1, LBO_A = M*16, LBO_B = N*16; if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -408,7 +424,7 @@ __global__ void kernel_b1(const uint8_t* A_g, const uint8_t* B_g, __shared__ __align__(128) uint8_t smA[B1_SMEM_A]; // 16384 bytes __shared__ __align__(128) uint8_t smB[B1_SMEM_B]; // 4096 bytes const int tid = threadIdx.x; - constexpr int K = B1_K, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; // 512, 128 + constexpr int K = B1_K, E = 1, LBO_A = M*16, LBO_B = N*16; // 512, 128 if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu index 6cce61abe..25d15a929 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu @@ -32,8 +32,8 @@ static constexpr int K = 16; static constexpr int WGSIZE = 128; static constexpr int E = 2; // bytes per f16 static constexpr int SBO = 128; -static constexpr int LBO_A = M * E * 8; // 1024 -static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int LBO_A = M * 16; // 1024 +static constexpr int LBO_B = N * 16; // 256 static constexpr int SMEM_A = M * K * E; // 2048 static constexpr int SMEM_B = K * N * E; // 512 static constexpr int D_REGS = N / 4; // 4 f16x2 registers per thread (N=16) @@ -61,10 +61,13 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { + // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's + // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - return (stride % T + (leading % 8) * T) * e - + (stride / T) * SBO - + (leading / 8) * LBO; + int u128 = (leading / T) * (LBO / 16) + + (stride % 8) + + (stride / 8) * (SBO / 16); + return u128 * 16 + (leading % T) * e; } #define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu index 0940fc352..2175f7c38 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu @@ -45,8 +45,8 @@ static constexpr int WGSIZE = 128; static constexpr int D_ELEMS = N / 2; // 8 f32 registers per thread static constexpr int E = 2; // bytes per f16 static constexpr int SBO = 128; -static constexpr int LBO_A = M * E * 8; // 1024 -static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int LBO_A = M * 16; // 1024 +static constexpr int LBO_B = N * 16; // 256 static constexpr int SMEM_A = M * K * E; // 2048 static constexpr int SMEM_B = K * N * E; // 512 @@ -73,10 +73,13 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { + // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's + // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - return (stride % T + (leading % 8) * T) * e - + (stride / T) * SBO - + (leading / 8) * LBO; + int u128 = (leading / T) * (LBO / 16) + + (stride % 8) + + (stride / 8) * (SBO / 16); + return u128 * 16 + (leading % T) * e; } #define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu index 12a9be3ae..68b8f5947 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu @@ -69,10 +69,13 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { + // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's + // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - return (stride % T + (leading % 8) * T) * e - + (stride / T) * SBO - + (leading / 8) * LBO; + int u128 = (leading / T) * (LBO / 16) + + (stride % 8) + + (stride / 8) * (SBO / 16); + return u128 * 16 + (leading % T) * e; } #define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") @@ -88,8 +91,8 @@ int smem_off(int leading, int stride, int e, int LBO) { template __global__ void kernel_f16_scaled(const half* A_g, const half* B_g, float init_val, float* D_g) { - constexpr int LBO_A = M * E_F16 * 8; - constexpr int LBO_B = N * E_F16 * 8; + constexpr int LBO_A = M * 16; + constexpr int LBO_B = N * 16; __shared__ __align__(128) char smA[M*K_F16*E_F16], smB[K_F16*N*E_F16]; const int tid = threadIdx.x; for (int i = tid; i < M*K_F16; i += WGSIZE) { int m=i/K_F16,k=i%K_F16; *(half*)(smA+smem_off(k,m,E_F16,LBO_A))=A_g[i]; } @@ -129,8 +132,8 @@ __global__ void kernel_f16_scaled(const half* A_g, const half* B_g, template __global__ void kernel_e4m3_scale(const uint8_t* A_g, const uint8_t* B_g, float* D_g) { - constexpr int LBO_A = M * E_FP8 * 8; - constexpr int LBO_B = N * E_FP8 * 8; + constexpr int LBO_A = M * 16; + constexpr int LBO_B = N * 16; __shared__ __align__(128) char smA[M*K_FP8*E_FP8], smB[K_FP8*N*E_FP8]; const int tid = threadIdx.x; for (int i = tid; i < M*K_FP8; i += WGSIZE) { int m=i/K_FP8,k=i%K_FP8; *(uint8_t*)(smA+smem_off(k,m,E_FP8,LBO_A))=A_g[i]; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu index 3915b1190..7bbb90c6e 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu @@ -57,10 +57,13 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { + // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's + // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - return (stride % T + (leading % 8) * T) * e - + (stride / T) * SBO - + (leading / 8) * LBO; + int u128 = (leading / T) * (LBO / 16) + + (stride % 8) + + (stride / 8) * (SBO / 16); + return u128 * 16 + (leading % T) * e; } #define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") @@ -72,7 +75,7 @@ int smem_off(int leading, int stride, int e, int LBO) { // --------------------------------------------------------------------------- __global__ void kernel_n16(const half* A_g, const half* B_g, float* D_g) { constexpr int N = 16, D_ELEMS = N/2; - constexpr int LBO_A = M*E*8, LBO_B = N*E*8; + constexpr int LBO_A = M*16, LBO_B = N*16; __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; const int tid = threadIdx.x; for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } @@ -99,7 +102,7 @@ __global__ void kernel_n16(const half* A_g, const half* B_g, float* D_g) { // --------------------------------------------------------------------------- __global__ void kernel_n32(const half* A_g, const half* B_g, float* D_g) { constexpr int N = 32, D_ELEMS = N/2; - constexpr int LBO_A = M*E*8, LBO_B = N*E*8; + constexpr int LBO_A = M*16, LBO_B = N*16; __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; const int tid = threadIdx.x; for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } @@ -130,7 +133,7 @@ __global__ void kernel_n32(const half* A_g, const half* B_g, float* D_g) { // --------------------------------------------------------------------------- __global__ void kernel_n64(const half* A_g, const half* B_g, float* D_g) { constexpr int N = 64, D_ELEMS = N/2; - constexpr int LBO_A = M*E*8, LBO_B = N*E*8; + constexpr int LBO_A = M*16, LBO_B = N*16; __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; const int tid = threadIdx.x; for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu index fb9fdfcda..15fa2209b 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu @@ -41,8 +41,8 @@ static constexpr int WGSIZE = 128; static constexpr int D_ELEMS = N / 2; // 8 D-regs per thread static constexpr int E = 2; // bytes per f16 static constexpr int SBO = 128; -static constexpr int LBO_A = M * E * 8; // 1024 -static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int LBO_A = M * 16; // 1024 +static constexpr int LBO_B = N * 16; // 256 static constexpr int SMEM_A = M * K * E; // 2048 static constexpr int SMEM_B = K * N * E; // 512 @@ -69,10 +69,13 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { + // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's + // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - return (stride % T + (leading % 8) * T) * e - + (stride / T) * SBO - + (leading / 8) * LBO; + int u128 = (leading / T) * (LBO / 16) + + (stride % 8) + + (stride / 8) * (SBO / 16); + return u128 * 16 + (leading % T) * e; } // --------------------------------------------------------------------------- diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu index 2163c9d07..62693447f 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu @@ -39,8 +39,8 @@ static constexpr int WGSIZE = 128; static constexpr int D_ELEMS = N / 2; static constexpr int E = 2; static constexpr int SBO = 128; -static constexpr int LBO_A = M * E * 8; // 1024 -static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int LBO_A = M * 16; // 1024 +static constexpr int LBO_B = N * 16; // 256 static constexpr int SMEM_A = M * K * E; // 2048 static constexpr int SMEM_B = K * N * E; // 512 @@ -67,10 +67,13 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { + // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's + // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - return (stride % T + (leading % 8) * T) * e - + (stride / T) * SBO - + (leading / 8) * LBO; + int u128 = (leading / T) * (LBO / 16) + + (stride % 8) + + (stride / 8) * (SBO / 16); + return u128 * 16 + (leading % T) * e; } #define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu index c3b56eae4..6dcb1e61e 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu @@ -69,8 +69,8 @@ static constexpr int WGSIZE = 128; static constexpr int D_ELEMS = N / 2; // 8 f32 registers per thread static constexpr int E = 2; // bytes per f16 static constexpr int SBO = 128; -static constexpr int LBO_A = M * E * 8; // 1024 -static constexpr int LBO_B = N * E * 8; // 256 +static constexpr int LBO_A = M * 16; // 1024 +static constexpr int LBO_B = N * 16; // 256 static constexpr int SMEM_A = M * K * E; // 2048 static constexpr int SMEM_B = K * N * E; // 512 @@ -98,10 +98,13 @@ __device__ __forceinline__ uint64_t make_gmma_desc( // Byte offset of logical element (leading, stride) - matches sim wgmma_smem_offset. __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { + // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's + // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - return (stride % T + (leading % 8) * T) * e - + (stride / T) * SBO - + (leading / 8) * LBO; + int u128 = (leading / T) * (LBO / 16) + + (stride % 8) + + (stride / 8) * (SBO / 16); + return u128 * 16 + (leading % T) * e; } // XOR swizzle - matches sim wgmma_apply_swizzle (mode 0=none,1=128B,2=64B,3=32B). From 76e0d1f383e8935fc63159a7276ed5e33aee91d0 Mon Sep 17 00:00:00 2001 From: Raunak Dani Date: Fri, 21 Aug 2026 01:49:04 -0400 Subject: [PATCH 33/35] Add mapa.shared::cluster.u32 functional-verification microbenchmarks Two companion ubenches that verify the Hopper+ mapa.shared::cluster.u32 address-remapping instruction and end-to-end distributed shared memory (DSMEM) communication between CTAs of a thread-block cluster. mapa_shared_cluster (2-CTA cluster): - Gate A rank sensitivity: mapa(local,0) != mapa(local,1). - Gate B remote store / local load: each CTA writes its magic into its peer's slot via st.shared::cluster[mapa(local,peer)], then reads its own slot with an ordinary shared load and must see the peer's magic. - Gate C self-map identity: mapa(local,own_rank) still names this CTA's slot. mapa_shared_cluster_extended (8-CTA cluster, mirror pairing 0<->7,1<->6,...): - Keeps Gates A/B/C across all 8 CTAs (every slot has exactly one remote writer), and adds - Gate A+ full rank round-trip: getctarank(mapa(local,k)) == k for every rank k, proving mapa lands on the correct CTA rather than just "some other CTA" - a distinction the 2-CTA test cannot make. Both poison-fill result buffers so a silent no-op mapa is caught, use cluster barriers to order the zero-fill before remote stores, and build to standalone executables that pass under GPGPU-Sim and on Hopper silicon. Co-authored-by: Cursor --- .../ubench/mem/mapa_shared_cluster/Makefile | 12 +- .../mapa_shared_cluster_extended.cu | 311 ++++++++++++++++++ 2 files changed, 319 insertions(+), 4 deletions(-) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster_extended.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile index a162019de..9bd7b10d1 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile +++ b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/Makefile @@ -1,10 +1,14 @@ BASE_DIR := $(shell pwd) BIN_DIR := $(BASE_DIR)/../../../bin/ -# mapa.shared::cluster.u32 careful functional microbenchmark. -# Remaps a shared-window address onto a cluster CTA rank; Gates A/B/C -# verify rank sensitivity and remote store / local load semantics. -ALL_BENCHES := mapa_shared_cluster +# mapa.shared::cluster.u32 functional microbenchmarks. Each remaps a +# shared-window address onto a cluster CTA rank and verifies rank sensitivity +# plus remote-store / local-load semantics: +# mapa_shared_cluster - 2-CTA cluster, Gates A/B/C +# mapa_shared_cluster_extended - 8-CTA cluster (mirror pairing) + full +# getctarank round-trip (Gate A+) +# Both build into standalone executables dropped in ../../../bin/. +ALL_BENCHES := mapa_shared_cluster mapa_shared_cluster_extended # mapa.shared::cluster is Hopper+ PTX (PTX ISA 7.8+, SM_90 and above). # Override on the command line, e.g. `make ARCH=sm_90a`, for a single GPU. diff --git a/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster_extended.cu b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster_extended.cu new file mode 100644 index 000000000..5e5fc004f --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/mem/mapa_shared_cluster/mapa_shared_cluster_extended.cu @@ -0,0 +1,311 @@ +#include +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// mapa_shared_cluster_extended.cu — wide functional gate for +// `mapa.shared::cluster.u32`, extending mapa_shared_cluster.cu from a 2-CTA +// cluster to a full 8-CTA cluster. +// +// WHAT mapa DOES (PTX ISA 7.8+, sm_90+) +// -------------------------------------- +// mapa.shared::cluster.u32 d, a, b; +// +// Remap shared-window address `a` onto cluster CTA rank `b`. Does NOT move data; +// a later ld/st.shared::cluster uses `d` to touch CTA b's copy of that offset. +// +// WHY AN 8-CTA VERSION IS A STRONGER TEST +// --------------------------------------- +// The 2-CTA test can only ever map to "the one other CTA", so it cannot tell a +// correct rank->CTA mapping apart from "always target the single peer". With 8 +// CTAs we pin the mapping down far more tightly: +// +// Pairing — MIRROR ACROSS THE CLUSTER +// rank r talks to rank (CLUSTER_SIZE-1 - r): 0<->7, 1<->6, 2<->5, 3<->4. +// peer() is an involution with peer(r) != r for all r, so the 8 CTAs form +// 4 disjoint pairs and every slot has exactly one remote writer. +// +// Gate A — RANK SENSITIVITY (address-only) +// For each CTA, mapa(local, peer) MUST differ from mapa(local, self). +// +// Gate A+ — FULL RANK ROUND-TRIP (the real upgrade) +// For every rank k in [0, CLUSTER_SIZE), getctarank(mapa(local, k)) == k. +// This proves mapa lands on the *correct* CTA for all 8 ranks, not merely +// that two addresses differ. An off-by-one or "always the peer" mapping is +// caught here. +// +// Gate B — REMOTE STORE, LOCAL LOAD (end-to-end) +// Each CTA writes its magic into its *peer* slot via +// st.shared::cluster [mapa(local, peer)] +// then reads its *own* slot with an ordinary shared load (no mapa). +// Expected local value == peer's magic. A no-op mapa writes its own slot +// instead, so the local load sees its own magic and fails. +// +// Gate C — SELF MAP IS IDENTITY-ISH +// mapa(local, own_rank) must still address this CTA's slot (local load +// through that mapped addr returns what the peer wrote into us, Gate B). +// +// Host checks all gates after D2H. Poison-fill so a silent no-op is obvious. +// +// NOTE (silicon): 8 CTAs is the portable maximum cluster size, so no +// non-portable opt-in is required. All CTAs of a cluster must be co-resident +// in one GPC on Hopper (sm_90); GPGPU-Sim models the 8-CTA cluster directly +// (needs >= 8 SMs, which the H100 config provides). +// +// make KERNEL=mapa_shared_cluster_extended +// ./run.sh mapa_shared_cluster_extended +// ./run.sh mapa_shared_cluster_extended silicon +// ───────────────────────────────────────────────────────────────────────────── + +#define CLUSTER_SIZE 8 +#define BLOCK_SIZE 32 +#define MAGIC_BASE 0xC000u +#define POISON 0xFFFFFFFFu + +// Mirror pairing across the cluster: 0<->7, 1<->6, 2<->5, 3<->4. +__device__ __host__ __forceinline__ unsigned mirror_peer(unsigned rank) { + return (CLUSTER_SIZE - 1u) - rank; +} + +__device__ __forceinline__ unsigned cluster_ctarank() { + unsigned rank; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(rank)); + return rank; +} + +// Instruction under test: remap shared-window address `addr` onto CTA `rank`. +__device__ __forceinline__ unsigned mapa_shared_cluster_u32(unsigned addr, + unsigned rank) { + unsigned mapped; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" + : "=r"(mapped) + : "r"(addr), "r"(rank)); + return mapped; +} + +// Inverse of mapa's rank encoding: which cluster rank owns this mapped address. +__device__ __forceinline__ unsigned getctarank_shared_cluster_u32(unsigned addr) { + unsigned rank; + asm volatile("getctarank.shared::cluster.u32 %0, %1;" + : "=r"(rank) + : "r"(addr)); + return rank; +} + +__device__ __forceinline__ void st_shared_cluster_u32(unsigned addr, + unsigned val) { + asm volatile("st.shared::cluster.u32 [%0], %1;" + : + : "r"(addr), "r"(val) + : "memory"); +} + +__device__ __forceinline__ unsigned ld_shared_cluster_u32(unsigned addr) { + unsigned val; + asm volatile("ld.shared::cluster.u32 %0, [%1];" + : "=r"(val) + : "r"(addr) + : "memory"); + return val; +} + +__global__ void __cluster_dims__(CLUSTER_SIZE, 1, 1) +mapa_shared_cluster_extended_kernel(unsigned *d_local_after, // Gate B + unsigned *d_self_via_mapa, // Gate C + unsigned *d_mapped_peer, // Gate A + unsigned *d_mapped_self, // Gate A + unsigned *d_local_addr, + unsigned *d_roundtrip_fail, // Gate A+ + unsigned *d_peer) { + __shared__ unsigned slot; + + const unsigned tid = threadIdx.x; + const unsigned rank = cluster_ctarank(); + const unsigned peer = mirror_peer(rank); + + // Shared-window address of our local slot (CUTLASS / PTX mapa input form). + const unsigned local_addr = + static_cast(__cvta_generic_to_shared(&slot)); + + // Start from a known empty slot so Gate B cannot see a self-written magic. + if (tid == 0) { + slot = 0u; + } + __syncthreads(); + + // Cluster barrier so every CTA's zeroing is globally visible before any + // remote store — otherwise a late zero could clobber a peer's magic. + asm volatile("barrier.cluster.arrive;" ::: "memory"); + asm volatile("barrier.cluster.wait;" ::: "memory"); + + if (tid == 0) { + // ── Gate B: write *into the peer CTA* via mapa ───────────────────── + const unsigned mapped_peer = mapa_shared_cluster_u32(local_addr, peer); + st_shared_cluster_u32(mapped_peer, MAGIC_BASE + rank); + } + + // Make remote stores visible before anyone reads their local slot. + asm volatile("barrier.cluster.arrive;" ::: "memory"); + asm volatile("barrier.cluster.wait;" ::: "memory"); + __syncthreads(); + + if (tid == 0) { + // Ordinary .shared::cta load of *our* slot — our mirror peer should have + // written MAGIC_BASE + peer into it through Gate B's remapped store. + const unsigned local_after = slot; + + // ── Gate C: self-map still names our own slot ────────────────────── + const unsigned mapped_self = + mapa_shared_cluster_u32(local_addr, rank); + const unsigned self_via_mapa = ld_shared_cluster_u32(mapped_self); + + // Recompute peer map for the host-visible address dump (same as Gate B). + const unsigned mapped_peer = mapa_shared_cluster_u32(local_addr, peer); + + // ── Gate A+: every rank must round-trip through mapa/getctarank ──── + unsigned rt_fail = 0u; + #pragma unroll + for (unsigned k = 0; k < CLUSTER_SIZE; ++k) { + const unsigned m = mapa_shared_cluster_u32(local_addr, k); + if (getctarank_shared_cluster_u32(m) != k) { + rt_fail++; + } + } + + d_local_after[rank] = local_after; + d_self_via_mapa[rank] = self_via_mapa; + d_mapped_peer[rank] = mapped_peer; + d_mapped_self[rank] = mapped_self; + d_local_addr[rank] = local_addr; + d_roundtrip_fail[rank] = rt_fail; + d_peer[rank] = peer; + } +} + +int main() { + setvbuf(stdout, NULL, _IONBF, 0); + + printf("=== mapa.shared::cluster.u32 extended (8-CTA) functional ubench ===\n"); + printf("cluster=%d CTAs, block=%d threads, magic_base=0x%X\n", + CLUSTER_SIZE, BLOCK_SIZE, MAGIC_BASE); + printf("pairing: rank r <-> (%d - r) [0<->%d, 1<->%d, ...]\n\n", + CLUSTER_SIZE - 1, CLUSTER_SIZE - 1, CLUSTER_SIZE - 2); + + const int n = CLUSTER_SIZE; + const size_t bytes = n * sizeof(unsigned); + + unsigned *h_local_after = (unsigned *)malloc(bytes); + unsigned *h_self_via_mapa = (unsigned *)malloc(bytes); + unsigned *h_mapped_peer = (unsigned *)malloc(bytes); + unsigned *h_mapped_self = (unsigned *)malloc(bytes); + unsigned *h_local_addr = (unsigned *)malloc(bytes); + unsigned *h_roundtrip = (unsigned *)malloc(bytes); + unsigned *h_peer = (unsigned *)malloc(bytes); + + unsigned *d_local_after = NULL, *d_self_via_mapa = NULL; + unsigned *d_mapped_peer = NULL, *d_mapped_self = NULL; + unsigned *d_local_addr = NULL, *d_roundtrip = NULL, *d_peer = NULL; + + cudaMalloc(&d_local_after, bytes); + cudaMalloc(&d_self_via_mapa, bytes); + cudaMalloc(&d_mapped_peer, bytes); + cudaMalloc(&d_mapped_self, bytes); + cudaMalloc(&d_local_addr, bytes); + cudaMalloc(&d_roundtrip, bytes); + cudaMalloc(&d_peer, bytes); + + cudaMemset(d_local_after, 0xFF, bytes); + cudaMemset(d_self_via_mapa, 0xFF, bytes); + cudaMemset(d_mapped_peer, 0xFF, bytes); + cudaMemset(d_mapped_self, 0xFF, bytes); + cudaMemset(d_local_addr, 0xFF, bytes); + cudaMemset(d_roundtrip, 0xFF, bytes); + cudaMemset(d_peer, 0xFF, bytes); + + // 8 CTAs is the portable maximum cluster size, so no non-portable opt-in + // is needed here. + mapa_shared_cluster_extended_kernel<<>>( + d_local_after, d_self_via_mapa, d_mapped_peer, d_mapped_self, + d_local_addr, d_roundtrip, d_peer); + cudaError_t err = cudaDeviceSynchronize(); + + int fails = 0; + if (err != cudaSuccess) { + printf("launch/sync error: %s\n", cudaGetErrorString(err)); + fails++; + } else { + cudaMemcpy(h_local_after, d_local_after, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_self_via_mapa, d_self_via_mapa, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_mapped_peer, d_mapped_peer, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_mapped_self, d_mapped_self, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_local_addr, d_local_addr, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_roundtrip, d_roundtrip, bytes, cudaMemcpyDeviceToHost); + cudaMemcpy(h_peer, d_peer, bytes, cudaMemcpyDeviceToHost); + + printf("per-CTA results (after D2H):\n"); + for (int r = 0; r < n; ++r) { + const unsigned expect_peer = mirror_peer((unsigned)r); + // After the remote store, my slot holds my mirror peer's magic. + const unsigned expect_local = MAGIC_BASE + expect_peer; + const unsigned expect_self = expect_local; // self-map -> same slot + + const int wrote = + (h_local_after[r] != POISON) && (h_self_via_mapa[r] != POISON) && + (h_mapped_peer[r] != POISON) && (h_mapped_self[r] != POISON) && + (h_local_addr[r] != POISON) && (h_roundtrip[r] != POISON); + + // Gate A: peer map must differ from self map (peer != self always). + const int peer_addr_differs = + (h_mapped_peer[r] != h_mapped_self[r]); + + // Gate A+: every rank round-tripped through mapa/getctarank. + const int roundtrip_ok = (h_roundtrip[r] == 0u); + + // Gate B / C value checks. + const int peer_ok = (h_peer[r] == expect_peer); + const int local_ok = (h_local_after[r] == expect_local); + const int self_ok = (h_self_via_mapa[r] == expect_self); + + if (!wrote) fails++; + if (!peer_addr_differs) fails++; + if (!roundtrip_ok) fails++; + if (!peer_ok) fails++; + if (!local_ok) fails++; + if (!self_ok) fails++; + + printf(" CTA rank %2d (peer %2d):\n", r, expect_peer); + printf(" local_addr = 0x%08X\n", h_local_addr[r]); + printf(" mapped_self = 0x%08X\n", h_mapped_self[r]); + printf(" mapped_peer = 0x%08X [%s]\n", + h_mapped_peer[r], + peer_addr_differs ? "differs from self ok" : "SAME AS SELF"); + printf(" roundtrip_fail = %u/%d [%s]\n", + h_roundtrip[r], CLUSTER_SIZE, + roundtrip_ok ? "all ranks map correctly" : "RANK MAP WRONG"); + printf(" local_after = 0x%08X (expected 0x%08X) [%s]\n", + h_local_after[r], expect_local, + local_ok ? "ok" : "MISMATCH"); + printf(" self_via_mapa = 0x%08X (expected 0x%08X) [%s]\n", + h_self_via_mapa[r], expect_self, + self_ok ? "ok" : "MISMATCH"); + } + } + + cudaFree(d_local_after); + cudaFree(d_self_via_mapa); + cudaFree(d_mapped_peer); + cudaFree(d_mapped_self); + cudaFree(d_local_addr); + cudaFree(d_roundtrip); + cudaFree(d_peer); + free(h_local_after); + free(h_self_via_mapa); + free(h_mapped_peer); + free(h_mapped_self); + free(h_local_addr); + free(h_roundtrip); + free(h_peer); + + printf("\nRESULT: %s\n", fails == 0 ? "PASSED" : "FAILED"); + return fails == 0 ? 0 : 1; +} From ff8812f6e5215f65b8974fa3c027b85b4ffe1387 Mon Sep 17 00:00:00 2001 From: jyasuumi <144069912+jyasuumi@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:58:24 -0400 Subject: [PATCH 34/35] Revert "wgmma ubench: match CuTe GMMA canonical smem layout" This reverts commit 350a7671ea75a83dc5a50febd2ba9fdd011caed8. --- .../ubench/wgmma/wgmma/wgmma_verify.cu | 84 ++++++++----------- .../ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu | 13 ++- .../ubench/wgmma/wgmma_multik/wgmma_multik.cu | 13 ++- .../ubench/wgmma/wgmma_scale/wgmma_scale.cu | 17 ++-- .../ubench/wgmma/wgmma_shapes/wgmma_shapes.cu | 15 ++-- .../ubench/wgmma/wgmma_sync/wgmma_fence.cu | 13 ++- .../ubench/wgmma/wgmma_sync/wgmma_sync.cu | 13 ++- .../wgmma_timing_edge/wgmma_timing_edge.cu | 13 ++- 8 files changed, 72 insertions(+), 109 deletions(-) diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu index 1f65cc5b2..c9db9b67f 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma/wgmma_verify.cu @@ -14,14 +14,11 @@ // b1 k256 D=s32 (GPGPU-Sim functional sim only; // real H100 needs bit-packed smem, not 1 byte/bit) // -// Smem layout: canonical CuTe GMMA Major-K tiling, identical to the simulator's -// wgmma_smem_offset() (gpgpu-sim src/cuda-sim/wgmma_layout.h). In uint128_t units: -// u128 = (leading/T)*(LBO/16) + (stride%8)*Wsw + (stride/8)*(SBO/16) -// off = u128*16 + (leading%T)*E (T = 16/E; leading=k, stride=mn) -// Wsw is the swizzle width in u128 (128B=8, 64B=4, 32B=2, none=1). SW=0 uses a -// dense layout (SBO=128, LBO=MN*16); SW>0 uses the sparse swizzled layout -// (SBO=128*Wsw, LBO=16). The Swizzle XOR is a bank-conflict optimization -// and is intentionally omitted on both producer and consumer. +// Smem layout (K-major, no swizzle unless SW!=0): +// T = 16 / E (elements per 128-bit column) +// off = (stride%T + leading%8 * T)*E + (stride/T)*SBO + (leading/8)*LBO +// SBO = 128 always. LBO_A = M*E*8, LBO_B = N*E*8. +// All shapes satisfy M*K*E = 2048 and K*N*E = 512. // // Two test cases per type: // all-ones – A=B=1 → D[m][n] = K (trivially verifiable) @@ -55,11 +52,10 @@ static constexpr int N = 16; static constexpr int WGSIZE = 128; static constexpr int D_ELEMS = N / 2; // 8 D-registers per thread -// SW=0 (dense) needs M*K*E / K*N*E bytes; the swizzled f16 layout (SW>0) is -// sparse and needs up to ~8 KB for A and ~2 KB for B, so size for the worst case. -static constexpr int SMEM_A = 8192; -static constexpr int SMEM_B = 2048; -static constexpr int SBO = 128; // stride byte offset for SW=0 (K-major) +// For all standard types: M*K*E = 2048, K*N*E = 512 (fixed by WGMMA design) +static constexpr int SMEM_A = 2048; +static constexpr int SMEM_B = 512; +static constexpr int SBO = 128; // stride byte offset (always 128 for K-major) // b1 (sim convention): 1 byte per bit, K=256 "elements" static constexpr int B1_K = 256; @@ -87,38 +83,28 @@ __device__ __forceinline__ uint64_t make_gmma_desc( return d; } -// Canonical CuTe GMMA Major-K smem byte offset, matching the simulator's -// wgmma_smem_offset() (src/cuda-sim/wgmma_layout.h). In units of uint128_t: -// u128 = (leading/T)*(LBO/16) + (stride%8)*Wsw + (stride/8)*(SBO/16) -// off = u128*16 + (leading%T)*E +// K-major smem byte offset (before swizzle). // For A (M×K): leading=k, stride=m. For B (K×N): leading=k, stride=n. -// Wsw (MN inner stride, in u128) is the swizzle width: 128B=8,64B=4,32B=2,none=1. -// The Swizzle XOR is a bank-conflict optimization only and is omitted on -// both producer and consumer (it does not change which element is multiplied). -__host__ __device__ __forceinline__ constexpr -int gmma_wsw(int sw) { return sw == 1 ? 8 : sw == 2 ? 4 : sw == 3 ? 2 : 1; } - -// Stride byte offset (SBO) and leading byte offset (LBO) for a canonical -// K-major operand tile whose MN extent is MN, matching the sim reader. -__host__ __device__ __forceinline__ constexpr -int gmma_sbo(int sw) { return 128 * gmma_wsw(sw); } -__host__ __device__ __forceinline__ constexpr -int gmma_lbo(int MN, int sw) { return sw ? 16 : MN * 16; } - __host__ __device__ __forceinline__ -int smem_off(int leading, int stride, int E, int LBO, int sw = 0) { +int smem_off(int leading, int stride, int E, int LBO) { int T = 16 / E; - int Wsw = gmma_wsw(sw); - int SBOb = gmma_sbo(sw); - int u128 = (leading / T) * (LBO / 16) - + (stride % 8) * Wsw - + (stride / 8) * (SBOb / 16); - return u128 * 16 + (leading % T) * E; + return (stride % T + (leading % 8) * T) * E + + (stride / T) * SBO + + (leading / 8) * LBO; +} + +// XOR-swizzle (self-inverse) applied to a byte offset. +__host__ __device__ __forceinline__ +int apply_sw(int off, int sw) { + if (!sw) return off; + int s = (sw == 1) ? 128 : (sw == 2) ? 64 : 32; + int b = off / s, i = off % s; + return b * s + (i ^ ((b & 1) ? (s >> 1) : 0)); } __host__ __device__ __forceinline__ int smem_off_sw(int leading, int stride, int E, int LBO, int sw) { - return smem_off(leading, stride, E, LBO, sw); + return apply_sw(smem_off(leading, stride, E, LBO), sw); } // --------------------------------------------------------------------------- @@ -141,9 +127,7 @@ __global__ void kernel_f16(const half* A_g, const half* B_g, float* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 16, E = 2; - constexpr int LBO_A = gmma_lbo(M, SW), LBO_B = gmma_lbo(N, SW); - constexpr int SBOe = gmma_sbo(SW); + constexpr int K = 16, E = 2, LBO_A = M*E*8, LBO_B = N*E*8; // 1024, 256 if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -157,8 +141,8 @@ __global__ void kernel_f16(const half* A_g, const half* B_g, __syncthreads(); if (!tid) { uint32_t c; READ_CLK(c); clk_g[1] = c; } - uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBOe, SW); - uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBOe, SW); + uint64_t da = make_gmma_desc(smem_addr(smA), LBO_A, SBO, SW); + uint64_t db = make_gmma_desc(smem_addr(smB), LBO_B, SBO, SW); float d0=0,d1=0,d2=0,d3=0,d4=0,d5=0,d6=0,d7=0; WGMMA_FENCE; asm volatile( @@ -182,7 +166,7 @@ __global__ void kernel_bf16(const __nv_bfloat16* A_g, const __nv_bfloat16* B_g, float* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 16, E = 2, LBO_A = M*16, LBO_B = N*16; + constexpr int K = 16, E = 2, LBO_A = M*E*8, LBO_B = N*E*8; if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -221,7 +205,7 @@ __global__ void kernel_tf32(const float* A_g, const float* B_g, float* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 8, E = 4, LBO_A = M*16, LBO_B = N*16; // 2048, 512 + constexpr int K = 8, E = 4, LBO_A = M*E*8, LBO_B = N*E*8; // 2048, 512 if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -261,7 +245,7 @@ __global__ void kernel_e4m3(const uint8_t* A_g, const uint8_t* B_g, float* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 32, E = 1, LBO_A = M*16, LBO_B = N*16; // 512, 128 + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; // 512, 128 if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -300,7 +284,7 @@ __global__ void kernel_e5m2(const uint8_t* A_g, const uint8_t* B_g, float* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 32, E = 1, LBO_A = M*16, LBO_B = N*16; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -339,7 +323,7 @@ __global__ void kernel_s8(const int8_t* A_g, const int8_t* B_g, int32_t* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 32, E = 1, LBO_A = M*16, LBO_B = N*16; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -378,7 +362,7 @@ __global__ void kernel_u8(const uint8_t* A_g, const uint8_t* B_g, int32_t* D_g, uint32_t* clk_g) { __shared__ __align__(128) char smA[SMEM_A], smB[SMEM_B]; const int tid = threadIdx.x; - constexpr int K = 32, E = 1, LBO_A = M*16, LBO_B = N*16; + constexpr int K = 32, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { @@ -424,7 +408,7 @@ __global__ void kernel_b1(const uint8_t* A_g, const uint8_t* B_g, __shared__ __align__(128) uint8_t smA[B1_SMEM_A]; // 16384 bytes __shared__ __align__(128) uint8_t smB[B1_SMEM_B]; // 4096 bytes const int tid = threadIdx.x; - constexpr int K = B1_K, E = 1, LBO_A = M*16, LBO_B = N*16; // 512, 128 + constexpr int K = B1_K, E = 1, LBO_A = M*E*8, LBO_B = N*E*8; // 512, 128 if (!tid) { uint32_t c; READ_CLK(c); clk_g[0] = c; } for (int i = tid; i < M*K; i += WGSIZE) { diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu index 25d15a929..6cce61abe 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_dtypes/wgmma_dtypes.cu @@ -32,8 +32,8 @@ static constexpr int K = 16; static constexpr int WGSIZE = 128; static constexpr int E = 2; // bytes per f16 static constexpr int SBO = 128; -static constexpr int LBO_A = M * 16; // 1024 -static constexpr int LBO_B = N * 16; // 256 +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 static constexpr int SMEM_A = M * K * E; // 2048 static constexpr int SMEM_B = K * N * E; // 512 static constexpr int D_REGS = N / 4; // 4 f16x2 registers per thread (N=16) @@ -61,13 +61,10 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { - // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's - // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - int u128 = (leading / T) * (LBO / 16) - + (stride % 8) - + (stride / 8) * (SBO / 16); - return u128 * 16 + (leading % T) * e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; } #define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu index 2175f7c38..0940fc352 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_multik/wgmma_multik.cu @@ -45,8 +45,8 @@ static constexpr int WGSIZE = 128; static constexpr int D_ELEMS = N / 2; // 8 f32 registers per thread static constexpr int E = 2; // bytes per f16 static constexpr int SBO = 128; -static constexpr int LBO_A = M * 16; // 1024 -static constexpr int LBO_B = N * 16; // 256 +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 static constexpr int SMEM_A = M * K * E; // 2048 static constexpr int SMEM_B = K * N * E; // 512 @@ -73,13 +73,10 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { - // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's - // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - int u128 = (leading / T) * (LBO / 16) - + (stride % 8) - + (stride / 8) * (SBO / 16); - return u128 * 16 + (leading % T) * e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; } #define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu index 68b8f5947..12a9be3ae 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_scale/wgmma_scale.cu @@ -69,13 +69,10 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { - // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's - // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - int u128 = (leading / T) * (LBO / 16) - + (stride % 8) - + (stride / 8) * (SBO / 16); - return u128 * 16 + (leading % T) * e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; } #define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") @@ -91,8 +88,8 @@ int smem_off(int leading, int stride, int e, int LBO) { template __global__ void kernel_f16_scaled(const half* A_g, const half* B_g, float init_val, float* D_g) { - constexpr int LBO_A = M * 16; - constexpr int LBO_B = N * 16; + constexpr int LBO_A = M * E_F16 * 8; + constexpr int LBO_B = N * E_F16 * 8; __shared__ __align__(128) char smA[M*K_F16*E_F16], smB[K_F16*N*E_F16]; const int tid = threadIdx.x; for (int i = tid; i < M*K_F16; i += WGSIZE) { int m=i/K_F16,k=i%K_F16; *(half*)(smA+smem_off(k,m,E_F16,LBO_A))=A_g[i]; } @@ -132,8 +129,8 @@ __global__ void kernel_f16_scaled(const half* A_g, const half* B_g, template __global__ void kernel_e4m3_scale(const uint8_t* A_g, const uint8_t* B_g, float* D_g) { - constexpr int LBO_A = M * 16; - constexpr int LBO_B = N * 16; + constexpr int LBO_A = M * E_FP8 * 8; + constexpr int LBO_B = N * E_FP8 * 8; __shared__ __align__(128) char smA[M*K_FP8*E_FP8], smB[K_FP8*N*E_FP8]; const int tid = threadIdx.x; for (int i = tid; i < M*K_FP8; i += WGSIZE) { int m=i/K_FP8,k=i%K_FP8; *(uint8_t*)(smA+smem_off(k,m,E_FP8,LBO_A))=A_g[i]; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu index 7bbb90c6e..3915b1190 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_shapes/wgmma_shapes.cu @@ -57,13 +57,10 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { - // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's - // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - int u128 = (leading / T) * (LBO / 16) - + (stride % 8) - + (stride / 8) * (SBO / 16); - return u128 * 16 + (leading % T) * e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; } #define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") @@ -75,7 +72,7 @@ int smem_off(int leading, int stride, int e, int LBO) { // --------------------------------------------------------------------------- __global__ void kernel_n16(const half* A_g, const half* B_g, float* D_g) { constexpr int N = 16, D_ELEMS = N/2; - constexpr int LBO_A = M*16, LBO_B = N*16; + constexpr int LBO_A = M*E*8, LBO_B = N*E*8; __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; const int tid = threadIdx.x; for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } @@ -102,7 +99,7 @@ __global__ void kernel_n16(const half* A_g, const half* B_g, float* D_g) { // --------------------------------------------------------------------------- __global__ void kernel_n32(const half* A_g, const half* B_g, float* D_g) { constexpr int N = 32, D_ELEMS = N/2; - constexpr int LBO_A = M*16, LBO_B = N*16; + constexpr int LBO_A = M*E*8, LBO_B = N*E*8; __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; const int tid = threadIdx.x; for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } @@ -133,7 +130,7 @@ __global__ void kernel_n32(const half* A_g, const half* B_g, float* D_g) { // --------------------------------------------------------------------------- __global__ void kernel_n64(const half* A_g, const half* B_g, float* D_g) { constexpr int N = 64, D_ELEMS = N/2; - constexpr int LBO_A = M*16, LBO_B = N*16; + constexpr int LBO_A = M*E*8, LBO_B = N*E*8; __shared__ __align__(128) char smA[M*K*E], smB[K*N*E]; const int tid = threadIdx.x; for (int i = tid; i < M*K; i += WGSIZE) { int m=i/K,k=i%K; *(half*)(smA+smem_off(k,m,E,LBO_A))=A_g[i]; } diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu index 15fa2209b..fb9fdfcda 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_fence.cu @@ -41,8 +41,8 @@ static constexpr int WGSIZE = 128; static constexpr int D_ELEMS = N / 2; // 8 D-regs per thread static constexpr int E = 2; // bytes per f16 static constexpr int SBO = 128; -static constexpr int LBO_A = M * 16; // 1024 -static constexpr int LBO_B = N * 16; // 256 +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 static constexpr int SMEM_A = M * K * E; // 2048 static constexpr int SMEM_B = K * N * E; // 512 @@ -69,13 +69,10 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { - // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's - // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - int u128 = (leading / T) * (LBO / 16) - + (stride % 8) - + (stride / 8) * (SBO / 16); - return u128 * 16 + (leading % T) * e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; } // --------------------------------------------------------------------------- diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu index 62693447f..2163c9d07 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_sync/wgmma_sync.cu @@ -39,8 +39,8 @@ static constexpr int WGSIZE = 128; static constexpr int D_ELEMS = N / 2; static constexpr int E = 2; static constexpr int SBO = 128; -static constexpr int LBO_A = M * 16; // 1024 -static constexpr int LBO_B = N * 16; // 256 +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 static constexpr int SMEM_A = M * K * E; // 2048 static constexpr int SMEM_B = K * N * E; // 512 @@ -67,13 +67,10 @@ __device__ __forceinline__ uint64_t make_gmma_desc( __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { - // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's - // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - int u128 = (leading / T) * (LBO / 16) - + (stride % 8) - + (stride / 8) * (SBO / 16); - return u128 * 16 + (leading % T) * e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; } #define WGMMA_FENCE asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory") diff --git a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu index 6dcb1e61e..c3b56eae4 100644 --- a/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu +++ b/src/cuda/GPU_Microbenchmark/ubench/wgmma/wgmma_timing_edge/wgmma_timing_edge.cu @@ -69,8 +69,8 @@ static constexpr int WGSIZE = 128; static constexpr int D_ELEMS = N / 2; // 8 f32 registers per thread static constexpr int E = 2; // bytes per f16 static constexpr int SBO = 128; -static constexpr int LBO_A = M * 16; // 1024 -static constexpr int LBO_B = N * 16; // 256 +static constexpr int LBO_A = M * E * 8; // 1024 +static constexpr int LBO_B = N * E * 8; // 256 static constexpr int SMEM_A = M * K * E; // 2048 static constexpr int SMEM_B = K * N * E; // 512 @@ -98,13 +98,10 @@ __device__ __forceinline__ uint64_t make_gmma_desc( // Byte offset of logical element (leading, stride) - matches sim wgmma_smem_offset. __host__ __device__ __forceinline__ int smem_off(int leading, int stride, int e, int LBO) { - // Canonical CuTe GMMA Major-K layout (SW=0), matching the simulator's - // wgmma_smem_offset(): u128 = (leading/T)*(LBO/16) + (stride%8) + (stride/8)*(SBO/16). int T = 16 / e; - int u128 = (leading / T) * (LBO / 16) - + (stride % 8) - + (stride / 8) * (SBO / 16); - return u128 * 16 + (leading % T) * e; + return (stride % T + (leading % 8) * T) * e + + (stride / T) * SBO + + (leading / 8) * LBO; } // XOR swizzle - matches sim wgmma_apply_swizzle (mode 0=none,1=128B,2=64B,3=32B). From 38a41d9be838ed25edd7273dca3fe63539e9548f Mon Sep 17 00:00:00 2001 From: jyasuumi <144069912+jyasuumi@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:33:45 -0400 Subject: [PATCH 35/35] ubench: add independent thread scheduling verification --- .../independent_thread_scheduling/Makefile | 13 ++ .../independent_thread_scheduling.cu | 188 ++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/Makefile create mode 100644 src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/independent_thread_scheduling.cu diff --git a/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/Makefile b/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/Makefile new file mode 100644 index 000000000..ce625f3df --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/Makefile @@ -0,0 +1,13 @@ +BASE_DIR := $(shell pwd) +BIN_DIR := $(BASE_DIR)/../../../bin/ + +SRC := independent_thread_scheduling.cu +EXE := independent_thread_scheduling + +# Independent thread scheduling was introduced with Volta (SM 70). Keep PTX +# for simulation and SASS for validation on NVIDIA hardware. +GENCODE_SM70 ?= -gencode=arch=compute_70,code=\"sm_70,compute_70\" +GENCODE_SM90 ?= -gencode=arch=compute_90,code=\"sm_90,compute_90\" +NVCC_FLAGS := $(GENCODE_SM70) $(GENCODE_SM90) -std=c++17 + +include ../../../common/common.mk diff --git a/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/independent_thread_scheduling.cu b/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/independent_thread_scheduling.cu new file mode 100644 index 000000000..13884d208 --- /dev/null +++ b/src/cuda/GPU_Microbenchmark/ubench/system/independent_thread_scheduling/independent_thread_scheduling.cu @@ -0,0 +1,188 @@ +#include + +#include +#include +#include + +namespace { + +constexpr int kWarpSize = 32; +constexpr int kUnset = -1; + +#define CUDA_CHECK(call) \ + do { \ + cudaError_t error = (call); \ + if (error != cudaSuccess) { \ + std::fprintf(stderr, "CUDA error at %s:%d: %s\n", __FILE__, __LINE__, \ + cudaGetErrorString(error)); \ + std::exit(EXIT_FAILURE); \ + } \ + } while (0) + +// An atomic read makes the polling operation visible to both real hardware +// and the timing simulator. Volta+ may schedule a runnable sibling lane while +// the current divergent path repeatedly executes this loop. +__device__ __forceinline__ int poll(const int *address) { + return *reinterpret_cast(address); +} + +// Lanes 1..31 can reach the wait before lane 0 reaches the publishing path. +// A non-preemptive PDOM stack can pin those waiters above lane 0 forever. +__global__ void forward_publish_kernel(int *output) { + __shared__ int ready; + __shared__ int payload; + const int lane = threadIdx.x; + + if (lane == 0) { + ready = 0; + payload = 0; + } + __syncthreads(); + + if (lane != 0) { + while (poll(&ready) == 0) { + } + output[lane] = payload + lane; + } else { + payload = 1000; + *reinterpret_cast(&ready) = 1; + output[0] = 1000; + } +} + +// Mirror the predicate and producer lane. This catches implementations that +// accidentally make progress only for one branch direction or lane ordering. +__global__ void reverse_publish_kernel(int *output) { + __shared__ int ready; + __shared__ int payload; + const int lane = threadIdx.x; + + if (lane == 0) { + ready = 0; + payload = 0; + } + __syncthreads(); + + if (lane != kWarpSize - 1) { + while (poll(&ready) == 0) { + } + output[lane] = payload - lane; + } else { + payload = 2000; + *reinterpret_cast(&ready) = 1; + output[lane] = 2000 - lane; + } +} + +// The active owner changes 32 times. Each lane must leave the polling path, +// publish its trace slot, and wake the next lane. This exercises many dynamic +// split masks rather than a single producer/consumer split. +__global__ void token_ring_kernel(int *output) { + __shared__ int token; + const int lane = threadIdx.x; + + if (lane == 0) *reinterpret_cast(&token) = 0; + __syncthreads(); + + while (poll(&token) != lane) { + } + output[lane] = 3000 + lane; + *reinterpret_cast(&token) = lane + 1; +} + +enum TestId { kRing, kForward, kReverse, kTestCount }; + +const char *const kTestNames[kTestCount] = { + "token-ring", "forward-publish", "reverse-publish"}; + +int expected_value(TestId test, int lane) { + switch (test) { + case kForward: + return 1000 + lane; + case kReverse: + return 2000 - lane; + case kRing: + return 3000 + lane; + default: + return kUnset; + } +} + +void launch(TestId test, int *device_output) { + switch (test) { + case kForward: + forward_publish_kernel<<<1, kWarpSize>>>(device_output); + break; + case kReverse: + reverse_publish_kernel<<<1, kWarpSize>>>(device_output); + break; + case kRing: + token_ring_kernel<<<1, kWarpSize>>>(device_output); + break; + default: + std::abort(); + } +} + +bool run_test(TestId test) { + int *device_output = nullptr; + int host_output[kWarpSize]; + CUDA_CHECK(cudaMalloc(&device_output, sizeof(host_output))); + CUDA_CHECK(cudaMemset(device_output, 0xff, sizeof(host_output))); + + launch(test, device_output); + CUDA_CHECK(cudaGetLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + CUDA_CHECK(cudaMemcpy(host_output, device_output, sizeof(host_output), + cudaMemcpyDeviceToHost)); + CUDA_CHECK(cudaFree(device_output)); + + int failures = 0; + for (int lane = 0; lane < kWarpSize; ++lane) { + const int expected = expected_value(test, lane); + if (host_output[lane] != expected) { + std::printf(" lane %2d: got %d, expected %d\n", lane, + host_output[lane], expected); + ++failures; + } + } + std::printf("[%s] %s\n", failures == 0 ? "PASS" : "FAIL", + kTestNames[test]); + return failures == 0; +} + +int find_test(const char *name) { + for (int i = 0; i < kTestCount; ++i) { + if (std::strcmp(name, kTestNames[i]) == 0) return i; + } + return -1; +} + +} // namespace + +int main(int argc, char **argv) { + int first = 0; + int last = kTestCount; + if (argc == 2) { + first = find_test(argv[1]); + if (first < 0) { + std::fprintf(stderr, "Unknown test '%s'. Available tests:\n", argv[1]); + for (const char *name : kTestNames) std::fprintf(stderr, " %s\n", name); + return EXIT_FAILURE; + } + last = first + 1; + } else if (argc != 1) { + std::fprintf(stderr, "Usage: %s [test-name]\n", argv[0]); + return EXIT_FAILURE; + } + + int failures = 0; + for (int test = first; test < last; ++test) { + if (!run_test(static_cast(test))) ++failures; + } + + std::printf("RESULT: %s (%d/%d tests passed)\n", + failures == 0 ? "PASSED" : "FAILED", + last - first - failures, last - first); + return failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +}