From 7317c7e3d50dbdb3f9891f2963492d23f9428b45 Mon Sep 17 00:00:00 2001 From: Robert Esclapez Garcia Date: Mon, 19 Oct 2026 09:32:24 -0700 Subject: [PATCH 1/6] [ROCm][RDNA] ROCM_RDNA_ATTN prefill: occupancy for 1024-VGPR archs The prefill kernel asks for kWavesPerSimd waves a SIMD (amdgpu_waves_per_eu), sized for archs with 1536 VGPRs a SIMD lane: 6 waves leave a wave 256 VGPRs. On archs with 1024 VGPRs the same hint caps a wave at 160 and LLVM spills: 394 of the 430 prefill entries spill, 352 by 32 VGPRs or more (up to 133), and prefill runs 12-16x slower than on a 1536-VGPR arch instead of ~2.5x. On 1024-VGPR archs the hint becomes 2/3 of the waves, in whole workgroups: the same VGPRs a wave as with 1536. launch_prefill() sizes the resident workgroups (firstRound) with the same target, telling the archs apart by hipDeviceAttributeMaxRegistersPerMultiprocessor. The hardware numbers involved are named (kSimdsPerWgp, kWgpLdsBytes, kBigLaneVgprs, kSmallLaneVgprs). Code for 1536-VGPR archs is unchanged: identical resource usage for all 430 entries. The kernels are still built and served only where they were; this makes a 1024-VGPR build of the same entries usable. On a 1024-VGPR arch (Radeon 890M), entries built through the tools' JIT: - 430 entries: 165 spill, none by 32 or more (max 8). - Every entry of 6 shapes (D 64..512, fp16/bf16, sliding window) matches Triton (D=512 bf16 to one bf16 ulp, as close to an fp32 reference as Triton); so do 456 more knob sets of 10/10/128 and 16/2/256. - 7 shapes x 33 cells (M 16..16k after 0/4k/16k cached tokens), rows as tuned on a 1536-VGPR arch: faster than TRITON_ATTN, ROCM_SEGMENTED_ATTN and ROCM_ATTN in every cell (geomean 6.6x, 3.1x, 9.0x; worst 1.08x, 1.04x, 1.38x); 2.1-6.5x faster than without this change, except D=64 (1.0x), whose entries spilled little. AI assistance was used for this patch. Co-authored-by: Claude Signed-off-by: Robert Esclapez Garcia --- csrc/rocm/rdna_prefill_attn.cu | 81 ++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/csrc/rocm/rdna_prefill_attn.cu b/csrc/rocm/rdna_prefill_attn.cu index 081c02c6ff08..067a9806a5b9 100644 --- a/csrc/rocm/rdna_prefill_attn.cu +++ b/csrc/rocm/rdna_prefill_attn.cu @@ -1,4 +1,4 @@ -// RDNA3.5 prefill attention for one sequence of M query tokens after a cached +// RDNA prefill attention for one sequence of M query tokens after a cached // prefix, or a batch through plan_prefill's work list, causal and optionally // windowed, over the paged KV cache // (num_pages, NUM_KV_HEADS, PAGE_SIZE, 2 * HEAD_DIM), as a template on one @@ -23,6 +23,7 @@ #include #include #include +#include #include "rdna_attn_common.cuh" #include "rdna_attn_launch.h" @@ -30,6 +31,13 @@ namespace rdna { +// An RDNA WGP has 4 SIMDs and 128 KiB of LDS (64 KiB at most for one +// workgroup). A SIMD lane has 1536 VGPRs or 1024, depending on the arch. +constexpr int kSimdsPerWgp = 4; +constexpr int kWgpLdsBytes = 128 * 1024; +constexpr int kBigLaneVgprs = 1536; +constexpr int kSmallLaneVgprs = 1024; + // The largest divisor of n up to cap. constexpr int largest_divisor_up_to(int n, int cap) { int g = cap < n ? cap : n; @@ -181,26 +189,44 @@ struct Prefill { (kDimParts > 1 ? kWaves * kWaveSubTiles * 256 * 4 : 0) + (kKeyWaves > 1 ? (2 * kKeyWaves * kBlockRows + kWaves * kWaveSize * 8) * 4 : 0); - static constexpr int kSimdWaves = kWaves >= 4 ? kWaves / 4 : 1; + // A workgroup's waves on each SIMD. + static constexpr int kSimdWaves = + kWaves >= kSimdsPerWgp ? kWaves / kSimdsPerWgp : 1; + // With 1536 VGPRs: 6 waves a SIMD (256 VGPRs each, the most a wave + // addresses), 8 for 16-wave workgroups, fewer if the LDS holds fewer + // workgroups; in whole workgroups. static constexpr int kWavesPerSimd = - std::min(kWaves >= 16 ? 8 : 6, 131072 / kLdsBytes * kWaves / 4) / + std::min(kWaves >= 16 ? 8 : 6, + kWgpLdsBytes / kLdsBytes * kWaves / kSimdsPerWgp) / kSimdWaves * kSimdWaves; + // With 1024 VGPRs, the waves that keep each wave the VGPRs it has with + // 1536, in whole workgroups. + static constexpr int kSmallVgprWaves = + kWavesPerSimd * kSmallLaneVgprs / kBigLaneVgprs; + static constexpr int kSmallVgprWavesPerSimd = + std::max(kSimdWaves, kSmallVgprWaves / kSimdWaves * kSimdWaves); + // The archs with 1024 VGPRs. +#if defined(__gfx1102__) || defined(__gfx1103__) || defined(__gfx1150__) || \ + defined(__gfx1152__) || defined(__gfx1153__) + static constexpr int kTargetWavesPerSimd = kSmallVgprWavesPerSimd; +#else + static constexpr int kTargetWavesPerSimd = kWavesPerSimd; +#endif // kBatch: a batch through the work list, apart so that the one-sequence // kernel keeps its registers. template - static __global__ void __launch_bounds__(kThreads) - __attribute__((amdgpu_waves_per_eu(kWavesPerSimd > 1 ? kWavesPerSimd - : 1))) - prefill_attn(const elem_t* __restrict__ q, const elem_t* __restrict__ kv, - const int* __restrict__ blockTable, elem_t* __restrict__ out, - const int* __restrict__ seqLens, int numTokens, float scale, - float* __restrict__ partialO, float* __restrict__ partialML, - int* __restrict__ counters, int numSegments, int firstRound, - const int* __restrict__ mmRanges, int numMmRanges, - const int4* __restrict__ plan, - const int* __restrict__ queryStart, int numEntries, - int blockTableStride, int headGroup) { + static __global__ void __launch_bounds__(kThreads) __attribute__(( + amdgpu_waves_per_eu(kTargetWavesPerSimd > 1 ? kTargetWavesPerSimd : 1))) + prefill_attn(const elem_t* __restrict__ q, const elem_t* __restrict__ kv, + const int* __restrict__ blockTable, elem_t* __restrict__ out, + const int* __restrict__ seqLens, int numTokens, float scale, + float* __restrict__ partialO, float* __restrict__ partialML, + int* __restrict__ counters, int numSegments, int firstRound, + const int* __restrict__ mmRanges, int numMmRanges, + const int4* __restrict__ plan, + const int* __restrict__ queryStart, int numEntries, + int blockTableStride, int headGroup) { __shared__ __attribute__(( aligned(16))) elem_t kTile[kBuffers][kKeyTile][kRowPitch]; __shared__ __attribute__((aligned(16))) elem_t @@ -882,15 +908,23 @@ template void launch_prefill(const PrefillArgs& a) { using P = Prefill; using elem_t = typename P::elem_t; - static const int numWgps = [] { - int device, n = 0; + // The WGPs, and the workgroups resident in each: as many as its LDS and its + // SIMDs' waves allow, with the waves the kernel was built for. A WGP with + // fewer VGPRs than 4 SIMDs x 32 lanes x 1536 is a 1024-VGPR one. + static const auto [numWgps, wgpWorkgroups] = [] { + int device, n = 0, regs = 0; (void)hipGetDevice(&device); (void)hipDeviceGetAttribute(&n, hipDeviceAttributeMultiprocessorCount, device); - return std::max(1, n); + (void)hipDeviceGetAttribute( + ®s, hipDeviceAttributeMaxRegistersPerMultiprocessor, device); + const bool smallVgprs = + regs > 0 && regs < kSimdsPerWgp * P::kWaveSize * kBigLaneVgprs; + const int waves = smallVgprs ? P::kSmallVgprWavesPerSimd : P::kWavesPerSimd; + const int resident = + std::min(kWgpLdsBytes / P::kLdsBytes, waves * kSimdsPerWgp / P::kWaves); + return std::pair{std::max(1, n), std::max(1, resident)}; }(); - constexpr int kWgpWorkgroups = std::max( - 1, std::min(131072 / P::kLdsBytes, P::kWavesPerSimd * 4 / P::kWaves)); if (a.plan) { const int headGroup = a.head_group ? a.head_group : P::kHeadGroup; hipLaunchKernelGGL( @@ -898,12 +932,13 @@ void launch_prefill(const PrefillArgs& a) { dim3(P::kThreads), 0, a.stream, static_cast(a.q), static_cast(a.kv), a.bt, static_cast(a.out), a.seq_lens, 0, a.scale, a.partial_o, a.partial_ml, a.counters, 1, - kWgpWorkgroups * numWgps, a.mm_ranges, a.num_mm_ranges, + wgpWorkgroups * numWgps, a.mm_ranges, a.num_mm_ranges, reinterpret_cast(a.plan), a.query_start, a.num_entries, a.bt_stride, headGroup); return; } - const int targetWorkgroups = 4 * numWgps; + // A workgroup per SIMD. + const int targetWorkgroups = kSimdsPerWgp * numWgps; const int numTiles = (a.num_tokens * P::kG + P::kBlockRows - 1) / P::kBlockRows; const int groups = numTiles * P::kNumKvHeads; @@ -924,7 +959,7 @@ void launch_prefill(const PrefillArgs& a) { dim3(P::kThreads), 0, a.stream, static_cast(a.q), static_cast(a.kv), a.bt, static_cast(a.out), a.seq_lens, a.num_tokens, a.scale, a.partial_o, a.partial_ml, a.counters, - segments, kWgpWorkgroups * numWgps, a.mm_ranges, a.num_mm_ranges, nullptr, + segments, wgpWorkgroups * numWgps, a.mm_ranges, a.num_mm_ranges, nullptr, nullptr, 0, 0, P::kHeadGroup); } From 9bccf1fbf02a44653705c619429c369db8fead38 Mon Sep 17 00:00:00 2001 From: Robert Esclapez Garcia Date: Thu, 22 Oct 2026 01:37:13 -0700 Subject: [PATCH 2/6] [Benchmarks] attention: one output buffer per layer The runner rotates the inputs over num_layers (sized so the KV working set leaves the cache) but every layer wrote the same output buffer, so its lines stayed in the write-back L2 and the output writes mostly never reached DRAM. Small prefills timed faster than their DRAM traffic allows: above the roofline, which counts the output. The output now rotates like the inputs. On a 1024-VGPR GPU, prefills of 32..64 tokens without a prefix: 4-44% slower with the output rotated, none above the roofline any more. AI assistance was used for this patch. Co-authored-by: Claude Signed-off-by: Robert Esclapez Garcia --- benchmarks/attention_benchmarks/runner.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/benchmarks/attention_benchmarks/runner.py b/benchmarks/attention_benchmarks/runner.py index a7f02564c4ca..16cacce0f9fe 100644 --- a/benchmarks/attention_benchmarks/runner.py +++ b/benchmarks/attention_benchmarks/runner.py @@ -506,7 +506,12 @@ def _run_single_benchmark( """ total_q = q_list[0].shape[0] v_dim = config.v_head_dim if config.v_head_dim is not None else config.head_dim - out = torch.empty(total_q, config.num_q_heads, v_dim, device=device, dtype=dtype) + # One output per layer, like the inputs: a single buffer rewritten by every + # layer would stay in the cache and its writes would not reach DRAM. + out_list = [ + torch.empty(total_q, config.num_q_heads, v_dim, device=device, dtype=dtype) + for _ in range(config.num_layers) + ] def benchmark_fn(): for i in range(config.num_layers): @@ -517,7 +522,7 @@ def benchmark_fn(): v_list[i], cache_list[i], attn_metadata, - output=out, + output=out_list[i], ) if config.ncu_profile: From e2259878aba9f74272b9ad4507a7e871f61ff570 Mon Sep 17 00:00:00 2001 From: Robert Esclapez Garcia Date: Sat, 24 Oct 2026 01:32:56 -0700 Subject: [PATCH 3/6] [ROCm][RDNA] ROCM_RDNA_ATTN: gfx1150's tables, used by gfx1152 and gfx1153 The configs are tuned on gfx1151, with 1536 VGPRs a SIMD lane. gfx1150, gfx1152 and gfx1153 have 1024 and a smaller GPU, and their best launch knobs differ; this gives them tables of their own, tuned on gfx1150, and selects the backend by default on gfx1150 too (gfx1152 and gfx1153, not tested yet, can select it). The batched tables start as gfx1151's (prefill batch cells pointing at gfx1150's prefill entries). - Per-family tables. The configs headers hold one complete table per Arch (rdna_attn_types.h): kGfx1151*Configs (the existing entries) and kGfx1150*Configs, the same shapes, which gfx1152 and gfx1153 also use, for the decode, batched decode, prefill and prefill batch tables. They are reached through decode_configs() etc. and the k*Configs[kNumArches] spans. CMake builds one object library per family for its archs in the arch list (rdna_attn_targets), and _rocm_C lists and launches the tables of the current GPU's family. - An entry in both tables is a distinct decode kernel per family (decode_attn); prefill's already have internal linkage. Same instructions on gfx1151. - Decode launch: the LDS residency correction (128 KiB a WGP) covers gfx1150..gfx1153, with their VGPR file (1024 in blocks of 16). - Tools: jit.py builds for the current GPU (RDNA_JIT_ARCH overrides it); tune.py records the family it measured on and --apply writes that family's table; utils.py has the roofline peaks of each GPU. - tune.py: a failed run of the current row is no baseline for accepting a decode candidate; reused points are built before the A/B; the decisions keep every A/B sample; prefill --reuse matches its points; a prefill step's winner loses at most --prefill-loss-limit (2%) to the current entry at any cell, and --reselect chooses stored decisions again without timing; prefill steps go up to M=8192 (vLLM's default chunk), --steps tunes some of them and --apply keeps the others. gfx1150's entries were tuned from gfx1151's on the gfx1150 boards whose BIOS keeps kernel launches fast (BIOS 1.08 doubles the launch latency and favours other knobs), each candidate checked against a float reference and timed through the backend, fp16 (bf16 shares the entries). Against gfx1151's entries in the same A/B: - prefill: 33 configurations x 9 steps of M (16..4096) x prefixes 0/1024/16384, 891 cells: 1.124x, none below 0.98x. The M=8192 step, tuned on its own, gains 3% or less on 17 configurations, which keep their M>=4096 entry; 16/8/256, 32/16/256 w1024 and 32/32/96 get one (1.44x, 1.06x, 1.03x). 334 entries (328 builds) against gfx1151's 434. - decode: 193 of 264 (configuration, M) accepted (1.01x-1.46x weighted, within the per-context loss limits); 1848 cells: 1.065x. Re-timed against gfx1151's entries in the kernel matrix, each twice on each of five gfx1150 boards: prefill 1.095x over 660 cells (M 16..8192, prefixes 0/4096), decode 1.061x over 1848 cells (M 1..8, contexts 128..32k). Tests: the backend's 398 tests pass on gfx1150 and gfx1151; gfx1150's tables cover gfx1151's shapes. gfx1151's code is the same as before (decode, batched decode and prefill units compared). AI assistance was used for this patch. Co-authored-by: Claude Signed-off-by: Robert Esclapez Garcia --- CMakeLists.txt | 10 +- .../kernels/rocm_rdna_hip_attn/README.md | 17 +- .../kernels/rocm_rdna_hip_attn/tools/jit.py | 26 +- .../kernels/rocm_rdna_hip_attn/tools/tune.py | 370 ++++++-- .../kernels/rocm_rdna_hip_attn/tools/utils.py | 31 +- cmake/rocm_rdna_attn.cmake | 51 +- csrc/rocm/rdna_attn_instances.cuh | 45 +- csrc/rocm/rdna_attn_launch.h | 16 +- csrc/rocm/rdna_attn_types.h | 9 +- csrc/rocm/rdna_decode_attn.cu | 26 +- csrc/rocm/rdna_decode_batch_configs.h | 700 ++++++++++++++- csrc/rocm/rdna_decode_configs.h | 698 ++++++++++++++- csrc/rocm/rdna_prefill_batch_configs.h | 828 +++++++++++++++++- csrc/rocm/rdna_prefill_configs.h | 360 +++++++- csrc/rocm/rdna_registry.cu | 155 +++- tests/kernels/attention/test_rocm_rdna_hip.py | 71 +- vllm/platforms/rocm.py | 5 +- vllm/v1/attention/backends/rocm_rdna_attn.py | 10 +- vllm/v1/attention/ops/rocm_rdna_hip.py | 4 +- 19 files changed, 3180 insertions(+), 252 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a716cccbd563..e0d9ce711785 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1576,12 +1576,12 @@ if(VLLM_GPU_LANG STREQUAL "HIP") "csrc/rocm/gdn_chunked.cu") endif() - # RDNA3.5 attention kernels (gfx1151 only). + # RDNA3.5 attention kernels (gfx1150, gfx1151, gfx1152, gfx1153). set(VLLM_ROCM_HAS_RDNA_ATTN OFF) - if("gfx1151" IN_LIST VLLM_GPU_ARCHES) + include(cmake/rocm_rdna_attn.cmake) + rdna_attn_targets(VLLM_RDNA_ATTN_TARGETS) + if(VLLM_RDNA_ATTN_TARGETS) set(VLLM_ROCM_HAS_RDNA_ATTN ON) - include(cmake/rocm_rdna_attn.cmake) - rdna_attn_target(_rocm_C_rdna_attn "gfx1151") list(APPEND VLLM_ROCM_EXT_SRC "csrc/rocm/rdna_registry.cu") endif() @@ -1634,7 +1634,7 @@ if(VLLM_GPU_LANG STREQUAL "HIP") target_compile_definitions(_rocm_C PRIVATE VLLM_ROCM_GDN_CHUNKED) endif() if(VLLM_ROCM_HAS_RDNA_ATTN) - target_link_libraries(_rocm_C PRIVATE _rocm_C_rdna_attn) + target_link_libraries(_rocm_C PRIVATE ${VLLM_RDNA_ATTN_TARGETS}) target_compile_definitions(_rocm_C PRIVATE VLLM_ROCM_RDNA_ATTN) endif() endif() diff --git a/benchmarks/kernels/rocm_rdna_hip_attn/README.md b/benchmarks/kernels/rocm_rdna_hip_attn/README.md index 6233b71605df..6654b62c6793 100644 --- a/benchmarks/kernels/rocm_rdna_hip_attn/README.md +++ b/benchmarks/kernels/rocm_rdna_hip_attn/README.md @@ -1,6 +1,7 @@ # RDNA3.5 attention: adding, tuning and checking a configuration -`ROCM_RDNA_ATTN` (gfx1151, selected by default there) runs decodes +`ROCM_RDNA_ATTN` (gfx1150..gfx1153, selected by default on gfx1150 and gfx1151) +runs decodes of up to 8 query tokens per sequence on `csrc/rocm/rdna_decode_attn.cu` and everything else on `csrc/rocm/rdna_prefill_attn.cu`. Its tables, in `csrc/rocm/`: @@ -12,14 +13,16 @@ of up to 8 query tokens per sequence on | `rdna_prefill_configs.h` | prefill builds, one per step of M | one sequence | | `rdna_prefill_batch_configs.h` | per batch cell, a prefill build and head group | a batch of sequences | -Calls without a build run on Triton. +Calls without a build run on Triton. Each file has a table per GPU family: +gfx1151's (1536 VGPRs a SIMD lane) and gfx1150's (1024), which gfx1152 and +gfx1153 also use. The tools tune and write the tables of the GPU they run on. A *configuration* is `(Hq, Hkv, D, window, page size)`, in fp16 and bf16. ## Requirements -- A gfx1151 GPU and a vLLM build of this tree; run from the tree root with - `PYTHONPATH=$PWD`. +- A gfx1150..gfx1153 GPU and a vLLM build of this tree for it; run from the + tree root with `PYTHONPATH=$PWD`. - `D` in 64, 96, 128, 256, 512, `Hq % Hkv == 0`, and a page size that is a multiple of 16 (the one vLLM uses for the model, e.g. 544 for Qwen3.5-0.8B). - Nothing else running on the GPU. @@ -78,7 +81,11 @@ PYTHONPATH=$PWD .venv/bin/python \ ``` An entry serves M from its `min_query_len` up to the next entry's. Steps start -at 16, 32, ..., 4096; the first entry also serves 9 to 15 tokens. +at 16, 32, ..., 8192 (vLLM's default chunk); the first entry also serves 9 to 15 +tokens. `--steps 8192` tunes only that step; `--apply` keeps the others. A step's +winner loses at most `--prefill-loss-limit` (2%) to the current entry at any +cell; `--reselect decisions.jsonl --decisions out.jsonl` chooses stored +decisions again from their A/B samples, without timing. ## Batched prefill diff --git a/benchmarks/kernels/rocm_rdna_hip_attn/tools/jit.py b/benchmarks/kernels/rocm_rdna_hip_attn/tools/jit.py index 0c3161cacbb3..8723a87d4498 100644 --- a/benchmarks/kernels/rocm_rdna_hip_attn/tools/jit.py +++ b/benchmarks/kernels/rocm_rdna_hip_attn/tools/jit.py @@ -31,7 +31,14 @@ # Not the configs headers: their entries do not change a build. _SOURCES = (_CSRC / "rdna_decode_attn.cu", *_COMMON) _PREFILL_SOURCES = (_CSRC / "rdna_prefill_attn.cu", *_COMMON) -_ARCH = os.environ.get("RDNA_JIT_ARCH", "gfx1151") + + +@functools.cache +def _arch() -> str: + """The target: RDNA_JIT_ARCH, else the current device's.""" + arch = os.environ.get("RDNA_JIT_ARCH") + return arch or torch.cuda.get_device_properties(0).gcnArchName.split(":")[0] + _WRAPPER = """#include "rdna_decode_attn.cu" using namespace rdna; @@ -201,7 +208,7 @@ def _build(v: R.KernelVariant | R.PrefillVariant) -> Path: *clang, "-x", "hip", - f"--offload-arch={_ARCH}", + f"--offload-arch={_arch()}", "-fno-gpu-rdc", "-O3", "-std=c++20", @@ -407,8 +414,10 @@ def failure(v: R.KernelVariant | R.PrefillVariant) -> str | None: return _failed.get(v) -def precompile(variants) -> None: - """Build variants in parallel and load them.""" +def precompile(variants) -> int: + """Build variants in parallel and load them, sealed or not; the number + built.""" + global _sealed todo = [ dataclasses.replace(v, min_query_len=R.PREFILL_MIN_M) if isinstance(v, R.PrefillVariant) @@ -416,8 +425,13 @@ def precompile(variants) -> None: for v in variants ] todo = [v for v in dict.fromkeys(todo) if v not in _loaded] - with ThreadPoolExecutor(max_workers=max(1, (os.cpu_count() or 8) - 2)) as pool: - list(pool.map(load, todo)) + sealed, _sealed = _sealed, False + try: + with ThreadPoolExecutor(max_workers=max(1, (os.cpu_count() or 8) - 2)) as pool: + list(pool.map(load, todo)) + finally: + _sealed = sealed + return len(todo) def seal() -> None: diff --git a/benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py b/benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py index f7e2840da937..ff2e78ab7cf7 100644 --- a/benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py +++ b/benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py @@ -11,12 +11,14 @@ over batch specs instead of contexts. Prefill (--prefill), per (Hq, Hkv, D, window): one row per step of M, chosen -in an interleaved A/B against TRITON_ATTN. +in an interleaved A/B against TRITON_ATTN among the finalists that lose at most +--prefill-loss-limit to the current row at any cell. Prefill batches (--prefill-batch): per cell, the build and head group. -`--apply` writes accepted decisions into csrc/rocm/rdna_*_configs.h, -refusing those measured on another kernel source or row. +`--apply` writes accepted decisions into the table of the GPU family they were +measured on in csrc/rocm/rdna_*_configs.h, refusing those measured on another +kernel source or row. PYTHONPATH=$PWD python \\ benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py \\ @@ -383,6 +385,7 @@ def tune(a) -> None: "dtype": a.dtype, "block_size": bs, "source": jit.source_digest(), + "family": gpu_family(), "contexts": cells, "batch": bool(batch), } @@ -593,6 +596,9 @@ def watchdog(): runs = [(v, k, tags) for v, (k, tags) in finalists.items()] if current is not None: runs.insert(0, (current, knobs_of(current), ["current"])) + # Points reused from --points were not built in this run. + if jit.precompile([v for v, _, _ in runs]): + time.sleep(5) # let clocks settle after the build times = [{c: [] for c in cells} for _ in runs] for _ in range(a.confirm_rounds): for c in cells: @@ -603,23 +609,29 @@ def watchdog(): t[c].append(float("inf")) med = [[statistics.median(t[c]) for c in cells] for t in times] base = med[0] if current is not None else None + # A current row that failed at every cell is no baseline: candidates are + # ranked against the roofline and none is accepted over it. + base_ok = base is None or any(map(math.isfinite, base)) results = [] - for (v, k, tags), us in zip(runs, med): + for (v, k, tags), us, t in zip(runs, med, times): if "current" in tags: continue # Against the row, or roof without one. ratio = ( [cell_ratio(b, u) for b, u in zip(base, us)] - if base + if base and base_ok else [r / u for r, u in zip(roof, us)] ) score = weighted(ctxs, ratio) - accepted = base is None or (score > a.min_gain and within_limits(ctxs, ratio)) + accepted = base is None or ( + base_ok and score > a.min_gain and within_limits(ctxs, ratio) + ) results.append( { "knobs": k, "for": tags, "us": us, + "samples": [t[c] for c in cells], "ratio": ratio, "score": score, "worst": min(ratio), @@ -638,6 +650,7 @@ def watchdog(): **fixed, "current": None if current is None else knobs_of(current), "current_us": base, + "current_samples": None if current is None else [times[0][c] for c in cells], "finalists": results, "accepted": best is not None, # The row is outside the search space and nothing beat it. @@ -673,11 +686,13 @@ def watchdog(): # ------------------------------------------------------------------- prefill -- # Steps of M; a row serves M from its step up to the next. -STEPS = (16, 32, 64, 128, 256, 512, 1024, 2048, 4096) +STEPS = (16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192) PREFIXES = (0, 1024, 16384) # A step keeps the previous step's row when within this factor of the winner. PREFILL_SLACK = 1.02 PREFILL_BASELINES = ("TRITON_ATTN",) +# Most a step's winner may lose at any cell against the current row. +PREFILL_LOSS_LIMIT = 0.02 def prefill_grid(d): @@ -733,6 +748,49 @@ def weighted_speedup(weights, ratio): return math.exp(sum(w * math.log(r) for w, r in zip(weights, ratio)) / sum(weights)) +def prefill_weights(hq, d, win, cells): + flops = [utils.attention_flops(hq, d, m, m + p, win) for m, p in cells] + total = sum(flops) + return {c: math.sqrt(f / total) for c, f in zip(cells, flops)} + + +def choose_prefill_steps(cells, weight, med, finalists, current, loss_limit): + """Per step of M, the finalist to keep: among those losing at most + `loss_limit` at any cell against the current row, prefer points never + slower than a baseline, then the speedup over it weighted by + sqrt(FLOPs); the previous step's winner stays within PREFILL_SLACK. + + `med` maps (point id or baseline, cell) to us; `finalists` and `current` + map M to point ids. Returns (M, winner, scored, baseline us) per step. + """ + out, prev = [], None + for m in sorted({c[0] for c in cells}): + sc = [c for c in cells if c[0] == m] + base = {c: min(med[(b, c)] for b in PREFILL_BASELINES) for c in sc} + cur = current.get(m) + allowed = [ + k + for k in sorted(finalists[m]) + if cur not in finalists[m] + or all(med[(cur, c)] / med[(k, c)] >= 1 - loss_limit for c in sc) + ] + w = [weight[c] for c in sc] + scored = {} + for k in allowed: + ratio = [base[c] / med[(k, c)] for c in sc] + scored[k] = (min(ratio) >= 1.0, weighted_speedup(w, ratio), ratio) + best = max(scored, key=lambda k: scored[k][:2]) + if ( + prev in scored + and (scored[prev][0] or not scored[best][0]) + and scored[prev][1] * PREFILL_SLACK >= scored[best][1] + ): + best = prev + out.append((m, best, scored, base)) + prev = best + return out + + def tune_prefill(a) -> None: import common import jit @@ -749,10 +807,9 @@ def tune_prefill(a) -> None: dtype = utils.torch_dtype(a.dtype) hq, hkv, d, win, bs = a.hq, a.hkv, a.head_dim, a.window, a.block_size tag = f"{hq}/{hkv}/{d}" + (f" w{win}" if win else "") + f" prefill {a.dtype}" - cells = [(m, p) for m in STEPS for p in a.prefixes] - flops = [utils.attention_flops(hq, d, m, m + p, win) for m, p in cells] - total = sum(flops) - weight = {c: math.sqrt(f / total) for c, f in zip(cells, flops)} + ms = sorted(a.steps) + cells = [(m, p) for m in ms for p in a.prefixes] + weight = prefill_weights(hq, d, win, cells) fixed = { "kind": "prefill", "hq": hq, @@ -762,7 +819,8 @@ def tune_prefill(a) -> None: "dtype": a.dtype, "block_size": bs, "source": jit.source_digest(prefill=True), - "cells": cells, + "family": gpu_family(), + "cells": [list(c) for c in cells], # as read back from --points } meta = { "vllm": vllm.__version__, @@ -776,9 +834,7 @@ def variant(knobs): def pid(knobs): return json.dumps(sorted(knobs.items())) - current = { - m: R.prefill_variant_for(hq, hkv, d, bs, 1, win, dtype, m) for m in STEPS - } + current = {m: R.prefill_variant_for(hq, hkv, d, bs, 1, win, dtype, m) for m in ms} raw = ( [json.loads(x) for x in Path(a.candidates).read_text().splitlines() if x] if a.candidates @@ -910,7 +966,7 @@ def step_cells(m): return [c for c in cells if c[0] == m] finalists: dict[int, set[str]] = {} - for m in STEPS: + for m in ms: sc = step_cells(m) fastest = {c: min(t[c] for t in ok.values()) for c in sc} ranked = sorted( @@ -924,8 +980,11 @@ def step_cells(m): finalists[m] = set(ranked[: a.top]) if current[m] is not None and pid(prefill_knobs_of(current[m])) in ok: finalists[m].add(pid(prefill_knobs_of(current[m]))) - for m, prev in zip(STEPS[1:], STEPS): + for m, prev in zip(ms[1:], ms): finalists[m] |= finalists[prev] + # Points reused from --points were not built in this run. + if jit.precompile([cands[k] for m in ms for k in finalists[m]]): + time.sleep(5) # let clocks settle after the build def harness(backend, m, p, knobs=None): from common import BenchmarkConfig @@ -943,7 +1002,7 @@ def harness(backend, m, p, knobs=None): dtype=dtype, sliding_window=win or None, use_cuda_graphs=False, - max_model_len=max(STEPS) * 2 + max(a.prefixes), + max_model_len=max(ms) * 2 + max(a.prefixes), ) if knobs is None: return run_attention_benchmark(cfg).median_time * 1e6 @@ -955,7 +1014,7 @@ def harness(backend, m, p, knobs=None): times: dict = {} for _ in range(a.confirm_rounds): - for m in STEPS: + for m in ms: for c in step_cells(m): for b in PREFILL_BASELINES: try: @@ -968,23 +1027,13 @@ def harness(backend, m, p, knobs=None): ) med = {key: statistics.median(t) for key, t in times.items()} - # 3. Per step, prefer points never slower than a baseline, then speedup. - steps, prev = [], None - for m in STEPS: + # 3. Per step, the winner within the loss limit against the current row. + cur_ids = {m: pid(prefill_knobs_of(v)) for m, v in current.items() if v is not None} + steps = [] + for m, best, scored, base in choose_prefill_steps( + cells, weight, med, finalists, cur_ids, a.prefill_loss_limit + ): sc = step_cells(m) - base = {c: min(med[(b, c)] for b in PREFILL_BASELINES) for c in sc} - w = [weight[c] for c in sc] - scored = {} - for k in finalists[m]: - ratio = [base[c] / med[(k, c)] for c in sc] - scored[k] = (min(ratio) >= 1.0, weighted_speedup(w, ratio), ratio) - best = max(scored, key=lambda k: scored[k][:2]) - if ( - prev in scored - and (scored[prev][0] or not scored[best][0]) - and scored[prev][1] * PREFILL_SLACK >= scored[best][1] - ): - best = prev cur = None if current[m] is None else prefill_knobs_of(current[m]) steps.append( { @@ -993,13 +1042,13 @@ def harness(backend, m, p, knobs=None): "winner": dict(json.loads(best)), "speedup": scored[best][1], "worst": min(scored[best][2]), + "finalists": [dict(json.loads(k)) for k in sorted(finalists[m])], "cells": { f"{c[0]}:{c[1]}": {"ours": med[(best, c)], "base": base[c]} for c in sc }, } ) - prev = best print( f"{tag} M>={m}: {prefill_label(steps[-1]['winner'])} " f"speedup {steps[-1]['speedup']:.2f}x, " @@ -1008,8 +1057,23 @@ def harness(backend, m, p, knobs=None): ) decision = { **fixed, - "current": {m: s["current"] for m, s in zip(STEPS, steps)}, + "current": {m: s["current"] for m, s in zip(ms, steps)}, "steps": steps, + # Every A/B run: the finalists of each step (the current row among + # them) and the baselines, per cell. + "ab": [ + { + "cell": list(c), + **( + {"backend": who} + if who in PREFILL_BASELINES + else {"knobs": dict(json.loads(who))} + ), + "samples": t, + } + for (who, c), t in times.items() + ], + "loss_limit": a.prefill_loss_limit, "accepted": True, **meta, "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), @@ -1018,6 +1082,46 @@ def harness(backend, m, p, knobs=None): out.write(json.dumps(decision) + "\n") +def reselect_prefill(path: str, out: str, loss_limit: float) -> None: + """Choose the steps of stored prefill decisions again from their A/B + samples, with `loss_limit`; no timing.""" + + def pid(knobs): + return json.dumps(sorted(knobs.items())) + + with open(out, "a") as fh: + for line in Path(path).read_text().splitlines(): + dec = json.loads(line) + if dec.get("kind") != "prefill" or "ab" not in dec: + continue + cells = [tuple(c) for c in dec["cells"]] + weight = prefill_weights(dec["hq"], dec["d"], dec["window"], cells) + med = { + (x.get("backend") or pid(x["knobs"]), tuple(x["cell"])): ( + statistics.median(x["samples"]) + ) + for x in dec["ab"] + } + finalists = {s["m"]: {pid(k) for k in s["finalists"]} for s in dec["steps"]} + current = {s["m"]: pid(s["current"]) for s in dec["steps"] if s["current"]} + chosen = choose_prefill_steps( + cells, weight, med, finalists, current, loss_limit + ) + for step, (m, best, scored, base) in zip(dec["steps"], chosen): + sc = [c for c in cells if c[0] == m] + step.update( + winner=dict(json.loads(best)), + speedup=scored[best][1], + worst=min(scored[best][2]), + cells={ + f"{c[0]}:{c[1]}": {"ours": med[(best, c)], "base": base[c]} + for c in sc + }, + ) + dec["loss_limit"] = loss_limit + fh.write(json.dumps(dec) + "\n") + + def apply_prefill(dec: dict, block_sizes: list[int]) -> int: """Write one prefill decision into the prefill configs.""" import jit @@ -1030,7 +1134,8 @@ def apply_prefill(dec: dict, block_sizes: list[int]) -> int: if dec["source"] != jit.source_digest(prefill=True): print(f"{tag}: refused, measured on another kernel source") return 0 - lines, rows = read_configs(PREFILL_CONFIGS_H) + family = dec.get("family") or gpu_family() + lines, rows = read_configs(configs_h("prefill"), family) col = {f: i for i, f in enumerate(R.PREFILL_CONFIG_FIELDS)} dtype = utils.torch_dtype(dec["dtype"]) for step in dec["steps"]: @@ -1050,28 +1155,51 @@ def mine(r): ) == (d, hq, hkv, win) sizes = sorted({r[col["page_size"]] for r in rows if mine(r)}) or block_sizes - rows = [r for r in rows if not mine(r)] - merged = [] - for step in dec["steps"]: - if not merged or merged[-1]["winner"] != step["winner"]: - merged.append(step) + kept = [r for r in rows if not mine(r)] + # A decided step's winner serves M from its step up to the next step of + # STEPS; outside them the entries stay. + decided = {s["m"]: s["winner"] for s in dec["steps"]} + ends = dict(zip(STEPS, (*STEPS[1:], None))) written = 0 - for bf16, bs, step in itertools.product((0, 1), sizes, merged): - v = R.PrefillVariant( - d, - hq, - hkv, - win, - bs, - step["m"], - **step["winner"], - dtype=torch.bfloat16 if bf16 else torch.float16, + for bf16, bs in itertools.product((0, 1), sizes): + group = sorted( + ( + r + for r in rows + if mine(r) and (r[col["dtype"]], r[col["page_size"]]) == (bf16, bs) + ), + key=lambda r: r[col["min_query_len"]], ) - rows.append(R.prefill_config_fields(v)) - written += 1 - write_configs(PREFILL_CONFIGS_H, lines, rows, R.PREFILL_CONFIG_FIELDS) - for step in merged: - print(f"{tag} M>={step['m']}: {prefill_label(step['winner'])}") + starts = {r[col["min_query_len"]] for r in group} + bounds = sorted(starts | set(decided) | {e for e in ends.values() if e}) + out = [] + for b in bounds: + m = max((m for m in decided if m <= b), default=None) + if m is not None and (ends[m] is None or b < ends[m]): + knobs = decided[m] + else: + below = [r for r in group if r[col["min_query_len"]] <= b] + if not below: + continue + knobs = {k: below[-1][col[k]] for k in utils.PREFILL_KNOBS} + if not out or out[-1][1] != knobs: + out.append((b, knobs)) + for b, knobs in out: + v = R.PrefillVariant( + d, + hq, + hkv, + win, + bs, + b, + **knobs, + dtype=torch.bfloat16 if bf16 else torch.float16, + ) + kept.append(R.prefill_config_fields(v)) + written += 1 + write_configs(configs_h("prefill"), lines, kept, R.PREFILL_CONFIG_FIELDS, family) + for m, knobs in sorted(decided.items()): + print(f"{tag} M>={m}: {prefill_label(knobs)}") return written @@ -1234,6 +1362,7 @@ def timed(launches, iters=25): "dtype": a.dtype, "block_size": bs, "source": jit.source_digest(prefill=True), + "family": gpu_family(), "rows": [ prefill_knobs_of(v) | {"min_query_len": v.min_query_len} for v in rows @@ -1261,7 +1390,8 @@ def apply_prefill_batch(dec: dict, block_sizes: list[int]) -> int: if now != dec["rows"]: print(f"{tag}: refused, the prefill builds changed since it was measured") return 0 - lines, table = read_configs(PREFILL_BATCH_CONFIGS_H) + family = dec.get("family") or gpu_family() + lines, table = read_configs(configs_h("prefill_batch"), family) pages = ( sorted( {k[3] for k in R._prefill_table() if k[:3] == (hq, hkv, d) and k[4] == win} @@ -1284,7 +1414,13 @@ def apply_prefill_batch(dec: dict, block_sizes: list[int]) -> int: c["head_group"], ] ) - write_configs(PREFILL_BATCH_CONFIGS_H, lines, table, R.PREFILL_BATCH_CONFIG_FIELDS) + write_configs( + configs_h("prefill_batch"), + lines, + table, + R.PREFILL_BATCH_CONFIG_FIELDS, + family, + ) print(f"{tag}: {len(dec['cells'])} cells at pages {pages}") return len(dec["cells"]) * len(pages) @@ -1293,33 +1429,69 @@ def apply_prefill_batch(dec: dict, block_sizes: list[int]) -> int: _CSRC = _HERE.parents[4] / "csrc" / "rocm" -DECODE_CONFIGS_H = _CSRC / "rdna_decode_configs.h" -DECODE_BATCH_CONFIGS_H = _CSRC / "rdna_decode_batch_configs.h" -PREFILL_CONFIGS_H = _CSRC / "rdna_prefill_configs.h" -PREFILL_BATCH_CONFIGS_H = _CSRC / "rdna_prefill_batch_configs.h" + + +def configs_h(kind: str) -> Path: + """The configs header of `kind`: decode, decode_batch, prefill or + prefill_batch.""" + return _CSRC / f"rdna_{kind}_configs.h" + + # One config entry per line. _ENTRY = re.compile(r"^\s*\{([\w:, ]+)\},$") +# The GPU family of each arch: its config tables (Arch in +# csrc/rocm/rdna_attn_types.h). +FAMILIES = { + "gfx1150": "Gfx1150", + "gfx1151": "Gfx1151", + "gfx1152": "Gfx1150", + "gfx1153": "Gfx1150", +} + + +def gpu_family() -> str: + """The family of the GPU tune.py measures on.""" + import torch + + return FAMILIES[torch.cuda.get_device_properties(0).gcnArchName.split(":")[0]] + + +def _table_lines(lines: list[str], family: str) -> range: + """The line indices of family's entries in a configs header.""" + start = 1 + next( + i + for i, line in enumerate(lines) + if re.search(rf"\bk{family}\w*Configs\[\] = \{{$", line) + ) + end = next(i for i in range(start, len(lines)) if lines[i].startswith("};")) + return range(start, end) + -def read_configs(path: Path) -> tuple[list[str], list[list[int]]]: - """The lines of a configs header, and its entries.""" +def read_configs(path: Path, family: str) -> tuple[list[str], list[list[int]]]: + """The lines of a configs header, and the entries of family's table.""" import jit lines = path.read_text().splitlines() - rows = [jit.parse_cpp_fields(m[1]) for m in map(_ENTRY.match, lines) if m] + at = _table_lines(lines, family) + rows = [jit.parse_cpp_fields(m[1]) for i in at if (m := _ENTRY.match(lines[i]))] return lines, rows def write_configs( - path: Path, lines: list[str], rows: list[list[int]], names: tuple[str, ...] + path: Path, + lines: list[str], + rows: list[list[int]], + names: tuple[str, ...], + family: str, ) -> None: - """Replace the entries of a configs header with `rows`, sorted; `names` - are their fields.""" + """Replace the entries of family's table in a configs header with `rows`, + sorted; `names` are their fields.""" import jit - at = [i for i, line in enumerate(lines) if _ENTRY.match(line)] + at = _table_lines(lines, family) entries = [f" {{{jit.cpp_fields(r, names)}}}," for r in sorted(rows)] - text = lines[: at[0]] + entries + lines[at[-1] + 1 :] + text = lines[: at.start] + entries + lines[at.stop :] path.write_text("\n".join(text) + "\n") @@ -1340,13 +1512,16 @@ def apply(path: str, block_sizes: list[int]) -> None: "dtype", "page_size", ), R.DECODE_CONFIG_FIELDS - # Keyed by (batched, shape). - headers = (DECODE_CONFIGS_H, DECODE_BATCH_CONFIGS_H) - lines, rows = [], {} - for batched, header in enumerate(headers): - text, entries = read_configs(header) - lines.append(text) - rows |= {(batched, *r[:7]): r for r in entries} + headers = (configs_h("decode"), configs_h("decode_batch")) + # Each (family, batched) table, keyed by shape. + tables: dict[tuple[str, int], dict[tuple, list[int]]] = {} + + def table(family: str, batched: int) -> dict[tuple, list[int]]: + if (family, batched) not in tables: + _, entries = read_configs(headers[batched], family) + tables[family, batched] = {tuple(r[:7]): r for r in entries} + return tables[family, batched] + source = jit.source_digest() written = 0 for line in Path(path).read_text().splitlines(): @@ -1383,7 +1558,8 @@ def apply(path: str, block_sizes: list[int]) -> None: if dec["current"] != (None if now is None else knobs_of(now)): print(f"{tag}: refused, the row changed since it was measured") continue - sizes = sorted({k[7] for k in rows if k[:6] == (batched, d, hq, hkv, win, m)}) + rows = table(dec.get("family") or gpu_family(), batched) + sizes = sorted({k[6] for k in rows if k[:5] == (d, hq, hkv, win, m)}) for bf16, bs in itertools.product((0, 1), sizes or block_sizes): v = R.KernelVariant( d, @@ -1398,15 +1574,13 @@ def apply(path: str, block_sizes: list[int]) -> None: batched=batched, ) row = R.config_fields(v) - rows[(batched, *row[:7])] = row + rows[tuple(row[:7])] = row written += 1 print(f"{tag}: {label(dec['winner'])}") - for batched, header in enumerate(headers): + for (family, batched), rows in tables.items(): + lines = headers[batched].read_text().splitlines() write_configs( - header, - lines[batched], - [r for k, r in rows.items() if k[0] == batched], - R.DECODE_CONFIG_FIELDS, + headers[batched], lines, list(rows.values()), R.DECODE_CONFIG_FIELDS, family ) print(f"# {written} rows written") @@ -1441,6 +1615,14 @@ def main() -> None: default=list(PREFIXES), help="with --prefill: cached tokens before each step's M", ) + p.add_argument( + "--steps", + type=int, + nargs="+", + default=list(STEPS), + choices=STEPS, + help="with --prefill: the steps of M to choose; --apply keeps the others", + ) p.add_argument("--hq", type=int) p.add_argument("--hkv", type=int) p.add_argument("--head-dim", type=int) @@ -1464,7 +1646,23 @@ def main() -> None: ) p.add_argument("--limit", type=int, default=0, help="first N points (smoke)") p.add_argument("--candidates", help="JSONL of knob sets instead of the grid") + p.add_argument( + "--prefill-loss-limit", + type=float, + default=PREFILL_LOSS_LIMIT, + help="with --prefill: most a winner may lose at a cell against the current row", + ) + p.add_argument( + "--reselect", + metavar="DECISIONS", + help="choose stored prefill decisions again from their A/B (to --decisions)", + ) a = p.parse_args() + if a.reselect: + if not a.decisions: + p.error("--reselect needs --decisions") + reselect_prefill(a.reselect, a.decisions, a.prefill_loss_limit) + return if a.apply: apply(a.apply, a.block_sizes) return diff --git a/benchmarks/kernels/rocm_rdna_hip_attn/tools/utils.py b/benchmarks/kernels/rocm_rdna_hip_attn/tools/utils.py index c028cbf20a52..fb0fb03c7a2b 100644 --- a/benchmarks/kernels/rocm_rdna_hip_attn/tools/utils.py +++ b/benchmarks/kernels/rocm_rdna_hip_attn/tools/utils.py @@ -4,20 +4,37 @@ """Roofline, knobs and --dtype shared by matrix.py and tune.py.""" import argparse +import os -# Peak DRAM bandwidth, GiB/s. -PEAK_GIBS = 230.0 +# Sustained peaks of each GPU: DRAM GiB/s, WMMA f16/bf16 TFLOPS, SIMDs. The +# GPU is the current device's (RDNA_ROOFLINE_ARCH overrides it, e.g. +# to analyse results elsewhere); any other GPU gets gfx1151's. +PEAKS = { + "gfx1151": (230.0, 43.0, 80), + "gfx1150": (81.0, 17.2, 32), +} -# Per-kernel dispatch overhead under a HIP graph. -DISPATCH_US = 1.48 -# Sustained WMMA f16/bf16 throughput on gfx1151. -PEAK_TFLOPS = 43.0 +def _arch() -> str: + arch = os.environ.get("RDNA_ROOFLINE_ARCH") + if arch: + return arch + try: + import torch + + return torch.cuda.get_device_properties(0).gcnArchName.split(":")[0] + except Exception: # noqa: BLE001 - no GPU here + return "gfx1151" + + +PEAK_GIBS, PEAK_TFLOPS, SIMDS = PEAKS.get(_arch(), PEAKS["gfx1151"]) + +# Per-kernel dispatch overhead under a HIP graph, the same on every GPU. +DISPATCH_US = 1.48 # Prefill model (roofline(realistic=True)): whole 16x16 score tiles, plus # SOFTMAX_CYCLES of VALU work per tile that does not overlap the WMMAs. WMMA_TILE = 16 -SIMDS = 80 WMMA_GHZ = PEAK_TFLOPS * 1e12 * 32 / (SIMDS * 2 * WMMA_TILE**3) / 1e9 VALU_OPS_PER_SCORE = 4.5 VALU_LANES = 32 diff --git a/cmake/rocm_rdna_attn.cmake b/cmake/rocm_rdna_attn.cmake index 6b0559cabfa9..a12a5ec65b90 100644 --- a/cmake/rocm_rdna_attn.cmake +++ b/cmake/rocm_rdna_attn.cmake @@ -1,17 +1,19 @@ # -# Builds the RDNA3.5 attention kernels into the object library ${TARGET} for -# ${ARCHES}. Each (kernel, head dim, dtype) is a generated translation unit -# that builds the entries of csrc/rocm/rdna__configs.h with -# that head dim and dtype and registers them, so the builds compile in -# parallel. The head dims and dtypes are the enumerators of -# csrc/rocm/rdna_attn_types.h; an entry without a unit makes -# rdna__configs() raise. +# Builds the RDNA3.5 attention kernels of one GPU family into the object +# library ${TARGET} for ${ARCHES}; FAMILY names the family's config tables +# (Arch::k${FAMILY} in csrc/rocm/rdna_attn_types.h). Each (kernel, head dim, +# dtype) is a generated translation unit that builds the entries of the +# family's table in csrc/rocm/rdna__configs.h with that head dim and +# dtype and registers them, so the builds compile in parallel. The head dims +# and dtypes are the enumerators of rdna_attn_types.h; an entry without a unit +# makes rdna__configs() raise. # -function(rdna_attn_target TARGET ARCHES) +function(rdna_attn_target TARGET FAMILY ARCHES) set(_KERNELS decode decode_batch prefill) set(_HEAD_DIMS 64 96 128 256 512) set(_DTYPES Fp16 Bf16) - set(_OUT ${CMAKE_CURRENT_BINARY_DIR}/rdna_attn) + string(TOLOWER ${FAMILY} _FAMILY_NAME) + set(_OUT ${CMAKE_CURRENT_BINARY_DIR}/rdna_attn/${_FAMILY_NAME}) set(_UNITS "") foreach(_KERNEL IN LISTS _KERNELS) foreach(_HEAD_DIM IN LISTS _HEAD_DIMS) @@ -21,8 +23,8 @@ function(rdna_attn_target TARGET ARCHES) file(CONFIGURE OUTPUT ${_UNIT} CONTENT "#include \"rdna_attn_instances.cuh\" using namespace rdna; -static const bool kRegistered = - register_${_KERNEL}(); +static const bool kRegistered = register_${_KERNEL}< + Arch::k${FAMILY}, HeadDim::k${_HEAD_DIM}, Dtype::k${_DTYPE}>(); ") list(APPEND _UNITS ${_UNIT}) endforeach() @@ -39,4 +41,31 @@ static const bool kRegistered = target_compile_options(${TARGET} PRIVATE $<$:${VLLM_ROCM_EXT_FLAGS}> -O3 -g0 -Wno-unused-variable) + # The archs the registry offers the family on. + foreach(_ARCH IN LISTS ARCHES) + string(TOUPPER ${_ARCH} _ARCH) + target_compile_definitions(${TARGET} INTERFACE VLLM_RDNA_ATTN_${_ARCH}) + endforeach() +endfunction() + +# The object libraries, in OUT, of the GPU families with an arch in +# VLLM_GPU_ARCHES: Gfx1151 (gfx1151) and Gfx1150 (gfx1150, gfx1152, gfx1153). +function(rdna_attn_targets OUT) + set(_TARGETS "") + foreach(_FAMILY_ARCHES "Gfx1151 gfx1151" "Gfx1150 gfx1150 gfx1152 gfx1153") + separate_arguments(_FAMILY_ARCHES) + list(POP_FRONT _FAMILY_ARCHES _FAMILY) + set(_ARCHES "") + foreach(_ARCH IN LISTS _FAMILY_ARCHES) + if(_ARCH IN_LIST VLLM_GPU_ARCHES) + list(APPEND _ARCHES ${_ARCH}) + endif() + endforeach() + if(_ARCHES) + string(TOLOWER ${_FAMILY} _NAME) + rdna_attn_target(_rocm_C_rdna_attn_${_NAME} ${_FAMILY} "${_ARCHES}") + list(APPEND _TARGETS _rocm_C_rdna_attn_${_NAME}) + endif() + endforeach() + set(${OUT} ${_TARGETS} PARENT_SCOPE) endfunction() diff --git a/csrc/rocm/rdna_attn_instances.cuh b/csrc/rocm/rdna_attn_instances.cuh index 75a61867bb87..b69e26bc0246 100644 --- a/csrc/rocm/rdna_attn_instances.cuh +++ b/csrc/rocm/rdna_attn_instances.cuh @@ -1,7 +1,8 @@ -// Builds the entries of a configs table with one head dim and dtype, and -// registers their launch functions. Each (kernel, head dim, dtype) is a -// translation unit generated by cmake/rocm_rdna_attn.cmake, so the -// builds compile in parallel. +// Builds the entries of one GPU family's decode, batched decode or prefill +// table with one head dim and dtype, and registers their launch functions. +// Each (kernel, family, head dim, dtype) is a translation unit generated by +// cmake/rocm_rdna_attn.cmake for the family's archs, so the builds compile in +// parallel. #pragma once #include @@ -12,58 +13,58 @@ namespace rdna { -template +template bool register_decode(std::index_sequence) { ( [] { - constexpr DecodeConfig c = kDecodeConfigs[I]; + constexpr DecodeConfig c = decode_configs()[I]; if constexpr (c.head_dim == kHeadDim && c.dtype == kDtype) - decode_launchers()[I] = &launch_decode; + decode_launchers(A)[I] = &launch_decode; }(), ...); return true; } -template +template bool register_decode() { - return register_decode( - std::make_index_sequence{}); + return register_decode( + std::make_index_sequence())>{}); } -template +template bool register_decode_batch(std::index_sequence) { ( [] { - constexpr DecodeConfig c = kDecodeBatchConfigs[I]; + constexpr DecodeConfig c = decode_batch_configs()[I]; if constexpr (c.head_dim == kHeadDim && c.dtype == kDtype) - decode_batch_launchers()[I] = &launch_decode; + decode_batch_launchers(A)[I] = &launch_decode; }(), ...); return true; } -template +template bool register_decode_batch() { - return register_decode_batch( - std::make_index_sequence{}); + return register_decode_batch( + std::make_index_sequence())>{}); } -template +template bool register_prefill(std::index_sequence) { ( [] { - constexpr PrefillConfig c = kPrefillConfigs[I]; + constexpr PrefillConfig c = prefill_configs()[I]; if constexpr (c.head_dim == kHeadDim && c.dtype == kDtype) - prefill_launchers()[I] = &launch_prefill; + prefill_launchers(A)[I] = &launch_prefill; }(), ...); return true; } -template +template bool register_prefill() { - return register_prefill( - std::make_index_sequence{}); + return register_prefill( + std::make_index_sequence())>{}); } } // namespace rdna diff --git a/csrc/rocm/rdna_attn_launch.h b/csrc/rocm/rdna_attn_launch.h index a1f46bfeaffe..d8c7ac39eca2 100644 --- a/csrc/rocm/rdna_attn_launch.h +++ b/csrc/rocm/rdna_attn_launch.h @@ -4,6 +4,8 @@ #include +#include "rdna_attn_types.h" + namespace rdna { // Raw pointers keep torch headers out of the kernels. Mirrored by @@ -33,12 +35,12 @@ struct LaunchArgs { using LaunchFn = void (*)(const LaunchArgs&); -// The launch function of each entry of kDecodeConfigs, null until its unit -// (cmake/rocm_rdna_attn.cmake) registers it. Defined in +// The launch function of each entry of GPU family `arch`'s decode table, null +// until its unit (cmake/rocm_rdna_attn.cmake) registers it. Defined in // rdna_registry.cu. -LaunchFn* decode_launchers(); -// Likewise for kDecodeBatchConfigs. -LaunchFn* decode_batch_launchers(); +LaunchFn* decode_launchers(Arch arch); +// Likewise for the batched decode tables. +LaunchFn* decode_batch_launchers(Arch arch); // One sequence of num_tokens query tokens. partial_o, partial_ml and counters // are the split-KV scratch; max_seq_len only sizes the split. mm_ranges holds @@ -69,7 +71,7 @@ struct PrefillArgs { using PrefillFn = void (*)(const PrefillArgs&); -// Likewise for kPrefillConfigs. -PrefillFn* prefill_launchers(); +// Likewise for the prefill tables. +PrefillFn* prefill_launchers(Arch arch); } // namespace rdna diff --git a/csrc/rocm/rdna_attn_types.h b/csrc/rocm/rdna_attn_types.h index b973134ef94f..bdebfaf62602 100644 --- a/csrc/rocm/rdna_attn_types.h +++ b/csrc/rocm/rdna_attn_types.h @@ -1,4 +1,5 @@ -// The head dims and element types the RDNA3.5 attention kernels are built for. +// The head dims, element types and GPU families the RDNA3.5 attention kernels +// are built for. #pragma once namespace rdna { @@ -13,4 +14,10 @@ enum class HeadDim : int { enum class Dtype : int { kFp16 = 0, kBf16 = 1 }; +// The GPU families with their own config tables, each built for its archs +// only (cmake/rocm_rdna_attn.cmake): gfx1151, with 1536 VGPRs a SIMD lane, and +// gfx1150, with 1024, whose tables gfx1152 and gfx1153 also use. +enum class Arch : int { kGfx1151 = 0, kGfx1150 = 1 }; +inline constexpr int kNumArches = 2; + } // namespace rdna diff --git a/csrc/rocm/rdna_decode_attn.cu b/csrc/rocm/rdna_decode_attn.cu index f322b6fa70dc..7086e1965a8d 100644 --- a/csrc/rocm/rdna_decode_attn.cu +++ b/csrc/rocm/rdna_decode_attn.cu @@ -1225,7 +1225,9 @@ struct DecodeBatch { int blockTableStride, accStride, statStride, counterStride; }; -template +// A is the GPU family the entry is built for: an entry in two families' +// tables is two kernels, each in its family's code objects. +template __global__ __launch_bounds__((Decode::kThreads)) void decode_attn( const typename Decode::elem_t* __restrict__ q, const typename Decode::elem_t* __restrict__ kv, @@ -1273,7 +1275,7 @@ __global__ __launch_bounds__((Decode::kThreads)) void decode_attn( } // Arguments are checked by rdna_registry.cu. -template +template void launch_decode(const LaunchArgs& a) { using K = Decode; using elem_t = typename K::elem_t; @@ -1288,17 +1290,21 @@ void launch_decode(const LaunchArgs& a) { (void)hipDeviceGetAttribute(&numWgps, hipDeviceAttributeMultiprocessorCount, device); (void)hipOccupancyMaxActiveBlocksPerMultiprocessor( - &perWgp, decode_attn, K::kThreads, 0); - // The runtime assumes 64 KiB of LDS per WGP; gfx1151 has 128 KiB. + &perWgp, decode_attn, K::kThreads, 0); + // The runtime assumes 64 KiB of LDS per WGP; gfx1150..gfx1153 have + // 128 KiB (64 KiB at most per workgroup). VGPRs a SIMD lane: 1536 in + // blocks of 24 on gfx1151, 1024 in blocks of 16 on the others. hipFuncAttributes attrs; hipDeviceProp_t prop; - if (hipFuncGetAttributes(&attrs, reinterpret_cast( - decode_attn)) == - hipSuccess && + if (hipFuncGetAttributes( + &attrs, reinterpret_cast( + decode_attn)) == hipSuccess && hipGetDeviceProperties(&prop, device) == hipSuccess && - strstr(prop.gcnArchName, "gfx1151")) { + strstr(prop.gcnArchName, "gfx115")) { + const bool gfx1151 = strstr(prop.gcnArchName, "gfx1151"); + const int vgprs = gfx1151 ? 1536 : 1024, block = gfx1151 ? 24 : 16; const int wavesPerSimd = - std::min(16, 1536 / ((attrs.numRegs + 23) / 24 * 24)); + std::min(16, vgprs / ((attrs.numRegs + block - 1) / block * block)); const int byWaves = 2 * (2 * wavesPerSimd / K::kWaves); const int byLds = attrs.sharedSizeBytes ? 128 * 1024 / (int)attrs.sharedSizeBytes @@ -1325,7 +1331,7 @@ void launch_decode(const LaunchArgs& a) { << 1; batch = {a.bt_stride, a.acc_stride, a.ml_stride, a.cnt_stride}; } - hipLaunchKernelGGL((decode_attn), grid, threads, 0, + hipLaunchKernelGGL((decode_attn), grid, threads, 0, a.stream, static_cast(a.q), static_cast(a.kv), a.bt, a.acc, a.m, a.l, a.cnt, static_cast(a.out), a.seq_lens, a.bt_width, diff --git a/csrc/rocm/rdna_decode_batch_configs.h b/csrc/rocm/rdna_decode_batch_configs.h index b32ef90ceaab..e9d7f3c0be88 100644 --- a/csrc/rocm/rdna_decode_batch_configs.h +++ b/csrc/rocm/rdna_decode_batch_configs.h @@ -2,14 +2,18 @@ // grid.y. A batch whose shape has no entry runs on Triton. #pragma once +#include + #include "rdna_decode_configs.h" namespace rdna { -// Written by benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py -// --apply, one entry per line in DecodeConfig order. +// One table per Arch (rdna_attn_types.h), each complete; a GPU family's +// entries are built for its archs only. Written by +// benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py --apply, one entry per +// line in DecodeConfig order. // clang-format off -inline constexpr DecodeConfig kDecodeBatchConfigs[] = { +inline constexpr DecodeConfig kGfx1151DecodeBatchConfigs[] = { {HeadDim::k64, 14, 2, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, {HeadDim::k64, 14, 2, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, {HeadDim::k64, 14, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 2, 8, 0, 1, 0, 0}, @@ -683,6 +687,694 @@ inline constexpr DecodeConfig kDecodeBatchConfigs[] = { {HeadDim::k512, 32, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, {HeadDim::k512, 32, 4, 0, 8, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, }; -// clang-format on + +inline constexpr DecodeConfig kGfx1150DecodeBatchConfigs[] = { + {HeadDim::k64, 14, 2, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 2, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 2, Dtype::kBf16, 16, 3, 1, 2, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 3, Dtype::kFp16, 16, 4, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 3, Dtype::kBf16, 16, 4, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 4, Dtype::kFp16, 16, 4, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 4, Dtype::kBf16, 16, 4, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 5, Dtype::kFp16, 16, 5, 1, 2, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 5, Dtype::kBf16, 16, 5, 1, 2, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 6, Dtype::kFp16, 16, 5, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 6, Dtype::kBf16, 16, 5, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 7, Dtype::kFp16, 16, 5, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k64, 14, 2, 0, 7, Dtype::kBf16, 16, 5, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k64, 14, 2, 0, 8, Dtype::kFp16, 16, 5, 2, 2, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 8, Dtype::kBf16, 16, 5, 2, 2, 8, 0, 0, 1, 0}, + {HeadDim::k64, 16, 2, 0, 1, Dtype::kFp16, 16, 2, 1, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 1, Dtype::kBf16, 16, 2, 1, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 2, Dtype::kBf16, 16, 3, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 3, Dtype::kFp16, 16, 2, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k64, 16, 2, 0, 3, Dtype::kBf16, 16, 2, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k64, 16, 2, 0, 4, Dtype::kFp16, 16, 3, 2, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 4, Dtype::kBf16, 16, 3, 2, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 6, Dtype::kFp16, 16, 1, 3, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 6, Dtype::kBf16, 16, 1, 3, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 7, Dtype::kFp16, 16, 2, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 7, Dtype::kBf16, 16, 2, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 1, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 1, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 4, Dtype::kFp16, 16, 1, 1, 4, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 4, Dtype::kBf16, 16, 1, 1, 4, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 1, 0, 0}, + {HeadDim::k96, 32, 32, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 1, 0, 0}, + {HeadDim::k96, 32, 32, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k128, 10, 10, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 4, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 4, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 1, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 1, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 2, Dtype::kBf16, 16, 3, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 3, Dtype::kFp16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 3, Dtype::kBf16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 4, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 4, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 6, Dtype::kFp16, 16, 5, 3, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 6, Dtype::kBf16, 16, 5, 3, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 7, Dtype::kFp16, 16, 3, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 7, Dtype::kBf16, 16, 3, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 8, Dtype::kFp16, 16, 3, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 8, Dtype::kBf16, 16, 3, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 24, 8, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 24, 8, 0, 7, Dtype::kFp16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 24, 8, 0, 7, Dtype::kBf16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 24, 8, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 24, 8, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 3, Dtype::kFp16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 3, Dtype::kBf16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 5, Dtype::kFp16, 16, 2, 5, 4, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 5, Dtype::kBf16, 16, 2, 5, 4, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 6, Dtype::kFp16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 6, Dtype::kBf16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 7, Dtype::kFp16, 16, 1, 7, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 7, Dtype::kBf16, 16, 1, 7, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 2, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 2, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 3, Dtype::kFp16, 16, 5, 3, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 3, Dtype::kBf16, 16, 5, 3, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 6, Dtype::kFp16, 16, 3, 6, 2, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 6, Dtype::kBf16, 16, 3, 6, 2, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 7, Dtype::kFp16, 16, 2, 7, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 7, Dtype::kBf16, 16, 2, 7, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 8, Dtype::kFp16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 8, Dtype::kBf16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 4, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 3, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 4, 0, 3, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 4, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 4, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 4, 0, 5, Dtype::kFp16, 16, 2, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 4, 0, 5, Dtype::kBf16, 16, 2, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 4, 0, 6, Dtype::kFp16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 4, 0, 6, Dtype::kBf16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 4, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 5, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 8, 0, 5, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 8, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 8, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 8, 0, 7, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 8, 0, 7, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 8, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 1, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 1, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 2, Dtype::kFp16, 16, 1, 1, 2, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 2, Dtype::kBf16, 16, 1, 1, 2, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 8, Dtype::kFp16, 16, 1, 1, 2, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 8, Dtype::kBf16, 16, 1, 1, 2, 4, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 4, Dtype::kFp16, 16, 3, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 4, Dtype::kBf16, 16, 3, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 5, Dtype::kFp16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 5, Dtype::kBf16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 7, Dtype::kFp16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 7, Dtype::kBf16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 8, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 8, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 1, Dtype::kFp16, 16, 8, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 0, 1, Dtype::kBf16, 16, 8, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 0, 2, Dtype::kFp16, 16, 5, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 0, 2, Dtype::kBf16, 16, 5, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 0, 3, Dtype::kFp16, 16, 8, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 3, Dtype::kBf16, 16, 8, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 4, Dtype::kFp16, 16, 5, 2, 2, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 4, Dtype::kBf16, 16, 5, 2, 2, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 5, Dtype::kFp16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 5, Dtype::kBf16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 6, Dtype::kFp16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 6, Dtype::kBf16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 7, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 7, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 8, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 0, 8, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kFp16, 16, 5, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kFp16, 32, 5, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kBf16, 16, 5, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kBf16, 32, 5, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kFp16, 16, 5, 1, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kFp16, 32, 5, 1, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kBf16, 16, 5, 1, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kBf16, 32, 5, 1, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kFp16, 16, 3, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kFp16, 32, 3, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kBf16, 16, 3, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kBf16, 32, 3, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kFp16, 16, 5, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kFp16, 32, 5, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kBf16, 16, 5, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kBf16, 32, 5, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kFp16, 16, 8, 5, 4, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kFp16, 32, 8, 5, 4, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kBf16, 16, 8, 5, 4, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kBf16, 32, 8, 5, 4, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kFp16, 16, 3, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kFp16, 32, 3, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kBf16, 16, 3, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kBf16, 32, 3, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kFp16, 16, 4, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kFp16, 32, 4, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kBf16, 16, 4, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kBf16, 32, 4, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kFp16, 16, 4, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kFp16, 32, 4, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kBf16, 16, 4, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kBf16, 32, 4, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kFp16, 16, 4, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kFp16, 544, 4, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kBf16, 16, 4, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kBf16, 544, 4, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kFp16, 544, 3, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kBf16, 16, 3, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kBf16, 544, 3, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kFp16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kFp16, 544, 4, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kBf16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kBf16, 544, 4, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kFp16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kFp16, 544, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kBf16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kBf16, 544, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kFp16, 544, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kBf16, 544, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kFp16, 16, 2, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kFp16, 544, 2, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kBf16, 16, 2, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kBf16, 544, 2, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kFp16, 544, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kBf16, 544, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kFp16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kFp16, 544, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kBf16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kBf16, 544, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kFp16, 32, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kBf16, 32, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kFp16, 16, 3, 1, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kFp16, 32, 3, 1, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kBf16, 16, 3, 1, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kBf16, 32, 3, 1, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kFp16, 16, 2, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kFp16, 32, 2, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kBf16, 16, 2, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kBf16, 32, 2, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kFp16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kFp16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kBf16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kBf16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kFp16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kFp16, 32, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kBf16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kBf16, 32, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kFp16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kFp16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kBf16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kBf16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kFp16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kFp16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kBf16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kBf16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kFp16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kFp16, 32, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kBf16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kBf16, 32, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 1, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 1, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 2, Dtype::kFp16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 2, Dtype::kBf16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 3, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 3, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 4, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 4, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 5, Dtype::kFp16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 5, Dtype::kBf16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 6, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 6, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 7, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 7, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 8, Dtype::kFp16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 8, Dtype::kBf16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 8, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 8, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 1, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 1, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 2, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 2, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 8, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 8, 4, 4096, 8, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kFp16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kFp16, 1056, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kBf16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kBf16, 1056, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kFp16, 16, 4, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kFp16, 1056, 4, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kBf16, 16, 4, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kBf16, 1056, 4, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kFp16, 1056, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kBf16, 1056, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kFp16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kFp16, 1056, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kBf16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kBf16, 1056, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kFp16, 16, 2, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kFp16, 1056, 2, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kBf16, 16, 2, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kBf16, 1056, 2, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kFp16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kFp16, 1056, 3, 3, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kBf16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kBf16, 1056, 3, 3, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kFp16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kFp16, 1056, 5, 4, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kBf16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kBf16, 1056, 5, 4, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kFp16, 16, 3, 4, 2, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kFp16, 1056, 3, 4, 2, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kBf16, 16, 3, 4, 2, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kBf16, 1056, 3, 4, 2, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kFp16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kBf16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kFp16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kBf16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 3, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 3, Dtype::kFp16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 3, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 3, Dtype::kBf16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 4, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 4, Dtype::kFp16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 4, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 4, Dtype::kBf16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kFp16, 16, 2, 1, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kFp16, 528, 2, 1, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kBf16, 16, 2, 1, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kBf16, 528, 2, 1, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kFp16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kFp16, 528, 2, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kBf16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kBf16, 528, 2, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kFp16, 16, 2, 1, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kFp16, 528, 2, 1, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kBf16, 16, 2, 1, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kBf16, 528, 2, 1, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kFp16, 16, 3, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kFp16, 528, 3, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kBf16, 16, 3, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kBf16, 528, 3, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 7, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 7, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 1, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 1, Dtype::kFp16, 784, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 1, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 1, Dtype::kBf16, 784, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 2, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 2, Dtype::kFp16, 784, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 2, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 2, Dtype::kBf16, 784, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kFp16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kFp16, 784, 2, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kBf16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kBf16, 784, 2, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kFp16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kFp16, 784, 2, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kBf16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kBf16, 784, 2, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kFp16, 784, 1, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kBf16, 784, 1, 5, 1, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kFp16, 16, 3, 3, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kFp16, 784, 3, 3, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kBf16, 16, 3, 3, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kBf16, 784, 3, 3, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kFp16, 16, 3, 3, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kFp16, 784, 3, 3, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kBf16, 16, 3, 3, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kBf16, 784, 3, 3, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 8, Dtype::kFp16, 16, 2, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 8, Dtype::kFp16, 784, 2, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 8, Dtype::kBf16, 16, 2, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 8, Dtype::kBf16, 784, 2, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 4, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 4, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 1, Dtype::kFp16, 16, 8, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 1, Dtype::kBf16, 16, 8, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 2, Dtype::kFp16, 16, 5, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 2, Dtype::kBf16, 16, 5, 1, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 3, Dtype::kFp16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 3, Dtype::kBf16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 4, Dtype::kFp16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 4, Dtype::kBf16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 5, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 5, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 6, Dtype::kFp16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 6, Dtype::kBf16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 7, Dtype::kFp16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 7, Dtype::kBf16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 8, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 8, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 2, 0, 1, Dtype::kFp16, 16, 4, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 1, Dtype::kBf16, 16, 4, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 2, Dtype::kBf16, 16, 3, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 3, Dtype::kFp16, 16, 4, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 3, Dtype::kBf16, 16, 4, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 4, Dtype::kFp16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 4, Dtype::kBf16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 5, Dtype::kFp16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 2, 0, 5, Dtype::kBf16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 2, 0, 6, Dtype::kFp16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 2, 0, 6, Dtype::kBf16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 2, 0, 7, Dtype::kFp16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 2, 0, 7, Dtype::kBf16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 2, 0, 8, Dtype::kFp16, 16, 5, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 8, Dtype::kBf16, 16, 5, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kFp16, 16, 5, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kFp16, 64, 5, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kBf16, 16, 5, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kBf16, 64, 5, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kFp16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kFp16, 64, 5, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kBf16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kBf16, 64, 5, 2, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kFp16, 64, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kBf16, 64, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kFp16, 64, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kBf16, 64, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kFp16, 16, 4, 5, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kFp16, 64, 4, 5, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kBf16, 16, 4, 5, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kBf16, 64, 4, 5, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kFp16, 16, 5, 6, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kFp16, 64, 5, 6, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kBf16, 16, 5, 6, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kBf16, 64, 5, 6, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 7, Dtype::kFp16, 16, 4, 7, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 7, Dtype::kFp16, 64, 4, 7, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 7, Dtype::kBf16, 16, 4, 7, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 7, Dtype::kBf16, 64, 4, 7, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kFp16, 16, 5, 8, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kFp16, 64, 5, 8, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kBf16, 16, 5, 8, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kBf16, 64, 5, 8, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kFp16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kFp16, 32, 4, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kBf16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kBf16, 32, 4, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kFp16, 16, 4, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kFp16, 32, 4, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kBf16, 16, 4, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kBf16, 32, 4, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kFp16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kFp16, 32, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kBf16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kBf16, 32, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kFp16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kFp16, 32, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kBf16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kBf16, 32, 5, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kFp16, 16, 2, 5, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kFp16, 32, 2, 5, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kBf16, 16, 2, 5, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kBf16, 32, 2, 5, 4, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kFp16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kBf16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kFp16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kBf16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kFp16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kBf16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kFp16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kFp16, 32, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kBf16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kBf16, 32, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 2, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 2, Dtype::kFp16, 32, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 2, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 2, Dtype::kBf16, 32, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kFp16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kFp16, 32, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kBf16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kBf16, 32, 2, 2, 2, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 5, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 5, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 8, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 8, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, +}; +// clang-format on + +// The table of GPU family A. +template +constexpr const auto& decode_batch_configs() { + if constexpr (A == Arch::kGfx1151) + return kGfx1151DecodeBatchConfigs; + else + return kGfx1150DecodeBatchConfigs; +} + +// The tables by Arch. +inline constexpr std::span kDecodeBatchConfigs[kNumArches] = + {kGfx1151DecodeBatchConfigs, kGfx1150DecodeBatchConfigs}; } // namespace rdna diff --git a/csrc/rocm/rdna_decode_configs.h b/csrc/rocm/rdna_decode_configs.h index 3d363804e11b..c38c58ed5256 100644 --- a/csrc/rocm/rdna_decode_configs.h +++ b/csrc/rocm/rdna_decode_configs.h @@ -2,6 +2,8 @@ // compiled; a call whose shape has no entry runs on Triton. #pragma once +#include + #include "rdna_attn_types.h" namespace rdna { @@ -27,10 +29,12 @@ struct DecodeConfig { int dot_product; // per-q-head dot products instead of WMMA }; -// Written by benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py -// --apply, one entry per line in DecodeConfig order. +// One table per Arch (rdna_attn_types.h), each complete; a GPU family's +// entries are built for its archs only. Written by +// benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py --apply, one entry per +// line in DecodeConfig order. // clang-format off -inline constexpr DecodeConfig kDecodeConfigs[] = { +inline constexpr DecodeConfig kGfx1151DecodeConfigs[] = { {HeadDim::k64, 14, 2, 0, 1, Dtype::kFp16, 16, 4, 1, 2, 4, 0, 1, 0, 0}, {HeadDim::k64, 14, 2, 0, 1, Dtype::kBf16, 16, 4, 1, 2, 4, 0, 1, 0, 0}, {HeadDim::k64, 14, 2, 0, 2, Dtype::kFp16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, @@ -704,6 +708,694 @@ inline constexpr DecodeConfig kDecodeConfigs[] = { {HeadDim::k512, 32, 4, 0, 8, Dtype::kBf16, 16, 2, 4, 4, 8, 0, 0, 0, 0}, {HeadDim::k512, 32, 4, 0, 8, Dtype::kBf16, 32, 2, 4, 4, 8, 0, 0, 0, 0}, }; + +inline constexpr DecodeConfig kGfx1150DecodeConfigs[] = { + {HeadDim::k64, 14, 2, 0, 1, Dtype::kFp16, 16, 4, 1, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 1, Dtype::kBf16, 16, 4, 1, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 3, Dtype::kFp16, 16, 4, 1, 4, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 3, Dtype::kBf16, 16, 4, 1, 4, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 4, Dtype::kFp16, 16, 4, 2, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 4, Dtype::kBf16, 16, 4, 2, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 5, Dtype::kFp16, 16, 4, 7, 2, 4, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 5, Dtype::kBf16, 16, 4, 7, 2, 4, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 6, Dtype::kFp16, 16, 1, 3, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 6, Dtype::kBf16, 16, 1, 3, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 7, Dtype::kFp16, 16, 4, 7, 2, 4, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 7, Dtype::kBf16, 16, 4, 7, 2, 4, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 1, Dtype::kFp16, 16, 4, 1, 4, 8, 0, 0, 1, 0}, + {HeadDim::k64, 16, 2, 0, 1, Dtype::kBf16, 16, 4, 1, 4, 8, 0, 0, 1, 0}, + {HeadDim::k64, 16, 2, 0, 2, Dtype::kFp16, 16, 4, 1, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 2, Dtype::kBf16, 16, 4, 1, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 3, Dtype::kFp16, 16, 4, 2, 4, 8, 0, 0, 1, 0}, + {HeadDim::k64, 16, 2, 0, 3, Dtype::kBf16, 16, 4, 2, 4, 8, 0, 0, 1, 0}, + {HeadDim::k64, 16, 2, 0, 4, Dtype::kFp16, 16, 4, 2, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 4, Dtype::kBf16, 16, 4, 2, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 5, Dtype::kFp16, 16, 2, 4, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 5, Dtype::kBf16, 16, 2, 4, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 6, Dtype::kFp16, 16, 2, 3, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 6, Dtype::kBf16, 16, 2, 3, 4, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 5, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 5, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 6, Dtype::kFp16, 16, 1, 3, 1, 2, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 6, Dtype::kBf16, 16, 1, 3, 1, 2, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 7, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 7, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 1, 0, 0}, + {HeadDim::k64, 32, 32, 0, 1, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 1, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 4, Dtype::kFp16, 16, 1, 1, 4, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 4, Dtype::kBf16, 16, 1, 1, 4, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k64, 32, 32, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k64, 32, 32, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k96, 32, 32, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k96, 32, 32, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k128, 10, 10, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 4, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 4, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 1, Dtype::kFp16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 1, Dtype::kBf16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 2, Dtype::kFp16, 16, 4, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 2, Dtype::kBf16, 16, 4, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 3, Dtype::kFp16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 3, Dtype::kBf16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 5, Dtype::kFp16, 16, 2, 4, 4, 8, 2, 1, 0, 0}, + {HeadDim::k128, 16, 2, 0, 5, Dtype::kBf16, 16, 2, 4, 4, 8, 2, 1, 0, 0}, + {HeadDim::k128, 16, 2, 0, 6, Dtype::kFp16, 16, 2, 4, 2, 8, 2, 1, 0, 0}, + {HeadDim::k128, 16, 2, 0, 6, Dtype::kBf16, 16, 2, 4, 2, 8, 2, 1, 0, 0}, + {HeadDim::k128, 16, 2, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 8, Dtype::kFp16, 16, 2, 4, 4, 8, 2, 1, 0, 0}, + {HeadDim::k128, 16, 2, 0, 8, Dtype::kBf16, 16, 2, 4, 4, 8, 2, 1, 0, 0}, + {HeadDim::k128, 16, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 7, Dtype::kFp16, 16, 1, 3, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 7, Dtype::kBf16, 16, 1, 3, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 8, Dtype::kFp16, 16, 1, 3, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 8, Dtype::kBf16, 16, 1, 3, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 2, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 2, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 3, Dtype::kFp16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 3, Dtype::kBf16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 4, 2, 1, 0, 0}, + {HeadDim::k128, 28, 4, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 4, 2, 1, 0, 0}, + {HeadDim::k128, 28, 4, 0, 6, Dtype::kFp16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 6, Dtype::kBf16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 7, Dtype::kFp16, 16, 1, 7, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 7, Dtype::kBf16, 16, 1, 7, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 4, 2, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 4, 2, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 1, Dtype::kFp16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 1, Dtype::kBf16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 2, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 2, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 3, Dtype::kFp16, 16, 2, 4, 4, 8, 2, 1, 0, 0}, + {HeadDim::k128, 32, 2, 0, 3, Dtype::kBf16, 16, 2, 4, 4, 8, 2, 1, 0, 0}, + {HeadDim::k128, 32, 2, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 5, Dtype::kFp16, 16, 2, 5, 4, 4, 2, 1, 0, 0}, + {HeadDim::k128, 32, 2, 0, 5, Dtype::kBf16, 16, 2, 5, 4, 4, 2, 1, 0, 0}, + {HeadDim::k128, 32, 2, 0, 6, Dtype::kFp16, 16, 2, 6, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 6, Dtype::kBf16, 16, 2, 6, 2, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 7, Dtype::kFp16, 16, 1, 8, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 7, Dtype::kBf16, 16, 1, 8, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 8, Dtype::kFp16, 16, 1, 8, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 8, Dtype::kBf16, 16, 1, 8, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 4, 0, 1, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 1, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 3, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 3, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 6, Dtype::kFp16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 6, Dtype::kBf16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 4, 2, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 4, 2, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 1, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 1, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k128, 32, 32, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k128, 32, 32, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 8, Dtype::kFp16, 16, 1, 1, 2, 4, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 8, Dtype::kBf16, 16, 1, 1, 2, 4, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 8, 2, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 8, 2, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 7, Dtype::kFp16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 7, Dtype::kBf16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 8, Dtype::kFp16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 8, Dtype::kBf16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 0, 1, Dtype::kFp16, 16, 5, 1, 1, 4, 0, 0, 0, 1}, + {HeadDim::k256, 8, 1, 0, 1, Dtype::kBf16, 16, 5, 1, 1, 4, 0, 0, 0, 1}, + {HeadDim::k256, 8, 1, 0, 2, Dtype::kFp16, 16, 4, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 0, 2, Dtype::kBf16, 16, 4, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 0, 3, Dtype::kFp16, 16, 8, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 0, 3, Dtype::kBf16, 16, 8, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 0, 4, Dtype::kFp16, 16, 3, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 4, Dtype::kBf16, 16, 3, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 5, Dtype::kFp16, 16, 4, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 5, Dtype::kBf16, 16, 4, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 6, Dtype::kFp16, 16, 3, 3, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 6, Dtype::kBf16, 16, 3, 3, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 7, Dtype::kFp16, 16, 4, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 7, Dtype::kBf16, 16, 4, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 8, Dtype::kFp16, 16, 4, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 8, Dtype::kBf16, 16, 4, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kFp16, 16, 2, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kFp16, 32, 2, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kBf16, 16, 2, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kBf16, 32, 2, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kFp16, 16, 3, 2, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kFp16, 32, 3, 2, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kBf16, 16, 3, 2, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kBf16, 32, 3, 2, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kFp16, 16, 5, 3, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kFp16, 32, 5, 3, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kBf16, 16, 5, 3, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kBf16, 32, 5, 3, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kFp16, 16, 3, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kFp16, 32, 3, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kBf16, 16, 3, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kBf16, 32, 3, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kFp16, 16, 3, 5, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kFp16, 32, 3, 5, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kBf16, 16, 3, 5, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kBf16, 32, 3, 5, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kFp16, 16, 5, 3, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kFp16, 32, 5, 3, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kBf16, 16, 5, 3, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kBf16, 32, 5, 3, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kFp16, 16, 5, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kFp16, 32, 5, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kBf16, 16, 5, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kBf16, 32, 5, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kFp16, 16, 4, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kFp16, 32, 4, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kBf16, 16, 4, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kBf16, 32, 4, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kFp16, 544, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kBf16, 544, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kFp16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kFp16, 544, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kBf16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kBf16, 544, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kFp16, 16, 3, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kFp16, 544, 3, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kBf16, 16, 3, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kBf16, 544, 3, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kFp16, 16, 2, 2, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kFp16, 544, 2, 2, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kBf16, 16, 2, 2, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kBf16, 544, 2, 2, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kFp16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kFp16, 544, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kBf16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kBf16, 544, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kFp16, 16, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kFp16, 544, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kBf16, 16, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kBf16, 544, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kFp16, 16, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kFp16, 544, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kBf16, 16, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kBf16, 544, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kFp16, 16, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kFp16, 544, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kBf16, 16, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kBf16, 544, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kFp16, 32, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kBf16, 32, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kFp16, 16, 3, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kFp16, 32, 3, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kBf16, 16, 3, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kBf16, 32, 3, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kFp16, 16, 2, 3, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kFp16, 32, 2, 3, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kBf16, 16, 2, 3, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kBf16, 32, 2, 3, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kFp16, 16, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kFp16, 32, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kBf16, 16, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kBf16, 32, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kFp16, 16, 3, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kFp16, 32, 3, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kBf16, 16, 3, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kBf16, 32, 3, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kFp16, 16, 1, 3, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kFp16, 32, 1, 3, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kBf16, 16, 1, 3, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kBf16, 32, 1, 3, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kFp16, 16, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kFp16, 32, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kBf16, 16, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kBf16, 32, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kFp16, 16, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kFp16, 32, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kBf16, 16, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kBf16, 32, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 4, 0, 1, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 1, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 2, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 2, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 5, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 5, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 4, 1024, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 4, 1024, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 4, 1024, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 4, 1024, 3, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 4, 1024, 3, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 4, 1024, 4, Dtype::kFp16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 4, 1024, 4, Dtype::kBf16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 4, 1024, 5, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 5, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 4, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 4, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 7, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 7, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 4, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 4, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 4, 4096, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 8, 4, 4096, 2, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 2, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 3, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 3, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 4, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 4, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 5, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 5, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 7, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 7, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kFp16, 16, 3, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kFp16, 1056, 3, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kBf16, 16, 3, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kBf16, 1056, 3, 2, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kFp16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kFp16, 1056, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kBf16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kBf16, 1056, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kFp16, 16, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kFp16, 1056, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kBf16, 16, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kBf16, 1056, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kFp16, 16, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kFp16, 1056, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kBf16, 16, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kBf16, 1056, 2, 2, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kFp16, 16, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kFp16, 1056, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kBf16, 16, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kBf16, 1056, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kFp16, 16, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kFp16, 1056, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kBf16, 16, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kBf16, 1056, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kFp16, 16, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kFp16, 1056, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kBf16, 16, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kBf16, 1056, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kFp16, 16, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kFp16, 1056, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kBf16, 16, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kBf16, 1056, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kFp16, 528, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kBf16, 528, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kFp16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kBf16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 3, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 3, Dtype::kFp16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 3, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 3, Dtype::kBf16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 4, Dtype::kFp16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 4, Dtype::kBf16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kFp16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kBf16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kFp16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kBf16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kFp16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kBf16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kFp16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kBf16, 528, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 16, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 16, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 1, Dtype::kFp16, 16, 1, 1, 2, 4, 0, 0, 0, 1}, + {HeadDim::k256, 16, 8, 1024, 1, Dtype::kBf16, 16, 1, 1, 2, 4, 0, 0, 0, 1}, + {HeadDim::k256, 16, 8, 1024, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 16, 8, 1024, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k256, 16, 8, 1024, 3, Dtype::kFp16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 3, Dtype::kBf16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 5, Dtype::kFp16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 5, Dtype::kBf16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 7, Dtype::kFp16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 7, Dtype::kBf16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 24, 4, 0, 1, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 1, Dtype::kFp16, 784, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 1, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 1, Dtype::kBf16, 784, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 2, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 2, Dtype::kFp16, 784, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 2, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 2, Dtype::kBf16, 784, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kFp16, 784, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kBf16, 784, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kFp16, 784, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kBf16, 784, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kFp16, 784, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kBf16, 784, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kFp16, 784, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kBf16, 784, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kFp16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kFp16, 784, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kBf16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kBf16, 784, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 8, Dtype::kFp16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 8, Dtype::kFp16, 784, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 8, Dtype::kBf16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 8, Dtype::kBf16, 784, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 2, Dtype::kFp16, 16, 1, 1, 2, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 2, Dtype::kBf16, 16, 1, 1, 2, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 4, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 4, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 8, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 8, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 1, Dtype::kFp16, 16, 3, 1, 1, 4, 0, 0, 0, 1}, + {HeadDim::k512, 8, 1, 0, 1, Dtype::kBf16, 16, 3, 1, 1, 4, 0, 0, 0, 1}, + {HeadDim::k512, 8, 1, 0, 2, Dtype::kFp16, 16, 3, 1, 1, 4, 0, 0, 0, 1}, + {HeadDim::k512, 8, 1, 0, 2, Dtype::kBf16, 16, 3, 1, 1, 4, 0, 0, 0, 1}, + {HeadDim::k512, 8, 1, 0, 3, Dtype::kFp16, 16, 5, 3, 2, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 1, 0, 3, Dtype::kBf16, 16, 5, 3, 2, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 1, 0, 4, Dtype::kFp16, 16, 5, 2, 2, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 1, 0, 4, Dtype::kBf16, 16, 5, 2, 2, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 1, 0, 5, Dtype::kFp16, 16, 5, 4, 2, 4, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 5, Dtype::kBf16, 16, 5, 4, 2, 4, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 6, Dtype::kFp16, 16, 5, 3, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 1, 0, 6, Dtype::kBf16, 16, 5, 3, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 1, 0, 7, Dtype::kFp16, 16, 4, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 7, Dtype::kBf16, 16, 4, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 8, Dtype::kFp16, 16, 4, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 1, 0, 8, Dtype::kBf16, 16, 4, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k512, 8, 2, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k512, 8, 2, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k512, 8, 2, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k512, 8, 2, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k512, 8, 2, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k512, 8, 2, 0, 4, Dtype::kFp16, 16, 3, 1, 2, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 4, Dtype::kBf16, 16, 3, 1, 2, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 5, Dtype::kFp16, 16, 4, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 5, Dtype::kBf16, 16, 4, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 6, Dtype::kFp16, 16, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 6, Dtype::kBf16, 16, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 7, Dtype::kFp16, 16, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 7, Dtype::kBf16, 16, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 8, Dtype::kFp16, 16, 3, 2, 2, 8, 8, 0, 0, 0}, + {HeadDim::k512, 8, 2, 0, 8, Dtype::kBf16, 16, 3, 2, 2, 8, 8, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kFp16, 16, 8, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kFp16, 64, 8, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kBf16, 16, 8, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kBf16, 64, 8, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kFp16, 16, 5, 2, 2, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kFp16, 64, 5, 2, 2, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kBf16, 16, 5, 2, 2, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kBf16, 64, 5, 2, 2, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kFp16, 16, 5, 3, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kFp16, 64, 5, 3, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kBf16, 16, 5, 3, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kBf16, 64, 5, 3, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kFp16, 16, 4, 4, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kFp16, 64, 4, 4, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kBf16, 16, 4, 4, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kBf16, 64, 4, 4, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kFp16, 16, 3, 5, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kFp16, 64, 3, 5, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kBf16, 16, 3, 5, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kBf16, 64, 3, 5, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kFp16, 16, 4, 6, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kFp16, 64, 4, 6, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kBf16, 16, 4, 6, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kBf16, 64, 4, 6, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 7, Dtype::kFp16, 16, 3, 7, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 7, Dtype::kFp16, 64, 3, 7, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 7, Dtype::kBf16, 16, 3, 7, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 7, Dtype::kBf16, 64, 3, 7, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kFp16, 16, 3, 8, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kFp16, 64, 3, 8, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kBf16, 16, 3, 8, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kBf16, 64, 3, 8, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kFp16, 32, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kBf16, 32, 1, 1, 1, 8, 0, 0, 0, 1}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kFp16, 16, 3, 2, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kFp16, 32, 3, 2, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kBf16, 16, 3, 2, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kBf16, 32, 3, 2, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kFp16, 16, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kFp16, 32, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kBf16, 16, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kBf16, 32, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kFp16, 16, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kFp16, 32, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kBf16, 16, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kBf16, 32, 3, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kFp16, 16, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kFp16, 32, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kBf16, 16, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kBf16, 32, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kFp16, 16, 2, 3, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kFp16, 32, 2, 3, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kBf16, 16, 2, 3, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kBf16, 32, 2, 3, 4, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kFp16, 16, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kFp16, 32, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kBf16, 16, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kBf16, 32, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kFp16, 16, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kFp16, 32, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kBf16, 16, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kBf16, 32, 2, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 0, 1}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kFp16, 32, 1, 1, 1, 4, 0, 0, 0, 1}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 0, 1}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kBf16, 32, 1, 1, 1, 4, 0, 0, 0, 1}, + {HeadDim::k512, 32, 4, 0, 2, Dtype::kFp16, 16, 1, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 2, Dtype::kFp16, 32, 1, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 2, Dtype::kBf16, 16, 1, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 2, Dtype::kBf16, 32, 1, 2, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 5, Dtype::kFp16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 5, Dtype::kBf16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kFp16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kBf16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kFp16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kBf16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 8, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 8, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, +}; // clang-format on +// The table of GPU family A. +template +constexpr const auto& decode_configs() { + if constexpr (A == Arch::kGfx1151) + return kGfx1151DecodeConfigs; + else + return kGfx1150DecodeConfigs; +} + +// The tables by Arch. +inline constexpr std::span kDecodeConfigs[kNumArches] = { + kGfx1151DecodeConfigs, kGfx1150DecodeConfigs}; + } // namespace rdna diff --git a/csrc/rocm/rdna_prefill_batch_configs.h b/csrc/rocm/rdna_prefill_batch_configs.h index 1a73ed9b999b..ac448948725d 100644 --- a/csrc/rocm/rdna_prefill_batch_configs.h +++ b/csrc/rocm/rdna_prefill_batch_configs.h @@ -2,6 +2,8 @@ // kPrefillConfigs entry and head group that run a prefill batch. #pragma once +#include + #include "rdna_attn_types.h" namespace rdna { @@ -18,10 +20,12 @@ struct PrefillBatchConfig { int head_group; // 0: the build's }; -// Written by benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py -// --apply, one entry per line in PrefillBatchConfig order. +// One table per Arch (rdna_attn_types.h), each complete; a GPU family's +// entries are built for its archs only. Written by +// benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py --apply, one entry per +// line in PrefillBatchConfig order. // clang-format off -inline constexpr PrefillBatchConfig kPrefillBatchConfigs[] = { +inline constexpr PrefillBatchConfig kGfx1151PrefillBatchConfigs[] = { {HeadDim::k64, 14, 2, 0, 16, 1, 0, 16, 2}, {HeadDim::k64, 14, 2, 0, 16, 1, 512, 16, 2}, {HeadDim::k64, 14, 2, 0, 16, 1, 2048, 16, 2}, @@ -822,6 +826,822 @@ inline constexpr PrefillBatchConfig kPrefillBatchConfigs[] = { {HeadDim::k512, 32, 4, 0, 32, 4096, 2048, 16, 1}, {HeadDim::k512, 32, 4, 0, 32, 4096, 8192, 16, 1}, }; -// clang-format on + +inline constexpr PrefillBatchConfig kGfx1150PrefillBatchConfigs[] = { + {HeadDim::k64, 14, 2, 0, 16, 1, 0, 16, 2}, + {HeadDim::k64, 14, 2, 0, 16, 1, 512, 16, 2}, + {HeadDim::k64, 14, 2, 0, 16, 1, 2048, 16, 2}, + {HeadDim::k64, 14, 2, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k64, 14, 2, 0, 16, 16, 0, 16, 2}, + {HeadDim::k64, 14, 2, 0, 16, 16, 512, 16, 1}, + {HeadDim::k64, 14, 2, 0, 16, 16, 2048, 128, 2}, + {HeadDim::k64, 14, 2, 0, 16, 16, 8192, 128, 2}, + {HeadDim::k64, 14, 2, 0, 16, 64, 0, 128, 1}, + {HeadDim::k64, 14, 2, 0, 16, 64, 512, 128, 1}, + {HeadDim::k64, 14, 2, 0, 16, 64, 2048, 128, 1}, + {HeadDim::k64, 14, 2, 0, 16, 64, 8192, 32, 2}, + {HeadDim::k64, 14, 2, 0, 16, 256, 0, 128, 2}, + {HeadDim::k64, 14, 2, 0, 16, 256, 512, 128, 1}, + {HeadDim::k64, 14, 2, 0, 16, 256, 2048, 128, 1}, + {HeadDim::k64, 14, 2, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k64, 14, 2, 0, 16, 1024, 512, 128, 2}, + {HeadDim::k64, 14, 2, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k64, 14, 2, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k64, 14, 2, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k64, 14, 2, 0, 16, 4096, 8192, 128, 1}, + {HeadDim::k64, 16, 2, 0, 16, 1, 0, 16, 2}, + {HeadDim::k64, 16, 2, 0, 16, 1, 512, 16, 2}, + {HeadDim::k64, 16, 2, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k64, 16, 2, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k64, 16, 2, 0, 16, 16, 0, 256, 1}, + {HeadDim::k64, 16, 2, 0, 16, 16, 512, 256, 2}, + {HeadDim::k64, 16, 2, 0, 16, 16, 2048, 256, 2}, + {HeadDim::k64, 16, 2, 0, 16, 16, 8192, 32, 2}, + {HeadDim::k64, 16, 2, 0, 16, 64, 0, 128, 2}, + {HeadDim::k64, 16, 2, 0, 16, 64, 512, 128, 1}, + {HeadDim::k64, 16, 2, 0, 16, 64, 2048, 128, 1}, + {HeadDim::k64, 16, 2, 0, 16, 64, 8192, 128, 1}, + {HeadDim::k64, 16, 2, 0, 16, 256, 0, 256, 2}, + {HeadDim::k64, 16, 2, 0, 16, 256, 512, 256, 1}, + {HeadDim::k64, 16, 2, 0, 16, 256, 2048, 128, 2}, + {HeadDim::k64, 16, 2, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k64, 16, 2, 0, 16, 1024, 512, 256, 1}, + {HeadDim::k64, 16, 2, 0, 16, 1024, 2048, 128, 2}, + {HeadDim::k64, 16, 2, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k64, 16, 2, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k64, 16, 2, 0, 16, 4096, 8192, 128, 1}, + {HeadDim::k64, 32, 8, 0, 16, 1, 0, 16, 8}, + {HeadDim::k64, 32, 8, 0, 16, 1, 512, 16, 8}, + {HeadDim::k64, 32, 8, 0, 16, 1, 2048, 16, 8}, + {HeadDim::k64, 32, 8, 0, 16, 1, 8192, 16, 2}, + {HeadDim::k64, 32, 8, 0, 16, 16, 0, 512, 8}, + {HeadDim::k64, 32, 8, 0, 16, 16, 512, 512, 8}, + {HeadDim::k64, 32, 8, 0, 16, 16, 2048, 512, 8}, + {HeadDim::k64, 32, 8, 0, 16, 16, 8192, 512, 8}, + {HeadDim::k64, 32, 8, 0, 16, 64, 0, 64, 8}, + {HeadDim::k64, 32, 8, 0, 16, 64, 512, 64, 4}, + {HeadDim::k64, 32, 8, 0, 16, 64, 2048, 64, 2}, + {HeadDim::k64, 32, 8, 0, 16, 64, 8192, 16, 2}, + {HeadDim::k64, 32, 8, 0, 16, 256, 0, 128, 8}, + {HeadDim::k64, 32, 8, 0, 16, 256, 512, 128, 4}, + {HeadDim::k64, 32, 8, 0, 16, 256, 2048, 128, 2}, + {HeadDim::k64, 32, 8, 0, 16, 256, 8192, 16, 1}, + {HeadDim::k64, 32, 8, 0, 16, 1024, 512, 64, 4}, + {HeadDim::k64, 32, 8, 0, 16, 1024, 2048, 128, 4}, + {HeadDim::k64, 32, 8, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k64, 32, 8, 0, 16, 4096, 2048, 128, 4}, + {HeadDim::k64, 32, 8, 0, 16, 4096, 8192, 128, 1}, + {HeadDim::k64, 32, 32, 0, 16, 1, 0, 128, 32}, + {HeadDim::k64, 32, 32, 0, 16, 1, 512, 128, 32}, + {HeadDim::k64, 32, 32, 0, 16, 1, 2048, 16, 32}, + {HeadDim::k64, 32, 32, 0, 16, 1, 8192, 16, 4}, + {HeadDim::k64, 32, 32, 0, 16, 16, 0, 128, 32}, + {HeadDim::k64, 32, 32, 0, 16, 16, 512, 16, 32}, + {HeadDim::k64, 32, 32, 0, 16, 16, 2048, 16, 32}, + {HeadDim::k64, 32, 32, 0, 16, 16, 8192, 16, 8}, + {HeadDim::k64, 32, 32, 0, 16, 64, 0, 256, 32}, + {HeadDim::k64, 32, 32, 0, 16, 64, 512, 256, 32}, + {HeadDim::k64, 32, 32, 0, 16, 64, 2048, 256, 32}, + {HeadDim::k64, 32, 32, 0, 16, 64, 8192, 256, 32}, + {HeadDim::k64, 32, 32, 0, 16, 256, 0, 256, 32}, + {HeadDim::k64, 32, 32, 0, 16, 256, 512, 256, 16}, + {HeadDim::k64, 32, 32, 0, 16, 256, 2048, 256, 4}, + {HeadDim::k64, 32, 32, 0, 16, 256, 8192, 256, 2}, + {HeadDim::k64, 32, 32, 0, 16, 1024, 512, 256, 8}, + {HeadDim::k64, 32, 32, 0, 16, 1024, 2048, 256, 1}, + {HeadDim::k64, 32, 32, 0, 16, 1024, 8192, 256, 1}, + {HeadDim::k64, 32, 32, 0, 16, 4096, 2048, 256, 1}, + {HeadDim::k64, 32, 32, 0, 16, 4096, 8192, 256, 1}, + {HeadDim::k96, 32, 32, 0, 16, 1, 0, 16, 32}, + {HeadDim::k96, 32, 32, 0, 16, 1, 512, 16, 32}, + {HeadDim::k96, 32, 32, 0, 16, 1, 2048, 16, 32}, + {HeadDim::k96, 32, 32, 0, 16, 1, 8192, 16, 4}, + {HeadDim::k96, 32, 32, 0, 16, 16, 0, 16, 32}, + {HeadDim::k96, 32, 32, 0, 16, 16, 512, 16, 32}, + {HeadDim::k96, 32, 32, 0, 16, 16, 2048, 16, 32}, + {HeadDim::k96, 32, 32, 0, 16, 16, 8192, 16, 1}, + {HeadDim::k96, 32, 32, 0, 16, 64, 0, 1024, 32}, + {HeadDim::k96, 32, 32, 0, 16, 64, 512, 1024, 32}, + {HeadDim::k96, 32, 32, 0, 16, 64, 2048, 1024, 32}, + {HeadDim::k96, 32, 32, 0, 16, 64, 8192, 1024, 32}, + {HeadDim::k96, 32, 32, 0, 16, 256, 0, 128, 8}, + {HeadDim::k96, 32, 32, 0, 16, 256, 512, 1024, 4}, + {HeadDim::k96, 32, 32, 0, 16, 256, 2048, 1024, 1}, + {HeadDim::k96, 32, 32, 0, 16, 256, 8192, 1024, 1}, + {HeadDim::k96, 32, 32, 0, 16, 1024, 512, 1024, 8}, + {HeadDim::k96, 32, 32, 0, 16, 1024, 2048, 1024, 1}, + {HeadDim::k96, 32, 32, 0, 16, 1024, 8192, 1024, 1}, + {HeadDim::k96, 32, 32, 0, 16, 4096, 2048, 1024, 1}, + {HeadDim::k96, 32, 32, 0, 16, 4096, 8192, 1024, 1}, + {HeadDim::k128, 10, 10, 0, 16, 1, 0, 16, 10}, + {HeadDim::k128, 10, 10, 0, 16, 1, 512, 16, 2}, + {HeadDim::k128, 10, 10, 0, 16, 1, 2048, 16, 10}, + {HeadDim::k128, 10, 10, 0, 16, 1, 8192, 16, 10}, + {HeadDim::k128, 10, 10, 0, 16, 16, 0, 16, 10}, + {HeadDim::k128, 10, 10, 0, 16, 16, 512, 16, 10}, + {HeadDim::k128, 10, 10, 0, 16, 16, 2048, 16, 10}, + {HeadDim::k128, 10, 10, 0, 16, 16, 8192, 16, 10}, + {HeadDim::k128, 10, 10, 0, 16, 64, 0, 16, 2}, + {HeadDim::k128, 10, 10, 0, 16, 64, 512, 128, 10}, + {HeadDim::k128, 10, 10, 0, 16, 64, 2048, 512, 2}, + {HeadDim::k128, 10, 10, 0, 16, 64, 8192, 16, 5}, + {HeadDim::k128, 10, 10, 0, 16, 256, 0, 16, 2}, + {HeadDim::k128, 10, 10, 0, 16, 256, 512, 512, 5}, + {HeadDim::k128, 10, 10, 0, 16, 256, 2048, 512, 1}, + {HeadDim::k128, 10, 10, 0, 16, 256, 8192, 512, 1}, + {HeadDim::k128, 10, 10, 0, 16, 1024, 512, 512, 5}, + {HeadDim::k128, 10, 10, 0, 16, 1024, 2048, 512, 1}, + {HeadDim::k128, 10, 10, 0, 16, 1024, 8192, 512, 1}, + {HeadDim::k128, 10, 10, 0, 16, 4096, 2048, 512, 1}, + {HeadDim::k128, 10, 10, 0, 16, 4096, 8192, 512, 2}, + {HeadDim::k128, 16, 2, 0, 16, 1, 0, 16, 2}, + {HeadDim::k128, 16, 2, 0, 16, 1, 512, 16, 2}, + {HeadDim::k128, 16, 2, 0, 16, 1, 2048, 16, 2}, + {HeadDim::k128, 16, 2, 0, 16, 1, 8192, 16, 2}, + {HeadDim::k128, 16, 2, 0, 16, 16, 0, 32, 2}, + {HeadDim::k128, 16, 2, 0, 16, 16, 512, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 16, 2048, 128, 2}, + {HeadDim::k128, 16, 2, 0, 16, 16, 8192, 32, 2}, + {HeadDim::k128, 16, 2, 0, 16, 64, 0, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 64, 512, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 64, 2048, 256, 1}, + {HeadDim::k128, 16, 2, 0, 16, 64, 8192, 256, 1}, + {HeadDim::k128, 16, 2, 0, 16, 256, 0, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 256, 512, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 256, 2048, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 256, 8192, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 1024, 512, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 1024, 2048, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 1024, 8192, 256, 1}, + {HeadDim::k128, 16, 2, 0, 16, 4096, 2048, 256, 1}, + {HeadDim::k128, 16, 2, 0, 16, 4096, 8192, 256, 1}, + {HeadDim::k128, 16, 8, 0, 16, 1, 0, 16, 8}, + {HeadDim::k128, 16, 8, 0, 16, 1, 512, 16, 4}, + {HeadDim::k128, 16, 8, 0, 16, 1, 2048, 16, 8}, + {HeadDim::k128, 16, 8, 0, 16, 1, 8192, 16, 8}, + {HeadDim::k128, 16, 8, 0, 16, 16, 0, 16, 8}, + {HeadDim::k128, 16, 8, 0, 16, 16, 512, 16, 8}, + {HeadDim::k128, 16, 8, 0, 16, 16, 2048, 16, 8}, + {HeadDim::k128, 16, 8, 0, 16, 16, 8192, 16, 2}, + {HeadDim::k128, 16, 8, 0, 16, 64, 0, 256, 4}, + {HeadDim::k128, 16, 8, 0, 16, 64, 512, 256, 4}, + {HeadDim::k128, 16, 8, 0, 16, 64, 2048, 256, 1}, + {HeadDim::k128, 16, 8, 0, 16, 64, 8192, 256, 2}, + {HeadDim::k128, 16, 8, 0, 16, 256, 0, 16, 8}, + {HeadDim::k128, 16, 8, 0, 16, 256, 512, 256, 2}, + {HeadDim::k128, 16, 8, 0, 16, 256, 2048, 256, 2}, + {HeadDim::k128, 16, 8, 0, 16, 256, 8192, 256, 1}, + {HeadDim::k128, 16, 8, 0, 16, 1024, 512, 256, 4}, + {HeadDim::k128, 16, 8, 0, 16, 1024, 2048, 256, 4}, + {HeadDim::k128, 16, 8, 0, 16, 1024, 8192, 256, 2}, + {HeadDim::k128, 16, 8, 0, 16, 4096, 2048, 256, 2}, + {HeadDim::k128, 16, 8, 0, 16, 4096, 8192, 256, 1}, + {HeadDim::k128, 24, 8, 0, 16, 1, 0, 16, 8}, + {HeadDim::k128, 24, 8, 0, 16, 1, 512, 16, 1}, + {HeadDim::k128, 24, 8, 0, 16, 1, 2048, 16, 8}, + {HeadDim::k128, 24, 8, 0, 16, 1, 8192, 16, 8}, + {HeadDim::k128, 24, 8, 0, 16, 16, 0, 16, 8}, + {HeadDim::k128, 24, 8, 0, 16, 16, 512, 16, 4}, + {HeadDim::k128, 24, 8, 0, 16, 16, 2048, 16, 4}, + {HeadDim::k128, 24, 8, 0, 16, 16, 8192, 16, 2}, + {HeadDim::k128, 24, 8, 0, 16, 64, 0, 512, 8}, + {HeadDim::k128, 24, 8, 0, 16, 64, 512, 512, 4}, + {HeadDim::k128, 24, 8, 0, 16, 64, 2048, 512, 1}, + {HeadDim::k128, 24, 8, 0, 16, 64, 8192, 512, 1}, + {HeadDim::k128, 24, 8, 0, 16, 256, 0, 512, 8}, + {HeadDim::k128, 24, 8, 0, 16, 256, 512, 512, 2}, + {HeadDim::k128, 24, 8, 0, 16, 256, 2048, 512, 1}, + {HeadDim::k128, 24, 8, 0, 16, 256, 8192, 512, 1}, + {HeadDim::k128, 24, 8, 0, 16, 1024, 512, 512, 2}, + {HeadDim::k128, 24, 8, 0, 16, 1024, 2048, 512, 1}, + {HeadDim::k128, 24, 8, 0, 16, 1024, 8192, 512, 1}, + {HeadDim::k128, 24, 8, 0, 16, 4096, 2048, 512, 1}, + {HeadDim::k128, 24, 8, 0, 16, 4096, 8192, 512, 1}, + {HeadDim::k128, 28, 4, 0, 16, 1, 0, 16, 4}, + {HeadDim::k128, 28, 4, 0, 16, 1, 512, 16, 4}, + {HeadDim::k128, 28, 4, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k128, 28, 4, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k128, 28, 4, 0, 16, 16, 0, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 16, 512, 256, 2}, + {HeadDim::k128, 28, 4, 0, 16, 16, 2048, 256, 2}, + {HeadDim::k128, 28, 4, 0, 16, 16, 8192, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 64, 0, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 64, 512, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 64, 2048, 256, 2}, + {HeadDim::k128, 28, 4, 0, 16, 64, 8192, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 256, 0, 256, 4}, + {HeadDim::k128, 28, 4, 0, 16, 256, 512, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 256, 2048, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 256, 8192, 256, 4}, + {HeadDim::k128, 28, 4, 0, 16, 1024, 512, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 1024, 2048, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 1024, 8192, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 4096, 2048, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 4096, 8192, 256, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1, 0, 16, 2}, + {HeadDim::k128, 32, 2, 0, 16, 1, 512, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1, 2048, 16, 2}, + {HeadDim::k128, 32, 2, 0, 16, 1, 8192, 16, 2}, + {HeadDim::k128, 32, 2, 0, 16, 16, 0, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 16, 512, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 16, 2048, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 16, 8192, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 64, 0, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 64, 512, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 64, 2048, 128, 2}, + {HeadDim::k128, 32, 2, 0, 16, 64, 8192, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 256, 0, 128, 2}, + {HeadDim::k128, 32, 2, 0, 16, 256, 512, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 256, 2048, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1024, 512, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k128, 32, 2, 0, 16, 4096, 8192, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 1, 0, 16, 2}, + {HeadDim::k128, 32, 4, 0, 16, 1, 512, 16, 4}, + {HeadDim::k128, 32, 4, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k128, 32, 4, 0, 16, 1, 8192, 16, 2}, + {HeadDim::k128, 32, 4, 0, 16, 16, 0, 16, 2}, + {HeadDim::k128, 32, 4, 0, 16, 16, 512, 32, 2}, + {HeadDim::k128, 32, 4, 0, 16, 16, 2048, 128, 2}, + {HeadDim::k128, 32, 4, 0, 16, 16, 8192, 16, 1}, + {HeadDim::k128, 32, 4, 0, 16, 64, 0, 128, 2}, + {HeadDim::k128, 32, 4, 0, 16, 64, 512, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 64, 2048, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 64, 8192, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 256, 0, 128, 2}, + {HeadDim::k128, 32, 4, 0, 16, 256, 512, 128, 2}, + {HeadDim::k128, 32, 4, 0, 16, 256, 2048, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 1024, 512, 128, 4}, + {HeadDim::k128, 32, 4, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 4096, 8192, 128, 1}, + {HeadDim::k128, 32, 8, 0, 16, 1, 0, 16, 8}, + {HeadDim::k128, 32, 8, 0, 16, 1, 512, 16, 8}, + {HeadDim::k128, 32, 8, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k128, 32, 8, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k128, 32, 8, 0, 16, 16, 0, 64, 4}, + {HeadDim::k128, 32, 8, 0, 16, 16, 512, 64, 4}, + {HeadDim::k128, 32, 8, 0, 16, 16, 2048, 64, 4}, + {HeadDim::k128, 32, 8, 0, 16, 16, 8192, 64, 8}, + {HeadDim::k128, 32, 8, 0, 16, 64, 0, 64, 4}, + {HeadDim::k128, 32, 8, 0, 16, 64, 512, 64, 2}, + {HeadDim::k128, 32, 8, 0, 16, 64, 2048, 64, 2}, + {HeadDim::k128, 32, 8, 0, 16, 64, 8192, 64, 1}, + {HeadDim::k128, 32, 8, 0, 16, 256, 0, 64, 4}, + {HeadDim::k128, 32, 8, 0, 16, 256, 512, 64, 2}, + {HeadDim::k128, 32, 8, 0, 16, 256, 2048, 64, 1}, + {HeadDim::k128, 32, 8, 0, 16, 256, 8192, 64, 1}, + {HeadDim::k128, 32, 8, 0, 16, 1024, 512, 64, 2}, + {HeadDim::k128, 32, 8, 0, 16, 1024, 2048, 64, 1}, + {HeadDim::k128, 32, 8, 0, 16, 1024, 8192, 64, 1}, + {HeadDim::k128, 32, 8, 0, 16, 4096, 2048, 64, 2}, + {HeadDim::k128, 32, 8, 0, 16, 4096, 8192, 64, 1}, + {HeadDim::k128, 32, 32, 0, 16, 1, 0, 16, 32}, + {HeadDim::k128, 32, 32, 0, 16, 1, 512, 16, 32}, + {HeadDim::k128, 32, 32, 0, 16, 1, 2048, 16, 32}, + {HeadDim::k128, 32, 32, 0, 16, 1, 8192, 16, 16}, + {HeadDim::k128, 32, 32, 0, 16, 16, 0, 16, 32}, + {HeadDim::k128, 32, 32, 0, 16, 16, 512, 16, 32}, + {HeadDim::k128, 32, 32, 0, 16, 16, 2048, 16, 16}, + {HeadDim::k128, 32, 32, 0, 16, 16, 8192, 16, 16}, + {HeadDim::k128, 32, 32, 0, 16, 64, 0, 512, 32}, + {HeadDim::k128, 32, 32, 0, 16, 64, 512, 512, 32}, + {HeadDim::k128, 32, 32, 0, 16, 64, 2048, 512, 16}, + {HeadDim::k128, 32, 32, 0, 16, 64, 8192, 512, 2}, + {HeadDim::k128, 32, 32, 0, 16, 256, 0, 16, 8}, + {HeadDim::k128, 32, 32, 0, 16, 256, 512, 512, 4}, + {HeadDim::k128, 32, 32, 0, 16, 256, 2048, 512, 2}, + {HeadDim::k128, 32, 32, 0, 16, 256, 8192, 512, 2}, + {HeadDim::k128, 32, 32, 0, 16, 1024, 512, 512, 4}, + {HeadDim::k128, 32, 32, 0, 16, 1024, 2048, 512, 4}, + {HeadDim::k128, 32, 32, 0, 16, 1024, 8192, 512, 1}, + {HeadDim::k128, 32, 32, 0, 16, 4096, 2048, 512, 4}, + {HeadDim::k128, 32, 32, 0, 16, 4096, 8192, 512, 1}, + {HeadDim::k128, 40, 8, 0, 16, 1, 0, 32, 8}, + {HeadDim::k128, 40, 8, 0, 16, 1, 512, 32, 8}, + {HeadDim::k128, 40, 8, 0, 16, 1, 2048, 32, 8}, + {HeadDim::k128, 40, 8, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k128, 40, 8, 0, 16, 16, 0, 128, 4}, + {HeadDim::k128, 40, 8, 0, 16, 16, 512, 32, 1}, + {HeadDim::k128, 40, 8, 0, 16, 16, 2048, 32, 1}, + {HeadDim::k128, 40, 8, 0, 16, 16, 8192, 32, 2}, + {HeadDim::k128, 40, 8, 0, 16, 64, 0, 128, 8}, + {HeadDim::k128, 40, 8, 0, 16, 64, 512, 128, 2}, + {HeadDim::k128, 40, 8, 0, 16, 64, 2048, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 64, 8192, 128, 2}, + {HeadDim::k128, 40, 8, 0, 16, 256, 0, 128, 8}, + {HeadDim::k128, 40, 8, 0, 16, 256, 512, 128, 2}, + {HeadDim::k128, 40, 8, 0, 16, 256, 2048, 128, 2}, + {HeadDim::k128, 40, 8, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 1024, 512, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 4096, 8192, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1, 0, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1, 512, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1, 2048, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1, 8192, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 16, 0, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 16, 512, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 16, 2048, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 16, 8192, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 64, 0, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 64, 512, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 64, 2048, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 64, 8192, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 256, 0, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 256, 512, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 256, 2048, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1024, 512, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 4096, 8192, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 1, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 1, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 16, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 16, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 64, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 64, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 256, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 256, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 1024, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 4096, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 1, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 1, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 16, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 16, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 64, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 64, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 256, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 256, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 1024, 512, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 4096, 512, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1, 0, 32, 2}, + {HeadDim::k256, 8, 2, 0, 16, 1, 512, 32, 2}, + {HeadDim::k256, 8, 2, 0, 16, 1, 2048, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1, 8192, 32, 2}, + {HeadDim::k256, 8, 2, 0, 16, 16, 0, 128, 2}, + {HeadDim::k256, 8, 2, 0, 16, 16, 512, 128, 2}, + {HeadDim::k256, 8, 2, 0, 16, 16, 2048, 128, 2}, + {HeadDim::k256, 8, 2, 0, 16, 16, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 64, 0, 128, 2}, + {HeadDim::k256, 8, 2, 0, 16, 64, 512, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 64, 2048, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 64, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 256, 0, 128, 2}, + {HeadDim::k256, 8, 2, 0, 16, 256, 512, 128, 2}, + {HeadDim::k256, 8, 2, 0, 16, 256, 2048, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1024, 512, 128, 2}, + {HeadDim::k256, 8, 2, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 4096, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1, 0, 32, 2}, + {HeadDim::k256, 8, 2, 0, 544, 1, 512, 32, 2}, + {HeadDim::k256, 8, 2, 0, 544, 1, 2048, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1, 8192, 32, 2}, + {HeadDim::k256, 8, 2, 0, 544, 16, 0, 128, 2}, + {HeadDim::k256, 8, 2, 0, 544, 16, 512, 128, 2}, + {HeadDim::k256, 8, 2, 0, 544, 16, 2048, 128, 2}, + {HeadDim::k256, 8, 2, 0, 544, 16, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 0, 544, 64, 0, 128, 2}, + {HeadDim::k256, 8, 2, 0, 544, 64, 512, 128, 1}, + {HeadDim::k256, 8, 2, 0, 544, 64, 2048, 128, 1}, + {HeadDim::k256, 8, 2, 0, 544, 64, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 0, 544, 256, 0, 128, 2}, + {HeadDim::k256, 8, 2, 0, 544, 256, 512, 128, 2}, + {HeadDim::k256, 8, 2, 0, 544, 256, 2048, 128, 1}, + {HeadDim::k256, 8, 2, 0, 544, 256, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1024, 512, 128, 2}, + {HeadDim::k256, 8, 2, 0, 544, 1024, 2048, 128, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1024, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 0, 544, 4096, 2048, 128, 1}, + {HeadDim::k256, 8, 2, 0, 544, 4096, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 512, 16, 1, 0, 16, 2}, + {HeadDim::k256, 8, 2, 512, 16, 1, 512, 64, 2}, + {HeadDim::k256, 8, 2, 512, 16, 16, 0, 512, 2}, + {HeadDim::k256, 8, 2, 512, 16, 16, 512, 16, 2}, + {HeadDim::k256, 8, 2, 512, 16, 64, 0, 512, 2}, + {HeadDim::k256, 8, 2, 512, 16, 64, 512, 64, 2}, + {HeadDim::k256, 8, 2, 512, 16, 256, 0, 512, 2}, + {HeadDim::k256, 8, 2, 512, 16, 256, 512, 64, 2}, + {HeadDim::k256, 8, 2, 512, 16, 1024, 512, 512, 2}, + {HeadDim::k256, 8, 2, 512, 16, 4096, 512, 512, 2}, + {HeadDim::k256, 8, 2, 512, 32, 1, 0, 16, 2}, + {HeadDim::k256, 8, 2, 512, 32, 1, 512, 64, 2}, + {HeadDim::k256, 8, 2, 512, 32, 16, 0, 512, 2}, + {HeadDim::k256, 8, 2, 512, 32, 16, 512, 16, 2}, + {HeadDim::k256, 8, 2, 512, 32, 64, 0, 512, 2}, + {HeadDim::k256, 8, 2, 512, 32, 64, 512, 64, 2}, + {HeadDim::k256, 8, 2, 512, 32, 256, 0, 512, 2}, + {HeadDim::k256, 8, 2, 512, 32, 256, 512, 64, 2}, + {HeadDim::k256, 8, 2, 512, 32, 1024, 512, 512, 2}, + {HeadDim::k256, 8, 2, 512, 32, 4096, 512, 512, 2}, + {HeadDim::k256, 8, 4, 0, 16, 1, 0, 16, 4}, + {HeadDim::k256, 8, 4, 0, 16, 1, 512, 16, 4}, + {HeadDim::k256, 8, 4, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k256, 8, 4, 0, 16, 1, 8192, 16, 4}, + {HeadDim::k256, 8, 4, 0, 16, 16, 0, 16, 4}, + {HeadDim::k256, 8, 4, 0, 16, 16, 512, 16, 4}, + {HeadDim::k256, 8, 4, 0, 16, 16, 2048, 16, 4}, + {HeadDim::k256, 8, 4, 0, 16, 16, 8192, 16, 2}, + {HeadDim::k256, 8, 4, 0, 16, 64, 0, 128, 4}, + {HeadDim::k256, 8, 4, 0, 16, 64, 512, 128, 4}, + {HeadDim::k256, 8, 4, 0, 16, 64, 2048, 128, 4}, + {HeadDim::k256, 8, 4, 0, 16, 64, 8192, 128, 2}, + {HeadDim::k256, 8, 4, 0, 16, 256, 0, 128, 4}, + {HeadDim::k256, 8, 4, 0, 16, 256, 512, 128, 2}, + {HeadDim::k256, 8, 4, 0, 16, 256, 2048, 128, 1}, + {HeadDim::k256, 8, 4, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k256, 8, 4, 0, 16, 1024, 512, 128, 4}, + {HeadDim::k256, 8, 4, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k256, 8, 4, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k256, 8, 4, 0, 16, 4096, 2048, 128, 2}, + {HeadDim::k256, 8, 4, 0, 16, 4096, 8192, 128, 2}, + {HeadDim::k256, 8, 4, 1024, 16, 1, 0, 64, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 1, 512, 64, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 16, 0, 64, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 16, 512, 64, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 64, 0, 512, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 64, 512, 64, 2}, + {HeadDim::k256, 8, 4, 1024, 16, 256, 0, 64, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 256, 512, 64, 2}, + {HeadDim::k256, 8, 4, 1024, 16, 1024, 512, 512, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 4096, 512, 512, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 1, 0, 64, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 1, 512, 16, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 1, 2048, 16, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 16, 0, 512, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 16, 512, 512, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 16, 2048, 512, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 64, 0, 512, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 64, 512, 256, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 64, 2048, 512, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 256, 0, 512, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 256, 512, 512, 2}, + {HeadDim::k256, 8, 4, 4096, 16, 256, 2048, 512, 1}, + {HeadDim::k256, 8, 4, 4096, 16, 1024, 512, 256, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 1024, 2048, 512, 1}, + {HeadDim::k256, 8, 4, 4096, 16, 4096, 2048, 512, 2}, + {HeadDim::k256, 16, 2, 0, 16, 1, 0, 16, 2}, + {HeadDim::k256, 16, 2, 0, 16, 1, 512, 16, 2}, + {HeadDim::k256, 16, 2, 0, 16, 1, 2048, 16, 2}, + {HeadDim::k256, 16, 2, 0, 16, 1, 8192, 16, 2}, + {HeadDim::k256, 16, 2, 0, 16, 16, 0, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 16, 512, 16, 1}, + {HeadDim::k256, 16, 2, 0, 16, 16, 2048, 16, 1}, + {HeadDim::k256, 16, 2, 0, 16, 16, 8192, 16, 2}, + {HeadDim::k256, 16, 2, 0, 16, 64, 0, 128, 2}, + {HeadDim::k256, 16, 2, 0, 16, 64, 512, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 64, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 64, 8192, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 256, 0, 128, 2}, + {HeadDim::k256, 16, 2, 0, 16, 256, 512, 128, 2}, + {HeadDim::k256, 16, 2, 0, 16, 256, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 1024, 512, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 4096, 8192, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 1, 0, 16, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 1, 512, 16, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 1, 2048, 16, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 1, 8192, 16, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 16, 0, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 16, 512, 16, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 16, 2048, 16, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 16, 8192, 16, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 64, 0, 128, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 64, 512, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 64, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 64, 8192, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 256, 0, 128, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 256, 512, 128, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 256, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 256, 8192, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 1024, 512, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 1024, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 1024, 8192, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 4096, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 4096, 8192, 128, 1}, + {HeadDim::k256, 16, 4, 0, 16, 1, 0, 32, 4}, + {HeadDim::k256, 16, 4, 0, 16, 1, 512, 32, 2}, + {HeadDim::k256, 16, 4, 0, 16, 1, 2048, 32, 2}, + {HeadDim::k256, 16, 4, 0, 16, 1, 8192, 32, 2}, + {HeadDim::k256, 16, 4, 0, 16, 16, 0, 64, 4}, + {HeadDim::k256, 16, 4, 0, 16, 16, 512, 64, 4}, + {HeadDim::k256, 16, 4, 0, 16, 16, 2048, 64, 4}, + {HeadDim::k256, 16, 4, 0, 16, 16, 8192, 32, 1}, + {HeadDim::k256, 16, 4, 0, 16, 64, 0, 64, 2}, + {HeadDim::k256, 16, 4, 0, 16, 64, 512, 64, 2}, + {HeadDim::k256, 16, 4, 0, 16, 64, 2048, 64, 1}, + {HeadDim::k256, 16, 4, 0, 16, 64, 8192, 64, 1}, + {HeadDim::k256, 16, 4, 0, 16, 256, 0, 64, 2}, + {HeadDim::k256, 16, 4, 0, 16, 256, 512, 64, 2}, + {HeadDim::k256, 16, 4, 0, 16, 256, 2048, 64, 2}, + {HeadDim::k256, 16, 4, 0, 16, 256, 8192, 64, 1}, + {HeadDim::k256, 16, 4, 0, 16, 1024, 512, 64, 2}, + {HeadDim::k256, 16, 4, 0, 16, 1024, 2048, 64, 2}, + {HeadDim::k256, 16, 4, 0, 16, 1024, 8192, 64, 1}, + {HeadDim::k256, 16, 4, 0, 16, 4096, 2048, 64, 1}, + {HeadDim::k256, 16, 4, 0, 16, 4096, 8192, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 1, 0, 32, 4}, + {HeadDim::k256, 16, 4, 0, 528, 1, 512, 32, 2}, + {HeadDim::k256, 16, 4, 0, 528, 1, 2048, 32, 2}, + {HeadDim::k256, 16, 4, 0, 528, 1, 8192, 32, 2}, + {HeadDim::k256, 16, 4, 0, 528, 16, 0, 64, 4}, + {HeadDim::k256, 16, 4, 0, 528, 16, 512, 64, 4}, + {HeadDim::k256, 16, 4, 0, 528, 16, 2048, 64, 4}, + {HeadDim::k256, 16, 4, 0, 528, 16, 8192, 32, 1}, + {HeadDim::k256, 16, 4, 0, 528, 64, 0, 64, 2}, + {HeadDim::k256, 16, 4, 0, 528, 64, 512, 64, 2}, + {HeadDim::k256, 16, 4, 0, 528, 64, 2048, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 64, 8192, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 256, 0, 64, 2}, + {HeadDim::k256, 16, 4, 0, 528, 256, 512, 64, 2}, + {HeadDim::k256, 16, 4, 0, 528, 256, 2048, 64, 2}, + {HeadDim::k256, 16, 4, 0, 528, 256, 8192, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 1024, 512, 64, 2}, + {HeadDim::k256, 16, 4, 0, 528, 1024, 2048, 64, 2}, + {HeadDim::k256, 16, 4, 0, 528, 1024, 8192, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 4096, 2048, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 4096, 8192, 64, 1}, + {HeadDim::k256, 16, 8, 0, 16, 1, 0, 512, 8}, + {HeadDim::k256, 16, 8, 0, 16, 1, 512, 512, 8}, + {HeadDim::k256, 16, 8, 0, 16, 1, 2048, 16, 8}, + {HeadDim::k256, 16, 8, 0, 16, 1, 8192, 16, 8}, + {HeadDim::k256, 16, 8, 0, 16, 16, 0, 512, 4}, + {HeadDim::k256, 16, 8, 0, 16, 16, 512, 512, 8}, + {HeadDim::k256, 16, 8, 0, 16, 16, 2048, 512, 4}, + {HeadDim::k256, 16, 8, 0, 16, 16, 8192, 512, 8}, + {HeadDim::k256, 16, 8, 0, 16, 64, 0, 128, 4}, + {HeadDim::k256, 16, 8, 0, 16, 64, 512, 128, 4}, + {HeadDim::k256, 16, 8, 0, 16, 64, 2048, 128, 1}, + {HeadDim::k256, 16, 8, 0, 16, 64, 8192, 128, 2}, + {HeadDim::k256, 16, 8, 0, 16, 256, 0, 512, 4}, + {HeadDim::k256, 16, 8, 0, 16, 256, 512, 128, 2}, + {HeadDim::k256, 16, 8, 0, 16, 256, 2048, 128, 1}, + {HeadDim::k256, 16, 8, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k256, 16, 8, 0, 16, 1024, 512, 128, 2}, + {HeadDim::k256, 16, 8, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k256, 16, 8, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k256, 16, 8, 0, 16, 4096, 2048, 128, 2}, + {HeadDim::k256, 16, 8, 0, 16, 4096, 8192, 128, 2}, + {HeadDim::k256, 16, 8, 1024, 16, 1, 0, 16, 8}, + {HeadDim::k256, 16, 8, 1024, 16, 1, 512, 16, 8}, + {HeadDim::k256, 16, 8, 1024, 16, 16, 0, 128, 4}, + {HeadDim::k256, 16, 8, 1024, 16, 16, 512, 128, 8}, + {HeadDim::k256, 16, 8, 1024, 16, 64, 0, 256, 4}, + {HeadDim::k256, 16, 8, 1024, 16, 64, 512, 256, 4}, + {HeadDim::k256, 16, 8, 1024, 16, 256, 0, 128, 4}, + {HeadDim::k256, 16, 8, 1024, 16, 256, 512, 256, 4}, + {HeadDim::k256, 16, 8, 1024, 16, 1024, 512, 256, 8}, + {HeadDim::k256, 16, 8, 1024, 16, 4096, 512, 256, 8}, + {HeadDim::k256, 24, 4, 0, 16, 1, 0, 16, 4}, + {HeadDim::k256, 24, 4, 0, 16, 1, 512, 16, 4}, + {HeadDim::k256, 24, 4, 0, 16, 1, 2048, 16, 4}, + {HeadDim::k256, 24, 4, 0, 16, 1, 8192, 16, 4}, + {HeadDim::k256, 24, 4, 0, 16, 16, 0, 64, 4}, + {HeadDim::k256, 24, 4, 0, 16, 16, 512, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 16, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 16, 8192, 16, 1}, + {HeadDim::k256, 24, 4, 0, 16, 64, 0, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 64, 512, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 64, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 64, 8192, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 256, 0, 64, 4}, + {HeadDim::k256, 24, 4, 0, 16, 256, 512, 64, 2}, + {HeadDim::k256, 24, 4, 0, 16, 256, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 256, 8192, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 1024, 512, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 1024, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 1024, 8192, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 4096, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 4096, 8192, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 1, 0, 16, 4}, + {HeadDim::k256, 24, 4, 0, 784, 1, 512, 16, 4}, + {HeadDim::k256, 24, 4, 0, 784, 1, 2048, 16, 4}, + {HeadDim::k256, 24, 4, 0, 784, 1, 8192, 16, 4}, + {HeadDim::k256, 24, 4, 0, 784, 16, 0, 64, 4}, + {HeadDim::k256, 24, 4, 0, 784, 16, 512, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 16, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 16, 8192, 16, 1}, + {HeadDim::k256, 24, 4, 0, 784, 64, 0, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 64, 512, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 64, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 64, 8192, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 256, 0, 64, 4}, + {HeadDim::k256, 24, 4, 0, 784, 256, 512, 64, 2}, + {HeadDim::k256, 24, 4, 0, 784, 256, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 256, 8192, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 1024, 512, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 1024, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 1024, 8192, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 4096, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 4096, 8192, 64, 1}, + {HeadDim::k256, 32, 16, 1024, 16, 1, 0, 16, 16}, + {HeadDim::k256, 32, 16, 1024, 16, 1, 512, 16, 16}, + {HeadDim::k256, 32, 16, 1024, 16, 16, 0, 16, 16}, + {HeadDim::k256, 32, 16, 1024, 16, 16, 512, 16, 16}, + {HeadDim::k256, 32, 16, 1024, 16, 64, 0, 128, 4}, + {HeadDim::k256, 32, 16, 1024, 16, 64, 512, 128, 2}, + {HeadDim::k256, 32, 16, 1024, 16, 256, 0, 16, 4}, + {HeadDim::k256, 32, 16, 1024, 16, 256, 512, 128, 2}, + {HeadDim::k256, 32, 16, 1024, 16, 1024, 512, 128, 16}, + {HeadDim::k256, 32, 16, 1024, 16, 4096, 512, 128, 16}, + {HeadDim::k512, 8, 1, 0, 16, 1, 0, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 1, 512, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 16, 0, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 16, 512, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 16, 2048, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 16, 8192, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 64, 0, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 64, 512, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 64, 2048, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 64, 8192, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 256, 0, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 256, 512, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 256, 2048, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 256, 8192, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 1024, 512, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 1024, 2048, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 1024, 8192, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 4096, 2048, 16, 1}, + {HeadDim::k512, 8, 1, 0, 16, 4096, 8192, 16, 1}, + {HeadDim::k512, 8, 2, 0, 16, 1, 0, 16, 1}, + {HeadDim::k512, 8, 2, 0, 16, 1, 512, 16, 2}, + {HeadDim::k512, 8, 2, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k512, 8, 2, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k512, 8, 2, 0, 16, 16, 0, 32, 1}, + {HeadDim::k512, 8, 2, 0, 16, 16, 512, 16, 1}, + {HeadDim::k512, 8, 2, 0, 16, 16, 2048, 16, 1}, + {HeadDim::k512, 8, 2, 0, 16, 16, 8192, 16, 1}, + {HeadDim::k512, 8, 2, 0, 16, 64, 0, 32, 1}, + {HeadDim::k512, 8, 2, 0, 16, 64, 512, 16, 1}, + {HeadDim::k512, 8, 2, 0, 16, 64, 2048, 32, 1}, + {HeadDim::k512, 8, 2, 0, 16, 64, 8192, 32, 1}, + {HeadDim::k512, 8, 2, 0, 16, 256, 0, 32, 2}, + {HeadDim::k512, 8, 2, 0, 16, 256, 512, 32, 2}, + {HeadDim::k512, 8, 2, 0, 16, 256, 2048, 32, 1}, + {HeadDim::k512, 8, 2, 0, 16, 256, 8192, 32, 1}, + {HeadDim::k512, 8, 2, 0, 16, 1024, 512, 32, 1}, + {HeadDim::k512, 8, 2, 0, 16, 1024, 2048, 32, 1}, + {HeadDim::k512, 8, 2, 0, 16, 1024, 8192, 32, 1}, + {HeadDim::k512, 8, 2, 0, 16, 4096, 2048, 32, 1}, + {HeadDim::k512, 8, 2, 0, 16, 4096, 8192, 32, 1}, + {HeadDim::k512, 16, 1, 0, 16, 1, 0, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 1, 512, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 16, 0, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 16, 512, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 16, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 16, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 64, 0, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 64, 512, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 64, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 64, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 256, 0, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 256, 512, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 256, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 256, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 1024, 512, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 1024, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 1024, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 4096, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 16, 4096, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 1, 0, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 1, 512, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 1, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 1, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 16, 0, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 16, 512, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 16, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 16, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 64, 0, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 64, 512, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 64, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 64, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 256, 0, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 256, 512, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 256, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 256, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 1024, 512, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 1024, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 1024, 8192, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 4096, 2048, 16, 1}, + {HeadDim::k512, 16, 1, 0, 64, 4096, 8192, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 1, 0, 16, 2}, + {HeadDim::k512, 16, 2, 0, 16, 1, 512, 16, 2}, + {HeadDim::k512, 16, 2, 0, 16, 1, 2048, 16, 2}, + {HeadDim::k512, 16, 2, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 16, 0, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 16, 512, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 16, 2048, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 16, 8192, 16, 2}, + {HeadDim::k512, 16, 2, 0, 16, 64, 0, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 64, 512, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 64, 2048, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 64, 8192, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 256, 0, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 256, 512, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 256, 2048, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 256, 8192, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 1024, 512, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 1024, 2048, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 1024, 8192, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 4096, 2048, 16, 1}, + {HeadDim::k512, 16, 2, 0, 16, 4096, 8192, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 1, 0, 16, 2}, + {HeadDim::k512, 16, 2, 0, 32, 1, 512, 16, 2}, + {HeadDim::k512, 16, 2, 0, 32, 1, 2048, 16, 2}, + {HeadDim::k512, 16, 2, 0, 32, 1, 8192, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 16, 0, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 16, 512, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 16, 2048, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 16, 8192, 16, 2}, + {HeadDim::k512, 16, 2, 0, 32, 64, 0, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 64, 512, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 64, 2048, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 64, 8192, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 256, 0, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 256, 512, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 256, 2048, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 256, 8192, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 1024, 512, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 1024, 2048, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 1024, 8192, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 4096, 2048, 16, 1}, + {HeadDim::k512, 16, 2, 0, 32, 4096, 8192, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 1, 0, 16, 4}, + {HeadDim::k512, 32, 4, 0, 16, 1, 512, 16, 4}, + {HeadDim::k512, 32, 4, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 1, 8192, 16, 4}, + {HeadDim::k512, 32, 4, 0, 16, 16, 0, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 16, 512, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 16, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 16, 8192, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 64, 0, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 64, 512, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 64, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 64, 8192, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 256, 0, 16, 4}, + {HeadDim::k512, 32, 4, 0, 16, 256, 512, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 256, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 256, 8192, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 1024, 512, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 1024, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 1024, 8192, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 4096, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 16, 4096, 8192, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 1, 0, 16, 4}, + {HeadDim::k512, 32, 4, 0, 32, 1, 512, 16, 4}, + {HeadDim::k512, 32, 4, 0, 32, 1, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 1, 8192, 16, 4}, + {HeadDim::k512, 32, 4, 0, 32, 16, 0, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 16, 512, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 16, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 16, 8192, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 64, 0, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 64, 512, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 64, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 64, 8192, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 256, 0, 16, 4}, + {HeadDim::k512, 32, 4, 0, 32, 256, 512, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 256, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 256, 8192, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 1024, 512, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 1024, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 1024, 8192, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 4096, 2048, 16, 1}, + {HeadDim::k512, 32, 4, 0, 32, 4096, 8192, 16, 1}, +}; +// clang-format on + +// The table of GPU family A. +template +constexpr const auto& prefill_batch_configs() { + if constexpr (A == Arch::kGfx1151) + return kGfx1151PrefillBatchConfigs; + else + return kGfx1150PrefillBatchConfigs; +} + +// The tables by Arch. +inline constexpr std::span + kPrefillBatchConfigs[kNumArches] = {kGfx1151PrefillBatchConfigs, + kGfx1150PrefillBatchConfigs}; } // namespace rdna diff --git a/csrc/rocm/rdna_prefill_configs.h b/csrc/rocm/rdna_prefill_configs.h index 5514ee1a4161..4a457299399e 100644 --- a/csrc/rocm/rdna_prefill_configs.h +++ b/csrc/rocm/rdna_prefill_configs.h @@ -2,6 +2,8 @@ // compiled; a call whose shape has no entry runs on Triton. #pragma once +#include + #include "rdna_attn_types.h" namespace rdna { @@ -28,10 +30,12 @@ struct PrefillConfig { int head_group; // kv heads whose tiles interleave in the grid }; -// Written by benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py -// --apply, one entry per line in PrefillConfig order. +// One table per Arch (rdna_attn_types.h), each complete; a GPU family's +// entries are built for its archs only. Written by +// benchmarks/kernels/rocm_rdna_hip_attn/tools/tune.py --apply, one entry per +// line in PrefillConfig order. // clang-format off -inline constexpr PrefillConfig kPrefillConfigs[] = { +inline constexpr PrefillConfig kGfx1151PrefillConfigs[] = { {HeadDim::k64, 14, 2, 0, Dtype::kFp16, 16, 16, 4, 32, 1, 0, 0, 2, 16, 8}, {HeadDim::k64, 14, 2, 0, Dtype::kFp16, 16, 32, 8, 64, 1, 0, 0, 2, 16, 1}, {HeadDim::k64, 14, 2, 0, Dtype::kFp16, 16, 64, 8, 64, 1, 0, 0, 2, 4, 1}, @@ -467,6 +471,356 @@ inline constexpr PrefillConfig kPrefillConfigs[] = { {HeadDim::k512, 32, 4, 0, Dtype::kBf16, 32, 32, 16, 16, 4, 1, 0, 1, 16, 2}, {HeadDim::k512, 32, 4, 0, Dtype::kBf16, 32, 256, 16, 16, 4, 1, 0, 1, 16, 1}, }; + +inline constexpr PrefillConfig kGfx1150PrefillConfigs[] = { + {HeadDim::k64, 14, 2, 0, Dtype::kFp16, 16, 16, 4, 64, 1, 0, 0, 2, 4, 8}, + {HeadDim::k64, 14, 2, 0, Dtype::kFp16, 16, 32, 8, 64, 1, 0, 0, 2, 16, 1}, + {HeadDim::k64, 14, 2, 0, Dtype::kFp16, 16, 64, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k64, 14, 2, 0, Dtype::kFp16, 16, 128, 8, 64, 1, 1, 0, 1, 1, 2}, + {HeadDim::k64, 14, 2, 0, Dtype::kFp16, 16, 512, 4, 32, 1, 0, 0, 1, 1, 1}, + {HeadDim::k64, 14, 2, 0, Dtype::kFp16, 16, 1024, 8, 64, 1, 1, 0, 1, 1, 1}, + {HeadDim::k64, 14, 2, 0, Dtype::kBf16, 16, 16, 4, 64, 1, 0, 0, 2, 4, 8}, + {HeadDim::k64, 14, 2, 0, Dtype::kBf16, 16, 32, 8, 64, 1, 0, 0, 2, 16, 1}, + {HeadDim::k64, 14, 2, 0, Dtype::kBf16, 16, 64, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k64, 14, 2, 0, Dtype::kBf16, 16, 128, 8, 64, 1, 1, 0, 1, 1, 2}, + {HeadDim::k64, 14, 2, 0, Dtype::kBf16, 16, 512, 4, 32, 1, 0, 0, 1, 1, 1}, + {HeadDim::k64, 14, 2, 0, Dtype::kBf16, 16, 1024, 8, 64, 1, 1, 0, 1, 1, 1}, + {HeadDim::k64, 16, 2, 0, Dtype::kFp16, 16, 16, 4, 64, 1, 0, 0, 2, 4, 8}, + {HeadDim::k64, 16, 2, 0, Dtype::kFp16, 16, 32, 8, 64, 1, 0, 0, 2, 16, 4}, + {HeadDim::k64, 16, 2, 0, Dtype::kFp16, 16, 64, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k64, 16, 2, 0, Dtype::kFp16, 16, 128, 8, 64, 1, 1, 0, 1, 1, 8}, + {HeadDim::k64, 16, 2, 0, Dtype::kFp16, 16, 256, 8, 64, 1, 1, 0, 1, 1, 1}, + {HeadDim::k64, 16, 2, 0, Dtype::kFp16, 16, 2048, 4, 32, 1, 0, 0, 1, 16, 1}, + {HeadDim::k64, 16, 2, 0, Dtype::kBf16, 16, 16, 4, 64, 1, 0, 0, 2, 4, 8}, + {HeadDim::k64, 16, 2, 0, Dtype::kBf16, 16, 32, 8, 64, 1, 0, 0, 2, 16, 4}, + {HeadDim::k64, 16, 2, 0, Dtype::kBf16, 16, 64, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k64, 16, 2, 0, Dtype::kBf16, 16, 128, 8, 64, 1, 1, 0, 1, 1, 8}, + {HeadDim::k64, 16, 2, 0, Dtype::kBf16, 16, 256, 8, 64, 1, 1, 0, 1, 1, 1}, + {HeadDim::k64, 16, 2, 0, Dtype::kBf16, 16, 2048, 4, 32, 1, 0, 0, 1, 16, 1}, + {HeadDim::k64, 32, 8, 0, Dtype::kFp16, 16, 16, 8, 64, 1, 0, 0, 2, 1, 8}, + {HeadDim::k64, 32, 8, 0, Dtype::kFp16, 16, 64, 8, 64, 1, 1, 0, 1, 1, 8}, + {HeadDim::k64, 32, 8, 0, Dtype::kFp16, 16, 128, 4, 32, 1, 1, 0, 1, 4, 8}, + {HeadDim::k64, 32, 8, 0, Dtype::kFp16, 16, 512, 8, 64, 1, 1, 0, 1, 1, 2}, + {HeadDim::k64, 32, 8, 0, Dtype::kBf16, 16, 16, 8, 64, 1, 0, 0, 2, 1, 8}, + {HeadDim::k64, 32, 8, 0, Dtype::kBf16, 16, 64, 8, 64, 1, 1, 0, 1, 1, 8}, + {HeadDim::k64, 32, 8, 0, Dtype::kBf16, 16, 128, 4, 32, 1, 1, 0, 1, 4, 8}, + {HeadDim::k64, 32, 8, 0, Dtype::kBf16, 16, 512, 8, 64, 1, 1, 0, 1, 1, 2}, + {HeadDim::k64, 32, 32, 0, Dtype::kFp16, 16, 16, 4, 32, 1, 0, 0, 2, 1, 1}, + {HeadDim::k64, 32, 32, 0, Dtype::kFp16, 16, 64, 4, 32, 1, 0, 0, 1, 1, 2}, + {HeadDim::k64, 32, 32, 0, Dtype::kFp16, 16, 128, 4, 64, 1, 0, 0, 1, 16, 8}, + {HeadDim::k64, 32, 32, 0, Dtype::kFp16, 16, 256, 8, 64, 1, 1, 0, 1, 1, 8}, + {HeadDim::k64, 32, 32, 0, Dtype::kFp16, 16, 1024, 4, 64, 1, 0, 0, 1, 16, 1}, + {HeadDim::k64, 32, 32, 0, Dtype::kFp16, 16, 2048, 8, 64, 1, 1, 0, 1, 1, 1}, + {HeadDim::k64, 32, 32, 0, Dtype::kBf16, 16, 16, 4, 32, 1, 0, 0, 2, 1, 1}, + {HeadDim::k64, 32, 32, 0, Dtype::kBf16, 16, 64, 4, 32, 1, 0, 0, 1, 1, 2}, + {HeadDim::k64, 32, 32, 0, Dtype::kBf16, 16, 128, 4, 64, 1, 0, 0, 1, 16, 8}, + {HeadDim::k64, 32, 32, 0, Dtype::kBf16, 16, 256, 8, 64, 1, 1, 0, 1, 1, 8}, + {HeadDim::k64, 32, 32, 0, Dtype::kBf16, 16, 1024, 4, 64, 1, 0, 0, 1, 16, 1}, + {HeadDim::k64, 32, 32, 0, Dtype::kBf16, 16, 2048, 8, 64, 1, 1, 0, 1, 1, 1}, + {HeadDim::k96, 32, 32, 0, Dtype::kFp16, 16, 16, 4, 32, 1, 0, 0, 2, 1, 2}, + {HeadDim::k96, 32, 32, 0, Dtype::kFp16, 16, 128, 4, 32, 1, 0, 0, 1, 1, 4}, + {HeadDim::k96, 32, 32, 0, Dtype::kFp16, 16, 1024, 8, 64, 1, 0, 0, 1, 16, 1}, + {HeadDim::k96, 32, 32, 0, Dtype::kFp16, 16, 8192, 8, 64, 1, 0, 0, 1, 16, 2}, + {HeadDim::k96, 32, 32, 0, Dtype::kBf16, 16, 16, 4, 32, 1, 0, 0, 2, 1, 2}, + {HeadDim::k96, 32, 32, 0, Dtype::kBf16, 16, 128, 4, 32, 1, 0, 0, 1, 1, 4}, + {HeadDim::k96, 32, 32, 0, Dtype::kBf16, 16, 1024, 8, 64, 1, 0, 0, 1, 16, 1}, + {HeadDim::k96, 32, 32, 0, Dtype::kBf16, 16, 8192, 8, 64, 1, 0, 0, 1, 16, 2}, + {HeadDim::k128, 10, 10, 0, Dtype::kFp16, 16, 16, 4, 32, 1, 0, 0, 2, 1, 8}, + {HeadDim::k128, 10, 10, 0, Dtype::kFp16, 16, 128, 4, 32, 1, 1, 0, 1, 1, 8}, + {HeadDim::k128, 10, 10, 0, Dtype::kFp16, 16, 256, 8, 64, 1, 1, 0, 2, 1, 2}, + {HeadDim::k128, 10, 10, 0, Dtype::kFp16, 16, 512, 8, 64, 1, 1, 0, 1, 4, 4}, + {HeadDim::k128, 10, 10, 0, Dtype::kBf16, 16, 16, 4, 32, 1, 0, 0, 2, 1, 8}, + {HeadDim::k128, 10, 10, 0, Dtype::kBf16, 16, 128, 4, 32, 1, 1, 0, 1, 1, 8}, + {HeadDim::k128, 10, 10, 0, Dtype::kBf16, 16, 256, 8, 64, 1, 1, 0, 2, 1, 2}, + {HeadDim::k128, 10, 10, 0, Dtype::kBf16, 16, 512, 8, 64, 1, 1, 0, 1, 4, 4}, + {HeadDim::k128, 16, 2, 0, Dtype::kFp16, 16, 16, 4, 32, 1, 0, 0, 2, 4, 8}, + {HeadDim::k128, 16, 2, 0, Dtype::kFp16, 16, 32, 8, 32, 1, 1, 0, 1, 4, 2}, + {HeadDim::k128, 16, 2, 0, Dtype::kFp16, 16, 64, 8, 64, 1, 1, 0, 2, 1, 4}, + {HeadDim::k128, 16, 2, 0, Dtype::kFp16, 16, 128, 4, 32, 1, 1, 0, 1, 16, 8}, + {HeadDim::k128, 16, 2, 0, Dtype::kFp16, 16, 256, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 16, 2, 0, Dtype::kBf16, 16, 16, 4, 32, 1, 0, 0, 2, 4, 8}, + {HeadDim::k128, 16, 2, 0, Dtype::kBf16, 16, 32, 8, 32, 1, 1, 0, 1, 4, 2}, + {HeadDim::k128, 16, 2, 0, Dtype::kBf16, 16, 64, 8, 64, 1, 1, 0, 2, 1, 4}, + {HeadDim::k128, 16, 2, 0, Dtype::kBf16, 16, 128, 4, 32, 1, 1, 0, 1, 16, 8}, + {HeadDim::k128, 16, 2, 0, Dtype::kBf16, 16, 256, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 16, 8, 0, Dtype::kFp16, 16, 16, 4, 32, 1, 0, 0, 2, 1, 4}, + {HeadDim::k128, 16, 8, 0, Dtype::kFp16, 16, 64, 4, 32, 1, 1, 0, 2, 1, 8}, + {HeadDim::k128, 16, 8, 0, Dtype::kFp16, 16, 128, 8, 64, 1, 1, 0, 1, 1, 8}, + {HeadDim::k128, 16, 8, 0, Dtype::kFp16, 16, 256, 8, 64, 1, 1, 0, 1, 16, 2}, + {HeadDim::k128, 16, 8, 0, Dtype::kFp16, 16, 1024, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 16, 8, 0, Dtype::kBf16, 16, 16, 4, 32, 1, 0, 0, 2, 1, 4}, + {HeadDim::k128, 16, 8, 0, Dtype::kBf16, 16, 64, 4, 32, 1, 1, 0, 2, 1, 8}, + {HeadDim::k128, 16, 8, 0, Dtype::kBf16, 16, 128, 8, 64, 1, 1, 0, 1, 1, 8}, + {HeadDim::k128, 16, 8, 0, Dtype::kBf16, 16, 256, 8, 64, 1, 1, 0, 1, 16, 2}, + {HeadDim::k128, 16, 8, 0, Dtype::kBf16, 16, 1024, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 24, 8, 0, Dtype::kFp16, 16, 16, 4, 16, 1, 0, 0, 1, 1, 4}, + {HeadDim::k128, 24, 8, 0, Dtype::kFp16, 16, 32, 4, 32, 1, 1, 0, 1, 1, 1}, + {HeadDim::k128, 24, 8, 0, Dtype::kFp16, 16, 128, 4, 32, 1, 1, 0, 1, 1, 8}, + {HeadDim::k128, 24, 8, 0, Dtype::kFp16, 16, 256, 4, 32, 1, 1, 0, 1, 1, 2}, + {HeadDim::k128, 24, 8, 0, Dtype::kFp16, 16, 512, 8, 64, 1, 1, 0, 1, 4, 1}, + {HeadDim::k128, 24, 8, 0, Dtype::kBf16, 16, 16, 4, 16, 1, 0, 0, 1, 1, 4}, + {HeadDim::k128, 24, 8, 0, Dtype::kBf16, 16, 32, 4, 32, 1, 1, 0, 1, 1, 1}, + {HeadDim::k128, 24, 8, 0, Dtype::kBf16, 16, 128, 4, 32, 1, 1, 0, 1, 1, 8}, + {HeadDim::k128, 24, 8, 0, Dtype::kBf16, 16, 256, 4, 32, 1, 1, 0, 1, 1, 2}, + {HeadDim::k128, 24, 8, 0, Dtype::kBf16, 16, 512, 8, 64, 1, 1, 0, 1, 4, 1}, + {HeadDim::k128, 28, 4, 0, Dtype::kFp16, 16, 16, 4, 32, 1, 0, 0, 2, 16, 2}, + {HeadDim::k128, 28, 4, 0, Dtype::kFp16, 16, 32, 4, 16, 1, 1, 0, 1, 16, 4}, + {HeadDim::k128, 28, 4, 0, Dtype::kFp16, 16, 64, 4, 32, 1, 1, 0, 1, 4, 4}, + {HeadDim::k128, 28, 4, 0, Dtype::kFp16, 16, 256, 8, 64, 1, 1, 0, 1, 4, 1}, + {HeadDim::k128, 28, 4, 0, Dtype::kBf16, 16, 16, 4, 32, 1, 0, 0, 2, 16, 2}, + {HeadDim::k128, 28, 4, 0, Dtype::kBf16, 16, 32, 4, 16, 1, 1, 0, 1, 16, 4}, + {HeadDim::k128, 28, 4, 0, Dtype::kBf16, 16, 64, 4, 32, 1, 1, 0, 1, 4, 4}, + {HeadDim::k128, 28, 4, 0, Dtype::kBf16, 16, 256, 8, 64, 1, 1, 0, 1, 4, 1}, + {HeadDim::k128, 32, 2, 0, Dtype::kFp16, 16, 16, 8, 64, 1, 1, 0, 1, 4, 8}, + {HeadDim::k128, 32, 2, 0, Dtype::kFp16, 16, 32, 8, 64, 1, 1, 0, 1, 4, 1}, + {HeadDim::k128, 32, 2, 0, Dtype::kFp16, 16, 64, 4, 32, 1, 1, 0, 1, 16, 2}, + {HeadDim::k128, 32, 2, 0, Dtype::kFp16, 16, 128, 8, 64, 1, 1, 0, 1, 4, 1}, + {HeadDim::k128, 32, 2, 0, Dtype::kFp16, 16, 4096, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 32, 2, 0, Dtype::kBf16, 16, 16, 8, 64, 1, 1, 0, 1, 4, 8}, + {HeadDim::k128, 32, 2, 0, Dtype::kBf16, 16, 32, 8, 64, 1, 1, 0, 1, 4, 1}, + {HeadDim::k128, 32, 2, 0, Dtype::kBf16, 16, 64, 4, 32, 1, 1, 0, 1, 16, 2}, + {HeadDim::k128, 32, 2, 0, Dtype::kBf16, 16, 128, 8, 64, 1, 1, 0, 1, 4, 1}, + {HeadDim::k128, 32, 2, 0, Dtype::kBf16, 16, 4096, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 32, 4, 0, Dtype::kFp16, 16, 16, 4, 32, 1, 1, 0, 2, 4, 4}, + {HeadDim::k128, 32, 4, 0, Dtype::kFp16, 16, 32, 4, 32, 1, 1, 0, 1, 16, 8}, + {HeadDim::k128, 32, 4, 0, Dtype::kFp16, 16, 128, 8, 64, 1, 1, 0, 1, 4, 2}, + {HeadDim::k128, 32, 4, 0, Dtype::kFp16, 16, 256, 8, 64, 1, 1, 0, 1, 4, 1}, + {HeadDim::k128, 32, 4, 0, Dtype::kBf16, 16, 16, 4, 32, 1, 1, 0, 2, 4, 4}, + {HeadDim::k128, 32, 4, 0, Dtype::kBf16, 16, 32, 4, 32, 1, 1, 0, 1, 16, 8}, + {HeadDim::k128, 32, 4, 0, Dtype::kBf16, 16, 128, 8, 64, 1, 1, 0, 1, 4, 2}, + {HeadDim::k128, 32, 4, 0, Dtype::kBf16, 16, 256, 8, 64, 1, 1, 0, 1, 4, 1}, + {HeadDim::k128, 32, 8, 0, Dtype::kFp16, 16, 16, 4, 16, 1, 0, 0, 1, 1, 8}, + {HeadDim::k128, 32, 8, 0, Dtype::kFp16, 16, 64, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 32, 8, 0, Dtype::kBf16, 16, 16, 4, 16, 1, 0, 0, 1, 1, 8}, + {HeadDim::k128, 32, 8, 0, Dtype::kBf16, 16, 64, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 32, 32, 0, Dtype::kFp16, 16, 16, 4, 16, 1, 0, 0, 1, 16, 8}, + {HeadDim::k128, 32, 32, 0, Dtype::kFp16, 16, 256, 8, 32, 1, 1, 0, 1, 16, 4}, + {HeadDim::k128, 32, 32, 0, Dtype::kFp16, 16, 512, 8, 64, 1, 1, 0, 1, 16, 2}, + {HeadDim::k128, 32, 32, 0, Dtype::kFp16, 16, 2048, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 32, 32, 0, Dtype::kBf16, 16, 16, 4, 16, 1, 0, 0, 1, 16, 8}, + {HeadDim::k128, 32, 32, 0, Dtype::kBf16, 16, 256, 8, 32, 1, 1, 0, 1, 16, 4}, + {HeadDim::k128, 32, 32, 0, Dtype::kBf16, 16, 512, 8, 64, 1, 1, 0, 1, 16, 2}, + {HeadDim::k128, 32, 32, 0, Dtype::kBf16, 16, 2048, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 40, 8, 0, Dtype::kFp16, 16, 16, 8, 32, 1, 1, 0, 1, 1, 4}, + {HeadDim::k128, 40, 8, 0, Dtype::kFp16, 16, 32, 4, 32, 1, 1, 0, 1, 1, 8}, + {HeadDim::k128, 40, 8, 0, Dtype::kFp16, 16, 128, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 40, 8, 0, Dtype::kFp16, 16, 256, 8, 64, 1, 1, 0, 1, 4, 2}, + {HeadDim::k128, 40, 8, 0, Dtype::kFp16, 16, 512, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 40, 8, 0, Dtype::kBf16, 16, 16, 8, 32, 1, 1, 0, 1, 1, 4}, + {HeadDim::k128, 40, 8, 0, Dtype::kBf16, 16, 32, 4, 32, 1, 1, 0, 1, 1, 8}, + {HeadDim::k128, 40, 8, 0, Dtype::kBf16, 16, 128, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k128, 40, 8, 0, Dtype::kBf16, 16, 256, 8, 64, 1, 1, 0, 1, 4, 2}, + {HeadDim::k128, 40, 8, 0, Dtype::kBf16, 16, 512, 8, 64, 1, 1, 0, 1, 16, 1}, + {HeadDim::k256, 8, 1, 0, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 1, 1, 4, 8}, + {HeadDim::k256, 8, 1, 0, Dtype::kFp16, 16, 32, 16, 16, 2, 1, 0, 1, 4, 2}, + {HeadDim::k256, 8, 1, 0, Dtype::kFp16, 16, 64, 8, 32, 2, 1, 0, 1, 4, 4}, + {HeadDim::k256, 8, 1, 0, Dtype::kFp16, 16, 128, 16, 16, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 8, 1, 0, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 1, 1, 4, 8}, + {HeadDim::k256, 8, 1, 0, Dtype::kBf16, 16, 32, 16, 16, 2, 1, 0, 1, 4, 2}, + {HeadDim::k256, 8, 1, 0, Dtype::kBf16, 16, 64, 8, 32, 2, 1, 0, 1, 4, 4}, + {HeadDim::k256, 8, 1, 0, Dtype::kBf16, 16, 128, 16, 16, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 8, 1, 512, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 1, 1, 4, 8}, + {HeadDim::k256, 8, 1, 512, Dtype::kFp16, 16, 64, 4, 16, 2, 1, 1, 1, 1, 2}, + {HeadDim::k256, 8, 1, 512, Dtype::kFp16, 16, 128, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 1, 512, Dtype::kFp16, 32, 16, 4, 16, 2, 1, 1, 1, 4, 8}, + {HeadDim::k256, 8, 1, 512, Dtype::kFp16, 32, 64, 4, 16, 2, 1, 1, 1, 1, 2}, + {HeadDim::k256, 8, 1, 512, Dtype::kFp16, 32, 128, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 1, 512, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 1, 1, 4, 8}, + {HeadDim::k256, 8, 1, 512, Dtype::kBf16, 16, 64, 4, 16, 2, 1, 1, 1, 1, 2}, + {HeadDim::k256, 8, 1, 512, Dtype::kBf16, 16, 128, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 1, 512, Dtype::kBf16, 32, 16, 4, 16, 2, 1, 1, 1, 4, 8}, + {HeadDim::k256, 8, 1, 512, Dtype::kBf16, 32, 64, 4, 16, 2, 1, 1, 1, 1, 2}, + {HeadDim::k256, 8, 1, 512, Dtype::kBf16, 32, 128, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 2, 0, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 8, 2, 0, Dtype::kFp16, 16, 32, 16, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 8, 2, 0, Dtype::kFp16, 16, 64, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 8, 2, 0, Dtype::kFp16, 16, 128, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 8, 2, 0, Dtype::kFp16, 544, 16, 4, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 8, 2, 0, Dtype::kFp16, 544, 32, 16, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 8, 2, 0, Dtype::kFp16, 544, 64, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 8, 2, 0, Dtype::kFp16, 544, 128, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 8, 2, 0, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 8, 2, 0, Dtype::kBf16, 16, 32, 16, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 8, 2, 0, Dtype::kBf16, 16, 64, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 8, 2, 0, Dtype::kBf16, 16, 128, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 8, 2, 0, Dtype::kBf16, 544, 16, 4, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 8, 2, 0, Dtype::kBf16, 544, 32, 16, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 8, 2, 0, Dtype::kBf16, 544, 64, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 8, 2, 0, Dtype::kBf16, 544, 128, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 8, 2, 512, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 1, 1, 4, 2}, + {HeadDim::k256, 8, 2, 512, Dtype::kFp16, 16, 64, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 2, 512, Dtype::kFp16, 16, 512, 8, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 8, 2, 512, Dtype::kFp16, 32, 16, 4, 16, 2, 1, 1, 1, 4, 2}, + {HeadDim::k256, 8, 2, 512, Dtype::kFp16, 32, 64, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 2, 512, Dtype::kFp16, 32, 512, 8, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 8, 2, 512, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 1, 1, 4, 2}, + {HeadDim::k256, 8, 2, 512, Dtype::kBf16, 16, 64, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 2, 512, Dtype::kBf16, 16, 512, 8, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 8, 2, 512, Dtype::kBf16, 32, 16, 4, 16, 2, 1, 1, 1, 4, 2}, + {HeadDim::k256, 8, 2, 512, Dtype::kBf16, 32, 64, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 2, 512, Dtype::kBf16, 32, 512, 8, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 8, 4, 0, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 0, 1, 4, 2}, + {HeadDim::k256, 8, 4, 0, Dtype::kFp16, 16, 64, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 8, 4, 0, Dtype::kFp16, 16, 128, 16, 16, 2, 1, 1, 1, 1, 1}, + {HeadDim::k256, 8, 4, 0, Dtype::kFp16, 16, 256, 16, 16, 2, 1, 0, 1, 1, 4}, + {HeadDim::k256, 8, 4, 0, Dtype::kFp16, 16, 512, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 8, 4, 0, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 0, 1, 4, 2}, + {HeadDim::k256, 8, 4, 0, Dtype::kBf16, 16, 64, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 8, 4, 0, Dtype::kBf16, 16, 128, 16, 16, 2, 1, 1, 1, 1, 1}, + {HeadDim::k256, 8, 4, 0, Dtype::kBf16, 16, 256, 16, 16, 2, 1, 0, 1, 1, 4}, + {HeadDim::k256, 8, 4, 0, Dtype::kBf16, 16, 512, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 8, 4, 1024, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 4, 1024, Dtype::kFp16, 16, 32, 4, 16, 2, 1, 1, 1, 1, 2}, + {HeadDim::k256, 8, 4, 1024, Dtype::kFp16, 16, 64, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 4, 1024, Dtype::kFp16, 16, 512, 8, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 8, 4, 1024, Dtype::kFp16, 16, 1024, 16, 16, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 8, 4, 1024, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 4, 1024, Dtype::kBf16, 16, 32, 4, 16, 2, 1, 1, 1, 1, 2}, + {HeadDim::k256, 8, 4, 1024, Dtype::kBf16, 16, 64, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 4, 1024, Dtype::kBf16, 16, 512, 8, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 8, 4, 1024, Dtype::kBf16, 16, 1024, 16, 16, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 8, 4, 4096, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 8}, + {HeadDim::k256, 8, 4, 4096, Dtype::kFp16, 16, 64, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 4, 4096, Dtype::kFp16, 16, 256, 8, 16, 2, 1, 1, 1, 16, 2}, + {HeadDim::k256, 8, 4, 4096, Dtype::kFp16, 16, 512, 16, 16, 2, 1, 0, 1, 16, 1}, + {HeadDim::k256, 8, 4, 4096, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 8}, + {HeadDim::k256, 8, 4, 4096, Dtype::kBf16, 16, 64, 8, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 8, 4, 4096, Dtype::kBf16, 16, 256, 8, 16, 2, 1, 1, 1, 16, 2}, + {HeadDim::k256, 8, 4, 4096, Dtype::kBf16, 16, 512, 16, 16, 2, 1, 0, 1, 16, 1}, + {HeadDim::k256, 16, 2, 0, Dtype::kFp16, 16, 16, 16, 16, 2, 1, 1, 1, 4, 4}, + {HeadDim::k256, 16, 2, 0, Dtype::kFp16, 16, 32, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 16, 2, 0, Dtype::kFp16, 16, 64, 8, 32, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 16, 2, 0, Dtype::kFp16, 16, 128, 16, 16, 2, 1, 0, 1, 1, 8}, + {HeadDim::k256, 16, 2, 0, Dtype::kFp16, 1056, 16, 16, 16, 2, 1, 1, 1, 4, 4}, + {HeadDim::k256, 16, 2, 0, Dtype::kFp16, 1056, 32, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 16, 2, 0, Dtype::kFp16, 1056, 64, 8, 32, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 16, 2, 0, Dtype::kFp16, 1056, 128, 16, 16, 2, 1, 0, 1, 1, 8}, + {HeadDim::k256, 16, 2, 0, Dtype::kBf16, 16, 16, 16, 16, 2, 1, 1, 1, 4, 4}, + {HeadDim::k256, 16, 2, 0, Dtype::kBf16, 16, 32, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 16, 2, 0, Dtype::kBf16, 16, 64, 8, 32, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 16, 2, 0, Dtype::kBf16, 16, 128, 16, 16, 2, 1, 0, 1, 1, 8}, + {HeadDim::k256, 16, 2, 0, Dtype::kBf16, 1056, 16, 16, 16, 2, 1, 1, 1, 4, 4}, + {HeadDim::k256, 16, 2, 0, Dtype::kBf16, 1056, 32, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 16, 2, 0, Dtype::kBf16, 1056, 64, 8, 32, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 16, 2, 0, Dtype::kBf16, 1056, 128, 16, 16, 2, 1, 0, 1, 1, 8}, + {HeadDim::k256, 16, 4, 0, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 16, 4, 0, Dtype::kFp16, 16, 32, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 16, 4, 0, Dtype::kFp16, 16, 64, 16, 16, 2, 1, 0, 1, 1, 4}, + {HeadDim::k256, 16, 4, 0, Dtype::kFp16, 16, 1024, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 16, 4, 0, Dtype::kFp16, 528, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 16, 4, 0, Dtype::kFp16, 528, 32, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 16, 4, 0, Dtype::kFp16, 528, 64, 16, 16, 2, 1, 0, 1, 1, 4}, + {HeadDim::k256, 16, 4, 0, Dtype::kFp16, 528, 1024, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 16, 4, 0, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 16, 4, 0, Dtype::kBf16, 16, 32, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 16, 4, 0, Dtype::kBf16, 16, 64, 16, 16, 2, 1, 0, 1, 1, 4}, + {HeadDim::k256, 16, 4, 0, Dtype::kBf16, 16, 1024, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 16, 4, 0, Dtype::kBf16, 528, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 16, 4, 0, Dtype::kBf16, 528, 32, 4, 16, 2, 1, 0, 1, 4, 8}, + {HeadDim::k256, 16, 4, 0, Dtype::kBf16, 528, 64, 16, 16, 2, 1, 0, 1, 1, 4}, + {HeadDim::k256, 16, 4, 0, Dtype::kBf16, 528, 1024, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 16, 8, 0, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 16, 8, 0, Dtype::kFp16, 16, 64, 16, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 16, 8, 0, Dtype::kFp16, 16, 128, 16, 16, 2, 1, 0, 1, 1, 4}, + {HeadDim::k256, 16, 8, 0, Dtype::kFp16, 16, 256, 16, 16, 2, 1, 0, 1, 4, 2}, + {HeadDim::k256, 16, 8, 0, Dtype::kFp16, 16, 512, 16, 16, 2, 1, 0, 1, 16, 1}, + {HeadDim::k256, 16, 8, 0, Dtype::kFp16, 16, 4096, 8, 32, 2, 1, 0, 1, 16, 2}, + {HeadDim::k256, 16, 8, 0, Dtype::kFp16, 16, 8192, 16, 16, 2, 1, 0, 1, 16, 2}, + {HeadDim::k256, 16, 8, 0, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 16, 8, 0, Dtype::kBf16, 16, 64, 16, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 16, 8, 0, Dtype::kBf16, 16, 128, 16, 16, 2, 1, 0, 1, 1, 4}, + {HeadDim::k256, 16, 8, 0, Dtype::kBf16, 16, 256, 16, 16, 2, 1, 0, 1, 4, 2}, + {HeadDim::k256, 16, 8, 0, Dtype::kBf16, 16, 512, 16, 16, 2, 1, 0, 1, 16, 1}, + {HeadDim::k256, 16, 8, 0, Dtype::kBf16, 16, 4096, 8, 32, 2, 1, 0, 1, 16, 2}, + {HeadDim::k256, 16, 8, 0, Dtype::kBf16, 16, 8192, 16, 16, 2, 1, 0, 1, 16, 2}, + {HeadDim::k256, 16, 8, 1024, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 16, 8, 1024, Dtype::kFp16, 16, 64, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 16, 8, 1024, Dtype::kFp16, 16, 128, 8, 16, 2, 1, 1, 1, 1, 2}, + {HeadDim::k256, 16, 8, 1024, Dtype::kFp16, 16, 256, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 16, 8, 1024, Dtype::kFp16, 16, 512, 16, 16, 2, 1, 0, 1, 4, 1}, + {HeadDim::k256, 16, 8, 1024, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 16, 8, 1024, Dtype::kBf16, 16, 64, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 16, 8, 1024, Dtype::kBf16, 16, 128, 8, 16, 2, 1, 1, 1, 1, 2}, + {HeadDim::k256, 16, 8, 1024, Dtype::kBf16, 16, 256, 16, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 16, 8, 1024, Dtype::kBf16, 16, 512, 16, 16, 2, 1, 0, 1, 4, 1}, + {HeadDim::k256, 24, 4, 0, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 24, 4, 0, Dtype::kFp16, 16, 64, 4, 16, 2, 1, 1, 1, 4, 2}, + {HeadDim::k256, 24, 4, 0, Dtype::kFp16, 16, 128, 16, 16, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 24, 4, 0, Dtype::kFp16, 16, 512, 16, 16, 2, 1, 0, 1, 16, 2}, + {HeadDim::k256, 24, 4, 0, Dtype::kFp16, 784, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 24, 4, 0, Dtype::kFp16, 784, 64, 4, 16, 2, 1, 1, 1, 4, 2}, + {HeadDim::k256, 24, 4, 0, Dtype::kFp16, 784, 128, 16, 16, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 24, 4, 0, Dtype::kFp16, 784, 512, 16, 16, 2, 1, 0, 1, 16, 2}, + {HeadDim::k256, 24, 4, 0, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 24, 4, 0, Dtype::kBf16, 16, 64, 4, 16, 2, 1, 1, 1, 4, 2}, + {HeadDim::k256, 24, 4, 0, Dtype::kBf16, 16, 128, 16, 16, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 24, 4, 0, Dtype::kBf16, 16, 512, 16, 16, 2, 1, 0, 1, 16, 2}, + {HeadDim::k256, 24, 4, 0, Dtype::kBf16, 784, 16, 4, 16, 2, 1, 1, 1, 1, 4}, + {HeadDim::k256, 24, 4, 0, Dtype::kBf16, 784, 64, 4, 16, 2, 1, 1, 1, 4, 2}, + {HeadDim::k256, 24, 4, 0, Dtype::kBf16, 784, 128, 16, 16, 2, 1, 0, 1, 1, 1}, + {HeadDim::k256, 24, 4, 0, Dtype::kBf16, 784, 512, 16, 16, 2, 1, 0, 1, 16, 2}, + {HeadDim::k256, 32, 16, 1024, Dtype::kFp16, 16, 16, 4, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 32, 16, 1024, Dtype::kFp16, 16, 128, 8, 16, 2, 1, 1, 1, 1, 1}, + {HeadDim::k256, 32, 16, 1024, Dtype::kFp16, 16, 256, 16, 16, 2, 1, 0, 1, 16, 1}, + {HeadDim::k256, 32, 16, 1024, Dtype::kFp16, 16, 8192, 8, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k256, 32, 16, 1024, Dtype::kBf16, 16, 16, 4, 16, 2, 1, 0, 1, 1, 2}, + {HeadDim::k256, 32, 16, 1024, Dtype::kBf16, 16, 128, 8, 16, 2, 1, 1, 1, 1, 1}, + {HeadDim::k256, 32, 16, 1024, Dtype::kBf16, 16, 256, 16, 16, 2, 1, 0, 1, 16, 1}, + {HeadDim::k256, 32, 16, 1024, Dtype::kBf16, 16, 8192, 8, 16, 2, 1, 1, 1, 4, 1}, + {HeadDim::k512, 8, 1, 0, Dtype::kFp16, 16, 16, 8, 16, 4, 1, 0, 1, 4, 4}, + {HeadDim::k512, 8, 1, 0, Dtype::kFp16, 16, 32, 16, 16, 4, 1, 0, 1, 4, 2}, + {HeadDim::k512, 8, 1, 0, Dtype::kFp16, 16, 64, 8, 16, 4, 1, 0, 1, 1, 2}, + {HeadDim::k512, 8, 1, 0, Dtype::kBf16, 16, 16, 8, 16, 4, 1, 0, 1, 4, 4}, + {HeadDim::k512, 8, 1, 0, Dtype::kBf16, 16, 32, 16, 16, 4, 1, 0, 1, 4, 2}, + {HeadDim::k512, 8, 1, 0, Dtype::kBf16, 16, 64, 8, 16, 4, 1, 0, 1, 1, 2}, + {HeadDim::k512, 8, 2, 0, Dtype::kFp16, 16, 16, 8, 16, 4, 1, 0, 1, 4, 4}, + {HeadDim::k512, 8, 2, 0, Dtype::kFp16, 16, 32, 16, 16, 4, 1, 0, 1, 4, 1}, + {HeadDim::k512, 8, 2, 0, Dtype::kFp16, 16, 64, 8, 16, 4, 1, 0, 1, 1, 2}, + {HeadDim::k512, 8, 2, 0, Dtype::kBf16, 16, 16, 8, 16, 4, 1, 0, 1, 4, 4}, + {HeadDim::k512, 8, 2, 0, Dtype::kBf16, 16, 32, 16, 16, 4, 1, 0, 1, 4, 1}, + {HeadDim::k512, 8, 2, 0, Dtype::kBf16, 16, 64, 8, 16, 4, 1, 0, 1, 1, 2}, + {HeadDim::k512, 16, 1, 0, Dtype::kFp16, 16, 16, 16, 16, 4, 1, 0, 1, 4, 2}, + {HeadDim::k512, 16, 1, 0, Dtype::kFp16, 16, 32, 8, 16, 4, 1, 0, 1, 1, 8}, + {HeadDim::k512, 16, 1, 0, Dtype::kFp16, 16, 2048, 8, 16, 4, 1, 0, 1, 16, 1}, + {HeadDim::k512, 16, 1, 0, Dtype::kFp16, 64, 16, 16, 16, 4, 1, 0, 1, 4, 2}, + {HeadDim::k512, 16, 1, 0, Dtype::kFp16, 64, 32, 8, 16, 4, 1, 0, 1, 1, 8}, + {HeadDim::k512, 16, 1, 0, Dtype::kFp16, 64, 2048, 8, 16, 4, 1, 0, 1, 16, 1}, + {HeadDim::k512, 16, 1, 0, Dtype::kBf16, 16, 16, 16, 16, 4, 1, 0, 1, 4, 2}, + {HeadDim::k512, 16, 1, 0, Dtype::kBf16, 16, 32, 8, 16, 4, 1, 0, 1, 1, 8}, + {HeadDim::k512, 16, 1, 0, Dtype::kBf16, 16, 2048, 8, 16, 4, 1, 0, 1, 16, 1}, + {HeadDim::k512, 16, 1, 0, Dtype::kBf16, 64, 16, 16, 16, 4, 1, 0, 1, 4, 2}, + {HeadDim::k512, 16, 1, 0, Dtype::kBf16, 64, 32, 8, 16, 4, 1, 0, 1, 1, 8}, + {HeadDim::k512, 16, 1, 0, Dtype::kBf16, 64, 2048, 8, 16, 4, 1, 0, 1, 16, 1}, + {HeadDim::k512, 16, 2, 0, Dtype::kFp16, 16, 16, 16, 16, 4, 1, 0, 1, 4, 1}, + {HeadDim::k512, 16, 2, 0, Dtype::kFp16, 16, 32, 8, 16, 4, 1, 0, 1, 1, 2}, + {HeadDim::k512, 16, 2, 0, Dtype::kFp16, 32, 16, 16, 16, 4, 1, 0, 1, 4, 1}, + {HeadDim::k512, 16, 2, 0, Dtype::kFp16, 32, 32, 8, 16, 4, 1, 0, 1, 1, 2}, + {HeadDim::k512, 16, 2, 0, Dtype::kBf16, 16, 16, 16, 16, 4, 1, 0, 1, 4, 1}, + {HeadDim::k512, 16, 2, 0, Dtype::kBf16, 16, 32, 8, 16, 4, 1, 0, 1, 1, 2}, + {HeadDim::k512, 16, 2, 0, Dtype::kBf16, 32, 16, 16, 16, 4, 1, 0, 1, 4, 1}, + {HeadDim::k512, 16, 2, 0, Dtype::kBf16, 32, 32, 8, 16, 4, 1, 0, 1, 1, 2}, + {HeadDim::k512, 32, 4, 0, Dtype::kFp16, 16, 16, 16, 16, 4, 1, 0, 1, 1, 1}, + {HeadDim::k512, 32, 4, 0, Dtype::kFp16, 16, 32, 8, 16, 4, 1, 0, 1, 16, 2}, + {HeadDim::k512, 32, 4, 0, Dtype::kFp16, 32, 16, 16, 16, 4, 1, 0, 1, 1, 1}, + {HeadDim::k512, 32, 4, 0, Dtype::kFp16, 32, 32, 8, 16, 4, 1, 0, 1, 16, 2}, + {HeadDim::k512, 32, 4, 0, Dtype::kBf16, 16, 16, 16, 16, 4, 1, 0, 1, 1, 1}, + {HeadDim::k512, 32, 4, 0, Dtype::kBf16, 16, 32, 8, 16, 4, 1, 0, 1, 16, 2}, + {HeadDim::k512, 32, 4, 0, Dtype::kBf16, 32, 16, 16, 16, 4, 1, 0, 1, 1, 1}, + {HeadDim::k512, 32, 4, 0, Dtype::kBf16, 32, 32, 8, 16, 4, 1, 0, 1, 16, 2}, +}; // clang-format on +// The table of GPU family A. +template +constexpr const auto& prefill_configs() { + if constexpr (A == Arch::kGfx1151) + return kGfx1151PrefillConfigs; + else + return kGfx1150PrefillConfigs; +} + +// The tables by Arch. +inline constexpr std::span kPrefillConfigs[kNumArches] = { + kGfx1151PrefillConfigs, kGfx1150PrefillConfigs}; + } // namespace rdna diff --git a/csrc/rocm/rdna_registry.cu b/csrc/rocm/rdna_registry.cu index ecabc5ccfed4..dd29f9bfae88 100644 --- a/csrc/rocm/rdna_registry.cu +++ b/csrc/rocm/rdna_registry.cu @@ -1,6 +1,7 @@ -// Torch ops over the RDNA3.5 attention kernels built into _rocm_C. +// Torch ops over the RDNA attention kernels built into _rocm_C. // rdna_*_configs list a table's entries and -// rdna_{decode,prefill}_attn launch a build by its index. +// rdna_{decode,prefill}_attn launch a build by its index, both in the tables +// of the current device's GPU family (rdna_attn_types.h). #include #include #include @@ -18,30 +19,88 @@ #include "rdna_prefill_batch_configs.h" #include "rdna_prefill_configs.h" -rdna::LaunchFn* rdna::decode_launchers() { - static LaunchFn launchers[std::size(kDecodeConfigs)] = {}; - return launchers; +rdna::LaunchFn* rdna::decode_launchers(Arch arch) { + static LaunchFn gfx1151[std::size(kGfx1151DecodeConfigs)] = {}; + static LaunchFn gfx1150[std::size(kGfx1150DecodeConfigs)] = {}; + return arch == Arch::kGfx1151 ? gfx1151 : gfx1150; } -rdna::LaunchFn* rdna::decode_batch_launchers() { - static LaunchFn launchers[std::size(kDecodeBatchConfigs)] = {}; - return launchers; +rdna::LaunchFn* rdna::decode_batch_launchers(Arch arch) { + static LaunchFn gfx1151[std::size(kGfx1151DecodeBatchConfigs)] = {}; + static LaunchFn gfx1150[std::size(kGfx1150DecodeBatchConfigs)] = {}; + return arch == Arch::kGfx1151 ? gfx1151 : gfx1150; } -rdna::PrefillFn* rdna::prefill_launchers() { - static PrefillFn launchers[std::size(kPrefillConfigs)] = {}; - return launchers; +rdna::PrefillFn* rdna::prefill_launchers(Arch arch) { + static PrefillFn gfx1151[std::size(kGfx1151PrefillConfigs)] = {}; + static PrefillFn gfx1150[std::size(kGfx1150PrefillConfigs)] = {}; + return arch == Arch::kGfx1151 ? gfx1151 : gfx1150; } namespace { -constexpr int64_t kNumDecodeConfigs = std::size(rdna::kDecodeConfigs); -constexpr int64_t kNumDecodeBatchConfigs = std::size(rdna::kDecodeBatchConfigs); -constexpr int64_t kNumPrefillConfigs = std::size(rdna::kPrefillConfigs); +using rdna::Arch; +using rdna::DecodeConfig; +using rdna::LaunchFn; +using rdna::PrefillBatchConfig; +using rdna::PrefillConfig; +using rdna::PrefillFn; -bool on_gfx1151() { - const std::string arch = at::cuda::getCurrentDeviceProperties()->gcnArchName; - return arch.find("gfx1151") != std::string::npos; +// A GPU family's tables and their launch functions. +struct Table { + const DecodeConfig* decode = nullptr; + int64_t num_decode = 0; + LaunchFn* decode_launchers = nullptr; + const DecodeConfig* decode_batch = nullptr; + int64_t num_decode_batch = 0; + LaunchFn* decode_batch_launchers = nullptr; + const PrefillConfig* prefill = nullptr; + int64_t num_prefill = 0; + PrefillFn* prefill_launchers = nullptr; + const PrefillBatchConfig* prefill_batch = nullptr; + int64_t num_prefill_batch = 0; +}; + +Table table_of(Arch arch) { + const int a = static_cast(arch); + const auto decode = rdna::kDecodeConfigs[a]; + const auto decode_batch = rdna::kDecodeBatchConfigs[a]; + const auto prefill = rdna::kPrefillConfigs[a]; + const auto prefill_batch = rdna::kPrefillBatchConfigs[a]; + return {decode.data(), + static_cast(decode.size()), + rdna::decode_launchers(arch), + decode_batch.data(), + static_cast(decode_batch.size()), + rdna::decode_batch_launchers(arch), + prefill.data(), + static_cast(prefill.size()), + rdna::prefill_launchers(arch), + prefill_batch.data(), + static_cast(prefill_batch.size())}; +} + +// The tables of the current device's GPU family; empty if this build carries +// no code for it. +Table device_table() { + const std::string name = at::cuda::getCurrentDeviceProperties()->gcnArchName; + const auto is = [&](const char* arch) { + return name.find(arch) != std::string::npos; + }; +#ifdef VLLM_RDNA_ATTN_GFX1150 + if (is("gfx1150")) return table_of(Arch::kGfx1150); +#endif +#ifdef VLLM_RDNA_ATTN_GFX1151 + if (is("gfx1151")) return table_of(Arch::kGfx1151); +#endif +#ifdef VLLM_RDNA_ATTN_GFX1152 + if (is("gfx1152")) return table_of(Arch::kGfx1150); +#endif +#ifdef VLLM_RDNA_ATTN_GFX1153 + if (is("gfx1153")) return table_of(Arch::kGfx1150); +#endif + (void)is; + return {}; } // The kernels expect (num_pages, NUM_KV_HEADS, PAGE_SIZE, 2 * HEAD_DIM). @@ -67,13 +126,12 @@ void check_device(const torch::Tensor& q, } // namespace -// Every build of a decode table, its DecodeConfig fields in order; none off -// gfx1151. Raises if a build has no instantiation unit. +// Every build of a decode table, its DecodeConfig fields in order; none on +// other devices. Raises if a build has no instantiation unit. std::vector decode_configs(const rdna::DecodeConfig* configs, int64_t n, rdna::LaunchFn* launchers, const char* kernel) { std::vector fields; - if (!on_gfx1151()) return fields; for (int64_t i = 0; i < n; ++i) { const rdna::DecodeConfig& c = configs[i]; TORCH_CHECK(launchers[i], kernel, " build ", i, " (head dim ", @@ -91,24 +149,25 @@ std::vector decode_configs(const rdna::DecodeConfig* configs, } std::vector rdna_decode_configs() { - return decode_configs(rdna::kDecodeConfigs, kNumDecodeConfigs, - rdna::decode_launchers(), "decode"); + const Table t = device_table(); + return decode_configs(t.decode, t.num_decode, t.decode_launchers, "decode"); } std::vector rdna_decode_batch_configs() { - return decode_configs(rdna::kDecodeBatchConfigs, kNumDecodeBatchConfigs, - rdna::decode_batch_launchers(), "batched decode"); + const Table t = device_table(); + return decode_configs(t.decode_batch, t.num_decode_batch, + t.decode_batch_launchers, "batched decode"); } -// Every prefill build, its PrefillConfig fields in order; none off gfx1151. -// Raises if a build has no instantiation unit. +// Every prefill build of the device's table set, its PrefillConfig fields in +// order; none on other devices. Raises if a build has no instantiation unit. std::vector rdna_prefill_configs() { std::vector fields; - if (!on_gfx1151()) return fields; - for (int64_t i = 0; i < kNumPrefillConfigs; ++i) { - const rdna::PrefillConfig& c = rdna::kPrefillConfigs[i]; - TORCH_CHECK(rdna::prefill_launchers()[i], "prefill build ", i, - " (head dim ", static_cast(c.head_dim), ", bf16 ", + const Table t = device_table(); + for (int64_t i = 0; i < t.num_prefill; ++i) { + const rdna::PrefillConfig& c = t.prefill[i]; + TORCH_CHECK(t.prefill_launchers[i], "prefill build ", i, " (head dim ", + static_cast(c.head_dim), ", bf16 ", c.dtype == rdna::Dtype::kBf16, ") has no unit in cmake/rocm_rdna_attn.cmake"); fields.insert( @@ -121,17 +180,17 @@ std::vector rdna_prefill_configs() { return fields; } -// Every prefill batch cell, its fields in order; none off gfx1151. Raises if -// a cell names no prefill build. +// Every prefill batch cell of the device's table set, its fields in order; +// none on other devices. Raises if a cell names no prefill build. std::vector rdna_prefill_batch_configs() { using namespace rdna; std::vector fields; - if (!on_gfx1151()) return fields; - for (const PrefillBatchConfig& b : kPrefillBatchConfigs) { + const Table t = device_table(); + for (int64_t i = 0; i < t.num_prefill_batch; ++i) { + const PrefillBatchConfig& b = t.prefill_batch[i]; for (Dtype dtype : {Dtype::kFp16, Dtype::kBf16}) { const bool built = std::any_of( - std::begin(kPrefillConfigs), std::end(kPrefillConfigs), - [&](const PrefillConfig& c) { + t.prefill, t.prefill + t.num_prefill, [&](const PrefillConfig& c) { return c.head_dim == b.head_dim && c.num_q_heads == b.num_q_heads && c.num_kv_heads == b.num_kv_heads && c.window == b.window && c.page_size == b.page_size && c.dtype == dtype && @@ -159,14 +218,13 @@ void rdna_decode_attn(int64_t config, torch::Tensor& q, torch::Tensor& kv_cache, const std::optional& mm_ranges, bool batched) { using namespace rdna; - LaunchFn* launchers = batched ? decode_batch_launchers() : decode_launchers(); - TORCH_CHECK( - config >= 0 && - config < (batched ? kNumDecodeBatchConfigs : kNumDecodeConfigs) && - launchers[config], - "no ", batched ? "batched " : "", "decode build ", config); - const DecodeConfig& c = - batched ? kDecodeBatchConfigs[config] : kDecodeConfigs[config]; + const Table t = device_table(); + LaunchFn* launchers = batched ? t.decode_batch_launchers : t.decode_launchers; + TORCH_CHECK(config >= 0 && + config < (batched ? t.num_decode_batch : t.num_decode) && + launchers[config], + "no ", batched ? "batched " : "", "decode build ", config); + const DecodeConfig& c = batched ? t.decode_batch[config] : t.decode[config]; const int maxm = c.max_query_len; TORCH_CHECK(q.is_contiguous() && out.is_contiguous(), "q and out must be contiguous"); @@ -269,10 +327,11 @@ void rdna_prefill_attn(int64_t config, torch::Tensor& q, const std::optional& query_start_loc, int64_t head_group) { using namespace rdna; + const Table t = device_table(); TORCH_CHECK( - config >= 0 && config < kNumPrefillConfigs && prefill_launchers()[config], + config >= 0 && config < t.num_prefill && t.prefill_launchers[config], "no prefill build ", config); - const PrefillConfig& c = kPrefillConfigs[config]; + const PrefillConfig& c = t.prefill[config]; TORCH_CHECK(q.is_contiguous() && out.is_contiguous(), "q and out must be contiguous"); TORCH_CHECK( @@ -349,7 +408,7 @@ void rdna_prefill_attn(int64_t config, torch::Tensor& q, "scratch too small for this build: needs ", rows * d, ", ", 2 * rows, ", ", target, " elements"); } - prefill_launchers()[config]( + t.prefill_launchers[config]( {q.data_ptr(), kv_cache.data_ptr(), block_table.data_ptr(), out.data_ptr(), seq_lens.data_ptr(), partial_o.data_ptr(), partial_ml.data_ptr(), counters.data_ptr(), (int)q.size(0), diff --git a/tests/kernels/attention/test_rocm_rdna_hip.py b/tests/kernels/attention/test_rocm_rdna_hip.py index a3996edaa2c2..c4f7e39dca13 100644 --- a/tests/kernels/attention/test_rocm_rdna_hip.py +++ b/tests/kernels/attention/test_rocm_rdna_hip.py @@ -5,9 +5,11 @@ import functools import itertools +from pathlib import Path import numpy as np import pytest +import regex as re import torch from vllm.platforms import current_platform @@ -22,11 +24,34 @@ DTYPES = list(TOL) -def _skip_unless_gfx1151(): - from vllm.platforms.rocm import on_gfx1151 +_CONFIG_ENTRY = re.compile(r"^\s*\{([\w:, ]+)\},$") - if not on_gfx1151(): - pytest.skip("kernel is built for gfx1151") + +@pytest.mark.parametrize( + "kind, shape_fields", + [("decode", 7), ("decode_batch", 7), ("prefill", 6), ("prefill_batch", 5)], +) +def test_gfx1150_configs_cover_the_same_shapes(kind, shape_fields): + """gfx1150's tables serve the shapes gfx1151's serve.""" + csrc = Path(__file__).resolve().parents[3] / "csrc" / "rocm" + text = (csrc / f"rdna_{kind}_configs.h").read_text() + shapes = {} + for table in text.split("inline constexpr ")[1:]: + if name := re.match(r"\w+ k(Gfx\d+)\w*Configs\[\] = \{", table): + entries = [ + m[1].split(", ") + for m in map(_CONFIG_ENTRY.match, table.split("\n")) + if m + ] + shapes[name[1]] = {tuple(e[:shape_fields]) for e in entries} + assert shapes["Gfx1151"] and shapes["Gfx1151"] == shapes["Gfx1150"] + + +def _skip_unless_gfx115x(): + from vllm.platforms.rocm import on_gfx115x + + if not on_gfx115x(): + pytest.skip("kernels are built for gfx1150..gfx1153") def _built(dtype=torch.float16, batch=0, **want): @@ -41,7 +66,7 @@ def _built(dtype=torch.float16, batch=0, **want): (variant, module). """ - _skip_unless_gfx1151() + _skip_unless_gfx115x() for v in hip.variants(): if ( v.dtype == dtype @@ -175,7 +200,7 @@ def test_decode_configs_are_well_formed(): def test_every_listed_variant_is_built_in(): - _skip_unless_gfx1151() + _skip_unless_gfx115x() # Raises if a build has no instantiation unit. assert torch.ops._rocm_C.rdna_decode_configs() assert torch.ops._rocm_C.rdna_decode_batch_configs() @@ -186,7 +211,7 @@ def test_every_listed_variant_is_built_in(): @pytest.mark.parametrize("unit", sorted({_unit(v) for v in hip.variants()})) def test_built_in_variants_match_reference(unit): """Every built decode variant matches the reference.""" - _skip_unless_gfx1151() + _skip_unless_gfx115x() for v in hip.variants(): if _unit(v) != unit: continue @@ -385,10 +410,11 @@ def test_non_decode_batches_fall_back_to_triton(nseq, tokens, max_q, reason): assert reason in impl._rejected -@pytest.mark.parametrize("gfx1151", [True, False]) +@pytest.mark.parametrize("arch", ["gfx1150", "gfx1151", "gfx1152", "gfx1201"]) @pytest.mark.parametrize("max_num_seqs", [1, 2]) -def test_rdna_first_only_on_gfx1151(gfx1151, max_num_seqs, monkeypatch): - """First on gfx1151 whatever max_num_seqs; elsewhere not a candidate.""" +def test_rdna_first_only_on_gfx1150_and_gfx1151(arch, max_num_seqs, monkeypatch): + """First on gfx1150 and gfx1151 whatever max_num_seqs; elsewhere not a + candidate.""" import types import vllm.config @@ -399,9 +425,10 @@ def test_rdna_first_only_on_gfx1151(gfx1151, max_num_seqs, monkeypatch): scheduler_config=types.SimpleNamespace(max_num_seqs=max_num_seqs) ) monkeypatch.setattr(vllm.config, "get_current_vllm_config_or_none", lambda: config) - monkeypatch.setattr(rocm, "on_gfx1151", lambda: gfx1151) + monkeypatch.setattr(rocm, "on_gfx1150", lambda: arch == "gfx1150") + monkeypatch.setattr(rocm, "on_gfx1151", lambda: arch == "gfx1151") order = rocm._get_backend_priorities(use_mla=False, use_sparse=False) - if gfx1151: + if arch in ("gfx1150", "gfx1151"): assert order[0] == B.ROCM_RDNA_ATTN else: assert B.ROCM_RDNA_ATTN not in order @@ -567,7 +594,7 @@ def test_prefill_configs_are_well_formed(): def test_every_listed_prefill_variant_is_built_in(): - _skip_unless_gfx1151() + _skip_unless_gfx115x() # Raises if a build has no instantiation unit. assert torch.ops._rocm_C.rdna_prefill_configs() missing = [v for v in _prefill_rows() if hip.load_prefill(v) is None] @@ -578,7 +605,7 @@ def test_every_listed_prefill_variant_is_built_in(): def test_built_in_prefill_variants_match_reference(unit): """Every built prefill variant matches the reference, with and without a prefix.""" - _skip_unless_gfx1151() + _skip_unless_gfx115x() for v in _prefill_rows(): if _prefill_unit(v) != unit: continue @@ -615,7 +642,7 @@ def test_built_in_prefill_variants_match_reference(unit): def test_prefill_past_the_last_step(): """M past the last step runs the last row.""" - _skip_unless_gfx1151() + _skip_unless_gfx115x() v = next( rows[-1] for rows in hip._prefill_table().values() @@ -642,7 +669,7 @@ def test_prefill_past_the_last_step(): def test_prefill_window_never_reads_freed_pages(): """Freed pages before the window do not reach the output.""" - _skip_unless_gfx1151() + _skip_unless_gfx115x() v = next(v for v in _prefill_rows() if v.window and v.dtype == torch.float16) module = hip.load_prefill(v) m, seq_len = v.min_query_len, v.min_query_len + 3 * v.window @@ -697,7 +724,7 @@ def test_prefill_plan_covers_every_tile_once(batch, window): """The work list covers every tile once, heaviest first, within the scratch's bounds.""" target = 160 - v = hip.prefill_variants(32, 8, 128, 16, 0, torch.float16)[2] + v = hip.prefill_variant_for(32, 8, 128, 16, 1, 0, torch.float16, 128) # The most counters and partials the bounds allow. lens = ( [(1, 32768)] * 19 + [(16, 16)] if batch == "at the bound" else _BATCHES[batch] @@ -817,7 +844,7 @@ def _serve_batch( ) def test_prefill_batch_mm_prefix_matches_reference(shape): """A batch with each sequence's own multimodal ranges.""" - _skip_unless_gfx1151() + _skip_unless_gfx115x() hq, hkv, d, window, dtype = shape v = hip.prefill_variants(hq, hkv, d, 16, window, dtype)[0] lens = [(600, 600), (1, 2000), (700, 3700), (300, 800)] @@ -845,7 +872,7 @@ def test_prefill_batch_mm_prefix_matches_reference(shape): ) def test_prefill_batch_matches_reference(unit): """A batch in one launch, each sequence against its reference.""" - _skip_unless_gfx1151() + _skip_unless_gfx115x() v = next(v for v in _prefill_rows() if _prefill_unit(v) == unit) for seed, (name, lens) in enumerate(_BATCHES.items()): lens = [(m, s + v.window if m != s else s) for m, s in lens] @@ -881,7 +908,7 @@ def test_prefill_batch_matches_reference(unit): def test_mixed_batch_matches_reference(unit): """Decodes then prefills, including a short one, each against its reference.""" - _skip_unless_gfx1151() + _skip_unless_gfx115x() v = next(v for v in _prefill_rows() if _prefill_unit(v) == unit) batches = [ [(1, 3000 + v.window), (1, 77), (1, 1024), (130, 2000 + v.window), (40, 40)], @@ -1006,7 +1033,7 @@ def test_prefill_batch_needs_this_steps_host_copy(): ) def test_prefill_mm_prefix_matches_reference(shape, m, prefix, mm_ranges): """Tokens of a multimodal range see the whole range (Gemma4's shapes).""" - _skip_unless_gfx1151() + _skip_unless_gfx115x() hq, hkv, d, window, dtype = shape v = hip.prefill_variant_for(hq, hkv, d, 16, 1, window, dtype, m) assert v is not None, shape @@ -1040,7 +1067,7 @@ def test_prefill_mm_prefix_matches_reference(shape, m, prefix, mm_ranges): def test_prefill_refuses_malformed_mm_ranges(bad): """Only one sequence's (ranges, 2) on the device reaches the kernel: the batch's (sequences, ranges, 2) would unmask other sequences' keys.""" - _skip_unless_gfx1151() + _skip_unless_gfx115x() v = hip.prefill_variant_for(32, 16, 256, 16, 1, 1024, torch.float16, 64) module = hip.load_prefill(v) scratch = hip.make_prefill_scratch([v], torch.device("cuda")) diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index ccb737b71927..2bb1dd32386d 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -494,8 +494,9 @@ def _get_backend_priorities( ] backends = [] - # Its validation rejects what it does not support. - if on_gfx1151(): + # Its validation rejects what it does not support. gfx1152 and gfx1153 can + # select it but are not tested yet. + if on_gfx1150() or on_gfx1151(): backends.append(AttentionBackendEnum.ROCM_RDNA_ATTN) if rocm_aiter_ops.is_mha_enabled(): backends.append(AttentionBackendEnum.ROCM_AITER_FA) diff --git a/vllm/v1/attention/backends/rocm_rdna_attn.py b/vllm/v1/attention/backends/rocm_rdna_attn.py index 571727a7774b..d50388a4f0c1 100644 --- a/vllm/v1/attention/backends/rocm_rdna_attn.py +++ b/vllm/v1/attention/backends/rocm_rdna_attn.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Attention for gfx1151. +"""Attention for gfx1150..gfx1153. Decodes of up to MAX_M query tokens per sequence run csrc/rocm/rdna_decode_attn.cu, other batches @@ -57,12 +57,12 @@ logger = init_logger(__name__) -def _on_gfx1151() -> bool: +def _on_gfx115x() -> bool: if not current_platform.is_rocm(): return False - from vllm.platforms.rocm import on_gfx1151 + from vllm.platforms.rocm import on_gfx115x - return on_gfx1151() + return on_gfx115x() # A kernel and its scratch buffers. @@ -285,7 +285,7 @@ def supports_attn_type(cls, attn_type: str) -> bool: @classmethod def supports_compute_capability(cls, capability: "DeviceCapability") -> bool: - return _on_gfx1151() + return _on_gfx115x() @classmethod def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]: diff --git a/vllm/v1/attention/ops/rocm_rdna_hip.py b/vllm/v1/attention/ops/rocm_rdna_hip.py index ed56cf706b38..7b1391f28f6e 100644 --- a/vllm/v1/attention/ops/rocm_rdna_hip.py +++ b/vllm/v1/attention/ops/rocm_rdna_hip.py @@ -4,7 +4,9 @@ Each entry of ``csrc/rocm/rdna_{decode,prefill}_configs.h`` is one build, with its shape and launch knobs as compile-time constants; those of -``rdna_decode_batch_configs.h`` serve a batch. +``rdna_decode_batch_configs.h`` serve a batch. Each header has a table per GPU +family: gfx1151's (1536 VGPRs a SIMD lane) and gfx1150's (1024), which gfx1152 +and gfx1153 also use. _rocm_C lists the builds of the current device's tables. """ import contextlib From 0a3b7dbdf28848d2a2eacbe4ecfab9d5794cd168 Mon Sep 17 00:00:00 2001 From: Robert Esclapez Garcia Date: Sat, 24 Oct 2026 01:33:30 -0700 Subject: [PATCH 4/6] [ROCm][RDNA] ROCM_RDNA_ATTN: gfx1150's batched tables gfx1150's batched decode and prefill batch tables were gfx1151's. Tuned on the low-latency gfx1150 boards (fp16 measured, bf16 shares the knobs) and written with tools/tune.py --apply: - batched decode: 171 of 264 (configuration, M) accepted against the current entry, 1.087x geomean over the accepted. - prefill batch: per (mean M, keys) cell, the fastest prefill entry and head group on uniform batches. D=512 keeps the previous cells: in mixed batches (many decodes and one long prefill) its tuned cells were up to 2.8x slower, as a cell's batches are uniform. Against the previous tables in the kernel matrix (matrix.py --batch, 33 configurations, each table set timed twice on three gfx1150 boards, median spread 0.1%): - batched decode (10 batches x M 1..8): 2640 cells, 1.047x, 916 faster and 16 slower by more than 2% (down to 0.95x); up to 18x where gfx1151's entries were far off (16/1/512 and 16/2/512, 64 sequences, M 6..8). - prefill batches, D < 512: 140 cells, 1.112x (0.90x-3.77x). - mixed batches, D < 512: 168 cells, 1.013x (0.83x-1.82x). AI assistance was used for this patch. Co-authored-by: Claude Signed-off-by: Robert Esclapez Garcia --- csrc/rocm/rdna_decode_batch_configs.h | 936 ++++++++++++------------ csrc/rocm/rdna_prefill_batch_configs.h | 940 ++++++++++++------------- 2 files changed, 938 insertions(+), 938 deletions(-) diff --git a/csrc/rocm/rdna_decode_batch_configs.h b/csrc/rocm/rdna_decode_batch_configs.h index e9d7f3c0be88..e7eb3b9859a3 100644 --- a/csrc/rocm/rdna_decode_batch_configs.h +++ b/csrc/rocm/rdna_decode_batch_configs.h @@ -691,52 +691,52 @@ inline constexpr DecodeConfig kGfx1151DecodeBatchConfigs[] = { inline constexpr DecodeConfig kGfx1150DecodeBatchConfigs[] = { {HeadDim::k64, 14, 2, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, {HeadDim::k64, 14, 2, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 14, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 2, 8, 0, 1, 0, 0}, - {HeadDim::k64, 14, 2, 0, 2, Dtype::kBf16, 16, 3, 1, 2, 8, 0, 1, 0, 0}, - {HeadDim::k64, 14, 2, 0, 3, Dtype::kFp16, 16, 4, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 14, 2, 0, 3, Dtype::kBf16, 16, 4, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 14, 2, 0, 4, Dtype::kFp16, 16, 4, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 14, 2, 0, 4, Dtype::kBf16, 16, 4, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 14, 2, 0, 5, Dtype::kFp16, 16, 5, 1, 2, 8, 0, 0, 1, 0}, - {HeadDim::k64, 14, 2, 0, 5, Dtype::kBf16, 16, 5, 1, 2, 8, 0, 0, 1, 0}, - {HeadDim::k64, 14, 2, 0, 6, Dtype::kFp16, 16, 5, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 14, 2, 0, 6, Dtype::kBf16, 16, 5, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 14, 2, 0, 7, Dtype::kFp16, 16, 5, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k64, 14, 2, 0, 7, Dtype::kBf16, 16, 5, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k64, 14, 2, 0, 8, Dtype::kFp16, 16, 5, 2, 2, 8, 0, 0, 1, 0}, - {HeadDim::k64, 14, 2, 0, 8, Dtype::kBf16, 16, 5, 2, 2, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 14, 2, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 7, Dtype::kFp16, 16, 2, 1, 2, 8, 0, 0, 0, 0}, + {HeadDim::k64, 14, 2, 0, 7, Dtype::kBf16, 16, 2, 1, 2, 8, 0, 0, 0, 0}, + {HeadDim::k64, 14, 2, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 14, 2, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 1, 0, 0}, {HeadDim::k64, 16, 2, 0, 1, Dtype::kFp16, 16, 2, 1, 4, 8, 0, 1, 0, 0}, {HeadDim::k64, 16, 2, 0, 1, Dtype::kBf16, 16, 2, 1, 4, 8, 0, 1, 0, 0}, - {HeadDim::k64, 16, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 16, 2, 0, 2, Dtype::kBf16, 16, 3, 1, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 16, 2, 0, 3, Dtype::kFp16, 16, 2, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k64, 16, 2, 0, 3, Dtype::kBf16, 16, 2, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k64, 16, 2, 0, 4, Dtype::kFp16, 16, 3, 2, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 16, 2, 0, 4, Dtype::kBf16, 16, 3, 2, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 3, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 3, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 16, 2, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 0, 1, 0}, {HeadDim::k64, 16, 2, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, {HeadDim::k64, 16, 2, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 16, 2, 0, 6, Dtype::kFp16, 16, 1, 3, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 16, 2, 0, 6, Dtype::kBf16, 16, 1, 3, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 16, 2, 0, 7, Dtype::kFp16, 16, 2, 4, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 16, 2, 0, 7, Dtype::kBf16, 16, 2, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 16, 2, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, {HeadDim::k64, 16, 2, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, {HeadDim::k64, 16, 2, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 1, 0, 0}, {HeadDim::k64, 32, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, {HeadDim::k64, 32, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 32, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 32, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 2, Dtype::kFp16, 16, 2, 4, 4, 2, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 2, Dtype::kBf16, 16, 2, 4, 4, 2, 0, 1, 0, 0}, {HeadDim::k64, 32, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k64, 32, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k64, 32, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 32, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 1, 0, 0}, - {HeadDim::k64, 32, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 32, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 32, 8, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 32, 8, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 32, 8, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 32, 8, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 32, 8, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, - {HeadDim::k64, 32, 8, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 1, 0}, + {HeadDim::k64, 32, 8, 0, 4, Dtype::kFp16, 16, 2, 4, 4, 2, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 4, Dtype::kBf16, 16, 2, 4, 4, 2, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 5, Dtype::kFp16, 16, 2, 4, 2, 2, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 5, Dtype::kBf16, 16, 2, 4, 2, 2, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 6, Dtype::kFp16, 16, 2, 4, 2, 2, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 6, Dtype::kBf16, 16, 2, 4, 2, 2, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 7, Dtype::kFp16, 16, 2, 4, 1, 2, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 7, Dtype::kBf16, 16, 2, 4, 1, 2, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 8, Dtype::kFp16, 16, 2, 4, 2, 2, 0, 1, 0, 0}, + {HeadDim::k64, 32, 8, 0, 8, Dtype::kBf16, 16, 2, 4, 2, 2, 0, 1, 0, 0}, {HeadDim::k64, 32, 32, 0, 1, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, {HeadDim::k64, 32, 32, 0, 1, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, {HeadDim::k64, 32, 32, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, @@ -759,56 +759,56 @@ inline constexpr DecodeConfig kGfx1150DecodeBatchConfigs[] = { {HeadDim::k96, 32, 32, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, {HeadDim::k96, 32, 32, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, {HeadDim::k96, 32, 32, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, - {HeadDim::k96, 32, 32, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, - {HeadDim::k96, 32, 32, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 2, 0, 0, 0, 0}, {HeadDim::k96, 32, 32, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, {HeadDim::k96, 32, 32, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, - {HeadDim::k96, 32, 32, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 1, 0, 0}, - {HeadDim::k96, 32, 32, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 1, 0, 0}, + {HeadDim::k96, 32, 32, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 2, 0, 0, 0, 0}, {HeadDim::k96, 32, 32, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, {HeadDim::k96, 32, 32, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, - {HeadDim::k96, 32, 32, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, - {HeadDim::k96, 32, 32, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, - {HeadDim::k128, 10, 10, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, - {HeadDim::k128, 10, 10, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k96, 32, 32, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 2, 0, 0, 0, 0}, + {HeadDim::k96, 32, 32, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 2, 0, 0, 0, 0}, + {HeadDim::k128, 10, 10, 0, 1, Dtype::kFp16, 16, 3, 1, 2, 4, 0, 0, 0, 0}, + {HeadDim::k128, 10, 10, 0, 1, Dtype::kBf16, 16, 3, 1, 2, 4, 0, 0, 0, 0}, {HeadDim::k128, 10, 10, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, {HeadDim::k128, 10, 10, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, - {HeadDim::k128, 10, 10, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, - {HeadDim::k128, 10, 10, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 3, Dtype::kFp16, 16, 3, 1, 2, 4, 0, 0, 0, 0}, + {HeadDim::k128, 10, 10, 0, 3, Dtype::kBf16, 16, 3, 1, 2, 4, 0, 0, 0, 0}, {HeadDim::k128, 10, 10, 0, 4, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, {HeadDim::k128, 10, 10, 0, 4, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, - {HeadDim::k128, 10, 10, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, - {HeadDim::k128, 10, 10, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, - {HeadDim::k128, 10, 10, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, - {HeadDim::k128, 10, 10, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 5, Dtype::kFp16, 16, 3, 1, 4, 4, 0, 0, 0, 0}, + {HeadDim::k128, 10, 10, 0, 5, Dtype::kBf16, 16, 3, 1, 4, 4, 0, 0, 0, 0}, + {HeadDim::k128, 10, 10, 0, 6, Dtype::kFp16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 10, 10, 0, 6, Dtype::kBf16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 10, 10, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 10, 10, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 10, 10, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, {HeadDim::k128, 10, 10, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, {HeadDim::k128, 16, 2, 0, 1, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, {HeadDim::k128, 16, 2, 0, 1, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k128, 16, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k128, 16, 2, 0, 2, Dtype::kBf16, 16, 3, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k128, 16, 2, 0, 3, Dtype::kFp16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k128, 16, 2, 0, 3, Dtype::kBf16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k128, 16, 2, 0, 4, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k128, 16, 2, 0, 4, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k128, 16, 2, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 16, 2, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 16, 2, 0, 6, Dtype::kFp16, 16, 5, 3, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 16, 2, 0, 6, Dtype::kBf16, 16, 5, 3, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 16, 2, 0, 7, Dtype::kFp16, 16, 3, 4, 4, 4, 0, 0, 1, 0}, - {HeadDim::k128, 16, 2, 0, 7, Dtype::kBf16, 16, 3, 4, 4, 4, 0, 0, 1, 0}, - {HeadDim::k128, 16, 2, 0, 8, Dtype::kFp16, 16, 3, 4, 4, 4, 0, 0, 1, 0}, - {HeadDim::k128, 16, 2, 0, 8, Dtype::kBf16, 16, 3, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 16, 2, 0, 3, Dtype::kFp16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 3, Dtype::kBf16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 8, 2, 1, 0, 0}, + {HeadDim::k128, 16, 2, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 8, 2, 1, 0, 0}, + {HeadDim::k128, 16, 2, 0, 6, Dtype::kFp16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 6, Dtype::kBf16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 2, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 16, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 16, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 16, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 16, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 16, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 16, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 16, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 16, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 16, 8, 0, 4, Dtype::kFp16, 16, 2, 4, 1, 2, 2, 1, 0, 0}, + {HeadDim::k128, 16, 8, 0, 4, Dtype::kBf16, 16, 2, 4, 1, 2, 2, 1, 0, 0}, {HeadDim::k128, 16, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 16, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 16, 8, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, @@ -823,88 +823,88 @@ inline constexpr DecodeConfig kGfx1150DecodeBatchConfigs[] = { {HeadDim::k128, 24, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 24, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 2, 4, 0, 0, 1, 0}, {HeadDim::k128, 24, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k128, 24, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 24, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 24, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 24, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 24, 8, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 24, 8, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 24, 8, 0, 7, Dtype::kFp16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 24, 8, 0, 7, Dtype::kBf16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 24, 8, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 24, 8, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, + {HeadDim::k128, 24, 8, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 2, 2, 1, 0, 0}, + {HeadDim::k128, 24, 8, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 2, 2, 1, 0, 0}, + {HeadDim::k128, 24, 8, 0, 5, Dtype::kFp16, 16, 1, 3, 1, 2, 2, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 5, Dtype::kBf16, 16, 1, 3, 1, 2, 2, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 6, Dtype::kFp16, 16, 1, 3, 1, 2, 2, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 6, Dtype::kBf16, 16, 1, 3, 1, 2, 2, 0, 1, 0}, + {HeadDim::k128, 24, 8, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 2, 0, 0, 0}, + {HeadDim::k128, 24, 8, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 2, 0, 0, 0}, + {HeadDim::k128, 24, 8, 0, 8, Dtype::kFp16, 16, 2, 4, 4, 2, 2, 1, 0, 0}, + {HeadDim::k128, 24, 8, 0, 8, Dtype::kBf16, 16, 2, 4, 4, 2, 2, 1, 0, 0}, {HeadDim::k128, 28, 4, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 28, 4, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 28, 4, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 0}, {HeadDim::k128, 28, 4, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 28, 4, 0, 3, Dtype::kFp16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k128, 28, 4, 0, 3, Dtype::kBf16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k128, 28, 4, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 28, 4, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 28, 4, 0, 5, Dtype::kFp16, 16, 2, 5, 4, 8, 0, 0, 0, 0}, - {HeadDim::k128, 28, 4, 0, 5, Dtype::kBf16, 16, 2, 5, 4, 8, 0, 0, 0, 0}, - {HeadDim::k128, 28, 4, 0, 6, Dtype::kFp16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, - {HeadDim::k128, 28, 4, 0, 6, Dtype::kBf16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, + {HeadDim::k128, 28, 4, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 2, 1, 0, 0}, + {HeadDim::k128, 28, 4, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 2, 1, 0, 0}, + {HeadDim::k128, 28, 4, 0, 4, Dtype::kFp16, 16, 3, 4, 1, 4, 2, 1, 0, 0}, + {HeadDim::k128, 28, 4, 0, 4, Dtype::kBf16, 16, 3, 4, 1, 4, 2, 1, 0, 0}, + {HeadDim::k128, 28, 4, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 6, Dtype::kFp16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 6, Dtype::kBf16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 28, 4, 0, 7, Dtype::kFp16, 16, 1, 7, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 28, 4, 0, 7, Dtype::kBf16, 16, 1, 7, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 28, 4, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 28, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 28, 4, 0, 8, Dtype::kFp16, 16, 2, 4, 4, 4, 2, 1, 0, 0}, + {HeadDim::k128, 28, 4, 0, 8, Dtype::kBf16, 16, 2, 4, 4, 4, 2, 1, 0, 0}, {HeadDim::k128, 32, 2, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 8, 0, 0, 0, 0}, {HeadDim::k128, 32, 2, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 2, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 2, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 3, Dtype::kFp16, 16, 5, 3, 4, 4, 0, 0, 1, 0}, - {HeadDim::k128, 32, 2, 0, 3, Dtype::kBf16, 16, 5, 3, 4, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 2, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 2, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 3, Dtype::kFp16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 3, Dtype::kBf16, 16, 1, 3, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 32, 2, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, {HeadDim::k128, 32, 2, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 6, Dtype::kFp16, 16, 3, 6, 2, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 6, Dtype::kBf16, 16, 3, 6, 2, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 7, Dtype::kFp16, 16, 2, 7, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 7, Dtype::kBf16, 16, 2, 7, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 8, Dtype::kFp16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 2, 0, 8, Dtype::kBf16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 5, Dtype::kFp16, 16, 5, 5, 2, 8, 2, 1, 0, 0}, + {HeadDim::k128, 32, 2, 0, 5, Dtype::kBf16, 16, 5, 5, 2, 8, 2, 1, 0, 0}, + {HeadDim::k128, 32, 2, 0, 6, Dtype::kFp16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 6, Dtype::kBf16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 7, Dtype::kFp16, 16, 1, 8, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 7, Dtype::kBf16, 16, 1, 8, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 2, 0, 8, Dtype::kFp16, 16, 1, 8, 1, 8, 0, 0, 0, 0}, + {HeadDim::k128, 32, 2, 0, 8, Dtype::kBf16, 16, 1, 8, 1, 8, 0, 0, 0, 0}, {HeadDim::k128, 32, 4, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 32, 4, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 32, 4, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 32, 4, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 32, 4, 0, 3, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 4, 0, 3, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 4, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 4, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 4, 0, 5, Dtype::kFp16, 16, 2, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 4, 0, 5, Dtype::kBf16, 16, 2, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 4, 0, 6, Dtype::kFp16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 4, 0, 6, Dtype::kBf16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, - {HeadDim::k128, 32, 4, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 32, 4, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 32, 4, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 32, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 3, Dtype::kFp16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 3, Dtype::kBf16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 4, Dtype::kFp16, 16, 3, 4, 2, 4, 2, 1, 0, 0}, + {HeadDim::k128, 32, 4, 0, 4, Dtype::kBf16, 16, 3, 4, 2, 4, 2, 1, 0, 0}, + {HeadDim::k128, 32, 4, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 6, Dtype::kFp16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 6, Dtype::kBf16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 4, 0, 7, Dtype::kFp16, 16, 2, 4, 2, 4, 2, 1, 0, 0}, + {HeadDim::k128, 32, 4, 0, 7, Dtype::kBf16, 16, 2, 4, 2, 4, 2, 1, 0, 0}, + {HeadDim::k128, 32, 4, 0, 8, Dtype::kFp16, 16, 3, 4, 1, 4, 2, 1, 0, 0}, + {HeadDim::k128, 32, 4, 0, 8, Dtype::kBf16, 16, 3, 4, 1, 4, 2, 1, 0, 0}, {HeadDim::k128, 32, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, {HeadDim::k128, 32, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, {HeadDim::k128, 32, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 32, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 32, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 32, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 32, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 32, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 32, 8, 0, 5, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 32, 8, 0, 5, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 32, 8, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 32, 8, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 32, 8, 0, 7, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 32, 8, 0, 7, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 32, 8, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 32, 8, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 32, 8, 0, 4, Dtype::kFp16, 16, 2, 4, 2, 2, 2, 1, 0, 0}, + {HeadDim::k128, 32, 8, 0, 4, Dtype::kBf16, 16, 2, 4, 2, 2, 2, 1, 0, 0}, + {HeadDim::k128, 32, 8, 0, 5, Dtype::kFp16, 16, 2, 4, 2, 2, 2, 1, 0, 0}, + {HeadDim::k128, 32, 8, 0, 5, Dtype::kBf16, 16, 2, 4, 2, 2, 2, 1, 0, 0}, + {HeadDim::k128, 32, 8, 0, 6, Dtype::kFp16, 16, 2, 4, 1, 2, 2, 1, 0, 0}, + {HeadDim::k128, 32, 8, 0, 6, Dtype::kBf16, 16, 2, 4, 1, 2, 2, 1, 0, 0}, + {HeadDim::k128, 32, 8, 0, 7, Dtype::kFp16, 16, 3, 2, 1, 8, 2, 0, 0, 0}, + {HeadDim::k128, 32, 8, 0, 7, Dtype::kBf16, 16, 3, 2, 1, 8, 2, 0, 0, 0}, + {HeadDim::k128, 32, 8, 0, 8, Dtype::kFp16, 16, 2, 4, 1, 2, 2, 1, 0, 0}, + {HeadDim::k128, 32, 8, 0, 8, Dtype::kBf16, 16, 2, 4, 1, 2, 2, 1, 0, 0}, {HeadDim::k128, 32, 32, 0, 1, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, {HeadDim::k128, 32, 32, 0, 1, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 0, 0}, {HeadDim::k128, 32, 32, 0, 2, Dtype::kFp16, 16, 1, 1, 2, 4, 0, 0, 0, 0}, {HeadDim::k128, 32, 32, 0, 2, Dtype::kBf16, 16, 1, 1, 2, 4, 0, 0, 0, 0}, {HeadDim::k128, 32, 32, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, {HeadDim::k128, 32, 32, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, - {HeadDim::k128, 32, 32, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, - {HeadDim::k128, 32, 32, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 32, 32, 0, 4, Dtype::kFp16, 16, 2, 1, 2, 2, 0, 0, 0, 0}, + {HeadDim::k128, 32, 32, 0, 4, Dtype::kBf16, 16, 2, 1, 2, 2, 0, 0, 0, 0}, {HeadDim::k128, 32, 32, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, {HeadDim::k128, 32, 32, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, {HeadDim::k128, 32, 32, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, @@ -919,216 +919,216 @@ inline constexpr DecodeConfig kGfx1150DecodeBatchConfigs[] = { {HeadDim::k128, 40, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 40, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k128, 40, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k128, 40, 8, 0, 4, Dtype::kFp16, 16, 3, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 40, 8, 0, 4, Dtype::kBf16, 16, 3, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 40, 8, 0, 5, Dtype::kFp16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 40, 8, 0, 5, Dtype::kBf16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 40, 8, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 40, 8, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 0, 0}, - {HeadDim::k128, 40, 8, 0, 7, Dtype::kFp16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 40, 8, 0, 7, Dtype::kBf16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k128, 40, 8, 0, 8, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k128, 40, 8, 0, 8, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 4, Dtype::kFp16, 16, 2, 2, 1, 8, 2, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 4, Dtype::kBf16, 16, 2, 2, 1, 8, 2, 0, 0, 0}, + {HeadDim::k128, 40, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 2, 2, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 2, 2, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 6, Dtype::kFp16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 6, Dtype::kBf16, 16, 1, 6, 1, 4, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 7, Dtype::kFp16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 7, Dtype::kBf16, 16, 1, 5, 1, 2, 0, 0, 1, 0}, + {HeadDim::k128, 40, 8, 0, 8, Dtype::kFp16, 16, 2, 4, 4, 2, 2, 1, 0, 0}, + {HeadDim::k128, 40, 8, 0, 8, Dtype::kBf16, 16, 2, 4, 4, 2, 2, 1, 0, 0}, {HeadDim::k256, 8, 1, 0, 1, Dtype::kFp16, 16, 8, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 1, 0, 1, Dtype::kBf16, 16, 8, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 0, 2, Dtype::kFp16, 16, 5, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 0, 2, Dtype::kBf16, 16, 5, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 0, 3, Dtype::kFp16, 16, 8, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 3, Dtype::kBf16, 16, 8, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 4, Dtype::kFp16, 16, 5, 2, 2, 4, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 4, Dtype::kBf16, 16, 5, 2, 2, 4, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 5, Dtype::kFp16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 5, Dtype::kBf16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 6, Dtype::kFp16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 6, Dtype::kBf16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 7, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 7, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 8, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 0, 8, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 512, 1, Dtype::kFp16, 16, 5, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 1, Dtype::kFp16, 32, 5, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 1, Dtype::kBf16, 16, 5, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 1, Dtype::kBf16, 32, 5, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 2, Dtype::kFp16, 16, 5, 1, 1, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 1, 512, 2, Dtype::kFp16, 32, 5, 1, 1, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 1, 512, 2, Dtype::kBf16, 16, 5, 1, 1, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 1, 512, 2, Dtype::kBf16, 32, 5, 1, 1, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 1, 512, 3, Dtype::kFp16, 16, 3, 3, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 3, Dtype::kFp16, 32, 3, 3, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 3, Dtype::kBf16, 16, 3, 3, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 3, Dtype::kBf16, 32, 3, 3, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 4, Dtype::kFp16, 16, 5, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 4, Dtype::kFp16, 32, 5, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 4, Dtype::kBf16, 16, 5, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 4, Dtype::kBf16, 32, 5, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 5, Dtype::kFp16, 16, 8, 5, 4, 4, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 512, 5, Dtype::kFp16, 32, 8, 5, 4, 4, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 512, 5, Dtype::kBf16, 16, 8, 5, 4, 4, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 512, 5, Dtype::kBf16, 32, 8, 5, 4, 4, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 512, 6, Dtype::kFp16, 16, 3, 3, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 6, Dtype::kFp16, 32, 3, 3, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 6, Dtype::kBf16, 16, 3, 3, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 6, Dtype::kBf16, 32, 3, 3, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 1, 512, 7, Dtype::kFp16, 16, 4, 4, 1, 4, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 512, 7, Dtype::kFp16, 32, 4, 4, 1, 4, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 512, 7, Dtype::kBf16, 16, 4, 4, 1, 4, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 512, 7, Dtype::kBf16, 32, 4, 4, 1, 4, 0, 0, 0, 0}, - {HeadDim::k256, 8, 1, 512, 8, Dtype::kFp16, 16, 4, 4, 1, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 1, 512, 8, Dtype::kFp16, 32, 4, 4, 1, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 1, 512, 8, Dtype::kBf16, 16, 4, 4, 1, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 1, 512, 8, Dtype::kBf16, 32, 4, 4, 1, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 2, 0, 1, Dtype::kFp16, 16, 4, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 1, Dtype::kFp16, 544, 4, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 1, Dtype::kBf16, 16, 4, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 1, Dtype::kBf16, 544, 4, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 2, Dtype::kFp16, 544, 3, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 2, Dtype::kBf16, 16, 3, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 2, Dtype::kBf16, 544, 3, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 3, Dtype::kFp16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 3, Dtype::kFp16, 544, 4, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 3, Dtype::kBf16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 3, Dtype::kBf16, 544, 4, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 4, Dtype::kFp16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 4, Dtype::kFp16, 544, 3, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 4, Dtype::kBf16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 4, Dtype::kBf16, 544, 3, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 0, 5, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 5, Dtype::kFp16, 544, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 5, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 5, Dtype::kBf16, 544, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 6, Dtype::kFp16, 16, 2, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 6, Dtype::kFp16, 544, 2, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 6, Dtype::kBf16, 16, 2, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 6, Dtype::kBf16, 544, 2, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 7, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 7, Dtype::kFp16, 544, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 7, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 7, Dtype::kBf16, 544, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 8, Dtype::kFp16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 8, Dtype::kFp16, 544, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 8, Dtype::kBf16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 0, 8, Dtype::kBf16, 544, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 2, 512, 1, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 1, Dtype::kFp16, 32, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 1, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 1, Dtype::kBf16, 32, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 2, Dtype::kFp16, 16, 3, 1, 2, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 2, 512, 2, Dtype::kFp16, 32, 3, 1, 2, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 2, 512, 2, Dtype::kBf16, 16, 3, 1, 2, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 2, 512, 2, Dtype::kBf16, 32, 3, 1, 2, 4, 4, 1, 0, 0}, - {HeadDim::k256, 8, 2, 512, 3, Dtype::kFp16, 16, 2, 2, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 3, Dtype::kFp16, 32, 2, 2, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 3, Dtype::kBf16, 16, 2, 2, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 3, Dtype::kBf16, 32, 2, 2, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 4, Dtype::kFp16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 4, Dtype::kFp16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 4, Dtype::kBf16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 4, Dtype::kBf16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 5, Dtype::kFp16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 5, Dtype::kFp16, 32, 2, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 5, Dtype::kBf16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 5, Dtype::kBf16, 32, 2, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 6, Dtype::kFp16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 6, Dtype::kFp16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 6, Dtype::kBf16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 6, Dtype::kBf16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 7, Dtype::kFp16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 7, Dtype::kFp16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 7, Dtype::kBf16, 16, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 7, Dtype::kBf16, 32, 2, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 8, Dtype::kFp16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 8, Dtype::kFp16, 32, 2, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 8, Dtype::kBf16, 16, 2, 2, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 2, 512, 8, Dtype::kBf16, 32, 2, 2, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 1, 0, 2, Dtype::kFp16, 16, 2, 1, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 2, Dtype::kBf16, 16, 2, 1, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 3, Dtype::kFp16, 16, 2, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 3, Dtype::kBf16, 16, 2, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 4, Dtype::kFp16, 16, 2, 1, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 4, Dtype::kBf16, 16, 2, 1, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 5, Dtype::kFp16, 16, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 5, Dtype::kBf16, 16, 2, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 6, Dtype::kFp16, 16, 2, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 6, Dtype::kBf16, 16, 2, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 7, Dtype::kFp16, 16, 2, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 7, Dtype::kBf16, 16, 2, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 8, Dtype::kFp16, 16, 2, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 0, 8, Dtype::kBf16, 16, 2, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kFp16, 16, 8, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kFp16, 32, 8, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kBf16, 16, 8, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 1, Dtype::kBf16, 32, 8, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kFp16, 16, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kFp16, 32, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kBf16, 16, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 2, Dtype::kBf16, 32, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kFp16, 16, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kFp16, 32, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kBf16, 16, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 3, Dtype::kBf16, 32, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kFp16, 16, 1, 2, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kFp16, 32, 1, 2, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kBf16, 16, 1, 2, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 4, Dtype::kBf16, 32, 1, 2, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kFp16, 16, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kFp16, 32, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kBf16, 16, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 5, Dtype::kBf16, 32, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kFp16, 32, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 6, Dtype::kBf16, 32, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kFp16, 16, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kFp16, 32, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kBf16, 16, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 7, Dtype::kBf16, 32, 3, 4, 4, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kFp16, 16, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kFp16, 32, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kBf16, 16, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 1, 512, 8, Dtype::kBf16, 32, 3, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kFp16, 16, 2, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kFp16, 544, 2, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kBf16, 16, 2, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 1, Dtype::kBf16, 544, 2, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kFp16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 2, Dtype::kBf16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kFp16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 3, Dtype::kBf16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kFp16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 4, Dtype::kBf16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kFp16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 5, Dtype::kBf16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kFp16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 6, Dtype::kBf16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kFp16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 7, Dtype::kBf16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kFp16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 0, 8, Dtype::kBf16, 544, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kFp16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kFp16, 32, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kBf16, 16, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 1, Dtype::kBf16, 32, 1, 2, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kFp16, 32, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 2, Dtype::kBf16, 32, 1, 1, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kFp16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kFp16, 32, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kBf16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 3, Dtype::kBf16, 32, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kFp16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kFp16, 32, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kBf16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 4, Dtype::kBf16, 32, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kFp16, 16, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kFp16, 32, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kBf16, 16, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 5, Dtype::kBf16, 32, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kFp16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kFp16, 32, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kBf16, 16, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 6, Dtype::kBf16, 32, 1, 2, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kFp16, 16, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kFp16, 32, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kBf16, 16, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 7, Dtype::kBf16, 32, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kFp16, 16, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kFp16, 32, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kBf16, 16, 1, 4, 1, 4, 4, 1, 0, 0}, + {HeadDim::k256, 8, 2, 512, 8, Dtype::kBf16, 32, 1, 4, 1, 4, 4, 1, 0, 0}, {HeadDim::k256, 8, 4, 0, 1, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 0, 1, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 0, 2, Dtype::kFp16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 0, 2, Dtype::kBf16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 2, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 2, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 0, 3, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 0, 3, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 0, 4, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 0, 4, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 0, 5, Dtype::kFp16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 0, 5, Dtype::kBf16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 5, Dtype::kFp16, 16, 2, 2, 2, 8, 4, 0, 0, 0}, + {HeadDim::k256, 8, 4, 0, 5, Dtype::kBf16, 16, 2, 2, 2, 8, 4, 0, 0, 0}, {HeadDim::k256, 8, 4, 0, 6, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 0, 6, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 0, 7, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 0, 7, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 0, 8, Dtype::kFp16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 0, 8, Dtype::kBf16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 1024, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 1024, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 4, Dtype::kFp16, 16, 2, 4, 2, 2, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 4, Dtype::kBf16, 16, 2, 4, 2, 2, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 1024, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 1024, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 6, Dtype::kFp16, 16, 2, 4, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 1024, 6, Dtype::kBf16, 16, 2, 4, 1, 2, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 8, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 1024, 8, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 1, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 1, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 2, Dtype::kFp16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 2, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 1, Dtype::kFp16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 8, 4, 4096, 1, Dtype::kBf16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 8, 4, 4096, 2, Dtype::kFp16, 16, 3, 4, 4, 2, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 2, Dtype::kBf16, 16, 3, 4, 4, 2, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 4096, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 4096, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 4, Dtype::kFp16, 16, 3, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 4, 4096, 4, Dtype::kBf16, 16, 3, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 4, 4096, 5, Dtype::kFp16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 8, 4, 4096, 5, Dtype::kBf16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, {HeadDim::k256, 8, 4, 4096, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 8, 4, 4096, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 8, 4, 4096, 8, Dtype::kFp16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 8, 4, 4096, 8, Dtype::kBf16, 16, 1, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 1, Dtype::kFp16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 2, 0, 1, Dtype::kFp16, 1056, 3, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 2, 0, 1, Dtype::kBf16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 2, 0, 1, Dtype::kBf16, 1056, 3, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 2, 0, 2, Dtype::kFp16, 16, 4, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 2, Dtype::kFp16, 1056, 4, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 2, Dtype::kBf16, 16, 4, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 2, Dtype::kBf16, 1056, 4, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 3, Dtype::kFp16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 3, Dtype::kFp16, 1056, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 3, Dtype::kBf16, 16, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 3, Dtype::kBf16, 1056, 2, 2, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 4, Dtype::kFp16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 4, Dtype::kFp16, 1056, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 4, Dtype::kBf16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 4, Dtype::kBf16, 1056, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 5, Dtype::kFp16, 16, 2, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 5, Dtype::kFp16, 1056, 2, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 5, Dtype::kBf16, 16, 2, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 5, Dtype::kBf16, 1056, 2, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 6, Dtype::kFp16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 6, Dtype::kFp16, 1056, 3, 3, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 6, Dtype::kBf16, 16, 3, 3, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 6, Dtype::kBf16, 1056, 3, 3, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 7, Dtype::kFp16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 7, Dtype::kFp16, 1056, 5, 4, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 7, Dtype::kBf16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 7, Dtype::kBf16, 1056, 5, 4, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 8, Dtype::kFp16, 16, 3, 4, 2, 4, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 8, Dtype::kFp16, 1056, 3, 4, 2, 4, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 8, Dtype::kBf16, 16, 3, 4, 2, 4, 0, 0, 0, 0}, - {HeadDim::k256, 16, 2, 0, 8, Dtype::kBf16, 1056, 3, 4, 2, 4, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 1, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 4, 0, 1, Dtype::kFp16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 4, 0, 1, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 4, 0, 1, Dtype::kBf16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 4, 0, 2, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 4, 0, 2, Dtype::kFp16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 4, 0, 2, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 4, 0, 2, Dtype::kBf16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k256, 8, 4, 4096, 7, Dtype::kFp16, 16, 2, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 8, 4, 4096, 7, Dtype::kBf16, 16, 2, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 8, 4, 4096, 8, Dtype::kFp16, 16, 3, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 8, 4, 4096, 8, Dtype::kBf16, 16, 3, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kFp16, 16, 3, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kFp16, 1056, 3, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kBf16, 16, 3, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 1, Dtype::kBf16, 1056, 3, 4, 2, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kFp16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 2, Dtype::kBf16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kFp16, 16, 3, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kFp16, 1056, 3, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kBf16, 16, 3, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 3, Dtype::kBf16, 1056, 3, 4, 4, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kFp16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 4, Dtype::kBf16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kFp16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 5, Dtype::kBf16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kFp16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 6, Dtype::kBf16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kFp16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 7, Dtype::kBf16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kFp16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 2, 0, 8, Dtype::kBf16, 1056, 1, 4, 1, 8, 4, 1, 0, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kFp16, 16, 3, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kFp16, 528, 3, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kBf16, 16, 3, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 1, Dtype::kBf16, 528, 3, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kFp16, 528, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 2, Dtype::kBf16, 528, 1, 4, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 4, 0, 3, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 4, 0, 3, Dtype::kFp16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 4, 0, 3, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, @@ -1137,50 +1137,50 @@ inline constexpr DecodeConfig kGfx1150DecodeBatchConfigs[] = { {HeadDim::k256, 16, 4, 0, 4, Dtype::kFp16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 4, 0, 4, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 4, 0, 4, Dtype::kBf16, 528, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 4, 0, 5, Dtype::kFp16, 16, 2, 1, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 5, Dtype::kFp16, 528, 2, 1, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 5, Dtype::kBf16, 16, 2, 1, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 5, Dtype::kBf16, 528, 2, 1, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 6, Dtype::kFp16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 6, Dtype::kFp16, 528, 2, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 6, Dtype::kBf16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 6, Dtype::kBf16, 528, 2, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 7, Dtype::kFp16, 16, 2, 1, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 7, Dtype::kFp16, 528, 2, 1, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 7, Dtype::kBf16, 16, 2, 1, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 7, Dtype::kBf16, 528, 2, 1, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 8, Dtype::kFp16, 16, 3, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 8, Dtype::kFp16, 528, 3, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 8, Dtype::kBf16, 16, 3, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k256, 16, 4, 0, 8, Dtype::kBf16, 528, 3, 4, 2, 8, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kFp16, 528, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 5, Dtype::kBf16, 528, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kFp16, 528, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 6, Dtype::kBf16, 528, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kFp16, 528, 1, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 7, Dtype::kBf16, 528, 1, 4, 1, 4, 0, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kFp16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kFp16, 528, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kBf16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 16, 4, 0, 8, Dtype::kBf16, 528, 1, 2, 1, 8, 4, 0, 0, 0}, {HeadDim::k256, 16, 8, 0, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 0, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 0, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 0, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 0, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 0, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 0, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 0, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 3, Dtype::kFp16, 16, 2, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 16, 8, 0, 3, Dtype::kBf16, 16, 2, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 16, 8, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 2, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 0, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 0, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 0, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 0, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 6, Dtype::kFp16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 16, 8, 0, 6, Dtype::kBf16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, {HeadDim::k256, 16, 8, 0, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 0, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 0, 8, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 0, 8, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 2, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 2, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 1024, 1, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 1024, 1, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 1024, 2, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 1024, 2, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 1024, 3, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 1024, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 3, Dtype::kFp16, 16, 2, 3, 4, 4, 4, 0, 0, 0}, + {HeadDim::k256, 16, 8, 1024, 3, Dtype::kBf16, 16, 2, 3, 4, 4, 4, 0, 0, 0}, {HeadDim::k256, 16, 8, 1024, 4, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 1024, 4, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 1024, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 16, 8, 1024, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 1024, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 16, 8, 1024, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 4, 0, 1, 0}, + {HeadDim::k256, 16, 8, 1024, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 4, 0, 1, 0}, {HeadDim::k256, 16, 8, 1024, 7, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, {HeadDim::k256, 16, 8, 1024, 7, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, {HeadDim::k256, 16, 8, 1024, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, @@ -1193,26 +1193,26 @@ inline constexpr DecodeConfig kGfx1150DecodeBatchConfigs[] = { {HeadDim::k256, 24, 4, 0, 2, Dtype::kFp16, 784, 2, 1, 2, 4, 0, 0, 1, 0}, {HeadDim::k256, 24, 4, 0, 2, Dtype::kBf16, 16, 2, 1, 2, 4, 0, 0, 1, 0}, {HeadDim::k256, 24, 4, 0, 2, Dtype::kBf16, 784, 2, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k256, 24, 4, 0, 3, Dtype::kFp16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 3, Dtype::kFp16, 784, 2, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 3, Dtype::kBf16, 16, 2, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 3, Dtype::kBf16, 784, 2, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 4, Dtype::kFp16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 4, Dtype::kFp16, 784, 2, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 4, Dtype::kBf16, 16, 2, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 4, Dtype::kBf16, 784, 2, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 5, Dtype::kFp16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 5, Dtype::kFp16, 784, 1, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 5, Dtype::kBf16, 16, 1, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 5, Dtype::kBf16, 784, 1, 5, 1, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 6, Dtype::kFp16, 16, 3, 3, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 6, Dtype::kFp16, 784, 3, 3, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 6, Dtype::kBf16, 16, 3, 3, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 6, Dtype::kBf16, 784, 3, 3, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 7, Dtype::kFp16, 16, 3, 3, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 7, Dtype::kFp16, 784, 3, 3, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 7, Dtype::kBf16, 16, 3, 3, 4, 8, 0, 0, 0, 0}, - {HeadDim::k256, 24, 4, 0, 7, Dtype::kBf16, 784, 3, 3, 4, 8, 0, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kFp16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kFp16, 784, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kBf16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 3, Dtype::kBf16, 784, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kFp16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kFp16, 784, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kBf16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 4, Dtype::kBf16, 784, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kFp16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kFp16, 784, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kBf16, 16, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 5, Dtype::kBf16, 784, 1, 2, 1, 8, 4, 0, 0, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kFp16, 16, 2, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kFp16, 784, 2, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kBf16, 16, 2, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 24, 4, 0, 6, Dtype::kBf16, 784, 2, 4, 2, 4, 4, 1, 0, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kFp16, 16, 1, 2, 1, 8, 4, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kFp16, 784, 1, 2, 1, 8, 4, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kBf16, 16, 1, 2, 1, 8, 4, 0, 1, 0}, + {HeadDim::k256, 24, 4, 0, 7, Dtype::kBf16, 784, 1, 2, 1, 8, 4, 0, 1, 0}, {HeadDim::k256, 24, 4, 0, 8, Dtype::kFp16, 16, 2, 3, 1, 4, 0, 0, 0, 0}, {HeadDim::k256, 24, 4, 0, 8, Dtype::kFp16, 784, 2, 3, 1, 4, 0, 0, 0, 0}, {HeadDim::k256, 24, 4, 0, 8, Dtype::kBf16, 16, 2, 3, 1, 4, 0, 0, 0, 0}, @@ -1225,138 +1225,138 @@ inline constexpr DecodeConfig kGfx1150DecodeBatchConfigs[] = { {HeadDim::k256, 32, 16, 1024, 3, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 32, 16, 1024, 4, Dtype::kFp16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, {HeadDim::k256, 32, 16, 1024, 4, Dtype::kBf16, 16, 1, 1, 2, 2, 0, 0, 1, 0}, - {HeadDim::k256, 32, 16, 1024, 5, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k256, 32, 16, 1024, 5, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k256, 32, 16, 1024, 5, Dtype::kFp16, 16, 3, 1, 2, 4, 4, 0, 0, 0}, + {HeadDim::k256, 32, 16, 1024, 5, Dtype::kBf16, 16, 3, 1, 2, 4, 4, 0, 0, 0}, {HeadDim::k256, 32, 16, 1024, 6, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 32, 16, 1024, 6, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 32, 16, 1024, 7, Dtype::kFp16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 32, 16, 1024, 7, Dtype::kBf16, 16, 1, 1, 1, 4, 0, 0, 1, 0}, {HeadDim::k256, 32, 16, 1024, 8, Dtype::kFp16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, {HeadDim::k256, 32, 16, 1024, 8, Dtype::kBf16, 16, 1, 1, 1, 2, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 1, Dtype::kFp16, 16, 8, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 1, 0, 1, Dtype::kBf16, 16, 8, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 1, 0, 2, Dtype::kFp16, 16, 5, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 2, Dtype::kBf16, 16, 5, 1, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 3, Dtype::kFp16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 3, Dtype::kBf16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 4, Dtype::kFp16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 4, Dtype::kBf16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 5, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 5, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 6, Dtype::kFp16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 6, Dtype::kBf16, 16, 5, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 7, Dtype::kFp16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 7, Dtype::kBf16, 16, 5, 4, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 8, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 1, 0, 8, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 2, 0, 1, Dtype::kFp16, 16, 4, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 2, 0, 1, Dtype::kBf16, 16, 4, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 2, 0, 2, Dtype::kFp16, 16, 3, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 2, 0, 2, Dtype::kBf16, 16, 3, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 2, 0, 3, Dtype::kFp16, 16, 4, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 2, 0, 3, Dtype::kBf16, 16, 4, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 2, 0, 4, Dtype::kFp16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 2, 0, 4, Dtype::kBf16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 2, 0, 5, Dtype::kFp16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 2, 0, 5, Dtype::kBf16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 2, 0, 6, Dtype::kFp16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 2, 0, 6, Dtype::kBf16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 2, 0, 7, Dtype::kFp16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 2, 0, 7, Dtype::kBf16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 8, 2, 0, 8, Dtype::kFp16, 16, 5, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 8, 2, 0, 8, Dtype::kBf16, 16, 5, 2, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 16, 1, 0, 1, Dtype::kFp16, 16, 5, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 1, Dtype::kFp16, 64, 5, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 1, Dtype::kBf16, 16, 5, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 1, Dtype::kBf16, 64, 5, 1, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 2, Dtype::kFp16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 2, Dtype::kFp16, 64, 5, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 2, Dtype::kBf16, 16, 5, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 2, Dtype::kBf16, 64, 5, 2, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 3, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 3, Dtype::kFp16, 64, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 3, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 3, Dtype::kBf16, 64, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 4, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 4, Dtype::kFp16, 64, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 4, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 4, Dtype::kBf16, 64, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 5, Dtype::kFp16, 16, 4, 5, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 5, Dtype::kFp16, 64, 4, 5, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 5, Dtype::kBf16, 16, 4, 5, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 5, Dtype::kBf16, 64, 4, 5, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 6, Dtype::kFp16, 16, 5, 6, 4, 4, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 6, Dtype::kFp16, 64, 5, 6, 4, 4, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 6, Dtype::kBf16, 16, 5, 6, 4, 4, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 6, Dtype::kBf16, 64, 5, 6, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 8, 1, 0, 1, Dtype::kFp16, 16, 3, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 1, Dtype::kBf16, 16, 3, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 2, Dtype::kFp16, 16, 3, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 2, Dtype::kBf16, 16, 3, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 3, Dtype::kFp16, 16, 2, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 3, Dtype::kBf16, 16, 2, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 4, Dtype::kFp16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 4, Dtype::kBf16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 5, Dtype::kFp16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 5, Dtype::kBf16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 6, Dtype::kFp16, 16, 2, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 6, Dtype::kBf16, 16, 2, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 7, Dtype::kFp16, 16, 2, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 7, Dtype::kBf16, 16, 2, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 8, Dtype::kFp16, 16, 2, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 1, 0, 8, Dtype::kBf16, 16, 2, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 8, 2, 0, 1, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 1, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 2, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 2, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 3, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 3, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 8, 2, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kFp16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kFp16, 64, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kBf16, 16, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 1, Dtype::kBf16, 64, 3, 1, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kFp16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kFp16, 64, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kBf16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 2, Dtype::kBf16, 64, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kFp16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kFp16, 64, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kBf16, 16, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 3, Dtype::kBf16, 64, 2, 4, 2, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kFp16, 16, 2, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kFp16, 64, 2, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kBf16, 16, 2, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 4, Dtype::kBf16, 64, 2, 4, 4, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kFp16, 16, 3, 5, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kFp16, 64, 3, 5, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kBf16, 16, 3, 5, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 5, Dtype::kBf16, 64, 3, 5, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kFp16, 16, 2, 6, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kFp16, 64, 2, 6, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kBf16, 16, 2, 6, 4, 4, 0, 0, 0, 0}, + {HeadDim::k512, 16, 1, 0, 6, Dtype::kBf16, 64, 2, 6, 4, 4, 0, 0, 0, 0}, {HeadDim::k512, 16, 1, 0, 7, Dtype::kFp16, 16, 4, 7, 1, 4, 0, 0, 0, 0}, {HeadDim::k512, 16, 1, 0, 7, Dtype::kFp16, 64, 4, 7, 1, 4, 0, 0, 0, 0}, {HeadDim::k512, 16, 1, 0, 7, Dtype::kBf16, 16, 4, 7, 1, 4, 0, 0, 0, 0}, {HeadDim::k512, 16, 1, 0, 7, Dtype::kBf16, 64, 4, 7, 1, 4, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 8, Dtype::kFp16, 16, 5, 8, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 8, Dtype::kFp16, 64, 5, 8, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 8, Dtype::kBf16, 16, 5, 8, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 1, 0, 8, Dtype::kBf16, 64, 5, 8, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 1, Dtype::kFp16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 16, 2, 0, 1, Dtype::kFp16, 32, 4, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 16, 2, 0, 1, Dtype::kBf16, 16, 4, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 16, 2, 0, 1, Dtype::kBf16, 32, 4, 1, 2, 4, 0, 0, 1, 0}, - {HeadDim::k512, 16, 2, 0, 2, Dtype::kFp16, 16, 4, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k512, 16, 2, 0, 2, Dtype::kFp16, 32, 4, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k512, 16, 2, 0, 2, Dtype::kBf16, 16, 4, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k512, 16, 2, 0, 2, Dtype::kBf16, 32, 4, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k512, 16, 2, 0, 3, Dtype::kFp16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 3, Dtype::kFp16, 32, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 3, Dtype::kBf16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 3, Dtype::kBf16, 32, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 4, Dtype::kFp16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 4, Dtype::kFp16, 32, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 4, Dtype::kBf16, 16, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 4, Dtype::kBf16, 32, 5, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 5, Dtype::kFp16, 16, 2, 5, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 5, Dtype::kFp16, 32, 2, 5, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 5, Dtype::kBf16, 16, 2, 5, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 5, Dtype::kBf16, 32, 2, 5, 4, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 6, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 6, Dtype::kFp16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 6, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 6, Dtype::kBf16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 7, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 7, Dtype::kFp16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 7, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 7, Dtype::kBf16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 8, Dtype::kFp16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 8, Dtype::kFp16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 8, Dtype::kBf16, 16, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 16, 2, 0, 8, Dtype::kBf16, 32, 5, 4, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 1, Dtype::kFp16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k512, 32, 4, 0, 1, Dtype::kFp16, 32, 2, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k512, 32, 4, 0, 1, Dtype::kBf16, 16, 2, 1, 1, 4, 0, 0, 1, 0}, - {HeadDim::k512, 32, 4, 0, 1, Dtype::kBf16, 32, 2, 1, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kFp16, 16, 1, 8, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kFp16, 64, 1, 8, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kBf16, 16, 1, 8, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 1, 0, 8, Dtype::kBf16, 64, 1, 8, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kFp16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 1, Dtype::kBf16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kFp16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 2, Dtype::kBf16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kFp16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 3, Dtype::kBf16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kFp16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 4, Dtype::kBf16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kFp16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 5, Dtype::kBf16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kFp16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 6, Dtype::kBf16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kFp16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 7, Dtype::kBf16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kFp16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 16, 2, 0, 8, Dtype::kBf16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kFp16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 1, Dtype::kBf16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, {HeadDim::k512, 32, 4, 0, 2, Dtype::kFp16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, {HeadDim::k512, 32, 4, 0, 2, Dtype::kFp16, 32, 2, 1, 4, 4, 0, 0, 1, 0}, {HeadDim::k512, 32, 4, 0, 2, Dtype::kBf16, 16, 2, 1, 4, 4, 0, 0, 1, 0}, {HeadDim::k512, 32, 4, 0, 2, Dtype::kBf16, 32, 2, 1, 4, 4, 0, 0, 1, 0}, - {HeadDim::k512, 32, 4, 0, 3, Dtype::kFp16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 3, Dtype::kFp16, 32, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 3, Dtype::kBf16, 16, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 3, Dtype::kBf16, 32, 2, 2, 2, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 4, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 4, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kFp16, 16, 1, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kFp16, 32, 1, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kBf16, 16, 1, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 3, Dtype::kBf16, 32, 1, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kFp16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kFp16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kBf16, 16, 1, 4, 1, 8, 8, 1, 0, 0}, + {HeadDim::k512, 32, 4, 0, 4, Dtype::kBf16, 32, 1, 4, 1, 8, 8, 1, 0, 0}, {HeadDim::k512, 32, 4, 0, 5, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, {HeadDim::k512, 32, 4, 0, 5, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, {HeadDim::k512, 32, 4, 0, 5, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, {HeadDim::k512, 32, 4, 0, 5, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 6, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 6, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 6, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 6, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 7, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, - {HeadDim::k512, 32, 4, 0, 7, Dtype::kBf16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kFp16, 16, 1, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kFp16, 32, 1, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kBf16, 16, 1, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 6, Dtype::kBf16, 32, 1, 3, 1, 4, 0, 0, 0, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kFp16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kFp16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kBf16, 16, 1, 4, 1, 4, 0, 0, 1, 0}, + {HeadDim::k512, 32, 4, 0, 7, Dtype::kBf16, 32, 1, 4, 1, 4, 0, 0, 1, 0}, {HeadDim::k512, 32, 4, 0, 8, Dtype::kFp16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, {HeadDim::k512, 32, 4, 0, 8, Dtype::kFp16, 32, 1, 4, 1, 8, 0, 0, 0, 0}, {HeadDim::k512, 32, 4, 0, 8, Dtype::kBf16, 16, 1, 4, 1, 8, 0, 0, 0, 0}, diff --git a/csrc/rocm/rdna_prefill_batch_configs.h b/csrc/rocm/rdna_prefill_batch_configs.h index ac448948725d..f4f85975f9b4 100644 --- a/csrc/rocm/rdna_prefill_batch_configs.h +++ b/csrc/rocm/rdna_prefill_batch_configs.h @@ -828,637 +828,637 @@ inline constexpr PrefillBatchConfig kGfx1151PrefillBatchConfigs[] = { }; inline constexpr PrefillBatchConfig kGfx1150PrefillBatchConfigs[] = { - {HeadDim::k64, 14, 2, 0, 16, 1, 0, 16, 2}, - {HeadDim::k64, 14, 2, 0, 16, 1, 512, 16, 2}, - {HeadDim::k64, 14, 2, 0, 16, 1, 2048, 16, 2}, - {HeadDim::k64, 14, 2, 0, 16, 1, 8192, 16, 1}, - {HeadDim::k64, 14, 2, 0, 16, 16, 0, 16, 2}, - {HeadDim::k64, 14, 2, 0, 16, 16, 512, 16, 1}, - {HeadDim::k64, 14, 2, 0, 16, 16, 2048, 128, 2}, - {HeadDim::k64, 14, 2, 0, 16, 16, 8192, 128, 2}, - {HeadDim::k64, 14, 2, 0, 16, 64, 0, 128, 1}, - {HeadDim::k64, 14, 2, 0, 16, 64, 512, 128, 1}, - {HeadDim::k64, 14, 2, 0, 16, 64, 2048, 128, 1}, - {HeadDim::k64, 14, 2, 0, 16, 64, 8192, 32, 2}, - {HeadDim::k64, 14, 2, 0, 16, 256, 0, 128, 2}, - {HeadDim::k64, 14, 2, 0, 16, 256, 512, 128, 1}, - {HeadDim::k64, 14, 2, 0, 16, 256, 2048, 128, 1}, - {HeadDim::k64, 14, 2, 0, 16, 256, 8192, 128, 1}, - {HeadDim::k64, 14, 2, 0, 16, 1024, 512, 128, 2}, - {HeadDim::k64, 14, 2, 0, 16, 1024, 2048, 128, 1}, - {HeadDim::k64, 14, 2, 0, 16, 1024, 8192, 128, 1}, - {HeadDim::k64, 14, 2, 0, 16, 4096, 2048, 128, 1}, - {HeadDim::k64, 14, 2, 0, 16, 4096, 8192, 128, 1}, - {HeadDim::k64, 16, 2, 0, 16, 1, 0, 16, 2}, - {HeadDim::k64, 16, 2, 0, 16, 1, 512, 16, 2}, + {HeadDim::k64, 14, 2, 0, 16, 1, 0, 512, 2}, + {HeadDim::k64, 14, 2, 0, 16, 1, 512, 512, 1}, + {HeadDim::k64, 14, 2, 0, 16, 1, 2048, 512, 1}, + {HeadDim::k64, 14, 2, 0, 16, 1, 8192, 32, 1}, + {HeadDim::k64, 14, 2, 0, 16, 16, 0, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 16, 512, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 16, 2048, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 16, 8192, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 64, 0, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 64, 512, 64, 2}, + {HeadDim::k64, 14, 2, 0, 16, 64, 2048, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 64, 8192, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 256, 0, 512, 1}, + {HeadDim::k64, 14, 2, 0, 16, 256, 512, 512, 1}, + {HeadDim::k64, 14, 2, 0, 16, 256, 2048, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 256, 8192, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 1024, 512, 512, 1}, + {HeadDim::k64, 14, 2, 0, 16, 1024, 2048, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 1024, 8192, 64, 1}, + {HeadDim::k64, 14, 2, 0, 16, 4096, 2048, 512, 2}, + {HeadDim::k64, 14, 2, 0, 16, 4096, 8192, 64, 1}, + {HeadDim::k64, 16, 2, 0, 16, 1, 0, 2048, 1}, + {HeadDim::k64, 16, 2, 0, 16, 1, 512, 16, 1}, {HeadDim::k64, 16, 2, 0, 16, 1, 2048, 16, 1}, - {HeadDim::k64, 16, 2, 0, 16, 1, 8192, 16, 1}, - {HeadDim::k64, 16, 2, 0, 16, 16, 0, 256, 1}, - {HeadDim::k64, 16, 2, 0, 16, 16, 512, 256, 2}, - {HeadDim::k64, 16, 2, 0, 16, 16, 2048, 256, 2}, - {HeadDim::k64, 16, 2, 0, 16, 16, 8192, 32, 2}, - {HeadDim::k64, 16, 2, 0, 16, 64, 0, 128, 2}, - {HeadDim::k64, 16, 2, 0, 16, 64, 512, 128, 1}, - {HeadDim::k64, 16, 2, 0, 16, 64, 2048, 128, 1}, - {HeadDim::k64, 16, 2, 0, 16, 64, 8192, 128, 1}, - {HeadDim::k64, 16, 2, 0, 16, 256, 0, 256, 2}, - {HeadDim::k64, 16, 2, 0, 16, 256, 512, 256, 1}, - {HeadDim::k64, 16, 2, 0, 16, 256, 2048, 128, 2}, - {HeadDim::k64, 16, 2, 0, 16, 256, 8192, 128, 1}, - {HeadDim::k64, 16, 2, 0, 16, 1024, 512, 256, 1}, - {HeadDim::k64, 16, 2, 0, 16, 1024, 2048, 128, 2}, - {HeadDim::k64, 16, 2, 0, 16, 1024, 8192, 128, 1}, - {HeadDim::k64, 16, 2, 0, 16, 4096, 2048, 128, 1}, - {HeadDim::k64, 16, 2, 0, 16, 4096, 8192, 128, 1}, - {HeadDim::k64, 32, 8, 0, 16, 1, 0, 16, 8}, - {HeadDim::k64, 32, 8, 0, 16, 1, 512, 16, 8}, + {HeadDim::k64, 16, 2, 0, 16, 1, 8192, 32, 1}, + {HeadDim::k64, 16, 2, 0, 16, 16, 0, 2048, 1}, + {HeadDim::k64, 16, 2, 0, 16, 16, 512, 2048, 1}, + {HeadDim::k64, 16, 2, 0, 16, 16, 2048, 2048, 1}, + {HeadDim::k64, 16, 2, 0, 16, 16, 8192, 2048, 1}, + {HeadDim::k64, 16, 2, 0, 16, 64, 0, 2048, 2}, + {HeadDim::k64, 16, 2, 0, 16, 64, 512, 2048, 2}, + {HeadDim::k64, 16, 2, 0, 16, 64, 2048, 64, 1}, + {HeadDim::k64, 16, 2, 0, 16, 64, 8192, 64, 2}, + {HeadDim::k64, 16, 2, 0, 16, 256, 0, 2048, 2}, + {HeadDim::k64, 16, 2, 0, 16, 256, 512, 2048, 2}, + {HeadDim::k64, 16, 2, 0, 16, 256, 2048, 64, 1}, + {HeadDim::k64, 16, 2, 0, 16, 256, 8192, 64, 2}, + {HeadDim::k64, 16, 2, 0, 16, 1024, 512, 2048, 2}, + {HeadDim::k64, 16, 2, 0, 16, 1024, 2048, 64, 1}, + {HeadDim::k64, 16, 2, 0, 16, 1024, 8192, 64, 1}, + {HeadDim::k64, 16, 2, 0, 16, 4096, 2048, 2048, 2}, + {HeadDim::k64, 16, 2, 0, 16, 4096, 8192, 64, 2}, + {HeadDim::k64, 32, 8, 0, 16, 1, 0, 128, 8}, + {HeadDim::k64, 32, 8, 0, 16, 1, 512, 128, 8}, {HeadDim::k64, 32, 8, 0, 16, 1, 2048, 16, 8}, - {HeadDim::k64, 32, 8, 0, 16, 1, 8192, 16, 2}, - {HeadDim::k64, 32, 8, 0, 16, 16, 0, 512, 8}, - {HeadDim::k64, 32, 8, 0, 16, 16, 512, 512, 8}, - {HeadDim::k64, 32, 8, 0, 16, 16, 2048, 512, 8}, - {HeadDim::k64, 32, 8, 0, 16, 16, 8192, 512, 8}, + {HeadDim::k64, 32, 8, 0, 16, 1, 8192, 16, 8}, + {HeadDim::k64, 32, 8, 0, 16, 16, 0, 64, 8}, + {HeadDim::k64, 32, 8, 0, 16, 16, 512, 64, 8}, + {HeadDim::k64, 32, 8, 0, 16, 16, 2048, 64, 8}, + {HeadDim::k64, 32, 8, 0, 16, 16, 8192, 64, 8}, {HeadDim::k64, 32, 8, 0, 16, 64, 0, 64, 8}, {HeadDim::k64, 32, 8, 0, 16, 64, 512, 64, 4}, {HeadDim::k64, 32, 8, 0, 16, 64, 2048, 64, 2}, - {HeadDim::k64, 32, 8, 0, 16, 64, 8192, 16, 2}, - {HeadDim::k64, 32, 8, 0, 16, 256, 0, 128, 8}, - {HeadDim::k64, 32, 8, 0, 16, 256, 512, 128, 4}, - {HeadDim::k64, 32, 8, 0, 16, 256, 2048, 128, 2}, - {HeadDim::k64, 32, 8, 0, 16, 256, 8192, 16, 1}, + {HeadDim::k64, 32, 8, 0, 16, 64, 8192, 64, 1}, + {HeadDim::k64, 32, 8, 0, 16, 256, 0, 64, 8}, + {HeadDim::k64, 32, 8, 0, 16, 256, 512, 64, 4}, + {HeadDim::k64, 32, 8, 0, 16, 256, 2048, 64, 1}, + {HeadDim::k64, 32, 8, 0, 16, 256, 8192, 64, 1}, {HeadDim::k64, 32, 8, 0, 16, 1024, 512, 64, 4}, - {HeadDim::k64, 32, 8, 0, 16, 1024, 2048, 128, 4}, - {HeadDim::k64, 32, 8, 0, 16, 1024, 8192, 128, 1}, - {HeadDim::k64, 32, 8, 0, 16, 4096, 2048, 128, 4}, - {HeadDim::k64, 32, 8, 0, 16, 4096, 8192, 128, 1}, - {HeadDim::k64, 32, 32, 0, 16, 1, 0, 128, 32}, - {HeadDim::k64, 32, 32, 0, 16, 1, 512, 128, 32}, - {HeadDim::k64, 32, 32, 0, 16, 1, 2048, 16, 32}, - {HeadDim::k64, 32, 32, 0, 16, 1, 8192, 16, 4}, - {HeadDim::k64, 32, 32, 0, 16, 16, 0, 128, 32}, + {HeadDim::k64, 32, 8, 0, 16, 1024, 2048, 64, 2}, + {HeadDim::k64, 32, 8, 0, 16, 1024, 8192, 64, 2}, + {HeadDim::k64, 32, 8, 0, 16, 4096, 2048, 64, 2}, + {HeadDim::k64, 32, 8, 0, 16, 4096, 8192, 64, 2}, + {HeadDim::k64, 32, 32, 0, 16, 1, 0, 16, 32}, + {HeadDim::k64, 32, 32, 0, 16, 1, 512, 64, 32}, + {HeadDim::k64, 32, 32, 0, 16, 1, 2048, 64, 32}, + {HeadDim::k64, 32, 32, 0, 16, 1, 8192, 16, 32}, + {HeadDim::k64, 32, 32, 0, 16, 16, 0, 64, 32}, {HeadDim::k64, 32, 32, 0, 16, 16, 512, 16, 32}, {HeadDim::k64, 32, 32, 0, 16, 16, 2048, 16, 32}, - {HeadDim::k64, 32, 32, 0, 16, 16, 8192, 16, 8}, + {HeadDim::k64, 32, 32, 0, 16, 16, 8192, 64, 32}, {HeadDim::k64, 32, 32, 0, 16, 64, 0, 256, 32}, - {HeadDim::k64, 32, 32, 0, 16, 64, 512, 256, 32}, - {HeadDim::k64, 32, 32, 0, 16, 64, 2048, 256, 32}, - {HeadDim::k64, 32, 32, 0, 16, 64, 8192, 256, 32}, + {HeadDim::k64, 32, 32, 0, 16, 64, 512, 256, 16}, + {HeadDim::k64, 32, 32, 0, 16, 64, 2048, 256, 16}, + {HeadDim::k64, 32, 32, 0, 16, 64, 8192, 256, 16}, {HeadDim::k64, 32, 32, 0, 16, 256, 0, 256, 32}, - {HeadDim::k64, 32, 32, 0, 16, 256, 512, 256, 16}, - {HeadDim::k64, 32, 32, 0, 16, 256, 2048, 256, 4}, - {HeadDim::k64, 32, 32, 0, 16, 256, 8192, 256, 2}, - {HeadDim::k64, 32, 32, 0, 16, 1024, 512, 256, 8}, - {HeadDim::k64, 32, 32, 0, 16, 1024, 2048, 256, 1}, + {HeadDim::k64, 32, 32, 0, 16, 256, 512, 256, 1}, + {HeadDim::k64, 32, 32, 0, 16, 256, 2048, 256, 1}, + {HeadDim::k64, 32, 32, 0, 16, 256, 8192, 256, 1}, + {HeadDim::k64, 32, 32, 0, 16, 1024, 512, 64, 1}, + {HeadDim::k64, 32, 32, 0, 16, 1024, 2048, 64, 1}, {HeadDim::k64, 32, 32, 0, 16, 1024, 8192, 256, 1}, - {HeadDim::k64, 32, 32, 0, 16, 4096, 2048, 256, 1}, + {HeadDim::k64, 32, 32, 0, 16, 4096, 2048, 256, 2}, {HeadDim::k64, 32, 32, 0, 16, 4096, 8192, 256, 1}, {HeadDim::k96, 32, 32, 0, 16, 1, 0, 16, 32}, {HeadDim::k96, 32, 32, 0, 16, 1, 512, 16, 32}, {HeadDim::k96, 32, 32, 0, 16, 1, 2048, 16, 32}, - {HeadDim::k96, 32, 32, 0, 16, 1, 8192, 16, 4}, + {HeadDim::k96, 32, 32, 0, 16, 1, 8192, 16, 32}, {HeadDim::k96, 32, 32, 0, 16, 16, 0, 16, 32}, {HeadDim::k96, 32, 32, 0, 16, 16, 512, 16, 32}, {HeadDim::k96, 32, 32, 0, 16, 16, 2048, 16, 32}, - {HeadDim::k96, 32, 32, 0, 16, 16, 8192, 16, 1}, - {HeadDim::k96, 32, 32, 0, 16, 64, 0, 1024, 32}, - {HeadDim::k96, 32, 32, 0, 16, 64, 512, 1024, 32}, + {HeadDim::k96, 32, 32, 0, 16, 16, 8192, 128, 32}, + {HeadDim::k96, 32, 32, 0, 16, 64, 0, 1024, 16}, + {HeadDim::k96, 32, 32, 0, 16, 64, 512, 1024, 16}, {HeadDim::k96, 32, 32, 0, 16, 64, 2048, 1024, 32}, {HeadDim::k96, 32, 32, 0, 16, 64, 8192, 1024, 32}, - {HeadDim::k96, 32, 32, 0, 16, 256, 0, 128, 8}, - {HeadDim::k96, 32, 32, 0, 16, 256, 512, 1024, 4}, - {HeadDim::k96, 32, 32, 0, 16, 256, 2048, 1024, 1}, + {HeadDim::k96, 32, 32, 0, 16, 256, 0, 1024, 4}, + {HeadDim::k96, 32, 32, 0, 16, 256, 512, 128, 1}, + {HeadDim::k96, 32, 32, 0, 16, 256, 2048, 128, 1}, {HeadDim::k96, 32, 32, 0, 16, 256, 8192, 1024, 1}, - {HeadDim::k96, 32, 32, 0, 16, 1024, 512, 1024, 8}, - {HeadDim::k96, 32, 32, 0, 16, 1024, 2048, 1024, 1}, + {HeadDim::k96, 32, 32, 0, 16, 1024, 512, 1024, 4}, + {HeadDim::k96, 32, 32, 0, 16, 1024, 2048, 1024, 2}, {HeadDim::k96, 32, 32, 0, 16, 1024, 8192, 1024, 1}, - {HeadDim::k96, 32, 32, 0, 16, 4096, 2048, 1024, 1}, + {HeadDim::k96, 32, 32, 0, 16, 4096, 2048, 1024, 2}, {HeadDim::k96, 32, 32, 0, 16, 4096, 8192, 1024, 1}, {HeadDim::k128, 10, 10, 0, 16, 1, 0, 16, 10}, - {HeadDim::k128, 10, 10, 0, 16, 1, 512, 16, 2}, + {HeadDim::k128, 10, 10, 0, 16, 1, 512, 16, 10}, {HeadDim::k128, 10, 10, 0, 16, 1, 2048, 16, 10}, {HeadDim::k128, 10, 10, 0, 16, 1, 8192, 16, 10}, - {HeadDim::k128, 10, 10, 0, 16, 16, 0, 16, 10}, + {HeadDim::k128, 10, 10, 0, 16, 16, 0, 128, 10}, {HeadDim::k128, 10, 10, 0, 16, 16, 512, 16, 10}, - {HeadDim::k128, 10, 10, 0, 16, 16, 2048, 16, 10}, + {HeadDim::k128, 10, 10, 0, 16, 16, 2048, 128, 10}, {HeadDim::k128, 10, 10, 0, 16, 16, 8192, 16, 10}, - {HeadDim::k128, 10, 10, 0, 16, 64, 0, 16, 2}, - {HeadDim::k128, 10, 10, 0, 16, 64, 512, 128, 10}, - {HeadDim::k128, 10, 10, 0, 16, 64, 2048, 512, 2}, - {HeadDim::k128, 10, 10, 0, 16, 64, 8192, 16, 5}, - {HeadDim::k128, 10, 10, 0, 16, 256, 0, 16, 2}, - {HeadDim::k128, 10, 10, 0, 16, 256, 512, 512, 5}, + {HeadDim::k128, 10, 10, 0, 16, 64, 0, 512, 10}, + {HeadDim::k128, 10, 10, 0, 16, 64, 512, 512, 10}, + {HeadDim::k128, 10, 10, 0, 16, 64, 2048, 512, 10}, + {HeadDim::k128, 10, 10, 0, 16, 64, 8192, 512, 1}, + {HeadDim::k128, 10, 10, 0, 16, 256, 0, 128, 1}, + {HeadDim::k128, 10, 10, 0, 16, 256, 512, 512, 1}, {HeadDim::k128, 10, 10, 0, 16, 256, 2048, 512, 1}, {HeadDim::k128, 10, 10, 0, 16, 256, 8192, 512, 1}, - {HeadDim::k128, 10, 10, 0, 16, 1024, 512, 512, 5}, + {HeadDim::k128, 10, 10, 0, 16, 1024, 512, 512, 1}, {HeadDim::k128, 10, 10, 0, 16, 1024, 2048, 512, 1}, {HeadDim::k128, 10, 10, 0, 16, 1024, 8192, 512, 1}, {HeadDim::k128, 10, 10, 0, 16, 4096, 2048, 512, 1}, - {HeadDim::k128, 10, 10, 0, 16, 4096, 8192, 512, 2}, - {HeadDim::k128, 16, 2, 0, 16, 1, 0, 16, 2}, + {HeadDim::k128, 10, 10, 0, 16, 4096, 8192, 512, 1}, + {HeadDim::k128, 16, 2, 0, 16, 1, 0, 128, 1}, {HeadDim::k128, 16, 2, 0, 16, 1, 512, 16, 2}, {HeadDim::k128, 16, 2, 0, 16, 1, 2048, 16, 2}, {HeadDim::k128, 16, 2, 0, 16, 1, 8192, 16, 2}, - {HeadDim::k128, 16, 2, 0, 16, 16, 0, 32, 2}, - {HeadDim::k128, 16, 2, 0, 16, 16, 512, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 16, 0, 128, 2}, + {HeadDim::k128, 16, 2, 0, 16, 16, 512, 128, 2}, {HeadDim::k128, 16, 2, 0, 16, 16, 2048, 128, 2}, - {HeadDim::k128, 16, 2, 0, 16, 16, 8192, 32, 2}, - {HeadDim::k128, 16, 2, 0, 16, 64, 0, 256, 2}, - {HeadDim::k128, 16, 2, 0, 16, 64, 512, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 16, 8192, 128, 2}, + {HeadDim::k128, 16, 2, 0, 16, 64, 0, 256, 1}, + {HeadDim::k128, 16, 2, 0, 16, 64, 512, 256, 1}, {HeadDim::k128, 16, 2, 0, 16, 64, 2048, 256, 1}, {HeadDim::k128, 16, 2, 0, 16, 64, 8192, 256, 1}, - {HeadDim::k128, 16, 2, 0, 16, 256, 0, 256, 2}, - {HeadDim::k128, 16, 2, 0, 16, 256, 512, 256, 2}, - {HeadDim::k128, 16, 2, 0, 16, 256, 2048, 256, 2}, - {HeadDim::k128, 16, 2, 0, 16, 256, 8192, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 256, 0, 256, 1}, + {HeadDim::k128, 16, 2, 0, 16, 256, 512, 256, 1}, + {HeadDim::k128, 16, 2, 0, 16, 256, 2048, 256, 1}, + {HeadDim::k128, 16, 2, 0, 16, 256, 8192, 256, 1}, {HeadDim::k128, 16, 2, 0, 16, 1024, 512, 256, 2}, - {HeadDim::k128, 16, 2, 0, 16, 1024, 2048, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 1024, 2048, 256, 1}, {HeadDim::k128, 16, 2, 0, 16, 1024, 8192, 256, 1}, - {HeadDim::k128, 16, 2, 0, 16, 4096, 2048, 256, 1}, - {HeadDim::k128, 16, 2, 0, 16, 4096, 8192, 256, 1}, + {HeadDim::k128, 16, 2, 0, 16, 4096, 2048, 256, 2}, + {HeadDim::k128, 16, 2, 0, 16, 4096, 8192, 32, 1}, {HeadDim::k128, 16, 8, 0, 16, 1, 0, 16, 8}, - {HeadDim::k128, 16, 8, 0, 16, 1, 512, 16, 4}, - {HeadDim::k128, 16, 8, 0, 16, 1, 2048, 16, 8}, - {HeadDim::k128, 16, 8, 0, 16, 1, 8192, 16, 8}, - {HeadDim::k128, 16, 8, 0, 16, 16, 0, 16, 8}, - {HeadDim::k128, 16, 8, 0, 16, 16, 512, 16, 8}, - {HeadDim::k128, 16, 8, 0, 16, 16, 2048, 16, 8}, - {HeadDim::k128, 16, 8, 0, 16, 16, 8192, 16, 2}, - {HeadDim::k128, 16, 8, 0, 16, 64, 0, 256, 4}, - {HeadDim::k128, 16, 8, 0, 16, 64, 512, 256, 4}, - {HeadDim::k128, 16, 8, 0, 16, 64, 2048, 256, 1}, - {HeadDim::k128, 16, 8, 0, 16, 64, 8192, 256, 2}, - {HeadDim::k128, 16, 8, 0, 16, 256, 0, 16, 8}, - {HeadDim::k128, 16, 8, 0, 16, 256, 512, 256, 2}, - {HeadDim::k128, 16, 8, 0, 16, 256, 2048, 256, 2}, - {HeadDim::k128, 16, 8, 0, 16, 256, 8192, 256, 1}, - {HeadDim::k128, 16, 8, 0, 16, 1024, 512, 256, 4}, - {HeadDim::k128, 16, 8, 0, 16, 1024, 2048, 256, 4}, - {HeadDim::k128, 16, 8, 0, 16, 1024, 8192, 256, 2}, - {HeadDim::k128, 16, 8, 0, 16, 4096, 2048, 256, 2}, - {HeadDim::k128, 16, 8, 0, 16, 4096, 8192, 256, 1}, + {HeadDim::k128, 16, 8, 0, 16, 1, 512, 64, 8}, + {HeadDim::k128, 16, 8, 0, 16, 1, 2048, 64, 2}, + {HeadDim::k128, 16, 8, 0, 16, 1, 8192, 16, 2}, + {HeadDim::k128, 16, 8, 0, 16, 16, 0, 128, 2}, + {HeadDim::k128, 16, 8, 0, 16, 16, 512, 128, 4}, + {HeadDim::k128, 16, 8, 0, 16, 16, 2048, 128, 8}, + {HeadDim::k128, 16, 8, 0, 16, 16, 8192, 128, 1}, + {HeadDim::k128, 16, 8, 0, 16, 64, 0, 128, 2}, + {HeadDim::k128, 16, 8, 0, 16, 64, 512, 128, 1}, + {HeadDim::k128, 16, 8, 0, 16, 64, 2048, 128, 1}, + {HeadDim::k128, 16, 8, 0, 16, 64, 8192, 128, 1}, + {HeadDim::k128, 16, 8, 0, 16, 256, 0, 128, 1}, + {HeadDim::k128, 16, 8, 0, 16, 256, 512, 128, 1}, + {HeadDim::k128, 16, 8, 0, 16, 256, 2048, 128, 1}, + {HeadDim::k128, 16, 8, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k128, 16, 8, 0, 16, 1024, 512, 128, 2}, + {HeadDim::k128, 16, 8, 0, 16, 1024, 2048, 128, 2}, + {HeadDim::k128, 16, 8, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k128, 16, 8, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k128, 16, 8, 0, 16, 4096, 8192, 128, 1}, {HeadDim::k128, 24, 8, 0, 16, 1, 0, 16, 8}, - {HeadDim::k128, 24, 8, 0, 16, 1, 512, 16, 1}, + {HeadDim::k128, 24, 8, 0, 16, 1, 512, 16, 8}, {HeadDim::k128, 24, 8, 0, 16, 1, 2048, 16, 8}, - {HeadDim::k128, 24, 8, 0, 16, 1, 8192, 16, 8}, + {HeadDim::k128, 24, 8, 0, 16, 1, 8192, 16, 2}, {HeadDim::k128, 24, 8, 0, 16, 16, 0, 16, 8}, - {HeadDim::k128, 24, 8, 0, 16, 16, 512, 16, 4}, - {HeadDim::k128, 24, 8, 0, 16, 16, 2048, 16, 4}, - {HeadDim::k128, 24, 8, 0, 16, 16, 8192, 16, 2}, - {HeadDim::k128, 24, 8, 0, 16, 64, 0, 512, 8}, - {HeadDim::k128, 24, 8, 0, 16, 64, 512, 512, 4}, + {HeadDim::k128, 24, 8, 0, 16, 16, 512, 16, 8}, + {HeadDim::k128, 24, 8, 0, 16, 16, 2048, 32, 8}, + {HeadDim::k128, 24, 8, 0, 16, 16, 8192, 512, 8}, + {HeadDim::k128, 24, 8, 0, 16, 64, 0, 512, 4}, + {HeadDim::k128, 24, 8, 0, 16, 64, 512, 512, 1}, {HeadDim::k128, 24, 8, 0, 16, 64, 2048, 512, 1}, {HeadDim::k128, 24, 8, 0, 16, 64, 8192, 512, 1}, - {HeadDim::k128, 24, 8, 0, 16, 256, 0, 512, 8}, - {HeadDim::k128, 24, 8, 0, 16, 256, 512, 512, 2}, + {HeadDim::k128, 24, 8, 0, 16, 256, 0, 32, 1}, + {HeadDim::k128, 24, 8, 0, 16, 256, 512, 512, 1}, {HeadDim::k128, 24, 8, 0, 16, 256, 2048, 512, 1}, {HeadDim::k128, 24, 8, 0, 16, 256, 8192, 512, 1}, - {HeadDim::k128, 24, 8, 0, 16, 1024, 512, 512, 2}, + {HeadDim::k128, 24, 8, 0, 16, 1024, 512, 512, 1}, {HeadDim::k128, 24, 8, 0, 16, 1024, 2048, 512, 1}, - {HeadDim::k128, 24, 8, 0, 16, 1024, 8192, 512, 1}, + {HeadDim::k128, 24, 8, 0, 16, 1024, 8192, 32, 1}, {HeadDim::k128, 24, 8, 0, 16, 4096, 2048, 512, 1}, - {HeadDim::k128, 24, 8, 0, 16, 4096, 8192, 512, 1}, + {HeadDim::k128, 24, 8, 0, 16, 4096, 8192, 32, 1}, {HeadDim::k128, 28, 4, 0, 16, 1, 0, 16, 4}, - {HeadDim::k128, 28, 4, 0, 16, 1, 512, 16, 4}, - {HeadDim::k128, 28, 4, 0, 16, 1, 2048, 16, 1}, - {HeadDim::k128, 28, 4, 0, 16, 1, 8192, 16, 1}, - {HeadDim::k128, 28, 4, 0, 16, 16, 0, 256, 1}, - {HeadDim::k128, 28, 4, 0, 16, 16, 512, 256, 2}, - {HeadDim::k128, 28, 4, 0, 16, 16, 2048, 256, 2}, + {HeadDim::k128, 28, 4, 0, 16, 1, 512, 16, 2}, + {HeadDim::k128, 28, 4, 0, 16, 1, 2048, 64, 4}, + {HeadDim::k128, 28, 4, 0, 16, 1, 8192, 16, 4}, + {HeadDim::k128, 28, 4, 0, 16, 16, 0, 256, 4}, + {HeadDim::k128, 28, 4, 0, 16, 16, 512, 256, 1}, + {HeadDim::k128, 28, 4, 0, 16, 16, 2048, 256, 1}, {HeadDim::k128, 28, 4, 0, 16, 16, 8192, 256, 1}, {HeadDim::k128, 28, 4, 0, 16, 64, 0, 256, 1}, {HeadDim::k128, 28, 4, 0, 16, 64, 512, 256, 1}, - {HeadDim::k128, 28, 4, 0, 16, 64, 2048, 256, 2}, + {HeadDim::k128, 28, 4, 0, 16, 64, 2048, 256, 1}, {HeadDim::k128, 28, 4, 0, 16, 64, 8192, 256, 1}, {HeadDim::k128, 28, 4, 0, 16, 256, 0, 256, 4}, {HeadDim::k128, 28, 4, 0, 16, 256, 512, 256, 1}, {HeadDim::k128, 28, 4, 0, 16, 256, 2048, 256, 1}, - {HeadDim::k128, 28, 4, 0, 16, 256, 8192, 256, 4}, + {HeadDim::k128, 28, 4, 0, 16, 256, 8192, 256, 1}, {HeadDim::k128, 28, 4, 0, 16, 1024, 512, 256, 1}, {HeadDim::k128, 28, 4, 0, 16, 1024, 2048, 256, 1}, {HeadDim::k128, 28, 4, 0, 16, 1024, 8192, 256, 1}, {HeadDim::k128, 28, 4, 0, 16, 4096, 2048, 256, 1}, - {HeadDim::k128, 28, 4, 0, 16, 4096, 8192, 256, 1}, - {HeadDim::k128, 32, 2, 0, 16, 1, 0, 16, 2}, - {HeadDim::k128, 32, 2, 0, 16, 1, 512, 16, 1}, - {HeadDim::k128, 32, 2, 0, 16, 1, 2048, 16, 2}, - {HeadDim::k128, 32, 2, 0, 16, 1, 8192, 16, 2}, - {HeadDim::k128, 32, 2, 0, 16, 16, 0, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 16, 512, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 16, 2048, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 16, 8192, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 64, 0, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 64, 512, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 64, 2048, 128, 2}, - {HeadDim::k128, 32, 2, 0, 16, 64, 8192, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 256, 0, 128, 2}, - {HeadDim::k128, 32, 2, 0, 16, 256, 512, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 256, 2048, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 256, 8192, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 1024, 512, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 1024, 2048, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 1024, 8192, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 4096, 2048, 128, 1}, - {HeadDim::k128, 32, 2, 0, 16, 4096, 8192, 128, 1}, - {HeadDim::k128, 32, 4, 0, 16, 1, 0, 16, 2}, - {HeadDim::k128, 32, 4, 0, 16, 1, 512, 16, 4}, - {HeadDim::k128, 32, 4, 0, 16, 1, 2048, 16, 1}, - {HeadDim::k128, 32, 4, 0, 16, 1, 8192, 16, 2}, - {HeadDim::k128, 32, 4, 0, 16, 16, 0, 16, 2}, - {HeadDim::k128, 32, 4, 0, 16, 16, 512, 32, 2}, - {HeadDim::k128, 32, 4, 0, 16, 16, 2048, 128, 2}, - {HeadDim::k128, 32, 4, 0, 16, 16, 8192, 16, 1}, - {HeadDim::k128, 32, 4, 0, 16, 64, 0, 128, 2}, - {HeadDim::k128, 32, 4, 0, 16, 64, 512, 128, 1}, + {HeadDim::k128, 28, 4, 0, 16, 4096, 8192, 64, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1, 0, 64, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1, 512, 64, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1, 2048, 64, 2}, + {HeadDim::k128, 32, 2, 0, 16, 1, 8192, 64, 2}, + {HeadDim::k128, 32, 2, 0, 16, 16, 0, 64, 1}, + {HeadDim::k128, 32, 2, 0, 16, 16, 512, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 16, 2048, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 16, 8192, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 64, 0, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 64, 512, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 64, 2048, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 64, 8192, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 256, 0, 16, 2}, + {HeadDim::k128, 32, 2, 0, 16, 256, 512, 16, 2}, + {HeadDim::k128, 32, 2, 0, 16, 256, 2048, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 256, 8192, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1024, 512, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1024, 2048, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 1024, 8192, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 4096, 2048, 16, 1}, + {HeadDim::k128, 32, 2, 0, 16, 4096, 8192, 16, 1}, + {HeadDim::k128, 32, 4, 0, 16, 1, 0, 16, 4}, + {HeadDim::k128, 32, 4, 0, 16, 1, 512, 16, 2}, + {HeadDim::k128, 32, 4, 0, 16, 1, 2048, 16, 2}, + {HeadDim::k128, 32, 4, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k128, 32, 4, 0, 16, 16, 0, 32, 2}, + {HeadDim::k128, 32, 4, 0, 16, 16, 512, 32, 4}, + {HeadDim::k128, 32, 4, 0, 16, 16, 2048, 32, 2}, + {HeadDim::k128, 32, 4, 0, 16, 16, 8192, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 64, 0, 128, 1}, + {HeadDim::k128, 32, 4, 0, 16, 64, 512, 128, 2}, {HeadDim::k128, 32, 4, 0, 16, 64, 2048, 128, 1}, {HeadDim::k128, 32, 4, 0, 16, 64, 8192, 128, 1}, {HeadDim::k128, 32, 4, 0, 16, 256, 0, 128, 2}, - {HeadDim::k128, 32, 4, 0, 16, 256, 512, 128, 2}, + {HeadDim::k128, 32, 4, 0, 16, 256, 512, 128, 1}, {HeadDim::k128, 32, 4, 0, 16, 256, 2048, 128, 1}, {HeadDim::k128, 32, 4, 0, 16, 256, 8192, 128, 1}, - {HeadDim::k128, 32, 4, 0, 16, 1024, 512, 128, 4}, + {HeadDim::k128, 32, 4, 0, 16, 1024, 512, 128, 1}, {HeadDim::k128, 32, 4, 0, 16, 1024, 2048, 128, 1}, {HeadDim::k128, 32, 4, 0, 16, 1024, 8192, 128, 1}, {HeadDim::k128, 32, 4, 0, 16, 4096, 2048, 128, 1}, {HeadDim::k128, 32, 4, 0, 16, 4096, 8192, 128, 1}, {HeadDim::k128, 32, 8, 0, 16, 1, 0, 16, 8}, - {HeadDim::k128, 32, 8, 0, 16, 1, 512, 16, 8}, + {HeadDim::k128, 32, 8, 0, 16, 1, 512, 16, 2}, {HeadDim::k128, 32, 8, 0, 16, 1, 2048, 16, 1}, {HeadDim::k128, 32, 8, 0, 16, 1, 8192, 16, 1}, - {HeadDim::k128, 32, 8, 0, 16, 16, 0, 64, 4}, - {HeadDim::k128, 32, 8, 0, 16, 16, 512, 64, 4}, - {HeadDim::k128, 32, 8, 0, 16, 16, 2048, 64, 4}, + {HeadDim::k128, 32, 8, 0, 16, 16, 0, 64, 8}, + {HeadDim::k128, 32, 8, 0, 16, 16, 512, 64, 8}, + {HeadDim::k128, 32, 8, 0, 16, 16, 2048, 64, 8}, {HeadDim::k128, 32, 8, 0, 16, 16, 8192, 64, 8}, - {HeadDim::k128, 32, 8, 0, 16, 64, 0, 64, 4}, - {HeadDim::k128, 32, 8, 0, 16, 64, 512, 64, 2}, - {HeadDim::k128, 32, 8, 0, 16, 64, 2048, 64, 2}, + {HeadDim::k128, 32, 8, 0, 16, 64, 0, 64, 1}, + {HeadDim::k128, 32, 8, 0, 16, 64, 512, 64, 1}, + {HeadDim::k128, 32, 8, 0, 16, 64, 2048, 64, 1}, {HeadDim::k128, 32, 8, 0, 16, 64, 8192, 64, 1}, - {HeadDim::k128, 32, 8, 0, 16, 256, 0, 64, 4}, - {HeadDim::k128, 32, 8, 0, 16, 256, 512, 64, 2}, + {HeadDim::k128, 32, 8, 0, 16, 256, 0, 64, 1}, + {HeadDim::k128, 32, 8, 0, 16, 256, 512, 64, 1}, {HeadDim::k128, 32, 8, 0, 16, 256, 2048, 64, 1}, {HeadDim::k128, 32, 8, 0, 16, 256, 8192, 64, 1}, - {HeadDim::k128, 32, 8, 0, 16, 1024, 512, 64, 2}, + {HeadDim::k128, 32, 8, 0, 16, 1024, 512, 64, 1}, {HeadDim::k128, 32, 8, 0, 16, 1024, 2048, 64, 1}, {HeadDim::k128, 32, 8, 0, 16, 1024, 8192, 64, 1}, - {HeadDim::k128, 32, 8, 0, 16, 4096, 2048, 64, 2}, + {HeadDim::k128, 32, 8, 0, 16, 4096, 2048, 64, 1}, {HeadDim::k128, 32, 8, 0, 16, 4096, 8192, 64, 1}, - {HeadDim::k128, 32, 32, 0, 16, 1, 0, 16, 32}, - {HeadDim::k128, 32, 32, 0, 16, 1, 512, 16, 32}, - {HeadDim::k128, 32, 32, 0, 16, 1, 2048, 16, 32}, + {HeadDim::k128, 32, 32, 0, 16, 1, 0, 16, 16}, + {HeadDim::k128, 32, 32, 0, 16, 1, 512, 16, 16}, + {HeadDim::k128, 32, 32, 0, 16, 1, 2048, 16, 16}, {HeadDim::k128, 32, 32, 0, 16, 1, 8192, 16, 16}, - {HeadDim::k128, 32, 32, 0, 16, 16, 0, 16, 32}, - {HeadDim::k128, 32, 32, 0, 16, 16, 512, 16, 32}, + {HeadDim::k128, 32, 32, 0, 16, 16, 0, 16, 16}, + {HeadDim::k128, 32, 32, 0, 16, 16, 512, 16, 16}, {HeadDim::k128, 32, 32, 0, 16, 16, 2048, 16, 16}, {HeadDim::k128, 32, 32, 0, 16, 16, 8192, 16, 16}, - {HeadDim::k128, 32, 32, 0, 16, 64, 0, 512, 32}, - {HeadDim::k128, 32, 32, 0, 16, 64, 512, 512, 32}, - {HeadDim::k128, 32, 32, 0, 16, 64, 2048, 512, 16}, - {HeadDim::k128, 32, 32, 0, 16, 64, 8192, 512, 2}, - {HeadDim::k128, 32, 32, 0, 16, 256, 0, 16, 8}, - {HeadDim::k128, 32, 32, 0, 16, 256, 512, 512, 4}, - {HeadDim::k128, 32, 32, 0, 16, 256, 2048, 512, 2}, - {HeadDim::k128, 32, 32, 0, 16, 256, 8192, 512, 2}, + {HeadDim::k128, 32, 32, 0, 16, 64, 0, 256, 32}, + {HeadDim::k128, 32, 32, 0, 16, 64, 512, 256, 16}, + {HeadDim::k128, 32, 32, 0, 16, 64, 2048, 256, 16}, + {HeadDim::k128, 32, 32, 0, 16, 64, 8192, 256, 32}, + {HeadDim::k128, 32, 32, 0, 16, 256, 0, 256, 4}, + {HeadDim::k128, 32, 32, 0, 16, 256, 512, 512, 1}, + {HeadDim::k128, 32, 32, 0, 16, 256, 2048, 512, 1}, + {HeadDim::k128, 32, 32, 0, 16, 256, 8192, 512, 1}, {HeadDim::k128, 32, 32, 0, 16, 1024, 512, 512, 4}, - {HeadDim::k128, 32, 32, 0, 16, 1024, 2048, 512, 4}, + {HeadDim::k128, 32, 32, 0, 16, 1024, 2048, 512, 2}, {HeadDim::k128, 32, 32, 0, 16, 1024, 8192, 512, 1}, - {HeadDim::k128, 32, 32, 0, 16, 4096, 2048, 512, 4}, + {HeadDim::k128, 32, 32, 0, 16, 4096, 2048, 512, 2}, {HeadDim::k128, 32, 32, 0, 16, 4096, 8192, 512, 1}, {HeadDim::k128, 40, 8, 0, 16, 1, 0, 32, 8}, {HeadDim::k128, 40, 8, 0, 16, 1, 512, 32, 8}, {HeadDim::k128, 40, 8, 0, 16, 1, 2048, 32, 8}, - {HeadDim::k128, 40, 8, 0, 16, 1, 8192, 16, 1}, - {HeadDim::k128, 40, 8, 0, 16, 16, 0, 128, 4}, - {HeadDim::k128, 40, 8, 0, 16, 16, 512, 32, 1}, - {HeadDim::k128, 40, 8, 0, 16, 16, 2048, 32, 1}, - {HeadDim::k128, 40, 8, 0, 16, 16, 8192, 32, 2}, - {HeadDim::k128, 40, 8, 0, 16, 64, 0, 128, 8}, + {HeadDim::k128, 40, 8, 0, 16, 1, 8192, 32, 8}, + {HeadDim::k128, 40, 8, 0, 16, 16, 0, 16, 4}, + {HeadDim::k128, 40, 8, 0, 16, 16, 512, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 16, 2048, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 16, 8192, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 64, 0, 128, 4}, {HeadDim::k128, 40, 8, 0, 16, 64, 512, 128, 2}, {HeadDim::k128, 40, 8, 0, 16, 64, 2048, 128, 1}, - {HeadDim::k128, 40, 8, 0, 16, 64, 8192, 128, 2}, - {HeadDim::k128, 40, 8, 0, 16, 256, 0, 128, 8}, - {HeadDim::k128, 40, 8, 0, 16, 256, 512, 128, 2}, - {HeadDim::k128, 40, 8, 0, 16, 256, 2048, 128, 2}, + {HeadDim::k128, 40, 8, 0, 16, 64, 8192, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 256, 0, 128, 4}, + {HeadDim::k128, 40, 8, 0, 16, 256, 512, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 256, 2048, 128, 1}, {HeadDim::k128, 40, 8, 0, 16, 256, 8192, 128, 1}, - {HeadDim::k128, 40, 8, 0, 16, 1024, 512, 128, 1}, + {HeadDim::k128, 40, 8, 0, 16, 1024, 512, 128, 2}, {HeadDim::k128, 40, 8, 0, 16, 1024, 2048, 128, 1}, {HeadDim::k128, 40, 8, 0, 16, 1024, 8192, 128, 1}, {HeadDim::k128, 40, 8, 0, 16, 4096, 2048, 128, 1}, - {HeadDim::k128, 40, 8, 0, 16, 4096, 8192, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 1, 0, 32, 1}, - {HeadDim::k256, 8, 1, 0, 16, 1, 512, 32, 1}, - {HeadDim::k256, 8, 1, 0, 16, 1, 2048, 32, 1}, - {HeadDim::k256, 8, 1, 0, 16, 1, 8192, 32, 1}, - {HeadDim::k256, 8, 1, 0, 16, 16, 0, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 16, 512, 32, 1}, + {HeadDim::k128, 40, 8, 0, 16, 4096, 8192, 16, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1, 0, 16, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1, 512, 16, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k256, 8, 1, 0, 16, 16, 0, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 16, 512, 16, 1}, {HeadDim::k256, 8, 1, 0, 16, 16, 2048, 32, 1}, - {HeadDim::k256, 8, 1, 0, 16, 16, 8192, 32, 1}, - {HeadDim::k256, 8, 1, 0, 16, 64, 0, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 64, 512, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 64, 2048, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 64, 8192, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 256, 0, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 256, 512, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 256, 2048, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 256, 8192, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 1024, 512, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 1024, 2048, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 1024, 8192, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 4096, 2048, 128, 1}, - {HeadDim::k256, 8, 1, 0, 16, 4096, 8192, 128, 1}, - {HeadDim::k256, 8, 1, 512, 16, 1, 0, 128, 1}, - {HeadDim::k256, 8, 1, 512, 16, 1, 512, 128, 1}, - {HeadDim::k256, 8, 1, 512, 16, 16, 0, 128, 1}, + {HeadDim::k256, 8, 1, 0, 16, 16, 8192, 16, 1}, + {HeadDim::k256, 8, 1, 0, 16, 64, 0, 64, 1}, + {HeadDim::k256, 8, 1, 0, 16, 64, 512, 64, 1}, + {HeadDim::k256, 8, 1, 0, 16, 64, 2048, 64, 1}, + {HeadDim::k256, 8, 1, 0, 16, 64, 8192, 64, 1}, + {HeadDim::k256, 8, 1, 0, 16, 256, 0, 16, 1}, + {HeadDim::k256, 8, 1, 0, 16, 256, 512, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 256, 2048, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 256, 8192, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1024, 512, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1024, 2048, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 1024, 8192, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 4096, 2048, 32, 1}, + {HeadDim::k256, 8, 1, 0, 16, 4096, 8192, 32, 1}, + {HeadDim::k256, 8, 1, 512, 16, 1, 0, 16, 1}, + {HeadDim::k256, 8, 1, 512, 16, 1, 512, 16, 1}, + {HeadDim::k256, 8, 1, 512, 16, 16, 0, 16, 1}, {HeadDim::k256, 8, 1, 512, 16, 16, 512, 128, 1}, - {HeadDim::k256, 8, 1, 512, 16, 64, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 64, 0, 16, 1}, {HeadDim::k256, 8, 1, 512, 16, 64, 512, 128, 1}, - {HeadDim::k256, 8, 1, 512, 16, 256, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 16, 256, 0, 16, 1}, {HeadDim::k256, 8, 1, 512, 16, 256, 512, 128, 1}, {HeadDim::k256, 8, 1, 512, 16, 1024, 512, 128, 1}, {HeadDim::k256, 8, 1, 512, 16, 4096, 512, 128, 1}, - {HeadDim::k256, 8, 1, 512, 32, 1, 0, 128, 1}, - {HeadDim::k256, 8, 1, 512, 32, 1, 512, 128, 1}, - {HeadDim::k256, 8, 1, 512, 32, 16, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 1, 0, 16, 1}, + {HeadDim::k256, 8, 1, 512, 32, 1, 512, 16, 1}, + {HeadDim::k256, 8, 1, 512, 32, 16, 0, 16, 1}, {HeadDim::k256, 8, 1, 512, 32, 16, 512, 128, 1}, - {HeadDim::k256, 8, 1, 512, 32, 64, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 64, 0, 16, 1}, {HeadDim::k256, 8, 1, 512, 32, 64, 512, 128, 1}, - {HeadDim::k256, 8, 1, 512, 32, 256, 0, 128, 1}, + {HeadDim::k256, 8, 1, 512, 32, 256, 0, 16, 1}, {HeadDim::k256, 8, 1, 512, 32, 256, 512, 128, 1}, {HeadDim::k256, 8, 1, 512, 32, 1024, 512, 128, 1}, {HeadDim::k256, 8, 1, 512, 32, 4096, 512, 128, 1}, - {HeadDim::k256, 8, 2, 0, 16, 1, 0, 32, 2}, - {HeadDim::k256, 8, 2, 0, 16, 1, 512, 32, 2}, - {HeadDim::k256, 8, 2, 0, 16, 1, 2048, 32, 1}, - {HeadDim::k256, 8, 2, 0, 16, 1, 8192, 32, 2}, - {HeadDim::k256, 8, 2, 0, 16, 16, 0, 128, 2}, - {HeadDim::k256, 8, 2, 0, 16, 16, 512, 128, 2}, - {HeadDim::k256, 8, 2, 0, 16, 16, 2048, 128, 2}, - {HeadDim::k256, 8, 2, 0, 16, 16, 8192, 128, 1}, - {HeadDim::k256, 8, 2, 0, 16, 64, 0, 128, 2}, - {HeadDim::k256, 8, 2, 0, 16, 64, 512, 128, 1}, - {HeadDim::k256, 8, 2, 0, 16, 64, 2048, 128, 1}, - {HeadDim::k256, 8, 2, 0, 16, 64, 8192, 128, 1}, - {HeadDim::k256, 8, 2, 0, 16, 256, 0, 128, 2}, - {HeadDim::k256, 8, 2, 0, 16, 256, 512, 128, 2}, - {HeadDim::k256, 8, 2, 0, 16, 256, 2048, 128, 1}, - {HeadDim::k256, 8, 2, 0, 16, 256, 8192, 128, 1}, - {HeadDim::k256, 8, 2, 0, 16, 1024, 512, 128, 2}, - {HeadDim::k256, 8, 2, 0, 16, 1024, 2048, 128, 1}, - {HeadDim::k256, 8, 2, 0, 16, 1024, 8192, 128, 1}, - {HeadDim::k256, 8, 2, 0, 16, 4096, 2048, 128, 1}, - {HeadDim::k256, 8, 2, 0, 16, 4096, 8192, 128, 1}, - {HeadDim::k256, 8, 2, 0, 544, 1, 0, 32, 2}, - {HeadDim::k256, 8, 2, 0, 544, 1, 512, 32, 2}, - {HeadDim::k256, 8, 2, 0, 544, 1, 2048, 32, 1}, - {HeadDim::k256, 8, 2, 0, 544, 1, 8192, 32, 2}, - {HeadDim::k256, 8, 2, 0, 544, 16, 0, 128, 2}, - {HeadDim::k256, 8, 2, 0, 544, 16, 512, 128, 2}, - {HeadDim::k256, 8, 2, 0, 544, 16, 2048, 128, 2}, - {HeadDim::k256, 8, 2, 0, 544, 16, 8192, 128, 1}, - {HeadDim::k256, 8, 2, 0, 544, 64, 0, 128, 2}, - {HeadDim::k256, 8, 2, 0, 544, 64, 512, 128, 1}, - {HeadDim::k256, 8, 2, 0, 544, 64, 2048, 128, 1}, - {HeadDim::k256, 8, 2, 0, 544, 64, 8192, 128, 1}, - {HeadDim::k256, 8, 2, 0, 544, 256, 0, 128, 2}, - {HeadDim::k256, 8, 2, 0, 544, 256, 512, 128, 2}, - {HeadDim::k256, 8, 2, 0, 544, 256, 2048, 128, 1}, - {HeadDim::k256, 8, 2, 0, 544, 256, 8192, 128, 1}, - {HeadDim::k256, 8, 2, 0, 544, 1024, 512, 128, 2}, - {HeadDim::k256, 8, 2, 0, 544, 1024, 2048, 128, 1}, - {HeadDim::k256, 8, 2, 0, 544, 1024, 8192, 128, 1}, - {HeadDim::k256, 8, 2, 0, 544, 4096, 2048, 128, 1}, - {HeadDim::k256, 8, 2, 0, 544, 4096, 8192, 128, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1, 0, 16, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1, 512, 16, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1, 2048, 16, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k256, 8, 2, 0, 16, 16, 0, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 16, 512, 32, 2}, + {HeadDim::k256, 8, 2, 0, 16, 16, 2048, 32, 2}, + {HeadDim::k256, 8, 2, 0, 16, 16, 8192, 32, 2}, + {HeadDim::k256, 8, 2, 0, 16, 64, 0, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 64, 512, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 64, 2048, 32, 2}, + {HeadDim::k256, 8, 2, 0, 16, 64, 8192, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 256, 0, 32, 2}, + {HeadDim::k256, 8, 2, 0, 16, 256, 512, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 256, 2048, 32, 2}, + {HeadDim::k256, 8, 2, 0, 16, 256, 8192, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1024, 512, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1024, 2048, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 1024, 8192, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 4096, 2048, 32, 1}, + {HeadDim::k256, 8, 2, 0, 16, 4096, 8192, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1, 0, 16, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1, 512, 16, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1, 2048, 16, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1, 8192, 16, 1}, + {HeadDim::k256, 8, 2, 0, 544, 16, 0, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 16, 512, 32, 2}, + {HeadDim::k256, 8, 2, 0, 544, 16, 2048, 32, 2}, + {HeadDim::k256, 8, 2, 0, 544, 16, 8192, 32, 2}, + {HeadDim::k256, 8, 2, 0, 544, 64, 0, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 64, 512, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 64, 2048, 32, 2}, + {HeadDim::k256, 8, 2, 0, 544, 64, 8192, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 256, 0, 32, 2}, + {HeadDim::k256, 8, 2, 0, 544, 256, 512, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 256, 2048, 32, 2}, + {HeadDim::k256, 8, 2, 0, 544, 256, 8192, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1024, 512, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1024, 2048, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 1024, 8192, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 4096, 2048, 32, 1}, + {HeadDim::k256, 8, 2, 0, 544, 4096, 8192, 32, 1}, {HeadDim::k256, 8, 2, 512, 16, 1, 0, 16, 2}, - {HeadDim::k256, 8, 2, 512, 16, 1, 512, 64, 2}, - {HeadDim::k256, 8, 2, 512, 16, 16, 0, 512, 2}, - {HeadDim::k256, 8, 2, 512, 16, 16, 512, 16, 2}, - {HeadDim::k256, 8, 2, 512, 16, 64, 0, 512, 2}, - {HeadDim::k256, 8, 2, 512, 16, 64, 512, 64, 2}, - {HeadDim::k256, 8, 2, 512, 16, 256, 0, 512, 2}, - {HeadDim::k256, 8, 2, 512, 16, 256, 512, 64, 2}, - {HeadDim::k256, 8, 2, 512, 16, 1024, 512, 512, 2}, - {HeadDim::k256, 8, 2, 512, 16, 4096, 512, 512, 2}, + {HeadDim::k256, 8, 2, 512, 16, 1, 512, 16, 2}, + {HeadDim::k256, 8, 2, 512, 16, 16, 0, 16, 1}, + {HeadDim::k256, 8, 2, 512, 16, 16, 512, 16, 1}, + {HeadDim::k256, 8, 2, 512, 16, 64, 0, 64, 1}, + {HeadDim::k256, 8, 2, 512, 16, 64, 512, 64, 1}, + {HeadDim::k256, 8, 2, 512, 16, 256, 0, 64, 1}, + {HeadDim::k256, 8, 2, 512, 16, 256, 512, 64, 1}, + {HeadDim::k256, 8, 2, 512, 16, 1024, 512, 64, 1}, + {HeadDim::k256, 8, 2, 512, 16, 4096, 512, 64, 1}, {HeadDim::k256, 8, 2, 512, 32, 1, 0, 16, 2}, - {HeadDim::k256, 8, 2, 512, 32, 1, 512, 64, 2}, - {HeadDim::k256, 8, 2, 512, 32, 16, 0, 512, 2}, - {HeadDim::k256, 8, 2, 512, 32, 16, 512, 16, 2}, - {HeadDim::k256, 8, 2, 512, 32, 64, 0, 512, 2}, - {HeadDim::k256, 8, 2, 512, 32, 64, 512, 64, 2}, - {HeadDim::k256, 8, 2, 512, 32, 256, 0, 512, 2}, - {HeadDim::k256, 8, 2, 512, 32, 256, 512, 64, 2}, - {HeadDim::k256, 8, 2, 512, 32, 1024, 512, 512, 2}, - {HeadDim::k256, 8, 2, 512, 32, 4096, 512, 512, 2}, + {HeadDim::k256, 8, 2, 512, 32, 1, 512, 16, 2}, + {HeadDim::k256, 8, 2, 512, 32, 16, 0, 16, 1}, + {HeadDim::k256, 8, 2, 512, 32, 16, 512, 16, 1}, + {HeadDim::k256, 8, 2, 512, 32, 64, 0, 64, 1}, + {HeadDim::k256, 8, 2, 512, 32, 64, 512, 64, 1}, + {HeadDim::k256, 8, 2, 512, 32, 256, 0, 64, 1}, + {HeadDim::k256, 8, 2, 512, 32, 256, 512, 64, 1}, + {HeadDim::k256, 8, 2, 512, 32, 1024, 512, 64, 1}, + {HeadDim::k256, 8, 2, 512, 32, 4096, 512, 64, 1}, {HeadDim::k256, 8, 4, 0, 16, 1, 0, 16, 4}, - {HeadDim::k256, 8, 4, 0, 16, 1, 512, 16, 4}, + {HeadDim::k256, 8, 4, 0, 16, 1, 512, 16, 1}, {HeadDim::k256, 8, 4, 0, 16, 1, 2048, 16, 1}, {HeadDim::k256, 8, 4, 0, 16, 1, 8192, 16, 4}, - {HeadDim::k256, 8, 4, 0, 16, 16, 0, 16, 4}, - {HeadDim::k256, 8, 4, 0, 16, 16, 512, 16, 4}, - {HeadDim::k256, 8, 4, 0, 16, 16, 2048, 16, 4}, - {HeadDim::k256, 8, 4, 0, 16, 16, 8192, 16, 2}, - {HeadDim::k256, 8, 4, 0, 16, 64, 0, 128, 4}, - {HeadDim::k256, 8, 4, 0, 16, 64, 512, 128, 4}, - {HeadDim::k256, 8, 4, 0, 16, 64, 2048, 128, 4}, - {HeadDim::k256, 8, 4, 0, 16, 64, 8192, 128, 2}, - {HeadDim::k256, 8, 4, 0, 16, 256, 0, 128, 4}, - {HeadDim::k256, 8, 4, 0, 16, 256, 512, 128, 2}, - {HeadDim::k256, 8, 4, 0, 16, 256, 2048, 128, 1}, - {HeadDim::k256, 8, 4, 0, 16, 256, 8192, 128, 1}, - {HeadDim::k256, 8, 4, 0, 16, 1024, 512, 128, 4}, - {HeadDim::k256, 8, 4, 0, 16, 1024, 2048, 128, 1}, - {HeadDim::k256, 8, 4, 0, 16, 1024, 8192, 128, 1}, - {HeadDim::k256, 8, 4, 0, 16, 4096, 2048, 128, 2}, - {HeadDim::k256, 8, 4, 0, 16, 4096, 8192, 128, 2}, - {HeadDim::k256, 8, 4, 1024, 16, 1, 0, 64, 4}, - {HeadDim::k256, 8, 4, 1024, 16, 1, 512, 64, 4}, - {HeadDim::k256, 8, 4, 1024, 16, 16, 0, 64, 4}, - {HeadDim::k256, 8, 4, 1024, 16, 16, 512, 64, 4}, - {HeadDim::k256, 8, 4, 1024, 16, 64, 0, 512, 4}, - {HeadDim::k256, 8, 4, 1024, 16, 64, 512, 64, 2}, - {HeadDim::k256, 8, 4, 1024, 16, 256, 0, 64, 4}, - {HeadDim::k256, 8, 4, 1024, 16, 256, 512, 64, 2}, - {HeadDim::k256, 8, 4, 1024, 16, 1024, 512, 512, 4}, - {HeadDim::k256, 8, 4, 1024, 16, 4096, 512, 512, 4}, - {HeadDim::k256, 8, 4, 4096, 16, 1, 0, 64, 4}, - {HeadDim::k256, 8, 4, 4096, 16, 1, 512, 16, 4}, - {HeadDim::k256, 8, 4, 4096, 16, 1, 2048, 16, 4}, - {HeadDim::k256, 8, 4, 4096, 16, 16, 0, 512, 4}, - {HeadDim::k256, 8, 4, 4096, 16, 16, 512, 512, 4}, - {HeadDim::k256, 8, 4, 4096, 16, 16, 2048, 512, 4}, + {HeadDim::k256, 8, 4, 0, 16, 16, 0, 256, 2}, + {HeadDim::k256, 8, 4, 0, 16, 16, 512, 256, 2}, + {HeadDim::k256, 8, 4, 0, 16, 16, 2048, 256, 2}, + {HeadDim::k256, 8, 4, 0, 16, 16, 8192, 256, 2}, + {HeadDim::k256, 8, 4, 0, 16, 64, 0, 256, 4}, + {HeadDim::k256, 8, 4, 0, 16, 64, 512, 256, 4}, + {HeadDim::k256, 8, 4, 0, 16, 64, 2048, 256, 4}, + {HeadDim::k256, 8, 4, 0, 16, 64, 8192, 256, 2}, + {HeadDim::k256, 8, 4, 0, 16, 256, 0, 256, 1}, + {HeadDim::k256, 8, 4, 0, 16, 256, 512, 256, 1}, + {HeadDim::k256, 8, 4, 0, 16, 256, 2048, 256, 1}, + {HeadDim::k256, 8, 4, 0, 16, 256, 8192, 256, 1}, + {HeadDim::k256, 8, 4, 0, 16, 1024, 512, 256, 1}, + {HeadDim::k256, 8, 4, 0, 16, 1024, 2048, 256, 1}, + {HeadDim::k256, 8, 4, 0, 16, 1024, 8192, 256, 1}, + {HeadDim::k256, 8, 4, 0, 16, 4096, 2048, 256, 1}, + {HeadDim::k256, 8, 4, 0, 16, 4096, 8192, 256, 1}, + {HeadDim::k256, 8, 4, 1024, 16, 1, 0, 16, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 1, 512, 16, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 16, 0, 1024, 2}, + {HeadDim::k256, 8, 4, 1024, 16, 16, 512, 1024, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 64, 0, 1024, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 64, 512, 1024, 4}, + {HeadDim::k256, 8, 4, 1024, 16, 256, 0, 1024, 1}, + {HeadDim::k256, 8, 4, 1024, 16, 256, 512, 1024, 1}, + {HeadDim::k256, 8, 4, 1024, 16, 1024, 512, 64, 1}, + {HeadDim::k256, 8, 4, 1024, 16, 4096, 512, 64, 1}, + {HeadDim::k256, 8, 4, 4096, 16, 1, 0, 16, 1}, + {HeadDim::k256, 8, 4, 4096, 16, 1, 512, 16, 1}, + {HeadDim::k256, 8, 4, 4096, 16, 1, 2048, 16, 1}, + {HeadDim::k256, 8, 4, 4096, 16, 16, 0, 512, 2}, + {HeadDim::k256, 8, 4, 4096, 16, 16, 512, 64, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 16, 2048, 64, 1}, {HeadDim::k256, 8, 4, 4096, 16, 64, 0, 512, 4}, - {HeadDim::k256, 8, 4, 4096, 16, 64, 512, 256, 4}, - {HeadDim::k256, 8, 4, 4096, 16, 64, 2048, 512, 4}, - {HeadDim::k256, 8, 4, 4096, 16, 256, 0, 512, 4}, - {HeadDim::k256, 8, 4, 4096, 16, 256, 512, 512, 2}, + {HeadDim::k256, 8, 4, 4096, 16, 64, 512, 512, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 64, 2048, 512, 2}, + {HeadDim::k256, 8, 4, 4096, 16, 256, 0, 64, 1}, + {HeadDim::k256, 8, 4, 4096, 16, 256, 512, 64, 1}, {HeadDim::k256, 8, 4, 4096, 16, 256, 2048, 512, 1}, - {HeadDim::k256, 8, 4, 4096, 16, 1024, 512, 256, 4}, + {HeadDim::k256, 8, 4, 4096, 16, 1024, 512, 64, 1}, {HeadDim::k256, 8, 4, 4096, 16, 1024, 2048, 512, 1}, - {HeadDim::k256, 8, 4, 4096, 16, 4096, 2048, 512, 2}, - {HeadDim::k256, 16, 2, 0, 16, 1, 0, 16, 2}, - {HeadDim::k256, 16, 2, 0, 16, 1, 512, 16, 2}, - {HeadDim::k256, 16, 2, 0, 16, 1, 2048, 16, 2}, - {HeadDim::k256, 16, 2, 0, 16, 1, 8192, 16, 2}, - {HeadDim::k256, 16, 2, 0, 16, 16, 0, 128, 1}, - {HeadDim::k256, 16, 2, 0, 16, 16, 512, 16, 1}, - {HeadDim::k256, 16, 2, 0, 16, 16, 2048, 16, 1}, + {HeadDim::k256, 8, 4, 4096, 16, 4096, 2048, 512, 1}, + {HeadDim::k256, 16, 2, 0, 16, 1, 0, 32, 1}, + {HeadDim::k256, 16, 2, 0, 16, 1, 512, 32, 1}, + {HeadDim::k256, 16, 2, 0, 16, 1, 2048, 32, 1}, + {HeadDim::k256, 16, 2, 0, 16, 1, 8192, 32, 1}, + {HeadDim::k256, 16, 2, 0, 16, 16, 0, 128, 2}, + {HeadDim::k256, 16, 2, 0, 16, 16, 512, 128, 2}, + {HeadDim::k256, 16, 2, 0, 16, 16, 2048, 16, 2}, {HeadDim::k256, 16, 2, 0, 16, 16, 8192, 16, 2}, - {HeadDim::k256, 16, 2, 0, 16, 64, 0, 128, 2}, - {HeadDim::k256, 16, 2, 0, 16, 64, 512, 128, 1}, - {HeadDim::k256, 16, 2, 0, 16, 64, 2048, 128, 1}, - {HeadDim::k256, 16, 2, 0, 16, 64, 8192, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 64, 0, 64, 1}, + {HeadDim::k256, 16, 2, 0, 16, 64, 512, 64, 1}, + {HeadDim::k256, 16, 2, 0, 16, 64, 2048, 64, 1}, + {HeadDim::k256, 16, 2, 0, 16, 64, 8192, 16, 1}, {HeadDim::k256, 16, 2, 0, 16, 256, 0, 128, 2}, - {HeadDim::k256, 16, 2, 0, 16, 256, 512, 128, 2}, + {HeadDim::k256, 16, 2, 0, 16, 256, 512, 128, 1}, {HeadDim::k256, 16, 2, 0, 16, 256, 2048, 128, 1}, {HeadDim::k256, 16, 2, 0, 16, 256, 8192, 128, 1}, {HeadDim::k256, 16, 2, 0, 16, 1024, 512, 128, 1}, - {HeadDim::k256, 16, 2, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 1024, 2048, 128, 2}, {HeadDim::k256, 16, 2, 0, 16, 1024, 8192, 128, 1}, - {HeadDim::k256, 16, 2, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 16, 4096, 2048, 128, 2}, {HeadDim::k256, 16, 2, 0, 16, 4096, 8192, 128, 1}, - {HeadDim::k256, 16, 2, 0, 1056, 1, 0, 16, 2}, - {HeadDim::k256, 16, 2, 0, 1056, 1, 512, 16, 2}, - {HeadDim::k256, 16, 2, 0, 1056, 1, 2048, 16, 2}, - {HeadDim::k256, 16, 2, 0, 1056, 1, 8192, 16, 2}, - {HeadDim::k256, 16, 2, 0, 1056, 16, 0, 128, 1}, - {HeadDim::k256, 16, 2, 0, 1056, 16, 512, 16, 1}, - {HeadDim::k256, 16, 2, 0, 1056, 16, 2048, 16, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 1, 0, 32, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 1, 512, 32, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 1, 2048, 32, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 1, 8192, 32, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 16, 0, 128, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 16, 512, 128, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 16, 2048, 16, 2}, {HeadDim::k256, 16, 2, 0, 1056, 16, 8192, 16, 2}, - {HeadDim::k256, 16, 2, 0, 1056, 64, 0, 128, 2}, - {HeadDim::k256, 16, 2, 0, 1056, 64, 512, 128, 1}, - {HeadDim::k256, 16, 2, 0, 1056, 64, 2048, 128, 1}, - {HeadDim::k256, 16, 2, 0, 1056, 64, 8192, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 64, 0, 64, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 64, 512, 64, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 64, 2048, 64, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 64, 8192, 16, 1}, {HeadDim::k256, 16, 2, 0, 1056, 256, 0, 128, 2}, - {HeadDim::k256, 16, 2, 0, 1056, 256, 512, 128, 2}, + {HeadDim::k256, 16, 2, 0, 1056, 256, 512, 128, 1}, {HeadDim::k256, 16, 2, 0, 1056, 256, 2048, 128, 1}, {HeadDim::k256, 16, 2, 0, 1056, 256, 8192, 128, 1}, {HeadDim::k256, 16, 2, 0, 1056, 1024, 512, 128, 1}, - {HeadDim::k256, 16, 2, 0, 1056, 1024, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 1024, 2048, 128, 2}, {HeadDim::k256, 16, 2, 0, 1056, 1024, 8192, 128, 1}, - {HeadDim::k256, 16, 2, 0, 1056, 4096, 2048, 128, 1}, + {HeadDim::k256, 16, 2, 0, 1056, 4096, 2048, 128, 2}, {HeadDim::k256, 16, 2, 0, 1056, 4096, 8192, 128, 1}, - {HeadDim::k256, 16, 4, 0, 16, 1, 0, 32, 4}, - {HeadDim::k256, 16, 4, 0, 16, 1, 512, 32, 2}, - {HeadDim::k256, 16, 4, 0, 16, 1, 2048, 32, 2}, - {HeadDim::k256, 16, 4, 0, 16, 1, 8192, 32, 2}, + {HeadDim::k256, 16, 4, 0, 16, 1, 0, 16, 4}, + {HeadDim::k256, 16, 4, 0, 16, 1, 512, 16, 4}, + {HeadDim::k256, 16, 4, 0, 16, 1, 2048, 16, 4}, + {HeadDim::k256, 16, 4, 0, 16, 1, 8192, 16, 4}, {HeadDim::k256, 16, 4, 0, 16, 16, 0, 64, 4}, {HeadDim::k256, 16, 4, 0, 16, 16, 512, 64, 4}, {HeadDim::k256, 16, 4, 0, 16, 16, 2048, 64, 4}, - {HeadDim::k256, 16, 4, 0, 16, 16, 8192, 32, 1}, - {HeadDim::k256, 16, 4, 0, 16, 64, 0, 64, 2}, - {HeadDim::k256, 16, 4, 0, 16, 64, 512, 64, 2}, - {HeadDim::k256, 16, 4, 0, 16, 64, 2048, 64, 1}, + {HeadDim::k256, 16, 4, 0, 16, 16, 8192, 64, 4}, + {HeadDim::k256, 16, 4, 0, 16, 64, 0, 64, 1}, + {HeadDim::k256, 16, 4, 0, 16, 64, 512, 64, 4}, + {HeadDim::k256, 16, 4, 0, 16, 64, 2048, 64, 4}, {HeadDim::k256, 16, 4, 0, 16, 64, 8192, 64, 1}, - {HeadDim::k256, 16, 4, 0, 16, 256, 0, 64, 2}, - {HeadDim::k256, 16, 4, 0, 16, 256, 512, 64, 2}, - {HeadDim::k256, 16, 4, 0, 16, 256, 2048, 64, 2}, + {HeadDim::k256, 16, 4, 0, 16, 256, 0, 64, 1}, + {HeadDim::k256, 16, 4, 0, 16, 256, 512, 64, 1}, + {HeadDim::k256, 16, 4, 0, 16, 256, 2048, 64, 1}, {HeadDim::k256, 16, 4, 0, 16, 256, 8192, 64, 1}, - {HeadDim::k256, 16, 4, 0, 16, 1024, 512, 64, 2}, - {HeadDim::k256, 16, 4, 0, 16, 1024, 2048, 64, 2}, + {HeadDim::k256, 16, 4, 0, 16, 1024, 512, 64, 1}, + {HeadDim::k256, 16, 4, 0, 16, 1024, 2048, 64, 1}, {HeadDim::k256, 16, 4, 0, 16, 1024, 8192, 64, 1}, {HeadDim::k256, 16, 4, 0, 16, 4096, 2048, 64, 1}, {HeadDim::k256, 16, 4, 0, 16, 4096, 8192, 64, 1}, - {HeadDim::k256, 16, 4, 0, 528, 1, 0, 32, 4}, - {HeadDim::k256, 16, 4, 0, 528, 1, 512, 32, 2}, - {HeadDim::k256, 16, 4, 0, 528, 1, 2048, 32, 2}, - {HeadDim::k256, 16, 4, 0, 528, 1, 8192, 32, 2}, + {HeadDim::k256, 16, 4, 0, 528, 1, 0, 16, 4}, + {HeadDim::k256, 16, 4, 0, 528, 1, 512, 16, 4}, + {HeadDim::k256, 16, 4, 0, 528, 1, 2048, 16, 4}, + {HeadDim::k256, 16, 4, 0, 528, 1, 8192, 16, 4}, {HeadDim::k256, 16, 4, 0, 528, 16, 0, 64, 4}, {HeadDim::k256, 16, 4, 0, 528, 16, 512, 64, 4}, {HeadDim::k256, 16, 4, 0, 528, 16, 2048, 64, 4}, - {HeadDim::k256, 16, 4, 0, 528, 16, 8192, 32, 1}, - {HeadDim::k256, 16, 4, 0, 528, 64, 0, 64, 2}, - {HeadDim::k256, 16, 4, 0, 528, 64, 512, 64, 2}, - {HeadDim::k256, 16, 4, 0, 528, 64, 2048, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 16, 8192, 64, 4}, + {HeadDim::k256, 16, 4, 0, 528, 64, 0, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 64, 512, 64, 4}, + {HeadDim::k256, 16, 4, 0, 528, 64, 2048, 64, 4}, {HeadDim::k256, 16, 4, 0, 528, 64, 8192, 64, 1}, - {HeadDim::k256, 16, 4, 0, 528, 256, 0, 64, 2}, - {HeadDim::k256, 16, 4, 0, 528, 256, 512, 64, 2}, - {HeadDim::k256, 16, 4, 0, 528, 256, 2048, 64, 2}, + {HeadDim::k256, 16, 4, 0, 528, 256, 0, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 256, 512, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 256, 2048, 64, 1}, {HeadDim::k256, 16, 4, 0, 528, 256, 8192, 64, 1}, - {HeadDim::k256, 16, 4, 0, 528, 1024, 512, 64, 2}, - {HeadDim::k256, 16, 4, 0, 528, 1024, 2048, 64, 2}, + {HeadDim::k256, 16, 4, 0, 528, 1024, 512, 64, 1}, + {HeadDim::k256, 16, 4, 0, 528, 1024, 2048, 64, 1}, {HeadDim::k256, 16, 4, 0, 528, 1024, 8192, 64, 1}, {HeadDim::k256, 16, 4, 0, 528, 4096, 2048, 64, 1}, {HeadDim::k256, 16, 4, 0, 528, 4096, 8192, 64, 1}, - {HeadDim::k256, 16, 8, 0, 16, 1, 0, 512, 8}, - {HeadDim::k256, 16, 8, 0, 16, 1, 512, 512, 8}, + {HeadDim::k256, 16, 8, 0, 16, 1, 0, 16, 8}, + {HeadDim::k256, 16, 8, 0, 16, 1, 512, 16, 4}, {HeadDim::k256, 16, 8, 0, 16, 1, 2048, 16, 8}, - {HeadDim::k256, 16, 8, 0, 16, 1, 8192, 16, 8}, - {HeadDim::k256, 16, 8, 0, 16, 16, 0, 512, 4}, - {HeadDim::k256, 16, 8, 0, 16, 16, 512, 512, 8}, - {HeadDim::k256, 16, 8, 0, 16, 16, 2048, 512, 4}, - {HeadDim::k256, 16, 8, 0, 16, 16, 8192, 512, 8}, - {HeadDim::k256, 16, 8, 0, 16, 64, 0, 128, 4}, - {HeadDim::k256, 16, 8, 0, 16, 64, 512, 128, 4}, - {HeadDim::k256, 16, 8, 0, 16, 64, 2048, 128, 1}, - {HeadDim::k256, 16, 8, 0, 16, 64, 8192, 128, 2}, - {HeadDim::k256, 16, 8, 0, 16, 256, 0, 512, 4}, - {HeadDim::k256, 16, 8, 0, 16, 256, 512, 128, 2}, + {HeadDim::k256, 16, 8, 0, 16, 1, 8192, 16, 1}, + {HeadDim::k256, 16, 8, 0, 16, 16, 0, 128, 4}, + {HeadDim::k256, 16, 8, 0, 16, 16, 512, 128, 4}, + {HeadDim::k256, 16, 8, 0, 16, 16, 2048, 64, 8}, + {HeadDim::k256, 16, 8, 0, 16, 16, 8192, 64, 8}, + {HeadDim::k256, 16, 8, 0, 16, 64, 0, 128, 8}, + {HeadDim::k256, 16, 8, 0, 16, 64, 512, 128, 8}, + {HeadDim::k256, 16, 8, 0, 16, 64, 2048, 64, 1}, + {HeadDim::k256, 16, 8, 0, 16, 64, 8192, 64, 1}, + {HeadDim::k256, 16, 8, 0, 16, 256, 0, 128, 1}, + {HeadDim::k256, 16, 8, 0, 16, 256, 512, 128, 1}, {HeadDim::k256, 16, 8, 0, 16, 256, 2048, 128, 1}, {HeadDim::k256, 16, 8, 0, 16, 256, 8192, 128, 1}, - {HeadDim::k256, 16, 8, 0, 16, 1024, 512, 128, 2}, + {HeadDim::k256, 16, 8, 0, 16, 1024, 512, 128, 1}, {HeadDim::k256, 16, 8, 0, 16, 1024, 2048, 128, 1}, {HeadDim::k256, 16, 8, 0, 16, 1024, 8192, 128, 1}, - {HeadDim::k256, 16, 8, 0, 16, 4096, 2048, 128, 2}, - {HeadDim::k256, 16, 8, 0, 16, 4096, 8192, 128, 2}, + {HeadDim::k256, 16, 8, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k256, 16, 8, 0, 16, 4096, 8192, 128, 1}, {HeadDim::k256, 16, 8, 1024, 16, 1, 0, 16, 8}, - {HeadDim::k256, 16, 8, 1024, 16, 1, 512, 16, 8}, - {HeadDim::k256, 16, 8, 1024, 16, 16, 0, 128, 4}, - {HeadDim::k256, 16, 8, 1024, 16, 16, 512, 128, 8}, - {HeadDim::k256, 16, 8, 1024, 16, 64, 0, 256, 4}, - {HeadDim::k256, 16, 8, 1024, 16, 64, 512, 256, 4}, - {HeadDim::k256, 16, 8, 1024, 16, 256, 0, 128, 4}, - {HeadDim::k256, 16, 8, 1024, 16, 256, 512, 256, 4}, - {HeadDim::k256, 16, 8, 1024, 16, 1024, 512, 256, 8}, - {HeadDim::k256, 16, 8, 1024, 16, 4096, 512, 256, 8}, + {HeadDim::k256, 16, 8, 1024, 16, 1, 512, 64, 8}, + {HeadDim::k256, 16, 8, 1024, 16, 16, 0, 64, 4}, + {HeadDim::k256, 16, 8, 1024, 16, 16, 512, 64, 8}, + {HeadDim::k256, 16, 8, 1024, 16, 64, 0, 64, 8}, + {HeadDim::k256, 16, 8, 1024, 16, 64, 512, 64, 2}, + {HeadDim::k256, 16, 8, 1024, 16, 256, 0, 128, 1}, + {HeadDim::k256, 16, 8, 1024, 16, 256, 512, 64, 1}, + {HeadDim::k256, 16, 8, 1024, 16, 1024, 512, 128, 1}, + {HeadDim::k256, 16, 8, 1024, 16, 4096, 512, 128, 1}, {HeadDim::k256, 24, 4, 0, 16, 1, 0, 16, 4}, - {HeadDim::k256, 24, 4, 0, 16, 1, 512, 16, 4}, + {HeadDim::k256, 24, 4, 0, 16, 1, 512, 128, 4}, {HeadDim::k256, 24, 4, 0, 16, 1, 2048, 16, 4}, {HeadDim::k256, 24, 4, 0, 16, 1, 8192, 16, 4}, - {HeadDim::k256, 24, 4, 0, 16, 16, 0, 64, 4}, - {HeadDim::k256, 24, 4, 0, 16, 16, 512, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 16, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 16, 0, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 16, 512, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 16, 2048, 128, 1}, {HeadDim::k256, 24, 4, 0, 16, 16, 8192, 16, 1}, - {HeadDim::k256, 24, 4, 0, 16, 64, 0, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 64, 512, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 64, 2048, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 64, 8192, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 256, 0, 64, 4}, - {HeadDim::k256, 24, 4, 0, 16, 256, 512, 64, 2}, - {HeadDim::k256, 24, 4, 0, 16, 256, 2048, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 256, 8192, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 1024, 512, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 1024, 2048, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 1024, 8192, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 4096, 2048, 64, 1}, - {HeadDim::k256, 24, 4, 0, 16, 4096, 8192, 64, 1}, + {HeadDim::k256, 24, 4, 0, 16, 64, 0, 128, 4}, + {HeadDim::k256, 24, 4, 0, 16, 64, 512, 128, 2}, + {HeadDim::k256, 24, 4, 0, 16, 64, 2048, 128, 4}, + {HeadDim::k256, 24, 4, 0, 16, 64, 8192, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 256, 0, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 256, 512, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 256, 2048, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 256, 8192, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 1024, 512, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 1024, 2048, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 1024, 8192, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 4096, 2048, 128, 1}, + {HeadDim::k256, 24, 4, 0, 16, 4096, 8192, 128, 1}, {HeadDim::k256, 24, 4, 0, 784, 1, 0, 16, 4}, - {HeadDim::k256, 24, 4, 0, 784, 1, 512, 16, 4}, + {HeadDim::k256, 24, 4, 0, 784, 1, 512, 128, 4}, {HeadDim::k256, 24, 4, 0, 784, 1, 2048, 16, 4}, {HeadDim::k256, 24, 4, 0, 784, 1, 8192, 16, 4}, - {HeadDim::k256, 24, 4, 0, 784, 16, 0, 64, 4}, - {HeadDim::k256, 24, 4, 0, 784, 16, 512, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 16, 2048, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 16, 0, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 16, 512, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 16, 2048, 128, 1}, {HeadDim::k256, 24, 4, 0, 784, 16, 8192, 16, 1}, - {HeadDim::k256, 24, 4, 0, 784, 64, 0, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 64, 512, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 64, 2048, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 64, 8192, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 256, 0, 64, 4}, - {HeadDim::k256, 24, 4, 0, 784, 256, 512, 64, 2}, - {HeadDim::k256, 24, 4, 0, 784, 256, 2048, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 256, 8192, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 1024, 512, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 1024, 2048, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 1024, 8192, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 4096, 2048, 64, 1}, - {HeadDim::k256, 24, 4, 0, 784, 4096, 8192, 64, 1}, + {HeadDim::k256, 24, 4, 0, 784, 64, 0, 128, 4}, + {HeadDim::k256, 24, 4, 0, 784, 64, 512, 128, 2}, + {HeadDim::k256, 24, 4, 0, 784, 64, 2048, 128, 4}, + {HeadDim::k256, 24, 4, 0, 784, 64, 8192, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 256, 0, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 256, 512, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 256, 2048, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 256, 8192, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 1024, 512, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 1024, 2048, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 1024, 8192, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 4096, 2048, 128, 1}, + {HeadDim::k256, 24, 4, 0, 784, 4096, 8192, 128, 1}, {HeadDim::k256, 32, 16, 1024, 16, 1, 0, 16, 16}, - {HeadDim::k256, 32, 16, 1024, 16, 1, 512, 16, 16}, - {HeadDim::k256, 32, 16, 1024, 16, 16, 0, 16, 16}, - {HeadDim::k256, 32, 16, 1024, 16, 16, 512, 16, 16}, - {HeadDim::k256, 32, 16, 1024, 16, 64, 0, 128, 4}, - {HeadDim::k256, 32, 16, 1024, 16, 64, 512, 128, 2}, - {HeadDim::k256, 32, 16, 1024, 16, 256, 0, 16, 4}, - {HeadDim::k256, 32, 16, 1024, 16, 256, 512, 128, 2}, - {HeadDim::k256, 32, 16, 1024, 16, 1024, 512, 128, 16}, - {HeadDim::k256, 32, 16, 1024, 16, 4096, 512, 128, 16}, + {HeadDim::k256, 32, 16, 1024, 16, 1, 512, 256, 16}, + {HeadDim::k256, 32, 16, 1024, 16, 16, 0, 256, 16}, + {HeadDim::k256, 32, 16, 1024, 16, 16, 512, 256, 16}, + {HeadDim::k256, 32, 16, 1024, 16, 64, 0, 256, 2}, + {HeadDim::k256, 32, 16, 1024, 16, 64, 512, 256, 4}, + {HeadDim::k256, 32, 16, 1024, 16, 256, 0, 128, 1}, + {HeadDim::k256, 32, 16, 1024, 16, 256, 512, 256, 1}, + {HeadDim::k256, 32, 16, 1024, 16, 1024, 512, 128, 1}, + {HeadDim::k256, 32, 16, 1024, 16, 4096, 512, 128, 1}, {HeadDim::k512, 8, 1, 0, 16, 1, 0, 16, 1}, {HeadDim::k512, 8, 1, 0, 16, 1, 512, 16, 1}, {HeadDim::k512, 8, 1, 0, 16, 1, 2048, 16, 1}, From d2cd6d7271d14bb7809191331ef35169cbe6c162 Mon Sep 17 00:00:00 2001 From: Robert Esclapez Garcia Date: Sat, 24 Oct 2026 05:14:04 -0700 Subject: [PATCH 5/6] [ROCm][RDNA] skinny GEMMs: no VGPR spills in wvSplitK on gfx11 and gfx12 wvSplitK_hf_sml_, wvSplitK_hf_ and wvSplitK_hf_big_ held every activation chunk of an unrolled step in registers next to the weights', and bf16 widened the weights to f32 before multiplying. With 1024-thread workgroups a wave gets 128 VGPRs on gfx1150 (192 on gfx1151), and the large tiles spilled: 102 of the 388 kernels of skinny_gemms.cu on gfx1150, 78 on gfx1151, among them the 32/1/32/16/8/1 bf16 tile most decode GEMMs of a 2B model use (139 VGPRs spilled). A spilling GEMM is slow, and its speed depends on the scratch the process allocated before: a kernel run before with 256 B/lane of scratch made it 1.3x slower, 4 KiB 2.1x. On gfx1x: - bf16 multiplies with v_dot2_f32_bf16, as fp16 with v_dot2_f32_f16, keeping the weights packed. - A tile holds the activation chunks while (N + YTILE) * UNRL * A_CHUNK / 2 operand VGPRs plus 64 fit the wave's VGPRs (1024 or 1536 a lane, split by the workgroup's waves per SIMD); else it reads each chunk from LDS where it is used, and only the weights' loads stay in flight. gfx9 code is unchanged. All 388 kernels of skinny_gemms.cu on gfx1150, gfx1151 and gfx1201 compile without VGPR spills; the ones left with scratch are the assert(false) stubs of the gfx950/MI3xx-only wvSplitKrc_ and wvSplitKQ_ (fp8) kernels. bench_rocm_skinny_gemm.py (76 shape x N cells per dtype, N 1..4, best of two runs per build), against gfx11: - gfx1150 bf16 1.050x (0.985x-2.63x, none slower by more than 2%), fp16 1.008x (0.984x-1.51x); N=1 and N=2 bf16 1.09x and 1.11x. - gfx1151 bf16 1.004x, fp16 1.000x. The 4096x2048 GEMM keeps 186-210 us whatever scratch a kernel run before used (242-651 us before). Cosmos-Reason2-2B, document-qa-long-context, decode on gfx1150 with TRITON_ATTN in both builds: 19.5 -> 22.6 tokens/s. test_rocm_skinny_gemms.py: 172 passed on gfx1150 and gfx1151 (the others are gfx950- or fp8-only). AI assistance was used for this patch. Co-authored-by: Claude Signed-off-by: Robert Esclapez Garcia --- csrc/rocm/skinny_gemms.cu | 268 +++++++++++++++++++++++++++----------- 1 file changed, 194 insertions(+), 74 deletions(-) diff --git a/csrc/rocm/skinny_gemms.cu b/csrc/rocm/skinny_gemms.cu index b214cfa804a9..6598ff9d8fdf 100644 --- a/csrc/rocm/skinny_gemms.cu +++ b/csrc/rocm/skinny_gemms.cu @@ -324,17 +324,36 @@ torch::Tensor LLMM1(at::Tensor& in_a, at::Tensor& in_b, V0 += (s.x + s.y); \ } #elif defined(__HIP__GFX1X__) - // gfx1x: v_dot2_f32_f16 (VOP3-P, dot10-insts, available on gfx11+gfx12) - #define DOT2C(V0, V2, V3) \ - if constexpr (std::is_same_v) { \ - asm("v_dot2_f32_f16 %0, %1, %2, %0" : "+v"(V0) : "v"(V2), "v"(V3)); \ - } else if constexpr (std::is_same_v) { \ - float2 s = __bfloat1622float2(*((__hip_bfloat162*)(&(V2)))) * \ - __bfloat1622float2(*((__hip_bfloat162*)(&(V3)))); \ - V0 += (s.x + s.y); \ + // gfx1x: v_dot2_f32_f16 and v_dot2_f32_bf16 (VOP3-P, gfx11 and gfx12). The + // bf16 dot keeps the weights packed; widening them to f32 first doubles + // their registers and makes the large tiles spill. + #define DOT2C(V0, V2, V3) \ + if constexpr (std::is_same_v) { \ + asm("v_dot2_f32_f16 %0, %1, %2, %0" : "+v"(V0) : "v"(V2), "v"(V3)); \ + } else if constexpr (std::is_same_v) { \ + asm("v_dot2_f32_bf16 %0, %1, %2, %0" : "+v"(V0) : "v"(V2), "v"(V3)); \ } #endif +#if defined(__HIP__GFX1X__) + // VGPRs a lane holds: 1024 on gfx1102, gfx1103 and gfx1150..gfx1153, else + // 1536. + #if defined(__gfx1102__) || defined(__gfx1103__) || defined(__gfx1150__) || \ + defined(__gfx1152__) || defined(__gfx1153__) +inline constexpr int kLaneVgprs = 1024; + #else +inline constexpr int kLaneVgprs = 1536; + #endif +// The VGPRs a wave of a THREADS-thread workgroup gets: its wave32 waves share +// a WGP's four SIMDs. +__device__ constexpr int wave_vgprs(int threads) { + const int vgprs = kLaneVgprs / ((threads / 32 + 3) / 4); + return vgprs < 256 ? vgprs : 256; +} +// Registers beside the operand tiles: sums, addresses, loop state. +inline constexpr int kOperandSlackVgprs = 64; +#endif + // To avoid LLVM silently upcasting to double __device__ inline unsigned int min__(uint32_t a, uint32_t b) { return min(a, b); @@ -483,10 +502,9 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) // are being worked on by each wave. //---------------------------------------------------- float sum[N][YTILE] = {}; - scalar8 sum4[N][YTILE] = {}; + [[maybe_unused]] scalar8 sum4[N][YTILE] = {}; for (uint32_t k1 = 0; k1 < K; k1 += THRDS * A_CHUNK * UNRL) { - bigType bigA[N][UNRL] = {}; bigType bigB[YTILE][UNRL]; // Fetch the weight matrix from memory! #pragma unroll @@ -498,29 +516,59 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) bigB[y][k2].h8 = (loadnt((scalar8*)(&B_[min__(y + m, M - 1) * Kbp]))); } - // Fetch activation matrix from either just LDS or from both LDS / memory + #if defined(__HIP__GFX1X__) + // Hold the activation chunks too while they fit; else read each + // where it is used, so only the weights' loads stay in flight. + constexpr bool hold_a = + (N + YTILE) * UNRL * (A_CHUNK / 2) + kOperandSlackVgprs <= + wave_vgprs(WvPrGrp * THRDS); + #else + constexpr bool hold_a = true; + #endif + if constexpr (hold_a) { + bigType bigA[N][UNRL] = {}; + // Fetch activation matrix from either just LDS or from both LDS / + // memory #pragma unroll - for (uint32_t k2 = 0; k2 < UNRL; k2++) { - uint32_t k = k1 + k2 * THRDS * A_CHUNK; - uint32_t k_ = k + threadIdx.x * A_CHUNK; - if (k_ >= K) break; - for (int n = 0; n < N; n++) { - bigA[n][k2] = *((const bigType*)(&(s[k_ + Kap * n]))); + for (uint32_t k2 = 0; k2 < UNRL; k2++) { + uint32_t k = k1 + k2 * THRDS * A_CHUNK; + uint32_t k_ = k + threadIdx.x * A_CHUNK; + if (k_ >= K) break; + for (int n = 0; n < N; n++) { + bigA[n][k2] = *((const bigType*)(&(s[k_ + Kap * n]))); + } } - } - // Do the matrix multiplication in interleaved manner - for (uint32_t k2 = 0; k2 < UNRL; k2++) { - for (uint32_t n = 0; n < N; n++) { - for (int y = 0; y < YTILE; y++) { - if constexpr (!use_mfma) + // Do the matrix multiplication in interleaved manner + for (uint32_t k2 = 0; k2 < UNRL; k2++) { + for (uint32_t n = 0; n < N; n++) { + for (int y = 0; y < YTILE; y++) { + if constexpr (!use_mfma) + for (uint32_t b = 0; b < A_CHUNK / 2; b++) { + DOT2C(sum[n][y], bigA[n][k2].f[b], bigB[y][k2].f[b]) + } + else + for (uint32_t b = 0; b < A_CHUNK / 4; b++) + sum4[n][y] = __builtin_amdgcn_mfma_f32_4x4x4bf16_1k( + bigA[n][k2].h4[b], bigB[y][k2].h4[b], sum4[n][y], 0, 0, + 0); + } + } + } + } else { + #pragma unroll + for (uint32_t k2 = 0; k2 < UNRL; k2++) { + uint32_t k_ = k1 + k2 * THRDS * A_CHUNK + threadIdx.x * A_CHUNK; + #pragma unroll + for (uint32_t n = 0; n < N; n++) { + bigType a = {}; + if (k_ < K) a = *((const bigType*)(&(s[k_ + Kap * n]))); + #pragma unroll + for (int y = 0; y < YTILE; y++) + #pragma unroll for (uint32_t b = 0; b < A_CHUNK / 2; b++) { - DOT2C(sum[n][y], bigA[n][k2].f[b], bigB[y][k2].f[b]) + DOT2C(sum[n][y], a.f[b], bigB[y][k2].f[b]) } - else - for (uint32_t b = 0; b < A_CHUNK / 4; b++) - sum4[n][y] = __builtin_amdgcn_mfma_f32_4x4x4bf16_1k( - bigA[n][k2].h4[b], bigB[y][k2].h4[b], sum4[n][y], 0, 0, 0); } } } @@ -713,10 +761,9 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) while (m < M) { float sum[N][YTILE] = {}; - scalar8 sum4[N][YTILE] = {}; + [[maybe_unused]] scalar8 sum4[N][YTILE] = {}; for (uint32_t k1 = 0; k1 < K; k1 += THRDS * A_CHUNK * UNRL) { - bigType bigA[N][UNRL] = {}; bigType bigB[YTILE][UNRL]; // Fetch the weight matrix from memory! #pragma unroll @@ -728,32 +775,67 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) bigB[y][k2].h8 = (loadnt((scalar8*)(&B_[min__(y + m, M - 1) * Kbp]))); } - // Fetch activation matrix from either just LDS or from both LDS / memory + #if defined(__HIP__GFX1X__) + // Hold the activation chunks too while they fit; else read each + // where it is used, so only the weights' loads stay in flight. + constexpr bool hold_a = + (N + YTILE) * UNRL * (A_CHUNK / 2) + kOperandSlackVgprs <= + wave_vgprs(WvPrGrp * THRDS); + #else + constexpr bool hold_a = true; + #endif + if constexpr (hold_a) { + bigType bigA[N][UNRL] = {}; + // Fetch activation matrix from either just LDS or from both LDS / + // memory #pragma unroll - for (uint32_t k2 = 0; k2 < UNRL; k2++) { - uint32_t k = k1 + k2 * THRDS * A_CHUNK; - uint32_t k_ = k + threadIdx.x * A_CHUNK; - if (k_ >= K) break; - for (int n = 0; n < N; n++) { - if (k_ + Kap * n < max_lds_len) - bigA[n][k2] = *((const bigType*)(&(s[k_ + Kap * n]))); - else - bigA[n][k2] = *((const bigType*)(&(A[k_ + Kap * n]))); + for (uint32_t k2 = 0; k2 < UNRL; k2++) { + uint32_t k = k1 + k2 * THRDS * A_CHUNK; + uint32_t k_ = k + threadIdx.x * A_CHUNK; + if (k_ >= K) break; + for (int n = 0; n < N; n++) { + if (k_ + Kap * n < max_lds_len) + bigA[n][k2] = *((const bigType*)(&(s[k_ + Kap * n]))); + else + bigA[n][k2] = *((const bigType*)(&(A[k_ + Kap * n]))); + } } - } - // Do the matrix multiplication in interleaved manner - for (uint32_t n = 0; n < N; n++) { + // Do the matrix multiplication in interleaved manner + for (uint32_t n = 0; n < N; n++) { + for (uint32_t k2 = 0; k2 < UNRL; k2++) { + for (int y = 0; y < YTILE; y++) { + if constexpr (!use_mfma) + for (uint32_t b = 0; b < A_CHUNK / 2; b++) { + DOT2C(sum[n][y], bigA[n][k2].f[b], bigB[y][k2].f[b]) + } + else + for (uint32_t b = 0; b < A_CHUNK / 4; b++) + sum4[n][y] = __builtin_amdgcn_mfma_f32_4x4x4bf16_1k( + bigA[n][k2].h4[b], bigB[y][k2].h4[b], sum4[n][y], 0, 0, + 0); + } + } + } + } else { + #pragma unroll for (uint32_t k2 = 0; k2 < UNRL; k2++) { - for (int y = 0; y < YTILE; y++) { - if constexpr (!use_mfma) + uint32_t k_ = k1 + k2 * THRDS * A_CHUNK + threadIdx.x * A_CHUNK; + #pragma unroll + for (uint32_t n = 0; n < N; n++) { + bigType a = {}; + if (k_ < K) { + if (k_ + Kap * n < max_lds_len) + a = *((const bigType*)(&(s[k_ + Kap * n]))); + else + a = *((const bigType*)(&(A[k_ + Kap * n]))); + } + #pragma unroll + for (int y = 0; y < YTILE; y++) + #pragma unroll for (uint32_t b = 0; b < A_CHUNK / 2; b++) { - DOT2C(sum[n][y], bigA[n][k2].f[b], bigB[y][k2].f[b]) + DOT2C(sum[n][y], a.f[b], bigB[y][k2].f[b]) } - else - for (uint32_t b = 0; b < A_CHUNK / 4; b++) - sum4[n][y] = __builtin_amdgcn_mfma_f32_4x4x4bf16_1k( - bigA[n][k2].h4[b], bigB[y][k2].h4[b], sum4[n][y], 0, 0, 0); } } } @@ -1010,7 +1092,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) // are being worked on by each wave. //---------------------------------------------------- float sum[N][YTILE] = {}; - scalar8 sum4[N][YTILE] = {}; + [[maybe_unused]] scalar8 sum4[N][YTILE] = {}; //---------------------------------------------------- // Fetch weight matrix B in interleaved K-split! @@ -1030,7 +1112,6 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) // TODO: Logic below will only work when K is multiple of 8 //---------------------------------------------------- for (uint32_t k1 = 0; k1 < K; k1 += THRDS * A_CHUNK * UNRL) { - bigType bigA[N][UNRL] = {}; bigType bigB[YTILE][UNRL]; #ifdef PCML @@ -1067,37 +1148,76 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) bigB[y][k2].h8 = (loadnt((scalar8*)(&B_[min__(y + m, M - 1) * Kbp]))); } - // Fetch activation matrix from either just LDS or from both LDS / memory + #if defined(__HIP__GFX1X__) + // Hold the activation chunks too while they fit; else read each + // where it is used, so only the weights' loads stay in flight. + constexpr bool hold_a = + (N + YTILE) * UNRL * (A_CHUNK / 2) + kOperandSlackVgprs <= + wave_vgprs(WvPrGrp * THRDS); + #else + constexpr bool hold_a = true; + #endif + if constexpr (hold_a) { + bigType bigA[N][UNRL] = {}; + // Fetch activation matrix from either just LDS or from both LDS / + // memory #pragma unroll - for (uint32_t k2 = 0; k2 < UNRL; k2++) { - uint32_t k = k1 + k2 * THRDS * A_CHUNK; - uint32_t k_ = k + threadIdx.x * A_CHUNK; - if (k_ >= K) break; - for (int n = 0; n < N; n++) { + for (uint32_t k2 = 0; k2 < UNRL; k2++) { + uint32_t k = k1 + k2 * THRDS * A_CHUNK; + uint32_t k_ = k + threadIdx.x * A_CHUNK; + if (k_ >= K) break; + for (int n = 0; n < N; n++) { #ifdef PCML - bigA[n][k2] = *((const bigType*)(&(s[k_ - kBase + kFit * n]))); + bigA[n][k2] = *((const bigType*)(&(s[k_ - kBase + kFit * n]))); #else - if (k_ + Kap * n < max_lds_len) - bigA[n][k2] = *((const bigType*)(&(s[k_ + Kap * n]))); - else - bigA[n][k2] = *((const bigType*)(&(A[k_ + Kap * n]))); + if (k_ + Kap * n < max_lds_len) + bigA[n][k2] = *((const bigType*)(&(s[k_ + Kap * n]))); + else + bigA[n][k2] = *((const bigType*)(&(A[k_ + Kap * n]))); #endif + } } - } - // Do the matrix multiplication in interleaved manner + // Do the matrix multiplication in interleaved manner + #pragma unroll + for (uint32_t k2 = 0; k2 < UNRL; k2++) { + for (uint32_t n = 0; n < N; n++) { + for (int y = 0; y < YTILE; y++) { + if constexpr (!use_mfma) + for (uint32_t b = 0; b < A_CHUNK / 2; b++) { + DOT2C(sum[n][y], bigA[n][k2].f[b], bigB[y][k2].f[b]) + } + else + for (uint32_t b = 0; b < A_CHUNK / 4; b++) + sum4[n][y] = __builtin_amdgcn_mfma_f32_4x4x4bf16_1k( + bigA[n][k2].h4[b], bigB[y][k2].h4[b], sum4[n][y], 0, 0, + 0); + } + } + } + } else { + #pragma unroll + for (uint32_t k2 = 0; k2 < UNRL; k2++) { + uint32_t k_ = k1 + k2 * THRDS * A_CHUNK + threadIdx.x * A_CHUNK; + #pragma unroll + for (uint32_t n = 0; n < N; n++) { + bigType a = {}; + if (k_ < K) { + #ifdef PCML + a = *((const bigType*)(&(s[k_ - kBase + kFit * n]))); + #else + if (k_ + Kap * n < max_lds_len) + a = *((const bigType*)(&(s[k_ + Kap * n]))); + else + a = *((const bigType*)(&(A[k_ + Kap * n]))); + #endif + } + #pragma unroll + for (int y = 0; y < YTILE; y++) #pragma unroll - for (uint32_t k2 = 0; k2 < UNRL; k2++) { - for (uint32_t n = 0; n < N; n++) { - for (int y = 0; y < YTILE; y++) { - if constexpr (!use_mfma) for (uint32_t b = 0; b < A_CHUNK / 2; b++) { - DOT2C(sum[n][y], bigA[n][k2].f[b], bigB[y][k2].f[b]) + DOT2C(sum[n][y], a.f[b], bigB[y][k2].f[b]) } - else - for (uint32_t b = 0; b < A_CHUNK / 4; b++) - sum4[n][y] = __builtin_amdgcn_mfma_f32_4x4x4bf16_1k( - bigA[n][k2].h4[b], bigB[y][k2].h4[b], sum4[n][y], 0, 0, 0); } } } From 9312929d4c1f613146a27c78445202dfaf6f8e7d Mon Sep 17 00:00:00 2001 From: Robert Esclapez Garcia Date: Sat, 24 Oct 2026 10:56:22 -0700 Subject: [PATCH 6/6] [ROCm][RDNA] int4 skinny GEMMs: no spilling tile on 1024-VGPR archs, only reachable tiles built The int4 dispatch macros expand every tile for every N = 1..5, so tiles the dispatch only picks for N = 1 were also built for N = 2..5, where they are never launched: 480 of the 1684 int4 kernels, and most of the ones that spill (310 of 334 on gfx1150, 285 of 293 on gfx1151). The N = 1 arms are now `if constexpr (__N == 1)`, so those are not built; the dispatch is unchanged. Of the kernels left, the only one that spills on gfx1150 is the MoE (YTILE=4, W=32, AC=32, UNRL=4) tile for N = 1 at K = 2048 (Qwen3.5-A3B gate_up): a 1024-thread workgroup leaves a wave 128 VGPRs on a 1024-VGPR arch. gfx1102, gfx1103 and gfx1150..gfx1153 take the (4, 4, 16, 16) tile there instead; gfx1151 keeps it (no spills with 192 VGPRs). Left spilling: that tile on gfx1150 (built for gfx1151, not launched on gfx1150), and on gfx1151 eight tiles at their VGPR limit as before: the dense (YTILE=2, W=16, AC=32, UNRL=8) N = 1 tile for K % 8192 == 0 (mostly with zero points) and the MoE (2, 16, 16, 4) N = 5 bf16 tile. The kernels' code is unchanged; the compiler emits different code for 15 (gfx1150) and 17 (gfx1151) of the 1204 kernels left in the smaller translation unit. Timed against gfx11 (int4 W4A16, group size 128, M = 1..4, best of two runs per build; dense: Qwen3-4B, Llama-8B and Qwen3.5-A3B shapes, fp16 and bf16; MoE: Qwen3.5-A3B, 256 experts, top 8, K = 2048): - gfx1150: MoE 1.19x (1.16x-1.24x); dense 0.99x-1.00x (the M = 3..4 cells vary by up to 20% between processes with either build). - gfx1151: dense 1.00x, MoE 1.00x (all cells within 1.5%). test_rdna_hybrid_w4a16.py (102 passed), test_hybrid_w4a16_moe.py (70 passed) and test_rocm_skinny_gemms.py (172 passed) on gfx1150 and gfx1151; the test_rdna_hybrid_w4a16.py tests needing dist_init were not run. AI assistance was used for this patch. Co-authored-by: Claude Signed-off-by: Robert Esclapez Garcia --- csrc/rocm/skinny_gemms_int4_kernels.cuh | 278 +++++++++++++----------- 1 file changed, 148 insertions(+), 130 deletions(-) diff --git a/csrc/rocm/skinny_gemms_int4_kernels.cuh b/csrc/rocm/skinny_gemms_int4_kernels.cuh index 0653a9bb8c41..c8b27db136cc 100644 --- a/csrc/rocm/skinny_gemms_int4_kernels.cuh +++ b/csrc/rocm/skinny_gemms_int4_kernels.cuh @@ -55,6 +55,20 @@ static bool on_gfx_int4() { return result; } +// Archs with 1024 VGPRs a SIMD lane, where a wave of a 1024-thread workgroup +// gets 128 VGPRs (192 with 1536). +static bool on_1024_vgpr_int4() { + static const bool result = [] { + const std::string_view arch = + at::cuda::getCurrentDeviceProperties()->gcnArchName; + for (std::string_view a : + {"gfx1102", "gfx1103", "gfx1150", "gfx1152", "gfx1153"}) + if (arch.starts_with(a)) return true; + return false; + }(); + return result; +} + #if defined(NDEBUG) #undef NDEBUG #include @@ -1309,64 +1323,67 @@ static int mindiv_int4(int N, int div1, int div2) { TORCH_CHECK(false, "Unsupported group_size=", group_size); \ } -#define WVSPLIT_INT4G_TILE(_sYT, __N, _HAS_ZP) \ - { \ - if (K_in * N_in > max_lds_len) { \ - /* Chunked branch. Same wave-packing rule as the fitting branch \ - below, for the same reason: it is a property of M, not of \ - chunking. */ \ - if (_sYT < 30) \ - WVSPLIT_INT4G_GS_CHUNKED(4, 2, __N, _HAS_ZP) \ - else if (mindiv_int4(M_in, CuCount * 4, 16) < 16) \ - WVSPLIT_INT4G_GS_CHUNKED(2, 2, __N, _HAS_ZP) \ - else \ - WVSPLIT_INT4G_GS_CHUNKED(4, 1, __N, _HAS_ZP) \ - } else if (__N >= 3 && _sYT >= 40 && K_in >= 4096 && \ - mindiv_int4(M_in, CuCount * 4, 16) < 16) \ - /* YTILE=4 is the better tuple in this range, but only when it \ - does not cost wave slots. mindiv_int4 trims WvPrGrp to even \ - out the m-rounds, and at YTILE=4 it can trim harder than at \ - YTILE=2: M=5376 gives 14 (2 of 16 wave-rows idle) against 15 at \ - YTILE=2. The K bound keeps this off shallow rows, where the k1 \ - loop is too short to amortise the extra m-tiles. The chunked \ - branch above needs no such bound: it only runs when K*N exceeds \ - LDS, so K is always past it. */ \ - WVSPLIT_INT4G_GS(2, 2, __N, _HAS_ZP) \ - else if (__N >= 4 && _sYT >= 480) \ - WVSPLIT_INT4G_GS(4, 1, __N, _HAS_ZP) \ - else if (__N >= 3 && _sYT >= 40) \ - WVSPLIT_INT4G_GS(4, 1, __N, _HAS_ZP) \ - else if (__N >= 3 && _sYT < 40 && (K_in <= 2048 || K_in >= 4096)) \ - WVSPLIT_INT4G_GS(2, 4, __N, _HAS_ZP) \ - else if (__N >= 3 && _sYT < 40) \ - WVSPLIT_INT4G_GS(2, 2, __N, _HAS_ZP) \ - else if (__N >= 2) \ - WVSPLIT_INT4G_GS(2, 2, __N, _HAS_ZP) \ - else if (on_gfx_int4<11, 12>() && __N == 1 && K_in == 4096) \ - /* Tuned for gfx1151 (Qwen3.5 W4A16 decode: GDN out_proj at \ - M=2048, K=4096, N=1). AC=32 doubles per-thread global load \ - granularity; the K=4096 row has enough work to amortize the \ - wider load. W stays at 16 (vs 32 in the bf16 K=2048 branch) \ - because int4 dequant inflates VGPR pressure -- AC=32 + W=32 \ - spills. Lifts kernel ~70% -> ~84% of LPDDR5X peak post- \ - overhead. K<=2048 already at ~87% by default and untouched. \ - Verify per shape with \ - benchmarks/kernels/sweep_int4g_kernel.py. */ \ - WVSPLITK_INT4G_GS_W_AC(1, 4, 16, 32, __N, _HAS_ZP) \ - else if (on_gfx_int4<11, 12>() && __N == 1 && K_in % (32 * 32 * 8) == 0) \ - /* Deep-K decode rows that divide this tuple's K step exactly \ - (THRDS*A_CHUNK*UNRL = 8192), e.g. gemma-4-31B down_proj at \ - K=8192 and K=16384, M=5376. AC=32 issues global_load_b128 \ - and UNRL=8 makes each row's contiguous run 4 KB. \ - \ \ - The exact-division guard is load-bearing: a K that leaves a \ - large ragged tail has too few k1 iterations to hide it behind \ - and prefers the default tuple. Geometry and row stride have \ - to be tuned together; re-verify both with \ - benchmarks/kernels/sweep_int4g_kernel.py. */ \ - WVSPLITK_INT4G_GS_W_AC(2, 8, 16, 32, __N, _HAS_ZP) \ - else /* N=1: YTILE=2 beats YTILE=1 across all CuCount values */ \ - WVSPLIT_INT4G_GS(2, 4, __N, _HAS_ZP) \ +#define WVSPLIT_INT4G_TILE(_sYT, __N, _HAS_ZP) \ + { \ + if (K_in * N_in > max_lds_len) { \ + /* Chunked branch. Same wave-packing rule as the fitting branch \ + below, for the same reason: it is a property of M, not of \ + chunking. */ \ + if (_sYT < 30) \ + WVSPLIT_INT4G_GS_CHUNKED(4, 2, __N, _HAS_ZP) \ + else if (mindiv_int4(M_in, CuCount * 4, 16) < 16) \ + WVSPLIT_INT4G_GS_CHUNKED(2, 2, __N, _HAS_ZP) \ + else \ + WVSPLIT_INT4G_GS_CHUNKED(4, 1, __N, _HAS_ZP) \ + } else if (__N >= 3 && _sYT >= 40 && K_in >= 4096 && \ + mindiv_int4(M_in, CuCount * 4, 16) < 16) \ + /* YTILE=4 is the better tuple in this range, but only when it \ + does not cost wave slots. mindiv_int4 trims WvPrGrp to even \ + out the m-rounds, and at YTILE=4 it can trim harder than at \ + YTILE=2: M=5376 gives 14 (2 of 16 wave-rows idle) against 15 at \ + YTILE=2. The K bound keeps this off shallow rows, where the k1 \ + loop is too short to amortise the extra m-tiles. The chunked \ + branch above needs no such bound: it only runs when K*N exceeds \ + LDS, so K is always past it. */ \ + WVSPLIT_INT4G_GS(2, 2, __N, _HAS_ZP) \ + else if (__N >= 4 && _sYT >= 480) \ + WVSPLIT_INT4G_GS(4, 1, __N, _HAS_ZP) \ + else if (__N >= 3 && _sYT >= 40) \ + WVSPLIT_INT4G_GS(4, 1, __N, _HAS_ZP) \ + else if (__N >= 3 && _sYT < 40 && (K_in <= 2048 || K_in >= 4096)) \ + WVSPLIT_INT4G_GS(2, 4, __N, _HAS_ZP) \ + else if (__N >= 3 && _sYT < 40) \ + WVSPLIT_INT4G_GS(2, 2, __N, _HAS_ZP) \ + else if (__N >= 2) \ + WVSPLIT_INT4G_GS(2, 2, __N, _HAS_ZP) \ + else if constexpr (__N == 1) { \ + /* N>=2 never gets here: the N=1 tuples are not built for it. */ \ + if (on_gfx_int4<11, 12>() && K_in == 4096) \ + /* Tuned for gfx1151 (Qwen3.5 W4A16 decode: GDN out_proj at \ + M=2048, K=4096, N=1). AC=32 doubles per-thread global load \ + granularity; the K=4096 row has enough work to amortize the \ + wider load. W stays at 16 (vs 32 in the bf16 K=2048 branch) \ + because int4 dequant inflates VGPR pressure -- AC=32 + W=32 \ + spills. Lifts kernel ~70% -> ~84% of LPDDR5X peak post- \ + overhead. K<=2048 already at ~87% by default and untouched. \ + Verify per shape with \ + benchmarks/kernels/sweep_int4g_kernel.py. */ \ + WVSPLITK_INT4G_GS_W_AC(1, 4, 16, 32, __N, _HAS_ZP) \ + else if (on_gfx_int4<11, 12>() && K_in % (32 * 32 * 8) == 0) \ + /* Deep-K decode rows that divide this tuple's K step exactly \ + (THRDS*A_CHUNK*UNRL = 8192), e.g. gemma-4-31B down_proj at \ + K=8192 and K=16384, M=5376. AC=32 issues global_load_b128 \ + and UNRL=8 makes each row's contiguous run 4 KB. \ + \ \ + The exact-division guard is load-bearing: a K that leaves a \ + large ragged tail has too few k1 iterations to hide it behind \ + and prefers the default tuple. Geometry and row stride have \ + to be tuned together; re-verify both with \ + benchmarks/kernels/sweep_int4g_kernel.py. */ \ + WVSPLITK_INT4G_GS_W_AC(2, 8, 16, 32, __N, _HAS_ZP) \ + else /* N=1: YTILE=2 beats YTILE=1 across all CuCount values */ \ + WVSPLIT_INT4G_GS(2, 4, __N, _HAS_ZP) \ + } \ } // Inner dispatch: shared by both symmetric and asymmetric paths @@ -1699,78 +1716,79 @@ __global__ void moe_wvSplitK_int4_hf_( } \ } -#define MOE_WVSPLIT_INT4G_TILE(_sYT, __N, _HAS_ZP) \ - { \ - if (K_in * __N > max_lds_len) { \ - if (_sYT < 30) \ - MOE_WVSPLIT_INT4G_GS(4, 2, __N, _HAS_ZP) \ - else \ - MOE_WVSPLIT_INT4G_GS(4, 1, __N, _HAS_ZP) \ - } else if (__N >= 4 && _sYT >= 480) \ - MOE_WVSPLIT_INT4G_GS(4, 1, __N, _HAS_ZP) \ - else if (__N >= 3 && _sYT >= 40) \ - MOE_WVSPLIT_INT4G_GS(4, 1, __N, _HAS_ZP) \ - else if (__N >= 3 && _sYT < 40 && (K_in <= 2048 || K_in >= 4096)) \ - MOE_WVSPLIT_INT4G_GS(2, 4, __N, _HAS_ZP) \ - else if (__N >= 3 && _sYT < 40) \ - MOE_WVSPLIT_INT4G_GS(2, 2, __N, _HAS_ZP) \ - else if (__N >= 2) \ - MOE_WVSPLIT_INT4G_GS(2, 2, __N, _HAS_ZP) \ - else /* N=1 split per K, picked from a full 4-axis (Y, U, W, AC) \ - Cartesian sweep on gfx1151. Re-verified in fresh procs \ - against the local kernel; the two K ranges land on \ - different optima: \ - K >= 1024 (gemm1 K=2048): Y=4, U=4, W=16, AC=16 \ - 64.17 us/call @ 182.6 GiB/s vs Y2U4's 66.40 us; \ - wider YTILE keeps each CU's m-tile fully fed at \ - gfx1151's 20-CU width when M is comparable to \ - CuCount * 16 * Y. \ - K < 1024 (gemm2 K=768): Y=4, U=2, W=32, AC=32 \ - ~10% faster than the previous (Y8 U2 W=AC=16) heur: \ - 2 wide-load batches per k-step instead of 4 narrow \ - ones, fewer waitcnt boundaries amortize HBM page-mode \ - costs. GFX9 wave64 stays on the previous heuristic \ - via MOE_WVSPLIT_INT4G_GS_W_AC's fallback. \ - LOW-VGPR override (gfx1x_int4 only): \ - On Qwen3.5-35B-A3B (gemm2 K=512), the (W=32, AC=32) \ - instantiation compiles to 157 VGPRs/wave; combined \ - with WG=1024 threads (32 wave32) this caps occupancy \ - at 32 active wave32/CU = 50% of peak. Switching to \ - (W=16, AC=16, U=2) drops VGPRs to 113 and WG=512 \ - threads (16 wave32) so 3 WGs fit per CU = 48 active \ - wave32/CU = 75% of peak. Better latency hiding for \ - memory-bound kernels. */ \ - { \ - if (on_gfx_int4<11, 12>() && K_in == 2048) \ - /* gfx1151 K=2048 N=1 (Qwen3.5 MoE gate_up at M=1024, \ - * E=256): 4-axis sweep via \ - * benchmarks/kernels/sweep_int4g_moe_kernel.py finds \ - * (Y=4, U=4, W=32, AC=32) at 44.6 us vs the (Y=4, U=4, \ - * W=16, AC=16) default at 46.5 us -- ~1.04x. Wider per- \ - * thread loads (AC=32) + max WG threads (W=32) amortize the \ - * per-row work the K=2048 case has plenty of. Narrow guard: \ - * only swept at M~1024; other M at K=2048 may benefit too \ - * but were not measured. */ \ - MOE_WVSPLIT_INT4G_GS_W_AC(4, 32, 32, 4, __N, _HAS_ZP) \ - else if (K_in >= 1024) \ - MOE_WVSPLIT_INT4G_GS(4, 4, __N, _HAS_ZP) \ - else if (on_gfx_int4<11, 12>() && K_in == 512) \ - /* gfx1151 K=512 N=1 (Qwen3.5 MoE down-proj). 4-axis sweep \ - * via benchmarks/kernels/sweep_int4g_moe_kernel.py finds \ - * (Y=4, U=1, W=32, AC=8) at 28.1 us vs the LOW-VGPR pick \ - * (Y=4, U=2, W=16, AC=16) at 31.4 us -- ~1.12x at \ - * fuse_silu_mul=False; ~1.18x at fuse_silu_mul=True (the \ - * down+silu trace shape). Pure-work %DRAM lifts ~62% \ - * -> ~78% post per-launch floor subtraction. W=32 + AC=8 \ - * keeps WG threads at 1024 (32 wv32) but the narrow per- \ - * iter loads avoid the (W=32, AC=32) VGPR-spill penalty the \ - * (W=32, AC=32) default below describes. */ \ - MOE_WVSPLIT_INT4G_GS_W_AC(4, 32, 8, 1, __N, _HAS_ZP) \ - else if (on_gfx_int4<11, 12>()) \ - MOE_WVSPLIT_INT4G_GS(4, 2, __N, _HAS_ZP) \ - else \ - MOE_WVSPLIT_INT4G_GS_W_AC(4, 32, 32, 2, __N, _HAS_ZP) \ - } \ +#define MOE_WVSPLIT_INT4G_TILE(_sYT, __N, _HAS_ZP) \ + { \ + if (K_in * __N > max_lds_len) { \ + if (_sYT < 30) \ + MOE_WVSPLIT_INT4G_GS(4, 2, __N, _HAS_ZP) \ + else \ + MOE_WVSPLIT_INT4G_GS(4, 1, __N, _HAS_ZP) \ + } else if (__N >= 4 && _sYT >= 480) \ + MOE_WVSPLIT_INT4G_GS(4, 1, __N, _HAS_ZP) \ + else if (__N >= 3 && _sYT >= 40) \ + MOE_WVSPLIT_INT4G_GS(4, 1, __N, _HAS_ZP) \ + else if (__N >= 3 && _sYT < 40 && (K_in <= 2048 || K_in >= 4096)) \ + MOE_WVSPLIT_INT4G_GS(2, 4, __N, _HAS_ZP) \ + else if (__N >= 3 && _sYT < 40) \ + MOE_WVSPLIT_INT4G_GS(2, 2, __N, _HAS_ZP) \ + else if (__N >= 2) \ + MOE_WVSPLIT_INT4G_GS(2, 2, __N, _HAS_ZP) \ + else if constexpr (__N == 1) { \ + /* N=1 split per K, picked from a full 4-axis (Y, U, W, AC) \ + Cartesian sweep on gfx1151. Re-verified in fresh procs \ + against the local kernel; the two K ranges land on \ + different optima: \ + K >= 1024 (gemm1 K=2048): Y=4, U=4, W=16, AC=16 \ + 64.17 us/call @ 182.6 GiB/s vs Y2U4's 66.40 us; \ + wider YTILE keeps each CU's m-tile fully fed at \ + gfx1151's 20-CU width when M is comparable to \ + CuCount * 16 * Y. \ + K < 1024 (gemm2 K=768): Y=4, U=2, W=32, AC=32 \ + ~10% faster than the previous (Y8 U2 W=AC=16) heur: \ + 2 wide-load batches per k-step instead of 4 narrow \ + ones, fewer waitcnt boundaries amortize HBM page-mode \ + costs. GFX9 wave64 stays on the previous heuristic \ + via MOE_WVSPLIT_INT4G_GS_W_AC's fallback. \ + LOW-VGPR override (gfx1x_int4 only): \ + On Qwen3.5-35B-A3B (gemm2 K=512), the (W=32, AC=32) \ + instantiation compiles to 157 VGPRs/wave; combined \ + with WG=1024 threads (32 wave32) this caps occupancy \ + at 32 active wave32/CU = 50% of peak. Switching to \ + (W=16, AC=16, U=2) drops VGPRs to 113 and WG=512 \ + threads (16 wave32) so 3 WGs fit per CU = 48 active \ + wave32/CU = 75% of peak. Better latency hiding for \ + memory-bound kernels. */ \ + if (on_gfx_int4<11, 12>() && !on_1024_vgpr_int4() && K_in == 2048) \ + /* gfx1151 K=2048 N=1 (Qwen3.5 MoE gate_up at M=1024, \ + * E=256): 4-axis sweep via \ + * benchmarks/kernels/sweep_int4g_moe_kernel.py finds \ + * (Y=4, U=4, W=32, AC=32) at 44.6 us vs the (Y=4, U=4, \ + * W=16, AC=16) default at 46.5 us -- ~1.04x. Wider per- \ + * thread loads (AC=32) + max WG threads (W=32) amortize the \ + * per-row work the K=2048 case has plenty of. Narrow guard: \ + * only swept at M~1024; other M at K=2048 may benefit too \ + * but were not measured. Not on 1024-VGPR archs: its 128 \ + * VGPRs a wave there spill; they take (4, 4, 16, 16). */ \ + MOE_WVSPLIT_INT4G_GS_W_AC(4, 32, 32, 4, __N, _HAS_ZP) \ + else if (K_in >= 1024) \ + MOE_WVSPLIT_INT4G_GS(4, 4, __N, _HAS_ZP) \ + else if (on_gfx_int4<11, 12>() && K_in == 512) \ + /* gfx1151 K=512 N=1 (Qwen3.5 MoE down-proj). 4-axis sweep \ + * via benchmarks/kernels/sweep_int4g_moe_kernel.py finds \ + * (Y=4, U=1, W=32, AC=8) at 28.1 us vs the LOW-VGPR pick \ + * (Y=4, U=2, W=16, AC=16) at 31.4 us -- ~1.12x at \ + * fuse_silu_mul=False; ~1.18x at fuse_silu_mul=True (the \ + * down+silu trace shape). Pure-work %DRAM lifts ~62% \ + * -> ~78% post per-launch floor subtraction. W=32 + AC=8 \ + * keeps WG threads at 1024 (32 wv32) but the narrow per- \ + * iter loads avoid the (W=32, AC=32) VGPR-spill penalty the \ + * (W=32, AC=32) default below describes. */ \ + MOE_WVSPLIT_INT4G_GS_W_AC(4, 32, 8, 1, __N, _HAS_ZP) \ + else if (on_gfx_int4<11, 12>()) \ + MOE_WVSPLIT_INT4G_GS(4, 2, __N, _HAS_ZP) \ + else \ + MOE_WVSPLIT_INT4G_GS_W_AC(4, 32, 32, 2, __N, _HAS_ZP) \ + } \ } #define MOE_WVSPLIT_INT4G_DISPATCH(_HAS_ZP) \