Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 88 additions & 27 deletions src/vsparse/_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,12 @@
they already did. Matrices with enough slices keep the original one-warp-per-
slice kernel, a separate compilation, because the chunk bounds cost ~25% at
the widest widths for a split of one where they do nothing.
- *Vector loads.* The aligned kernels gather one dense-operand row per nonzero,
and when the width is a multiple of four they do it with ``float4`` loads
(:data:`_VECTOR_WIDTH`), so ``width / 4`` lanes cover the width. That is a
third compilation axis, and it shifts what "lanes across the width" means for
the accumulator count and the split threshold, which are both expressed in the
vector units actually loaded.
"""

from __future__ import annotations
Expand Down Expand Up @@ -127,6 +133,15 @@
_MIN_CHUNK_NNZ = 256
_MIN_SPLIT_WW = 8

#: The aligned kernels gather each dense-operand row with ``float4`` loads when
#: the width is a multiple of this: a quarter of the load instructions, which
#: measured 1.5-2.4x faster at widths 8-64 and ~10-30% at 128-256. CuPy
#: allocations are 256-byte aligned, so with ``width * 4`` a multiple of 16
#: bytes every row is too. Other widths keep scalar loads. The misaligned
#: kernels stay scalar: ``atomicAdd(float4*)`` needs sm_90 and gained <10%,
#: since the atomics, not the ``B`` loads, dominate them.
_VECTOR_WIDTH = 4


def cuda_is_available() -> bool:
"""Whether CuPy is importable and a CUDA device is actually present."""
Expand Down Expand Up @@ -165,6 +180,26 @@ def _require_cupy() -> Any:
if (g_code == {G_SQRT}) return x > 0.0f ? sqrtf(x) : 0.0f;
return x;
}}

// The aligned kernels' inner loop is written once over a vector type VT: a
// plain float, or a float4 when the width is a multiple of four (see VEC).
__device__ __forceinline__ void fma_into(float& a, const float d, const float b) {{ a += d * b; }}
__device__ __forceinline__ void fma_into(float4& a, const float d, const float4 b) {{
a.x += d * b.x; a.y += d * b.y; a.z += d * b.z; a.w += d * b.w;
}}
__device__ __forceinline__ float zero_of(float*) {{ return 0.0f; }}
__device__ __forceinline__ float4 zero_of(float4*) {{ return make_float4(0.f, 0.f, 0.f, 0.f); }}
__device__ __forceinline__ float shfl_down(const float a, const int off) {{
return __shfl_down_sync(FULL_MASK, a, off);
}}
__device__ __forceinline__ float4 shfl_down(const float4 a, const int off) {{
return make_float4(__shfl_down_sync(FULL_MASK, a.x, off), __shfl_down_sync(FULL_MASK, a.y, off),
__shfl_down_sync(FULL_MASK, a.z, off), __shfl_down_sync(FULL_MASK, a.w, off));
}}
__device__ __forceinline__ void add_into(float& a, const float b) {{ a += b; }}
__device__ __forceinline__ void add_into(float4& a, const float4 b) {{
a.x += b.x; a.y += b.y; a.z += b.z; a.w += b.w;
}}
"""

#: The transforms ``apply_g`` implements. A ``g_code`` missing from here would
Expand Down Expand Up @@ -343,31 +378,37 @@ def _require_cupy() -> Any:
#: once. The dense operand is indexed by the per-nonzero minor index.
_BODY_ALIGNED = """
{hoist}
float* const orow = out + {orow_index} * (long long)width;
for (int cbase = 0; cbase < width; cbase += ww * ACC_MAX) {{
float acc[ACC_MAX];
// VT is float, or float4 when VEC == 4: `width` is then a multiple of
// four and every row of B and of the output is 16-byte aligned, so each
// gather is one 128-bit load instead of four 32-bit ones. `wv` is the
// width in VT's, and `ww` counts lanes across those.
VT* const orow = (VT*)(out + {orow_index} * (long long)width);
const VT* const Bv = (const VT*)B;
const int wv = width / VEC;
for (int cbase = 0; cbase < wv; cbase += ww * ACC_MAX) {{
VT acc[ACC_MAX];
#pragma unroll
for (int a = 0; a < ACC_MAX; ++a) acc[a] = 0.0f;
for (int a = 0; a < ACC_MAX; ++a) acc[a] = zero_of((VT*)0);
{walk_open}
const float* const brow = B + (long long)mi * width;
const VT* const brow = Bv + (long long)mi * wv;
#pragma unroll
for (int a = 0; a < ACC_MAX; ++a) {{
const int c = cbase + a * ww + wl;
if (c < width) acc[a] += d * brow[c];
if (c < wv) fma_into(acc[a], d, brow[c]);
}}{walk_close}

// Lanes sharing a `wl` sit `ww` apart, so folding by ww, 2*ww, ...
// lands each column's total in the nl == 0 group.
#pragma unroll
for (int a = 0; a < ACC_MAX; ++a) {{
for (int off = ww; off < {warp}; off <<= 1)
acc[a] += __shfl_down_sync(FULL_MASK, acc[a], off);
add_into(acc[a], shfl_down(acc[a], off));
}}
if (nl == 0) {{
#pragma unroll
for (int a = 0; a < ACC_MAX; ++a) {{
const int c = cbase + a * ww + wl;
if (c < width) orow[c] = acc[a];
if (c < wv) orow[c] = acc[a];
}}
}}
}}
Expand Down Expand Up @@ -399,23 +440,26 @@ def _require_cupy() -> Any:
}


def _build_source(acc: int, chunked: bool, idx_t: str, ptr_t: str) -> str:
"""The full CUDA source for ``acc`` width accumulators per lane.
def _build_source(acc: int, chunked: bool, vec: int, idx_t: str, ptr_t: str) -> str:
"""The full CUDA source for ``acc`` accumulators per lane, ``vec`` floats wide.

``idx_t``/``ptr_t`` are the C types of the device ``indices`` and of the two
pointer arrays (see :func:`_index_dtype` and :func:`_pointer_dtype`).

All four kernels, assembled from the fragments above, for either the whole
or the chunked way of handing slices to warps. ``acc`` only matters to the
aligned kernels; the misaligned ones loop over the width instead.
aligned kernels; the misaligned ones loop over the width instead and
ignore ``vec``.
"""
slices, walk, orow_index = (
(_SLICES_CHUNKED, _WALK_CHUNKED, "(maj * split + chunk)")
if chunked
else (_SLICES_WHOLE, _WALK_WHOLE, "maj")
)
vt = "float4" if vec == 4 else "float"
parts = [
f"#define ACC_MAX {acc}\ntypedef {idx_t} idx_t;\ntypedef {ptr_t} ptr_t;\n",
f"#define ACC_MAX {acc}\n#define VEC {vec}\n#define VT {vt}\n"
f"typedef {idx_t} idx_t;\ntypedef {ptr_t} ptr_t;\n",
_PREAMBLE,
]
for name, (body, scales) in _KERNELS.items():
Expand All @@ -437,7 +481,7 @@ def _build_source(acc: int, chunked: bool, idx_t: str, ptr_t: str) -> str:
np.dtype(np.int64): "long long",
}

_module_cache: dict[tuple[int, bool, np.dtype, np.dtype], Any] = {}
_module_cache: dict[tuple[int, bool, int, np.dtype, np.dtype], Any] = {}


def _index_dtype(n_minor: int) -> np.dtype:
Expand All @@ -450,13 +494,13 @@ def _pointer_dtype(max_entry: int) -> np.dtype:
return np.dtype(np.int32 if max_entry <= np.iinfo(np.int32).max else np.int64)


def _module(acc: int, chunked: bool, idx_dtype: np.dtype, ptr_dtype: np.dtype) -> Any:
def _module(acc: int, chunked: bool, vec: int, idx_dtype: np.dtype, ptr_dtype: np.dtype) -> Any:
"""The compiled :class:`cupy.RawModule` for a variant, built once per process.

Variants are compiled lazily, so a process only pays for the accumulator
counts, slice schemes and index/pointer widths its matrices actually reach.
"""
key = (acc, chunked, idx_dtype, ptr_dtype)
key = (acc, chunked, vec, idx_dtype, ptr_dtype)
if key not in _module_cache:
cp = _require_cupy()
# --use_fast_math: measured 24% off the parafac2 kernel (every recipe
Expand All @@ -466,7 +510,7 @@ def _module(acc: int, chunked: bool, idx_dtype: np.dtype, ptr_dtype: np.dtype) -
# already committed to. Its denormal flushing is if anything the safer
# behaviour here: a denormal `gene_scale` fails the `> 0.0f` guard and
# its column is skipped, rather than dividing into an infinity.
source = _build_source(acc, chunked, _C_TYPES[idx_dtype], _C_TYPES[ptr_dtype])
source = _build_source(acc, chunked, vec, _C_TYPES[idx_dtype], _C_TYPES[ptr_dtype])
_module_cache[key] = cp.RawModule(code=source, options=("--use_fast_math",))
return _module_cache[key]

Expand All @@ -484,9 +528,13 @@ def _accumulators(width: int, ww: int) -> int:
return min(_ACC_MAX, max(1, -(-width // ww)))


def _split_factor(n_major: int, nnz: int, sm_count: int, ww: int) -> int:
"""Chunks to cut each major slice into; 1 when the grid is already full or ``ww`` is narrow."""
if n_major <= 0 or ww < _MIN_SPLIT_WW:
def _split_factor(n_major: int, nnz: int, sm_count: int, columns: int) -> int:
"""Chunks to cut each major slice into; 1 when the grid is already full or ``columns`` is narrow.

``columns`` is the dense columns the width lanes cover in one load: ``ww``
for scalar loads, ``4 * ww`` for float4 ones.
"""
if n_major <= 0 or columns < _MIN_SPLIT_WW:
return 1
wanted = -(-(sm_count * _TARGET_WARPS_PER_SM) // n_major)
affordable = nnz // (n_major * _MIN_CHUNK_NNZ)
Expand All @@ -502,27 +550,39 @@ class _Launch(NamedTuple):
groups: int
acc: int
split: int
vec: int

@property
def chunked(self) -> bool:
"""Whether slices are cut into chunks, which selects the chunked kernels."""
return self.split > 1


def _launch_config(n_major: int, width: int, nnz: int, sm_count: int) -> _Launch:
"""The launch for a slice count, a dense width, and a device of ``sm_count`` SMs."""
def _launch_config(
n_major: int, width: int, nnz: int, sm_count: int, *, aligned: bool = False
) -> _Launch:
"""The launch for a slice count, a dense width, and a device of ``sm_count`` SMs.

``aligned`` marks the register-accumulating kernels, which load ``float4``
when ``width`` is a multiple of four (:data:`_VECTOR_WIDTH`), so ``width / 4``
lanes cover the width. The misaligned kernels ``atomicAdd`` scalars and
always take ``vec == 1``.
"""
vec = _VECTOR_WIDTH if aligned and width % _VECTOR_WIDTH == 0 else 1
lanes = width // vec
ww = 1
while ww < width and ww < _WARP:
while ww < lanes and ww < _WARP:
ww <<= 1
split = _split_factor(n_major, nnz, sm_count, ww)
split = _split_factor(n_major, nnz, sm_count, ww * vec)
blocks = min(_MAX_BLOCKS, max(1, -(-(n_major * split) // _WARPS_PER_BLOCK)))
return _Launch(
(blocks,),
(_WARP, _WARPS_PER_BLOCK),
ww,
_WARP // ww,
_accumulators(width, ww),
_accumulators(lanes, ww),
split,
vec,
)


Expand Down Expand Up @@ -647,17 +707,18 @@ def _kernel_args(self, g_code: int) -> tuple[Any, ...]:
def _run(self, name: str, B: Any, out: Any, n_major: int) -> None:
cp = _require_cupy()
width = int(B.shape[1])
cfg = _launch_config(n_major, width, self.nnz, self.sm_count)
aligned = _KERNELS[name][0] is _BODY_ALIGNED
cfg = _launch_config(n_major, width, self.nnz, self.sm_count, aligned=aligned)
# An aligned kernel stores rather than adds, so split chunks of one
# slice can't share its output row: each writes a partial row of its
# own (zero-initialised, so a skipped slice stays zero), and the
# partials are summed below. The sum is a fixed-order reduction, so
# the aligned direction stays deterministic. A misaligned kernel
# atomicAdds into `out` whatever the split, and needs no scratch.
scratch = _KERNELS[name][0] is _BODY_ALIGNED and cfg.chunked
scratch = aligned and cfg.chunked
target = cp.zeros((n_major * cfg.split, width), dtype=DEVICE_DTYPE) if scratch else out
kernel = _module(
cfg.acc, cfg.chunked, self.indices.dtype, self.major_ptr.dtype
cfg.acc, cfg.chunked, cfg.vec, self.indices.dtype, self.major_ptr.dtype
).get_function(name)
kernel(
cfg.grid,
Expand Down
25 changes: 24 additions & 1 deletion tests/test_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ def skewed_counts() -> sp.csr_array:

@pytest.mark.parametrize("split", [2, 3, 7, 64])
@pytest.mark.parametrize("cls", [VCSRArray, VCSCArray])
@pytest.mark.parametrize("width", [1, 8, 32, 33, 65, 97, 130])
@pytest.mark.parametrize("width", [1, 4, 8, 12, 32, 33, 65, 97, 130, 256])
@pytest.mark.parametrize("matrix", ["random", "skewed"])
def test_split_slices_match_cpu(forced_split, mat, matrix, cls, width, split):
"""Chunked kernels agree with the host in all four directions, at every accumulator count.
Expand Down Expand Up @@ -380,3 +380,26 @@ def test_split_factor_decisions():
assert few.chunked and not plan(full, 16, nnz, 84).chunked
# The grid always covers every (slice, chunk) pair, up to the block cap.
assert few.grid[0] * few.block[1] >= min(2_000 * few.split, _cuda._MAX_BLOCKS * few.block[1])


@pytest.mark.parametrize(
("width", "vec", "ww", "acc"),
[(1, 1, 1, 1), (6, 1, 8, 1), (4, 4, 1, 1), (8, 4, 2, 1), (16, 4, 4, 1), (128, 4, 32, 1),
(256, 4, 32, 2), (512, 4, 32, 4), (130, 1, 32, 4), (33, 1, 32, 2)],
) # fmt: skip
def test_aligned_launch_uses_float4_for_multiples_of_four(width, vec, ww, acc):
"""Aligned launches load ``float4`` when the width allows it; misaligned ones never do."""
from vsparse import _cuda

cfg = _cuda._launch_config(1000, width, 10**6, 84, aligned=True)
assert (cfg.vec, cfg.ww, cfg.acc) == (vec, ww, acc)
assert _cuda._launch_config(1000, width, 10**6, 84).vec == 1


def test_vector_loads_do_not_change_when_a_narrow_matrix_splits():
"""The split threshold counts columns, not lanes: width 8 splits as scalar or float4."""
from vsparse import _cuda

nnz = 6_000_000
assert _cuda._launch_config(2_000, 8, nnz, 84, aligned=True).split > 1
assert _cuda._launch_config(2_000, 4, nnz, 84, aligned=True).split == 1
Loading