Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions src/vsparse/_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -589,6 +589,30 @@ def _launch_config(
# -- the device-resident view -------------------------------------------------


def _sorted_within_runs(cp: Any, indices: Any, value_ptr: Any, n_minor: int) -> Any:
"""``indices`` with every value-run ascending, sorting only if one is not.

Arrays built the normal way already satisfy this, and the kernels gain
nothing from re-sorting them. A few paths break it silently -- notably
``VCSC[perm_rows]`` -- and for a VCSC at width 1 to 2 that costs up to
~3x, since the dense operand is indexed by cell id and ascending ids share
cache lines. The check is one pass over ``indices``; the sort (a key sort
over ``run_id * n_minor + index``, runs being contiguous) runs only when
the check fails.
"""
if len(indices) < 2:
return indices
ptr = cp.asarray(value_ptr, dtype=cp.int64)
descent = indices[1:] < indices[:-1]
# Ascent is not required across runs: mask each run's first element.
descent[ptr[1:-1] - 1] = False
if not bool(descent.any()):
return indices
run_id = cp.repeat(cp.arange(len(ptr) - 1, dtype=cp.int64), cp.diff(ptr))
keys = cp.sort(run_id * n_minor + indices.astype(cp.int64))
return (keys % n_minor).astype(indices.dtype)


class CudaNormalizedView:
"""A :class:`~vsparse._norm_common.NormalizedViewBase` resident on a CUDA device.

Expand Down Expand Up @@ -640,7 +664,9 @@ def __init__(self, view: NormalizedViewBase) -> None:
# (`value_ptr`); both share one dtype (one kernel type), sized for the
# larger count, nnz: int32 unless that exceeds 2**31 - 1.
n_minor = self.shape[1] if self._format == "csr" else self.shape[0]
self.indices = cp.asarray(arr.indices, dtype=_index_dtype(n_minor))
self.indices = _sorted_within_runs(
cp, cp.asarray(arr.indices, dtype=_index_dtype(n_minor)), arr.value_ptr, n_minor
)
ptr_dtype = _pointer_dtype(len(arr.indices))
self.major_ptr = cp.asarray(arr.major_ptr, dtype=ptr_dtype)
self.value_ptr = cp.asarray(arr.value_ptr, dtype=ptr_dtype)
Expand Down Expand Up @@ -795,8 +821,8 @@ def to_cupy_sparse(self, format: str | None = None, *, sort_indices: bool = True
should pay the one-time conversion and pass ``"csr"``.
sort_indices : bool, default True
Whether to sort each slice's indices ascending before returning.
A VCS slice stores its indices grouped by the value they share,
never in index order, so the expanded matrix is *not* canonical
A VCS slice stores its indices grouped by the value they share
(ascending within each run, but not across runs), so the expanded matrix is *not* canonical
unless this sorts it -- and cuSPARSE measured ~6x slower on the
unsorted form. That penalty is why this defaults to ``True``
where the host's
Expand Down
46 changes: 46 additions & 0 deletions tests/test_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -403,3 +403,49 @@ def test_vector_loads_do_not_change_when_a_narrow_matrix_splits():
nnz = 6_000_000
assert _cuda._launch_config(2_000, 8, nnz, 84, aligned=True).split > 1
assert _cuda._launch_config(2_000, 4, nnz, 84, aligned=True).split == 1


def _runs_ascending(arr) -> bool:
"""Whether every value-run of ``arr.indices`` is ascending."""
idx, ptr = np.asarray(arr.indices, dtype=np.int64), np.asarray(arr.value_ptr)
bad = np.flatnonzero(idx[1:] < idx[:-1]) + 1 # position of each descent's second element
return bool(np.isin(bad, ptr).all()) # a descent is fine only on a run boundary


@pytest.mark.parametrize("cls", [VCSRArray, VCSCArray])
def test_stored_run_order_survives_ordinary_paths(mat, cls):
"""Canonical construction, transpose and plain selection keep every run ascending.

The kernels (VCSC at width 1 to 2 especially) rely on this order, and
nothing else pins it.
"""
arr = cls.from_scipy(mat)
assert _runs_ascending(arr)
assert _runs_ascending(arr.T)
assert _runs_ascending(arr[10:200])
assert _runs_ascending(arr[:, 5:70])


@pytest.mark.parametrize("width", [1, 2, 8])
def test_to_gpu_restores_run_order_after_row_permutation(mat, width):
"""``VCSC[perm_rows]`` breaks run order on the host; the upload repairs it."""
import cupy as cp

perm = np.random.default_rng(1).permutation(mat.shape[0])
arr = VCSCArray.from_scipy(mat)[perm]
gpu = arr.normalized("raw").to_gpu()
_check_gpu_runs_ascending(gpu)
B = np.random.default_rng(2).standard_normal((mat.shape[1], width))
Bt = np.random.default_rng(3).standard_normal((width, mat.shape[0]))
nv = arr.normalized("raw")
assert_close(gpu @ cp.asarray(B, dtype=cp.float32), nv @ B)
assert_close(cp.asarray(Bt, dtype=cp.float32) @ gpu, Bt @ nv)


def _check_gpu_runs_ascending(gpu) -> None:
import cupy as cp

idx = cp.asnumpy(gpu.indices).astype(np.int64)
ptr = cp.asnumpy(gpu.value_ptr)
bad = np.flatnonzero(idx[1:] < idx[:-1]) + 1
assert np.isin(bad, ptr).all()
Loading