From 8a7766e47107af70eeda123acdf2bd427267857f Mon Sep 17 00:00:00 2001 From: Hadi Honarvar Date: Sun, 25 Oct 2026 03:38:34 -0700 Subject: [PATCH] [Bugfix][ROCm] Stage pageable H2D copies in async_copy_to_gpu through pinned memory On ROCm (RX 7900 XTX, gfx1100) with Model Runner V2, the non_blocking copy of query_start_loc from a pageable numpy array intermittently leaves the first elements of input_buffers.query_start_loc stale, even after a full device synchronize. Attention then reads the KV cache at wrong offsets and Qwen3 answers garbage from the first or second request on. Stage pageable input through the caching host allocator before the non_blocking copy, which is what upstream main does since vllm-project/vllm#56888, where async_copy_to_gpu was folded into async_tensor_h2d. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Hadi Honarvar --- vllm/v1/worker/gpu/buffer_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vllm/v1/worker/gpu/buffer_utils.py b/vllm/v1/worker/gpu/buffer_utils.py index e4497de43a77..21afca84f7e6 100644 --- a/vllm/v1/worker/gpu/buffer_utils.py +++ b/vllm/v1/worker/gpu/buffer_utils.py @@ -7,7 +7,7 @@ import torch from vllm.triton_utils import tl, triton -from vllm.utils.platform_utils import is_uva_available +from vllm.utils.platform_utils import is_pin_memory_available, is_uva_available from vllm.utils.torch_utils import ( async_tensor_h2d, get_accelerator_view_from_cpu_tensor, @@ -36,9 +36,11 @@ def async_copy_to_gpu( assert device is not None out = torch.empty_like(x, device=device) - # Copy directly to GPU — explicit pin_memory() causes sporadic stalls - # under high concurrency due to CUDA driver contention. The driver - # handles the transfer efficiently without manual pinning. + if not x.is_pinned() and is_pin_memory_available(): + # A non_blocking copy from pageable memory is not reliable on ROCm: + # stage it through the caching host allocator, as async_tensor_h2d + # does on main. + x = torch.empty(x.shape, dtype=x.dtype, device="cpu", pin_memory=True).copy_(x) return out.copy_(x, non_blocking=True)