diff --git a/AGENTS.md b/AGENTS.md index f4e20f4873ec..4e090acdde73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -184,7 +184,8 @@ See [CI overview](docs/source/developer-guide/ci-overview.md) for full details. New C++ unit tests should use `add_tllm_gtest` with explicit `SOURCES`, `LINK_LIBRARIES`, `OWNER`, and scheduling `LABELS`. Request `FULL_STACK` explicitly -for integration tests and `REQUIRES_GPU` for tests that need a GPU. `google-tests` +for integration tests and include `gpu` in `LABELS` for tests that need a GPU. +CI controls test concurrency through `parallel_run_ctest`. `google-tests` continues to build all C++ tests; component aggregate targets build migrated tests. ### Advisory semantic review diff --git a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/CMakeLists.txt b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/CMakeLists.txt index 360e6f1a2205..19c7a4050fad 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/CMakeLists.txt +++ b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/CMakeLists.txt @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations under # the License. +add_subdirectory(bounce) + # ============================================================================ # NIXL Wrapper Library # ============================================================================ @@ -54,37 +56,22 @@ if(NIXL_ROOT) # CacheTransceiverConfig (agent_bounce_buffer_enable + # kv_cache_bounce_size_mb, Python (v2) transceiver only). # ------------------------------------------------------------------------ - find_package(PkgConfig) - if(PKG_CONFIG_FOUND) - pkg_check_modules(ZMQ libzmq) # optional: bounce is skipped if libzmq is - # absent - endif() - if(ZMQ_FOUND) - target_sources( + if(TARGET tllm_executor_bounce_control) + target_sources(${NIXL_WRAPPER_TARGET} PRIVATE bounce/BounceTransport.cpp) + target_link_libraries( ${NIXL_WRAPPER_TARGET} - PRIVATE bounce/BounceTransferPlan.cpp - bounce/BuddyAllocator.cpp - bounce/CreditScheduler.cpp - bounce/BounceMessage.cpp - bounce/BounceArena.cpp - bounce/ExecPool.cpp - bounce/GatherScatterKernel.cu - bounce/ZmqControlChannel.cpp - bounce/BounceTransport.cpp) - target_include_directories( - ${NIXL_WRAPPER_TARGET} PRIVATE ${CMAKE_BINARY_DIR}/_deps/cppzmq-src - ${ZMQ_INCLUDE_DIRS}) - target_link_libraries(${NIXL_WRAPPER_TARGET} PRIVATE ${ZMQ_LIBRARIES} - ${NIXL_BUILD_LIBRARY}) + PRIVATE + $ + ${NIXL_BUILD_LIBRARY}) target_compile_definitions(${NIXL_WRAPPER_TARGET} PRIVATE TLLM_BOUNCE_V2=1) # The .cu device symbols must resolve into this shared lib. set_target_properties(${NIXL_WRAPPER_TARGET} PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) - message(STATUS "NIXL bounce v2: enabled (libzmq found)") + message(STATUS "NIXL bounce v2: enabled (libzmq and cppzmq found)") else() message( STATUS - "NIXL bounce v2: disabled - libzmq/pkg-config not found; an explicit agent_bounce_buffer_enable opt-in fails at agent construction (on a bounce build, runtime setup failures degrade to standard NIXL with a WARNING)" + "NIXL bounce v2: disabled - libzmq or cppzmq unavailable; an explicit agent_bounce_buffer_enable opt-in fails at agent construction (on a bounce build, runtime setup failures degrade to standard NIXL with a WARNING)" ) endif() diff --git a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/bounce/CMakeLists.txt b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/bounce/CMakeLists.txt new file mode 100644 index 000000000000..3c48d9d5f470 --- /dev/null +++ b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/bounce/CMakeLists.txt @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +add_library( + tllm_executor_bounce_core STATIC EXCLUDE_FROM_ALL + BuddyAllocator.cpp CreditScheduler.cpp BounceTransferPlan.cpp + BounceMessage.cpp) +add_library(tllm::executor_bounce_core ALIAS tllm_executor_bounce_core) +target_link_libraries(tllm_executor_bounce_core PUBLIC tllm::common_logging + Threads::Threads) +target_compile_features(tllm_executor_bounce_core PUBLIC cxx_std_20) +target_compile_options(tllm_executor_bounce_core PRIVATE -Wno-error) +set_target_properties(tllm_executor_bounce_core + PROPERTIES POSITION_INDEPENDENT_CODE ON) + +add_library(tllm_executor_bounce_cuda STATIC EXCLUDE_FROM_ALL + ExecPool.cpp GatherScatterKernel.cu) +add_library(tllm::executor_bounce_cuda ALIAS tllm_executor_bounce_cuda) +target_link_libraries(tllm_executor_bounce_cuda + PUBLIC tllm::executor_bounce_core tllm::common_cuda) +target_compile_features(tllm_executor_bounce_cuda PUBLIC cxx_std_20) +target_compile_options(tllm_executor_bounce_cuda + PRIVATE $<$:-Wno-error>) +set_target_properties( + tllm_executor_bounce_cuda + PROPERTIES POSITION_INDEPENDENT_CODE ON CUDA_STANDARD 20 + CUDA_STANDARD_REQUIRED ON CUDA_RESOLVE_DEVICE_SYMBOLS ON) + +add_library(tllm_executor_bounce_arena STATIC EXCLUDE_FROM_ALL BounceArena.cpp) +add_library(tllm::executor_bounce_arena ALIAS tllm_executor_bounce_arena) +target_link_libraries( + tllm_executor_bounce_arena + PUBLIC tllm::common_cuda + PRIVATE tllm::kv_cache_legacy) +target_compile_features(tllm_executor_bounce_arena PUBLIC cxx_std_20) +target_compile_options(tllm_executor_bounce_arena PRIVATE -Wno-error) +set_target_properties(tllm_executor_bounce_arena + PROPERTIES POSITION_INDEPENDENT_CODE ON) + +find_package(PkgConfig QUIET) +if(PkgConfig_FOUND) + pkg_check_modules(TLLM_BOUNCE_ZMQ QUIET IMPORTED_TARGET GLOBAL libzmq) +endif() +if(TLLM_BOUNCE_ZMQ_FOUND AND EXISTS ${CMAKE_BINARY_DIR}/_deps/cppzmq-src) + add_library(tllm_executor_bounce_control STATIC EXCLUDE_FROM_ALL + ZmqControlChannel.cpp) + add_library(tllm::executor_bounce_control ALIAS tllm_executor_bounce_control) + target_link_libraries( + tllm_executor_bounce_control PUBLIC tllm::executor_bounce_core + PkgConfig::TLLM_BOUNCE_ZMQ) + target_include_directories(tllm_executor_bounce_control SYSTEM + PUBLIC ${CMAKE_BINARY_DIR}/_deps/cppzmq-src) + target_compile_features(tllm_executor_bounce_control PUBLIC cxx_std_20) + target_compile_options(tllm_executor_bounce_control PRIVATE -Wno-error) + set_target_properties(tllm_executor_bounce_control + PROPERTIES POSITION_INDEPENDENT_CODE ON) +endif() diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 5ba58455a7a8..75ed0e730cad 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -41,7 +41,7 @@ foreach(owner common runtime executor batch-manager kernel) endforeach() function(add_tllm_gtest test_name) - set(options FULL_STACK NO_GTEST_MAIN REQUIRES_GPU) + set(options FULL_STACK NO_GTEST_MAIN) set(one_value_args TIMEOUT OWNER) set(multi_value_args SOURCES LINK_LIBRARIES LABELS) cmake_parse_arguments(PARSE_ARGV 1 ARG "${options}" "${one_value_args}" @@ -82,9 +82,6 @@ function(add_tllm_gtest test_name) PRIVATE TOP_LEVEL_DIR="${TOP_LEVEL_DIR}") set(test_properties ENVIRONMENT "CUDA_MODULE_LOADING=LAZY") - if(ARG_REQUIRES_GPU) - list(APPEND test_properties RESOURCE_LOCK gpu) - endif() if(ARG_TIMEOUT) list(APPEND test_properties TIMEOUT "${ARG_TIMEOUT}") endif() diff --git a/cpp/tests/unit_tests/batch_manager/CMakeLists.txt b/cpp/tests/unit_tests/batch_manager/CMakeLists.txt index 69f9e6a6aa02..0f2879b1dd77 100644 --- a/cpp/tests/unit_tests/batch_manager/CMakeLists.txt +++ b/cpp/tests/unit_tests/batch_manager/CMakeLists.txt @@ -64,8 +64,7 @@ add_tllm_gtest( LINK_LIBRARIES tllm::kv_cache_legacy LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( bufferIndexHolderTest @@ -76,8 +75,7 @@ add_tllm_gtest( LINK_LIBRARIES tllm::kv_cache_legacy LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( capacitySchedulerTest @@ -88,8 +86,7 @@ add_tllm_gtest( LINK_LIBRARIES tllm::batch_scheduler LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( contextProgressTest @@ -100,10 +97,11 @@ add_tllm_gtest( LINK_LIBRARIES tllm::batch_scheduler LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") -add_gtest(contextTransferCoordinatorTest contextTransferCoordinatorTest.cpp) +add_tllm_gtest( + contextTransferCoordinatorTest SOURCES contextTransferCoordinatorTest.cpp + FULL_STACK LABELS "unit;full-stack;host") add_tllm_gtest( evictionPolicyTest @@ -125,8 +123,7 @@ add_tllm_gtest( LINK_LIBRARIES tllm::kv_cache_legacy LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") # GDS partial-copy cases load the NIXL wrapper at runtime. if(TARGET tensorrt_llm_nixl_wrapper) @@ -146,8 +143,7 @@ add_tllm_gtest( LINK_LIBRARIES tllm::kv_cache_legacy LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( kvCacheManagerV2ConfigTest @@ -171,8 +167,7 @@ add_tllm_gtest( tllm::kv_cache $ LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( kvCacheManagerV2ColdPageTest @@ -184,8 +179,7 @@ add_tllm_gtest( tllm::kv_cache $ LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( coldPageCodecTest @@ -196,8 +190,7 @@ add_tllm_gtest( LINK_LIBRARIES tllm::kv_cache LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( kvCacheManagerV2TypedIndexTest @@ -221,8 +214,7 @@ add_tllm_gtest( tllm::kv_cache $ LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( kvCacheManagerV2HostMemBackingTest @@ -234,8 +226,7 @@ add_tllm_gtest( tllm::kv_cache $ LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( kvCacheManagerV2StagingBufferTest @@ -247,8 +238,7 @@ add_tllm_gtest( tllm::kv_cache $ LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( kvCacheManagerV2StatsTest @@ -260,8 +250,7 @@ add_tllm_gtest( tllm::kv_cache $ LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( kvCacheManagerV2SlotAllocatorTest @@ -297,8 +286,7 @@ add_tllm_gtest( tllm::kv_cache $ LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( kvCacheManagerV2DigestPoolTest @@ -321,8 +309,7 @@ add_tllm_gtest( LINK_LIBRARIES tllm::kv_cache_legacy LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( llmRequestTest @@ -333,8 +320,7 @@ add_tllm_gtest( LINK_LIBRARIES tllm::kv_cache_legacy LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") add_tllm_gtest( microBatchSchedulerTest @@ -345,12 +331,13 @@ add_tllm_gtest( LINK_LIBRARIES tllm::batch_scheduler LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") -add_gtest(peftCacheManagerTest peftCacheManagerTest.cpp) +add_tllm_gtest(peftCacheManagerTest SOURCES peftCacheManagerTest.cpp FULL_STACK + LABELS "unit;full-stack;gpu") -add_gtest(rnnCacheFormatterTest rnnCacheFormatterTest.cpp) +add_tllm_gtest(rnnCacheFormatterTest SOURCES rnnCacheFormatterTest.cpp + FULL_STACK LABELS "unit;full-stack;host") add_tllm_gtest( agentTreeTest @@ -372,5 +359,4 @@ add_tllm_gtest( LINK_LIBRARIES tllm::kv_cache_legacy LABELS - "unit;batch-manager;gpu" - REQUIRES_GPU) + "unit;batch-manager;gpu") diff --git a/cpp/tests/unit_tests/common/CMakeLists.txt b/cpp/tests/unit_tests/common/CMakeLists.txt index 4630d6324a38..4174665d7284 100644 --- a/cpp/tests/unit_tests/common/CMakeLists.txt +++ b/cpp/tests/unit_tests/common/CMakeLists.txt @@ -13,10 +13,30 @@ # License for the specific language governing permissions and limitations under # the License. -add_gtest(cudaProfilerUtilsTest cudaProfilerUtilsTest.cpp) -add_gtest(cudaUtilsTest cudaUtilsTest.cpp) -add_gtest(attentionWorkspaceTest attentionWorkspaceTest.cpp) -add_gtest(memoryUtilsTest memoryUtilsTest.cu) +add_tllm_gtest(cudaProfilerUtilsTest SOURCES cudaProfilerUtilsTest.cpp + FULL_STACK LABELS "unit;full-stack;host") +add_tllm_gtest( + cudaUtilsTest + OWNER + common + SOURCES + cudaUtilsTest.cpp + LINK_LIBRARIES + tllm::runtime_buffers + LABELS + "unit;common;gpu") +add_tllm_gtest(attentionWorkspaceTest SOURCES attentionWorkspaceTest.cpp + FULL_STACK LABELS "unit;full-stack;gpu") +add_tllm_gtest( + memoryUtilsTest + OWNER + common + SOURCES + memoryUtilsTest.cu + LINK_LIBRARIES + tllm::common_cuda + LABELS + "unit;common;host") add_tllm_gtest( optionalRefTest OWNER @@ -78,7 +98,17 @@ add_tllm_gtest( tllm::common_logging LABELS "unit;common;host") -add_gtest(cudaDriverWrapperTest cudaDriverWrapperTest.cpp) +add_tllm_gtest( + cudaDriverWrapperTest + OWNER + common + SOURCES + cudaDriverWrapperTest.cpp + LINK_LIBRARIES + tllm::common_cuda + CUDA::cuda_driver + LABELS + "unit;common;gpu") add_tllm_gtest( loggerTest OWNER diff --git a/cpp/tests/unit_tests/executor/CMakeLists.txt b/cpp/tests/unit_tests/executor/CMakeLists.txt index 5ea2f47cb368..53fd9da6ded3 100644 --- a/cpp/tests/unit_tests/executor/CMakeLists.txt +++ b/cpp/tests/unit_tests/executor/CMakeLists.txt @@ -33,8 +33,26 @@ add_tllm_gtest( tllm::executor_types LABELS "unit;executor;host") -add_gtest(requestTest requestTest.cpp) -add_gtest(responseTest responseTest.cpp) +add_tllm_gtest( + requestTest + OWNER + executor + SOURCES + requestTest.cpp + LINK_LIBRARIES + tllm::executor_values + LABELS + "unit;executor;host") +add_tllm_gtest( + responseTest + OWNER + executor + SOURCES + responseTest.cpp + LINK_LIBRARIES + tllm::executor_values + LABELS + "unit;executor;host") add_tllm_gtest( executorConfigTest @@ -46,13 +64,50 @@ add_tllm_gtest( tllm::executor_types LABELS "unit;executor;host") -add_gtest(executorTensorTest tensorTest.cpp) -add_gtest(serializeUtilsTest serializeUtilsTest.cpp) -add_gtest(loraConfigTest loraConfigTest.cpp) -add_gtest(coalesceTest coalesceTest.cpp) -add_gtest(genUniqueAgentNameTest genUniqueAgentNameTest.cpp) +add_tllm_gtest( + executorTensorTest + OWNER + executor + SOURCES + tensorTest.cpp + LINK_LIBRARIES + tllm::executor_values + LABELS + "unit;executor;gpu") +add_tllm_gtest(serializeUtilsTest SOURCES serializeUtilsTest.cpp FULL_STACK + LABELS "unit;full-stack;host") +add_tllm_gtest( + loraConfigTest + OWNER + executor + SOURCES + loraConfigTest.cpp + LINK_LIBRARIES + tllm::executor_values + LABELS + "unit;executor;gpu") +add_tllm_gtest( + coalesceTest + OWNER + executor + SOURCES + coalesceTest.cpp + LINK_LIBRARIES + tllm::executor_transfer_agent + LABELS + "unit;executor;host") +add_tllm_gtest(genUniqueAgentNameTest SOURCES genUniqueAgentNameTest.cpp + FULL_STACK LABELS "unit;full-stack;host") target_link_libraries(genUniqueAgentNameTest PRIVATE ${Python3_LIBRARIES}) -add_gtest(ucxCommTest ucxCommTest.cpp TIMEOUT 60) +add_tllm_gtest( + ucxCommTest + SOURCES + ucxCommTest.cpp + FULL_STACK + LABELS + "unit;full-stack;gpu" + TIMEOUT + 60) target_link_libraries(ucxCommTest PRIVATE ${Python3_LIBRARIES}) target_link_libraries(serializeUtilsTest PRIVATE ${Python3_LIBRARIES}) @@ -68,8 +123,10 @@ if(EXISTS "/etc/redhat-release") endif() if(NIXL_ROOT OR (MOONCAKE_ROOT AND NOT IS_ROCKY8)) - add_gtest(agentCommTest agentCommTest.cpp) - add_gtest(transferAgentTest transferAgentTest.cpp) + add_tllm_gtest(agentCommTest SOURCES agentCommTest.cpp FULL_STACK LABELS + "unit;full-stack;gpu") + add_tllm_gtest(transferAgentTest SOURCES transferAgentTest.cpp FULL_STACK + LABELS "unit;full-stack;gpu") if(NIXL_ROOT) target_link_libraries(transferAgentTest PRIVATE tensorrt_llm_nixl_wrapper) diff --git a/cpp/tests/unit_tests/executor/bounce/CMakeLists.txt b/cpp/tests/unit_tests/executor/bounce/CMakeLists.txt index 76edc38937d9..ceaa55d53818 100644 --- a/cpp/tests/unit_tests/executor/bounce/CMakeLists.txt +++ b/cpp/tests/unit_tests/executor/bounce/CMakeLists.txt @@ -13,67 +13,117 @@ # License for the specific language governing permissions and limitations under # the License. -# Unit tests for the NIXL bounce-buffer v2 KV-cache transfer. The code under -# test lives in tensorrt_llm/executor/cache_transmission/nixl_utils/bounce/; the -# bounce core .cpp files are compiled directly into each test (they are also -# built into the NIXL wrapper for production). Three tiers, each independently -# gated: - pure-logic / GPU tests (no NIXL, no zmq, no network) - control-plane -# tests (need zmq + the fetched cppzmq header) - real-NIXL e2e tests (need -# NIXL_ROOT + zmq; two real agents, real RDMA) +# Bounce tests cover host logic, CUDA execution, ZeroMQ control channels and +# real NIXL transport. Transport tests require NIXL_ROOT and ZeroMQ. set(BOUNCE_DIR ${PROJECT_SOURCE_DIR}/tensorrt_llm/executor/cache_transmission/nixl_utils/bounce ) # ---- Pure-logic + GPU unit tests (no NIXL / zmq / network) ---- -add_gtest(creditSchedulerTest creditSchedulerTest.cpp) -target_sources(creditSchedulerTest PRIVATE ${BOUNCE_DIR}/CreditScheduler.cpp - ${BOUNCE_DIR}/BuddyAllocator.cpp) -add_gtest(bounceTransferPlanTest bounceTransferPlanTest.cpp) -target_sources(bounceTransferPlanTest - PRIVATE ${BOUNCE_DIR}/BounceTransferPlan.cpp) -add_gtest(bounceMessageCodecTest bounceMessageCodecTest.cpp) -target_sources(bounceMessageCodecTest PRIVATE ${BOUNCE_DIR}/BounceMessage.cpp) +add_tllm_gtest( + creditSchedulerTest + OWNER + executor + SOURCES + creditSchedulerTest.cpp + LINK_LIBRARIES + tllm::executor_bounce_core + LABELS + "unit;executor;bounce;host") +add_tllm_gtest( + bounceTransferPlanTest + OWNER + executor + SOURCES + bounceTransferPlanTest.cpp + LINK_LIBRARIES + tllm::executor_bounce_core + LABELS + "unit;executor;bounce;host") +add_tllm_gtest( + bounceMessageCodecTest + OWNER + executor + SOURCES + bounceMessageCodecTest.cpp + LINK_LIBRARIES + tllm::executor_bounce_core + LABELS + "unit;executor;bounce;host") # Header-only: knob resolution (agent_bounce_params dict > TRTLLM_NIXL_BOUNCE_* # env > default), byte-size suffixes, garbage fallback, derived-timeout # invariants. -add_gtest(bounceConfigTest bounceConfigTest.cpp) -add_gtest(buddyAllocatorTest buddyAllocatorTest.cpp) -target_sources(buddyAllocatorTest PRIVATE ${BOUNCE_DIR}/BuddyAllocator.cpp) +add_tllm_gtest( + bounceConfigTest + OWNER + executor + SOURCES + bounceConfigTest.cpp + LINK_LIBRARIES + tllm::common_logging + LABELS + "unit;executor;host") +add_tllm_gtest( + buddyAllocatorTest + OWNER + executor + SOURCES + buddyAllocatorTest.cpp + LINK_LIBRARIES + tllm::executor_bounce_core + LABELS + "unit;executor;bounce;host") # GPU test: execution-context pool (stream/event/scratch/hostPinned) # acquire/release. -add_gtest(execPoolTest execPoolTest.cpp) -target_sources(execPoolTest PRIVATE ${BOUNCE_DIR}/ExecPool.cpp) +add_tllm_gtest( + execPoolTest + OWNER + executor + SOURCES + execPoolTest.cpp + LINK_LIBRARIES + tllm::executor_bounce_cuda + LABELS + "unit;executor;bounce;gpu") # GPU test: gather/scatter CUDA kernel round-trip (needs a CUDA device to run). -add_gtest(gatherScatterKernelTest gatherScatterKernelTest.cpp) -target_sources(gatherScatterKernelTest - PRIVATE ${BOUNCE_DIR}/GatherScatterKernel.cu) +add_tllm_gtest( + gatherScatterKernelTest + OWNER + executor + SOURCES + gatherScatterKernelTest.cpp + LINK_LIBRARIES + tllm::executor_bounce_cuda + LABELS + "unit;executor;bounce;gpu") # GPU test: shared data-arena alloc / address mapping / shutdown. -add_gtest(bounceArenaTest bounceArenaTest.cpp) -target_sources(bounceArenaTest PRIVATE ${BOUNCE_DIR}/BounceArena.cpp) +add_tllm_gtest( + bounceArenaTest + OWNER + executor + SOURCES + bounceArenaTest.cpp + LINK_LIBRARIES + tllm::executor_bounce_arena + LABELS + "unit;executor;bounce;gpu") -# ---- Control-plane unit test — needs only zmq (+ the fetched cppzmq header), -# no NIXL. ZMQ_FOUND is reused by the real-NIXL block below. ---- -find_package(PkgConfig) -if(PKG_CONFIG_FOUND) - pkg_check_modules(ZMQ libzmq) -endif() -# Both zmq tiers need libzmq AND the fetched cppzmq header: without the latter -# the sources compile-fail on zmq.hpp even though libzmq itself was found. -if(ZMQ_FOUND AND EXISTS ${CMAKE_BINARY_DIR}/_deps/cppzmq-src) +# ---- Control-plane unit test: ZeroMQ and cppzmq, without NIXL. ---- +if(TARGET tllm_executor_bounce_control) set(BOUNCE_CPPZMQ_AVAILABLE TRUE) + add_tllm_gtest( + zmqControlChannelTest + OWNER + executor + SOURCES + zmqControlChannelTest.cpp + LINK_LIBRARIES + tllm::executor_bounce_control + LABELS + "unit;executor;bounce;host") else() set(BOUNCE_CPPZMQ_AVAILABLE FALSE) endif() -if(BOUNCE_CPPZMQ_AVAILABLE) - add_gtest(zmqControlChannelTest zmqControlChannelTest.cpp) - target_sources( - zmqControlChannelTest PRIVATE ${BOUNCE_DIR}/ZmqControlChannel.cpp - ${BOUNCE_DIR}/BounceMessage.cpp) - target_include_directories( - zmqControlChannelTest PRIVATE ${CMAKE_BINARY_DIR}/_deps/cppzmq-src - ${ZMQ_INCLUDE_DIRS}) - target_link_libraries(zmqControlChannelTest PRIVATE ${ZMQ_LIBRARIES}) -endif() # ---- Real-NIXL tests (need NIXL_ROOT). The bounce DATA PLANE is always real # NIXL — there is no loopback fake — so every test that drives the reactor @@ -84,7 +134,8 @@ if(NIXL_ROOT) # when NIXL is on); all of them share this source set + include/link sets. # Same gate as the zmq-only tier above. if(BOUNCE_CPPZMQ_AVAILABLE) - set(BOUNCE_REACTOR_SRCS + set(BOUNCE_REACTOR_SRCS ${BOUNCE_DIR}/BounceTransport.cpp) + set(BOUNCE_TSAN_SRCS ${BOUNCE_DIR}/BounceTransport.cpp ${BOUNCE_DIR}/BounceTransferPlan.cpp ${BOUNCE_DIR}/BuddyAllocator.cpp @@ -94,16 +145,25 @@ if(NIXL_ROOT) ${BOUNCE_DIR}/ExecPool.cpp ${BOUNCE_DIR}/GatherScatterKernel.cu ${BOUNCE_DIR}/ZmqControlChannel.cpp) - set(BOUNCE_TEST_INCS ${CMAKE_BINARY_DIR}/_deps/cppzmq-src - ${ZMQ_INCLUDE_DIRS}) + set(BOUNCE_TEST_INCS ${CMAKE_BINARY_DIR}/_deps/cppzmq-src) set(BOUNCE_TEST_LIBS tensorrt_llm_nixl_wrapper NIXL::nixl - ${NIXL_BUILD_LIBRARY} ${ZMQ_LIBRARIES}) + ${NIXL_BUILD_LIBRARY} PkgConfig::TLLM_BOUNCE_ZMQ) # One reactor test target: the bounce sources compiled in, plus the shared # include/link sets. function(add_bounce_reactor_test name src) - add_gtest(${name} ${src}) - target_sources(${name} PRIVATE ${BOUNCE_REACTOR_SRCS}) + cmake_parse_arguments(ARG "INSTRUMENTED" "" "" ${ARGN}) + add_tllm_gtest(${name} SOURCES ${src} FULL_STACK LABELS + "unit;full-stack;gpu") + if(ARG_INSTRUMENTED) + target_sources(${name} PRIVATE ${BOUNCE_TSAN_SRCS}) + else() + target_sources(${name} PRIVATE ${BOUNCE_REACTOR_SRCS}) + target_link_libraries( + ${name} + PRIVATE tllm::executor_bounce_core tllm::executor_bounce_cuda + tllm::executor_bounce_arena tllm::executor_bounce_control) + endif() target_include_directories(${name} PRIVATE ${BOUNCE_TEST_INCS}) target_link_libraries(${name} PRIVATE ${BOUNCE_TEST_LIBS}) endfunction() @@ -125,7 +185,7 @@ if(NIXL_ROOT) "Build a ThreadSanitizer variant of the bounce reactor test" OFF) if(BOUNCE_TSAN) add_bounce_reactor_test(bounceTransportTsanTest - bounceTransportFailureTest.cpp) + bounceTransportFailureTest.cpp INSTRUMENTED) target_compile_options( bounceTransportTsanTest PRIVATE $<$:-fsanitize=thread -g> @@ -138,7 +198,8 @@ if(NIXL_ROOT) # wrapper; this test only uses the public agent API (the zmq include is # needed only because the shared bounceTestNixlNode.h helpers pull in # ZmqControlChannel.h at compile time). - add_gtest(bounceAgentE2ETest bounceAgentE2ETest.cpp) + add_tllm_gtest(bounceAgentE2ETest SOURCES bounceAgentE2ETest.cpp FULL_STACK + LABELS "unit;full-stack;gpu") target_include_directories(bounceAgentE2ETest PRIVATE ${BOUNCE_TEST_INCS}) target_link_libraries( bounceAgentE2ETest PRIVATE tensorrt_llm_nixl_wrapper NIXL::nixl diff --git a/cpp/tests/unit_tests/multi_gpu/CMakeLists.txt b/cpp/tests/unit_tests/multi_gpu/CMakeLists.txt index 263c393b9a75..76af1a5550c7 100644 --- a/cpp/tests/unit_tests/multi_gpu/CMakeLists.txt +++ b/cpp/tests/unit_tests/multi_gpu/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & +# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & # AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -15,22 +15,45 @@ add_subdirectory(kernels) -add_gtest(cacheTransceiverTest cacheTransceiverTest.cpp) +add_tllm_gtest(cacheTransceiverTest SOURCES cacheTransceiverTest.cpp FULL_STACK + LABELS "unit;full-stack;gpu") target_link_libraries(cacheTransceiverTest PRIVATE ${Python3_LIBRARIES}) -add_gtest( +add_tllm_gtest( kvCacheManagerV2HostMemNumaTest - "kvCacheManagerV2HostMemNumaTest.cpp;../batch_manager/kvCacheManagerV2PoisonCheck.cpp" -) + OWNER + batch-manager + SOURCES + kvCacheManagerV2HostMemNumaTest.cpp + LINK_LIBRARIES + $ + FULL_STACK + LABELS + "unit;full-stack;batch-manager;gpu") target_include_directories( kvCacheManagerV2HostMemNumaTest PRIVATE ${PROJECT_SOURCE_DIR}/tensorrt_llm/batch_manager) # Queries libnuma and get_mempolicy directly to see where pages landed. target_link_libraries(kvCacheManagerV2HostMemNumaTest PRIVATE numa) -add_gtest(mpiUtilsTest mpiUtilsTest.cpp) -add_gtest(userBufferTest userBufferTest.cpp) -add_gtest(ncclUtilsTest ncclUtilsTest.cpp) +add_tllm_gtest( + mpiUtilsTest + OWNER + runtime + SOURCES + mpiUtilsTest.cpp + LINK_LIBRARIES + tllm::runtime_distributed + tllm::runtime_buffers + LABELS + "unit;runtime;host") +if(ENABLE_MULTI_DEVICE) + target_link_libraries(mpiUtilsTest PRIVATE ${NCCL_LIB}) +endif() +add_tllm_gtest(userBufferTest SOURCES userBufferTest.cpp FULL_STACK LABELS + "unit;full-stack;gpu") +add_tllm_gtest(ncclUtilsTest SOURCES ncclUtilsTest.cpp FULL_STACK LABELS + "unit;full-stack;gpu") target_link_libraries(ncclUtilsTest PRIVATE ${Python3_LIBRARIES}) if(BUILD_PYT) target_compile_definitions(cacheTransceiverTest PUBLIC BUILD_PYT) diff --git a/cpp/tests/unit_tests/multi_gpu/kernels/CMakeLists.txt b/cpp/tests/unit_tests/multi_gpu/kernels/CMakeLists.txt index db7554f49950..c883cd35c5aa 100644 --- a/cpp/tests/unit_tests/multi_gpu/kernels/CMakeLists.txt +++ b/cpp/tests/unit_tests/multi_gpu/kernels/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & +# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & # AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -14,9 +14,12 @@ # the License. if(NOT ENABLE_MULTI_DEVICE EQUAL 0) - add_gtest(allReduceKernelTest allReduce/allReduceKernelTest.cu) - add_gtest(allReduceFusionTest allReduce/allReduceFusionTest.cu) - add_gtest(gemmAllReduceTest allReduce/gemmAllReduceTest.cu) + add_tllm_gtest(allReduceKernelTest SOURCES allReduce/allReduceKernelTest.cu + FULL_STACK LABELS "unit;full-stack;gpu") + add_tllm_gtest(allReduceFusionTest SOURCES allReduce/allReduceFusionTest.cu + FULL_STACK LABELS "unit;full-stack;gpu") + add_tllm_gtest(gemmAllReduceTest SOURCES allReduce/gemmAllReduceTest.cu + FULL_STACK LABELS "unit;full-stack;gpu") if(USING_OSS_CUTLASS_ALLREDUCE_GEMM) target_link_libraries(gemmAllReduceTest PRIVATE ar_gemm_src) target_compile_definitions(gemmAllReduceTest diff --git a/cpp/tests/unit_tests/runtime/CMakeLists.txt b/cpp/tests/unit_tests/runtime/CMakeLists.txt index 045d71dfadf9..1cea71176a22 100644 --- a/cpp/tests/unit_tests/runtime/CMakeLists.txt +++ b/cpp/tests/unit_tests/runtime/CMakeLists.txt @@ -22,8 +22,7 @@ add_tllm_gtest( LINK_LIBRARIES tllm::runtime_buffers LABELS - "unit;runtime;gpu" - REQUIRES_GPU) + "unit;runtime;gpu") add_tllm_gtest( cudaMemPoolTest OWNER @@ -33,11 +32,12 @@ add_tllm_gtest( LINK_LIBRARIES tllm::runtime_memory LABELS - "unit;runtime;gpu" - REQUIRES_GPU) -add_gtest(gdrcopyTest gdrcopyTest.cpp) -add_gtest(hostAccessibleDeviceAllocatorTest - hostAccessibleDeviceAllocatorTest.cu) + "unit;runtime;gpu") +add_tllm_gtest(gdrcopyTest SOURCES gdrcopyTest.cpp FULL_STACK LABELS + "unit;full-stack;gpu") +add_tllm_gtest( + hostAccessibleDeviceAllocatorTest SOURCES + hostAccessibleDeviceAllocatorTest.cu FULL_STACK LABELS "unit;full-stack;gpu") add_tllm_gtest( iBufferTest OWNER @@ -48,7 +48,8 @@ add_tllm_gtest( tllm::runtime_buffers LABELS "unit;runtime;host") -add_gtest(ipcNvlsMemoryTest ipcNvlsMemoryTest.cpp) +add_tllm_gtest(ipcNvlsMemoryTest SOURCES ipcNvlsMemoryTest.cpp FULL_STACK + LABELS "unit;full-stack;gpu") add_tllm_gtest( iTensorTest OWNER @@ -59,9 +60,12 @@ add_tllm_gtest( tllm::runtime_buffers LABELS "unit;runtime;host") -add_gtest(loraCacheTest loraCacheTest.cpp) -add_gtest(loraUtilsTest loraUtilsTest.cpp) -add_gtest(moeLoadBalancerTest moeLoadBalancerTest.cpp) +add_tllm_gtest(loraCacheTest SOURCES loraCacheTest.cpp FULL_STACK LABELS + "unit;full-stack;gpu") +add_tllm_gtest(loraUtilsTest SOURCES loraUtilsTest.cpp FULL_STACK LABELS + "unit;full-stack;gpu") +add_tllm_gtest(moeLoadBalancerTest SOURCES moeLoadBalancerTest.cpp FULL_STACK + LABELS "unit;full-stack;gpu") add_tllm_gtest( runtimeMpiUtilsTest OWNER @@ -72,9 +76,29 @@ add_tllm_gtest( tllm::runtime_distributed LABELS "unit;runtime;host") -add_gtest(localizationTest localizationTest.cu) -add_gtest(localityDomainPublicConfigTest localityDomainPublicConfigTest.cpp) -add_gtest(sanitizerTest sanitizerTest.cpp) +add_tllm_gtest(localizationTest SOURCES localizationTest.cu FULL_STACK LABELS + "unit;full-stack;gpu") +add_tllm_gtest( + localityDomainPublicConfigTest + OWNER + runtime + SOURCES + localityDomainPublicConfigTest.cpp + LINK_LIBRARIES + tllm::common_headers + CUDA::toolkit + LABELS + "unit;runtime;host") +add_tllm_gtest( + sanitizerTest + OWNER + runtime + SOURCES + sanitizerTest.cpp + LINK_LIBRARIES + tllm::runtime_buffers + LABELS + "unit;runtime;gpu") add_tllm_gtest( tllmBuffersTest OWNER @@ -84,10 +108,11 @@ add_tllm_gtest( LINK_LIBRARIES tllm::runtime_buffers LABELS - "unit;runtime;gpu" - REQUIRES_GPU) -add_gtest(transposeKVKernelTest transposeKVKernelTest.cpp) -add_gtest(utilsTest utilsTest.cpp) + "unit;runtime;gpu") +add_tllm_gtest(transposeKVKernelTest SOURCES transposeKVKernelTest.cpp + FULL_STACK LABELS "unit;full-stack;gpu") +add_tllm_gtest(utilsTest SOURCES utilsTest.cpp FULL_STACK LABELS + "unit;full-stack;gpu") add_tllm_gtest( virtualMemoryTest OWNER @@ -97,9 +122,9 @@ add_tllm_gtest( LINK_LIBRARIES tllm::runtime_buffers LABELS - "unit;runtime;gpu" - REQUIRES_GPU) -add_gtest(workerPoolTest workerPoolTest.cpp) + "unit;runtime;gpu") +add_tllm_gtest(workerPoolTest SOURCES workerPoolTest.cpp FULL_STACK LABELS + "unit;full-stack;host") add_tllm_gtest( worldConfigTest OWNER @@ -112,6 +137,15 @@ add_tllm_gtest( "unit;runtime;host") if(${BUILD_PYT}) - add_gtest(torchTest torchTest.cpp) - target_link_libraries(torchTest PUBLIC ${TORCH_LIBRARIES}) + add_tllm_gtest( + torchTest + OWNER + runtime + SOURCES + torchTest.cpp + LINK_LIBRARIES + tllm::runtime_buffers + LABELS + "unit;runtime;gpu") + target_link_libraries(torchTest PRIVATE ${TORCH_LIBRARIES}) endif() diff --git a/cpp/tests/unit_tests/thop/CMakeLists.txt b/cpp/tests/unit_tests/thop/CMakeLists.txt index e078db4a2186..70733afed359 100644 --- a/cpp/tests/unit_tests/thop/CMakeLists.txt +++ b/cpp/tests/unit_tests/thop/CMakeLists.txt @@ -14,7 +14,14 @@ # the License. if(${BUILD_PYT}) - add_gtest(thUtilsTest thUtilsTest.cpp eagerMoeWorkspaceTest.cpp) + add_tllm_gtest( + thUtilsTest + SOURCES + thUtilsTest.cpp + eagerMoeWorkspaceTest.cpp + FULL_STACK + LABELS + "unit;full-stack;gpu") target_link_libraries(thUtilsTest PUBLIC th_utils ${Python3_LIBRARIES} ${TORCH_LIBRARIES}) endif() diff --git a/cpp/tests/unit_tests/thop/eagerMoeWorkspaceTest.cpp b/cpp/tests/unit_tests/thop/eagerMoeWorkspaceTest.cpp index 36dbb43fe362..0e2e354ff1f1 100644 --- a/cpp/tests/unit_tests/thop/eagerMoeWorkspaceTest.cpp +++ b/cpp/tests/unit_tests/thop/eagerMoeWorkspaceTest.cpp @@ -16,6 +16,7 @@ #include "tensorrt_llm/thop/moe/cutlass/eagerMoeWorkspace.h" #include "tensorrt_llm/thop/moe/cutlass/eagerMoeWorkspaceScope.h" +#include #include #include #include @@ -153,6 +154,8 @@ TEST(EagerMoeWorkspace, ReclamationRetriesAfterReleasingScratchUnderMemoryPressu GTEST_SKIP() << "Uses native allocator quota and OOM counters"; } constexpr int64_t kMiB = 1024 * 1024; + // Allocator quota queries require PyTorch's CUDA initialization. + at::globalContext().lazyInitDevice(c10::DeviceType::CUDA); auto const device = c10::cuda::current_device(); auto const previousLimit = c10::cuda::CUDACachingAllocator::getMemoryFraction(device); auto restoreLimit @@ -205,6 +208,8 @@ TEST(EagerMoeWorkspace, ConservativeReservationOomStillAllowsSmallerActualDemand GTEST_SKIP() << "Uses native allocator quota and OOM counters"; } constexpr int64_t kMiB = 1024 * 1024; + // Allocator quota queries require PyTorch's CUDA initialization. + at::globalContext().lazyInitDevice(c10::DeviceType::CUDA); auto const device = c10::cuda::current_device(); auto const previousLimit = c10::cuda::CUDACachingAllocator::getMemoryFraction(device); auto restoreLimit