From 577b50b7df92b6191aee29ce97d8a737bfce6a5b Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:04:42 +0000 Subject: [PATCH 1/4] fix: initialize CUDA before querying eager MoE test allocator quotas The two EagerMoeWorkspace memory-pressure tests query the native CUDA caching allocator memory fraction before making their first tensor allocation. Reading the current CUDA device does not initialize the PyTorch allocator, so either test can fail in a fresh process with an "Allocator not initialized for device" internal assertion. Running the complete Google Test suite in one process masks this ordering dependency because earlier tests allocate CUDA tensors. CTest executes the registered cases in separate processes and exposes the missing setup. Explicitly initialize the CUDA device through ATen's global context in both memory-pressure tests before reading or changing allocator quotas. Include ATen/Context.h directly for the initialization API. Keep the quota restoration and OOM assertions unchanged. Validation: - Reproduced ConservativeReservationOomStillAllowsSmallerActualDemand failing in isolation before the fix. - Rebuilt thUtilsTest in the project devcontainer. - All 13 EagerMoeWorkspace and EagerMoeWorkspaceScope tests passed through CTest on an A100, with each test running in a fresh process. - clang-format --dry-run --Werror and git diff --check passed. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- cpp/tests/unit_tests/thop/eagerMoeWorkspaceTest.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cpp/tests/unit_tests/thop/eagerMoeWorkspaceTest.cpp b/cpp/tests/unit_tests/thop/eagerMoeWorkspaceTest.cpp index 36dbb43fe362..0e2e354ff1f1 100644 --- a/cpp/tests/unit_tests/thop/eagerMoeWorkspaceTest.cpp +++ b/cpp/tests/unit_tests/thop/eagerMoeWorkspaceTest.cpp @@ -16,6 +16,7 @@ #include "tensorrt_llm/thop/moe/cutlass/eagerMoeWorkspace.h" #include "tensorrt_llm/thop/moe/cutlass/eagerMoeWorkspaceScope.h" +#include #include #include #include @@ -153,6 +154,8 @@ TEST(EagerMoeWorkspace, ReclamationRetriesAfterReleasingScratchUnderMemoryPressu GTEST_SKIP() << "Uses native allocator quota and OOM counters"; } constexpr int64_t kMiB = 1024 * 1024; + // Allocator quota queries require PyTorch's CUDA initialization. + at::globalContext().lazyInitDevice(c10::DeviceType::CUDA); auto const device = c10::cuda::current_device(); auto const previousLimit = c10::cuda::CUDACachingAllocator::getMemoryFraction(device); auto restoreLimit @@ -205,6 +208,8 @@ TEST(EagerMoeWorkspace, ConservativeReservationOomStillAllowsSmallerActualDemand GTEST_SKIP() << "Uses native allocator quota and OOM counters"; } constexpr int64_t kMiB = 1024 * 1024; + // Allocator quota queries require PyTorch's CUDA initialization. + at::globalContext().lazyInitDevice(c10::DeviceType::CUDA); auto const device = c10::cuda::current_device(); auto const previousLimit = c10::cuda::CUDACachingAllocator::getMemoryFraction(device); auto restoreLimit From 802275aa7d6ecce3f7d21a60af3a5716ac657369 Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:39:39 +0000 Subject: [PATCH 2/4] fix: add lora_model_config initialization in TestResourceManager Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- tests/unittest/_torch/executor/test_resource_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittest/_torch/executor/test_resource_manager.py b/tests/unittest/_torch/executor/test_resource_manager.py index 4b18ca1d116f..d608d644c885 100644 --- a/tests/unittest/_torch/executor/test_resource_manager.py +++ b/tests/unittest/_torch/executor/test_resource_manager.py @@ -1263,6 +1263,7 @@ def _reload_ckpt_source(self, server_ckpt_source, request_ckpt_source): peft_cache_config=self.create_peft_cache_config(), lora_config=LoraConfig(lora_ckpt_source=server_ckpt_source), model_config=self.model_config, + lora_model_config=_text_lora_model_config(self.model_config), ) peft_cache_manager.impl = MagicMock() peft_cache_manager.impl.is_task_cached.return_value = False From 408f4f6f6b7afcd351f2cfa7c8e5c6e81a6e2815 Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:50:33 +0000 Subject: [PATCH 3/4] fix: add py_lora_ckpt_source to runtime request in _load function Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- tests/unittest/_torch/peft/test_lora_ownership.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unittest/_torch/peft/test_lora_ownership.py b/tests/unittest/_torch/peft/test_lora_ownership.py index 8600e92e871b..7ff67517b423 100644 --- a/tests/unittest/_torch/peft/test_lora_ownership.py +++ b/tests/unittest/_torch/peft/test_lora_ownership.py @@ -225,7 +225,11 @@ def _load(runtime: _Runtime, entrypoint: str, path: Path) -> None: ) else: runtime.request = SimpleNamespace( - lora_task_id="7", py_lora_path=str(path), lora_weights=None, lora_config=None + lora_task_id="7", + py_lora_path=str(path), + lora_weights=None, + lora_config=None, + py_lora_ckpt_source=runtime.config.lora_ckpt_source, ) runtime.cache.add_request_peft(runtime.request) From 6b8a85bffe1206212bb2c34f7ef52ffb8942d11e Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Sun, 25 Oct 2026 06:00:22 +0000 Subject: [PATCH 4/4] Revert "[None][infra] Waive 70 failed cases for main in pre-merge 66148 (function-level waives for test_lora_ownership) (#20953)" This reverts commit b7d6e624d1d5d297bbea8d6f4d383ca6e2d46711. --- tests/integration/test_lists/waives.txt | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index ae5be426fef6..51cdbaecae3a 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -200,8 +200,6 @@ unittest/_torch/executor/kv_cache/test_kv_cache_v2_vocab_size.py::test_branch_on unittest/_torch/executor/kv_cache/test_kv_cache_v2_vocab_size.py::test_unresolvable_vocab_size_only_warns_when_multimodal_keys_are_needed[False] SKIP (https://nvbugs/6818710) unittest/_torch/executor/kv_cache/test_kv_cache_v2_vocab_size.py::test_unresolvable_vocab_size_only_warns_when_multimodal_keys_are_needed[True] SKIP (https://nvbugs/6818710) unittest/_torch/executor/test_pytorch_model_engine.py::PyTorchModelEngineTestCase::test_prepare_inputs_with_helix_parallelism SKIP (https://nvbugs/6818710) -unittest/_torch/executor/test_resource_manager.py::TestResourceManager::test_add_request_peft_reload_defaults_to_config_ckpt_source SKIP (https://nvbugs/6988757) -unittest/_torch/executor/test_resource_manager.py::TestResourceManager::test_add_request_peft_reloads_with_request_ckpt_source SKIP (https://nvbugs/6988758) unittest/_torch/kernels/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_backend[mnk2-cublaslt] SKIP (https://nvbugs/6581067) unittest/_torch/kernels/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_backend[mnk2-cutlass] SKIP (https://nvbugs/6581067) unittest/_torch/kernels/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_backend[mnk3-cublaslt] SKIP (https://nvbugs/6581067) @@ -218,24 +216,6 @@ unittest/_torch/moe/test_moe_backend.py::test_moe_backend[act=Relu2-e60_k4_h2048 unittest/_torch/moe/test_moe_backend.py::test_trtllm_bf16_unquantized_moe[deepseekv3-relu2-8-separated_routing] SKIP (https://nvbugs/6778221) unittest/_torch/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=TTP-comm=NVLINK_ONE_SIDED-e8_k1_h512_i512-seq=8-dtype=torch.float16-backend=CUTLASS-quant=NVFP4-routing=Renormalize] SKIP (https://nvbugs/6843482) unittest/_torch/multi_gpu/test_linear.py::test_row_linear_norm_fusion[2-hidden:16-seqlen:2] SKIP (https://nvbugs/6501404) -unittest/_torch/peft/test_lora_ownership.py::test_decoder_aliases_load_through_worker_and_lazy_cache SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_decoder_target_filtering_remains_distinct_from_owner_validation[lazy-eager] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_decoder_target_filtering_remains_distinct_from_owner_validation[lazy-graphs] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_duplicate_decoder_aliases_are_rejected_before_cache[lazy-hf-eager] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_duplicate_decoder_aliases_are_rejected_before_cache[lazy-hf-graphs] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_duplicate_decoder_aliases_are_rejected_before_cache[lazy-nemo-eager] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_duplicate_decoder_aliases_are_rejected_before_cache[lazy-nemo-graphs] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_hf_module_aliases_cannot_overwrite_one_canonical_destination[lazy-eager] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_hf_module_aliases_cannot_overwrite_one_canonical_destination[lazy-graphs] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_mixed_checkpoint_rejects_unmapped_encoder_weights[lazy-eager] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_mixed_checkpoint_rejects_unmapped_encoder_weights[lazy-graphs] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_mixed_moe_representations_are_rejected_at_every_loading_entrypoint SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_moe_representations_load_when_only_one_is_present SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_non_layered_projector_rejected_at_startup_and_runtime SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_out_of_scope_adapter_is_rejected_before_cache_and_uid_can_be_reused SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_saved_decoder_embeddings_and_head_remain_supported[lazy-eager] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_saved_decoder_embeddings_and_head_remain_supported[lazy-graphs] SKIP (https://nvbugs/6988759) -unittest/_torch/peft/test_lora_ownership.py::test_saved_encoder_weights_cannot_bypass_owner_validation SKIP (https://nvbugs/6988759) unittest/_torch/ray_orchestrator/multi_gpu/test_accuracy_with_allreduce_strategy.py::test_accuracy_with_allreduce_strategy[AUTO-Qwen3/Qwen3-8B] SKIP (https://nvbugs/6923016) unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py::test_llm_update_weights_qwen35_35b_bf16 SKIP (https://nvbugs/6907190) unittest/_torch/visual_gen/multi_gpu/test_attn2d_attention.py::TestAttn2DAttentionGQAAndCrossAttention::test_attn2d_cross_attention SKIP (https://nvbugs/6890768)