From ec7b666b2b200cf4e76cc4846b9807f4ac1a1355 Mon Sep 17 00:00:00 2001 From: Weimin Wang <301118019+weiminwang-nv@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:13:26 +0800 Subject: [PATCH 1/4] [None][infra] Keep CPU test pods off nodes that cannot mount the model scratch CPU test pods mount the model scratch NFS 10.117.145.14. The builders in the ipp6/ipp9 DCs and the sc-ipp-blossom-prod-k8arm-* builders are not in its export list, so these pods stay in ContainerCreating until the 1000s agent launch timeout. Exclude these nodes for CPU test pods only. Build, Slurm and agent pods do not mount the model scratch and are not changed. Signed-off-by: Weimin Wang <301118019+weiminwang-nv@users.noreply.github.com> --- jenkins/L0_Test.groovy | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index fbb900a59ffc..0616d36fd89a 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4024,6 +4024,23 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod operator: NotIn values: - "rl300-0021.ipp2a1.colossus"''' : "" + // CPU test pods mount the model scratch from 10.117.145.14, which is not exported to the builders + // in the ipp6/ipp9 DCs or to the k8arm builders. Pods placed there stay in ContainerCreating. + // Build pods do not mount the model scratch, so they can still use these nodes. + def scratchUnreachableNodeAffinity = type == "cpu" ? ''' + - key: "region" + operator: NotIn + values: + - "ipp6" + - "ipp9" + - key: "kubernetes.io/hostname" + operator: NotIn + values: + - "sc-ipp-blossom-prod-k8arm-01" + - "sc-ipp-blossom-prod-k8arm-02" + - "sc-ipp-blossom-prod-k8arm-03" + - "sc-ipp-blossom-prod-k8arm-04" + - "sc-ipp-blossom-prod-k8arm-05"''' : "" def nodeLabel = trtllm_utils.generateNodeLabel(nodeLabelPrefix) def pvcVolume = """ - name: sw-tensorrt-pvc @@ -4078,6 +4095,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod - "core" - "qa_only" ${blockedNodeAffinity} +${scratchUnreachableNodeAffinity} nodeSelector: ${selectors} imagePullSecrets: - name: ${ARTIFACTORY_IMAGE_PULL_SECRET} From db3a077a0629ecc55e589806c390eaa30711751b Mon Sep 17 00:00:00 2001 From: Weimin Wang <301118019+weiminwang-nv@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:33:01 +0800 Subject: [PATCH 2/4] [None][infra] Stop excluding the k8arm builders for CPU test pods The model scratch NFS is now exported to the k8arm builders (10.116.173.0/24), and CPU test pods start normally on them. Keep only the ipp6/ipp9 exclusion. Signed-off-by: Weimin Wang <301118019+weiminwang-nv@users.noreply.github.com> --- jenkins/L0_Test.groovy | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 0616d36fd89a..74cc70215f11 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4025,22 +4025,14 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod values: - "rl300-0021.ipp2a1.colossus"''' : "" // CPU test pods mount the model scratch from 10.117.145.14, which is not exported to the builders - // in the ipp6/ipp9 DCs or to the k8arm builders. Pods placed there stay in ContainerCreating. + // in the ipp6/ipp9 DCs. Pods placed there stay in ContainerCreating. // Build pods do not mount the model scratch, so they can still use these nodes. def scratchUnreachableNodeAffinity = type == "cpu" ? ''' - key: "region" operator: NotIn values: - "ipp6" - - "ipp9" - - key: "kubernetes.io/hostname" - operator: NotIn - values: - - "sc-ipp-blossom-prod-k8arm-01" - - "sc-ipp-blossom-prod-k8arm-02" - - "sc-ipp-blossom-prod-k8arm-03" - - "sc-ipp-blossom-prod-k8arm-04" - - "sc-ipp-blossom-prod-k8arm-05"''' : "" + - "ipp9"''' : "" def nodeLabel = trtllm_utils.generateNodeLabel(nodeLabelPrefix) def pvcVolume = """ - name: sw-tensorrt-pvc From 4641b7001d08b099f2504f8073ff26bf129cdb83 Mon Sep 17 00:00:00 2001 From: Weimin Wang <301118019+weiminwang-nv@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:55:45 +0800 Subject: [PATCH 3/4] [None][infra] Remove old hostname blocklists from the builder pod templates Remove the node rules that were added for network problems that no longer show up: - rl300-0021 was kept off arm64 build and test pods (L0_Test, Build, BuildDockerImage) after pod DNS/JNLP failures. - sc-ipp-blossom-prod-k8w-105/114/115/121/123/124 were kept off amd64 docker image build pods because of unstable network access. - arm64 wheel builds in BuildDockerImage were limited to a hostname list because of a ucxx build problem. Three names in the list no longer match any node. The ucxx downloads now go through the GitHub mirror. A network probe on these nodes (agent launch, DNS, HTTP, github.com download, pip download) shows no difference from other builders. Signed-off-by: Weimin Wang <301118019+weiminwang-nv@users.noreply.github.com> --- jenkins/Build.groovy | 7 ---- jenkins/BuildDockerImage.groovy | 69 +-------------------------------- jenkins/L0_Test.groovy | 7 ---- 3 files changed, 2 insertions(+), 81 deletions(-) diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index 608fa8f1d776..1bc2f89c8263 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -257,12 +257,6 @@ def createKubernetesPodConfig(image, type, arch = "amd64") nodeLabelPrefix = "cpu" break } - // Temporarily avoid an arm64 builder with repeated pod DNS/JNLP failures seen in Build-SBSA #5564. - def blockedNodeAffinity = arch == "arm64" ? ''' - - key: "kubernetes.io/hostname" - operator: NotIn - values: - - "rl300-0021.ipp2a1.colossus"''' : "" def nodeLabel = trtllm_utils.generateNodeLabel(nodeLabelPrefix) def pvcVolume = """ - name: sw-tensorrt-pvc @@ -299,7 +293,6 @@ def createKubernetesPodConfig(image, type, arch = "amd64") values: - "core" - "qa_only" -${blockedNodeAffinity} nodeSelector: ${selectors} imagePullSecrets: - name: ${ARTIFACTORY_IMAGE_PULL_SECRET} diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index efcdc1deff9c..685e01167402 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -117,7 +117,7 @@ def globalVars = [ @Field def imageKeyToTag = [:] -def createKubernetesPodConfig(type, arch = "amd64", build_wheel = false) +def createKubernetesPodConfig(type, arch = "amd64") { def targetCould = "kubernetes-cpu" def containerConfig = "" @@ -127,66 +127,6 @@ def createKubernetesPodConfig(type, arch = "amd64", build_wheel = false) kubernetes.io/os: linux kubernetes.io/arch: ${arch}""" - if (build_wheel && arch == "arm64") { - // For aarch64, we need to use hostname to fix the ucxx issue when building wheels - selectors += """ - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: In - values: - - "rl300-0008.ipp2u1.colossus" - - "rl300-0014.ipp2u1.colossus" - - "rl300-0023.ipp2u1.colossus" - - "rl300-0024.ipp2u1.colossus" - - "rl300-0030.ipp2u1.colossus" - - "rl300-0040.ipp2u1.colossus" - - "rl300-0041.ipp2u1.colossus" - - "rl300-0042.ipp2u1.colossus" - - "rl300-0043.ipp2u1.colossus" - - "rl300-0044.ipp2u1.colossus" - - "rl300-0045.ipp2u1.colossus" - - "rl300-0046.ipp2u1.colossus" - - "rl300-0047.ipp2u1.colossus" - """ - } else if (arch == "arm64") { - // Temporarily avoid an arm64 builder with repeated pod DNS/JNLP failures seen in Build-SBSA #5564. - selectors += """ - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: NotIn - values: - - "rl300-0021.ipp2a1.colossus" - """ - } - - if (arch == "amd64") { - // For x86_64, we block some nodes to avoid unstable network access. - selectors += """ - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: NotIn - values: - - "sc-ipp-blossom-prod-k8w-105" - - "sc-ipp-blossom-prod-k8w-114" - - "sc-ipp-blossom-prod-k8w-115" - - "sc-ipp-blossom-prod-k8w-121" - - "sc-ipp-blossom-prod-k8w-123" - - "sc-ipp-blossom-prod-k8w-124" - """ - } - def archSuffix = arch == "arm64" ? "arm" : "amd" def jnlpImage = "artifactory.pdx.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" @@ -770,7 +710,6 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { args: "", torchInstallType: "skip", arch: "amd64", - build_wheel: false, dependent: [:], dockerfileStage: "tritondevel", ] @@ -792,14 +731,12 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { target: "trtllm", action: release_action, customTag: LLM_BRANCH_TAG + "-x86_64", - build_wheel: true, dockerfileStage: "tritonrelease", ], (stageNames.internalReleaseSBSA): [ target: "trtllm", action: release_action, customTag: LLM_BRANCH_TAG + "-sbsa", - build_wheel: true, arch: "arm64", dockerfileStage: "tritonrelease", ], @@ -827,7 +764,6 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { target: "ngc-release", action: release_action, args: "DOCKER_BUILD_OPTS='--load --platform linux/amd64'", - build_wheel: true, dependent: [ target: "ngc-devel", dockerfileStage: "devel", @@ -839,7 +775,6 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { action: release_action, args: "DOCKER_BUILD_OPTS='--load --platform linux/arm64'", arch: "arm64", - build_wheel: true, dependent: [ target: "ngc-devel", dockerfileStage: "devel", @@ -890,7 +825,7 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { config[defaultKey] = defaultValue } } - config.podConfig = createKubernetesPodConfig("build", config.arch, config.build_wheel) + config.podConfig = createKubernetesPodConfig("build", config.arch) } echo "Build configs:" println buildConfigs diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 74cc70215f11..4d3ff0801547 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4018,12 +4018,6 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod imagePullPolicy: Always """ } - // Temporarily avoid an arm64 CPU builder with repeated pod DNS/JNLP failures seen in Build-SBSA #5564. - def blockedNodeAffinity = targetCloud == "kubernetes-cpu" && arch == "arm64" ? ''' - - key: "kubernetes.io/hostname" - operator: NotIn - values: - - "rl300-0021.ipp2a1.colossus"''' : "" // CPU test pods mount the model scratch from 10.117.145.14, which is not exported to the builders // in the ipp6/ipp9 DCs. Pods placed there stay in ContainerCreating. // Build pods do not mount the model scratch, so they can still use these nodes. @@ -4086,7 +4080,6 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod values: - "core" - "qa_only" -${blockedNodeAffinity} ${scratchUnreachableNodeAffinity} nodeSelector: ${selectors} imagePullSecrets: From a7ef8f01fe53176a332ee43281efd5d877c3f6c5 Mon Sep 17 00:00:00 2001 From: Weimin Wang <301118019+weiminwang-nv@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:22:14 +0800 Subject: [PATCH 4/4] [None][infra] Keep the build_wheel argument and reword the region comment Keep the createKubernetesPodConfig() signature and the build_wheel config in BuildDockerImage.groovy unchanged. Reword the comment on the region rule for CPU test pods. Signed-off-by: Weimin Wang <301118019+weiminwang-nv@users.noreply.github.com> --- jenkins/BuildDockerImage.groovy | 9 +++++++-- jenkins/L0_Test.groovy | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index 685e01167402..ef228716dc35 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -117,7 +117,7 @@ def globalVars = [ @Field def imageKeyToTag = [:] -def createKubernetesPodConfig(type, arch = "amd64") +def createKubernetesPodConfig(type, arch = "amd64", build_wheel = false) { def targetCould = "kubernetes-cpu" def containerConfig = "" @@ -710,6 +710,7 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { args: "", torchInstallType: "skip", arch: "amd64", + build_wheel: false, dependent: [:], dockerfileStage: "tritondevel", ] @@ -731,12 +732,14 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { target: "trtllm", action: release_action, customTag: LLM_BRANCH_TAG + "-x86_64", + build_wheel: true, dockerfileStage: "tritonrelease", ], (stageNames.internalReleaseSBSA): [ target: "trtllm", action: release_action, customTag: LLM_BRANCH_TAG + "-sbsa", + build_wheel: true, arch: "arm64", dockerfileStage: "tritonrelease", ], @@ -764,6 +767,7 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { target: "ngc-release", action: release_action, args: "DOCKER_BUILD_OPTS='--load --platform linux/amd64'", + build_wheel: true, dependent: [ target: "ngc-devel", dockerfileStage: "devel", @@ -775,6 +779,7 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { action: release_action, args: "DOCKER_BUILD_OPTS='--load --platform linux/arm64'", arch: "arm64", + build_wheel: true, dependent: [ target: "ngc-devel", dockerfileStage: "devel", @@ -825,7 +830,7 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { config[defaultKey] = defaultValue } } - config.podConfig = createKubernetesPodConfig("build", config.arch) + config.podConfig = createKubernetesPodConfig("build", config.arch, config.build_wheel) } echo "Build configs:" println buildConfigs diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 4d3ff0801547..db5c0170f4fa 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4018,9 +4018,9 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod imagePullPolicy: Always """ } - // CPU test pods mount the model scratch from 10.117.145.14, which is not exported to the builders - // in the ipp6/ipp9 DCs. Pods placed there stay in ContainerCreating. - // Build pods do not mount the model scratch, so they can still use these nodes. + // CPU test pods mount the model scratch from 10.117.145.14, which can't be accessed from ipp6/ipp9 DC. + // Pods placed there stay in ContainerCreating. Build stage pods do not mount the model scratch, + // so they can still use these nodes. def scratchUnreachableNodeAffinity = type == "cpu" ? ''' - key: "region" operator: NotIn