diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index 608fa8f1d776..1bc2f89c8263 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -257,12 +257,6 @@ def createKubernetesPodConfig(image, type, arch = "amd64") nodeLabelPrefix = "cpu" break } - // Temporarily avoid an arm64 builder with repeated pod DNS/JNLP failures seen in Build-SBSA #5564. - def blockedNodeAffinity = arch == "arm64" ? ''' - - key: "kubernetes.io/hostname" - operator: NotIn - values: - - "rl300-0021.ipp2a1.colossus"''' : "" def nodeLabel = trtllm_utils.generateNodeLabel(nodeLabelPrefix) def pvcVolume = """ - name: sw-tensorrt-pvc @@ -299,7 +293,6 @@ def createKubernetesPodConfig(image, type, arch = "amd64") values: - "core" - "qa_only" -${blockedNodeAffinity} nodeSelector: ${selectors} imagePullSecrets: - name: ${ARTIFACTORY_IMAGE_PULL_SECRET} diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index efcdc1deff9c..ef228716dc35 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -127,66 +127,6 @@ def createKubernetesPodConfig(type, arch = "amd64", build_wheel = false) kubernetes.io/os: linux kubernetes.io/arch: ${arch}""" - if (build_wheel && arch == "arm64") { - // For aarch64, we need to use hostname to fix the ucxx issue when building wheels - selectors += """ - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: In - values: - - "rl300-0008.ipp2u1.colossus" - - "rl300-0014.ipp2u1.colossus" - - "rl300-0023.ipp2u1.colossus" - - "rl300-0024.ipp2u1.colossus" - - "rl300-0030.ipp2u1.colossus" - - "rl300-0040.ipp2u1.colossus" - - "rl300-0041.ipp2u1.colossus" - - "rl300-0042.ipp2u1.colossus" - - "rl300-0043.ipp2u1.colossus" - - "rl300-0044.ipp2u1.colossus" - - "rl300-0045.ipp2u1.colossus" - - "rl300-0046.ipp2u1.colossus" - - "rl300-0047.ipp2u1.colossus" - """ - } else if (arch == "arm64") { - // Temporarily avoid an arm64 builder with repeated pod DNS/JNLP failures seen in Build-SBSA #5564. - selectors += """ - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: NotIn - values: - - "rl300-0021.ipp2a1.colossus" - """ - } - - if (arch == "amd64") { - // For x86_64, we block some nodes to avoid unstable network access. - selectors += """ - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: NotIn - values: - - "sc-ipp-blossom-prod-k8w-105" - - "sc-ipp-blossom-prod-k8w-114" - - "sc-ipp-blossom-prod-k8w-115" - - "sc-ipp-blossom-prod-k8w-121" - - "sc-ipp-blossom-prod-k8w-123" - - "sc-ipp-blossom-prod-k8w-124" - """ - } - def archSuffix = arch == "arm64" ? "arm" : "amd" def jnlpImage = "artifactory.pdx.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index fbb900a59ffc..db5c0170f4fa 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4018,12 +4018,15 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod imagePullPolicy: Always """ } - // Temporarily avoid an arm64 CPU builder with repeated pod DNS/JNLP failures seen in Build-SBSA #5564. - def blockedNodeAffinity = targetCloud == "kubernetes-cpu" && arch == "arm64" ? ''' - - key: "kubernetes.io/hostname" + // CPU test pods mount the model scratch from 10.117.145.14, which can't be accessed from ipp6/ipp9 DC. + // Pods placed there stay in ContainerCreating. Build stage pods do not mount the model scratch, + // so they can still use these nodes. + def scratchUnreachableNodeAffinity = type == "cpu" ? ''' + - key: "region" operator: NotIn values: - - "rl300-0021.ipp2a1.colossus"''' : "" + - "ipp6" + - "ipp9"''' : "" def nodeLabel = trtllm_utils.generateNodeLabel(nodeLabelPrefix) def pvcVolume = """ - name: sw-tensorrt-pvc @@ -4077,7 +4080,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod values: - "core" - "qa_only" -${blockedNodeAffinity} +${scratchUnreachableNodeAffinity} nodeSelector: ${selectors} imagePullSecrets: - name: ${ARTIFACTORY_IMAGE_PULL_SECRET}