[TRTLLM-17014][infra] migrate build artifacts to artifactory.nvidia.com

Publish build_info.txt, the packaged builds (*.tar.gz) and waive_list/ to
sw-tensorrt-llm-generic-local on artifactory.nvidia.com instead of
sw-tensorrt-generic on urm.nvidia.com, and read them from there first.

- Readers fall back to urm only when a file is absent (HTTP 404), so builds
  published before the switch stay readable. A rejected credential fails
  fast with a message that names the credential and says that its token
  needs renewal or lacks the permission the request needs, and never falls
  back.
- jenkins/scripts/fetch_build_artifact.sh is the one read helper: the
  pipelines, the Slurm jobs, the image build, coverage-based test selection
  and BOLT profile generation run it, and it checks each download's SHA-1.
  jenkins/scripts/check_artifactory_access.sh checks the CI identities in
  each environment.
- LLM_ARTIFACTS_PRIMARY=urm on the Jenkins controllers returns publishing
  and first reads to urm without a code change. A parent pipeline passes
  its value to the jobs it starts.
- Build reuse copies within one instance, and rebuilds when the earlier
  build is only on the other one.
- While under test, jenkins/L0_MergeRequest.groovy loads
  trtllm-jenkins-shared-lib from the branch of the library change that
  adds the artifactory.nvidia.com destination to uploadArtifacts.
- Until trtllm-artifactory-credentials holds read permission on
  sw-tensorrt-llm-generic-local, reads of these artifacts use
  aws-artifactory-credentials, the publishing credential. Container-image
  pulls keep trtllm-artifactory-credentials.
- specs/001-migrate-builds-aws-artifactory/ carries the design documents
  for review while the pull request is a draft. They are removed before it
  is marked ready.

Signed-off-by: Sharabesh Jayaraman <338298191+sharabeshj-nvda@users.noreply.github.com>

Author: sharabeshj-nvda

docker/Dockerfile.multi

@@ -159,16 +159,33 @@ ARG GIT_COMMIT
 ARG GITHUB_MIRROR=""
 ARG BUILD_WHEEL_ARGS="--clean"
 ARG BUILD_WHEEL_SCRIPT="scripts/build_wheel.py"
+# Used when BUILD_WHEEL_SCRIPT takes the wheel from a packaged build
+# (scripts/get_wheel_from_package.py): the location read first (aws or urm; empty
+# means aws) and the name of the credential behind the artifactory_netrc secret,
+# which appears only in error messages.
+ARG LLM_ARTIFACTS_PRIMARY=""
+ARG ARTIFACTORY_CREDENTIAL_ID=""
 RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=${CCACHE_DIR} \
     --mount=type=secret,id=github_clone_token \
+    --mount=type=secret,id=artifactory_netrc \
+    --mount=type=bind,source=jenkins/scripts/fetch_build_artifact.sh,target=/src/tensorrt_llm/jenkins/scripts/fetch_build_artifact.sh \
     if [ -n "$GITHUB_MIRROR" ]; then \
       export PIP_INDEX_URL="https://artifactory.pdx.nvidia.com/artifactory/api/pypi/pypi-remote/simple"; \
     fi && \
     # Authenticates `git submodule update` and the CMake FetchContent clones
     # (3rdparty/fetch_content.json) that this build pulls from github.com.
     . docker/common/github_auth.sh && \
+    # get_wheel_from_package.py reads the packaged build through the bind-mounted
+    # jenkins/scripts/fetch_build_artifact.sh, which takes the Artifactory access token
+    # for artifactory.nvidia.com from this netrc. The secret is present only when the
+    # build passes it (ARTIFACTORY_NETRC in docker/Makefile).
+    if [ -f /run/secrets/artifactory_netrc ]; then \
+      export ARTIFACTORY_NETRC=/run/secrets/artifactory_netrc; \
+    fi && \
     TRTLLM_BUILD_SOURCE_COMMIT=${GIT_COMMIT} \
     GITHUB_MIRROR=${GITHUB_MIRROR} \
+    LLM_ARTIFACTS_PRIMARY=${LLM_ARTIFACTS_PRIMARY} \
+    ARTIFACTORY_CREDENTIAL_ID=${ARTIFACTORY_CREDENTIAL_ID} \
     python3 ${BUILD_WHEEL_SCRIPT} ${BUILD_WHEEL_ARGS}
 
 FROM ${DEVEL_IMAGE} AS release

docker/Makefile

@@ -55,8 +55,19 @@ GITHUB_MIRROR      ?=
 # environment, never passed as a build arg -- build-arg values stay readable in
 # `docker history`. Empty means anonymous, rate-limited access.
 GITHUB_CLONE_TOKEN ?=
+# Optional netrc file with an Artifactory access token for artifactory.nvidia.com,
+# needed when the wheel stage takes the wheel from a packaged build
+# (BUILD_WHEEL_SCRIPT=scripts/get_wheel_from_package.py, which reads it through
+# jenkins/scripts/fetch_build_artifact.sh). Passed as a BuildKit secret, never as a
+# build arg, for the same reason as GITHUB_CLONE_TOKEN.
+ARTIFACTORY_NETRC  ?=
+# The location the packaged build is read from first (aws or urm; empty means aws),
+# and the name of the CI credential behind ARTIFACTORY_NETRC, used only in messages.
+LLM_ARTIFACTS_PRIMARY ?=
+ARTIFACTORY_CREDENTIAL_ID ?=
 COMMA              := ,
 GITHUB_SECRET_ARG  := $(if $(GITHUB_CLONE_TOKEN),--secret id=github_clone_token$(COMMA)env=GITHUB_CLONE_TOKEN)
+ARTIFACTORY_SECRET_ARG := $(if $(ARTIFACTORY_NETRC),--secret id=artifactory_netrc$(COMMA)src=$(ARTIFACTORY_NETRC))
 PYTHON_VERSION     ?=
 NGC_STAGING_REPO   ?= nvcr.io/nvstaging/tensorrt-llm
 NGC_REPO           ?= nvcr.io/nvidia/tensorrt-llm
@@ -95,12 +106,15 @@ base_pull:
 	docker buildx build $(DOCKER_BUILD_OPTS) $(DOCKER_BUILD_ARGS) \
 		--progress $(DOCKER_PROGRESS) \
 		$(GITHUB_SECRET_ARG) \
+		$(ARTIFACTORY_SECRET_ARG) \
 		$(if $(BASE_IMAGE), --build-arg BASE_IMAGE=$(BASE_IMAGE)) \
 		$(if $(BASE_TAG), --build-arg BASE_TAG=$(BASE_TAG)) \
 		$(if $(TRITON_IMAGE), --build-arg TRITON_IMAGE=$(TRITON_IMAGE)) \
 		$(if $(TRITON_BASE_TAG), --build-arg TRITON_BASE_TAG=$(TRITON_BASE_TAG)) \
 		$(if $(BUILD_WHEEL_ARGS), --build-arg BUILD_WHEEL_ARGS="$(BUILD_WHEEL_ARGS)") \
 		$(if $(BUILD_WHEEL_SCRIPT), --build-arg BUILD_WHEEL_SCRIPT="$(BUILD_WHEEL_SCRIPT)") \
+		$(if $(LLM_ARTIFACTS_PRIMARY), --build-arg LLM_ARTIFACTS_PRIMARY="$(LLM_ARTIFACTS_PRIMARY)") \
+		$(if $(ARTIFACTORY_CREDENTIAL_ID), --build-arg ARTIFACTORY_CREDENTIAL_ID="$(ARTIFACTORY_CREDENTIAL_ID)") \
 		$(if $(TORCH_INSTALL_TYPE), --build-arg TORCH_INSTALL_TYPE="$(TORCH_INSTALL_TYPE)") \
 		$(if $(CUDA_VERSION), --build-arg CUDA_VER="$(CUDA_VERSION)") \
 		$(if $(CUDNN_VERSION), --build-arg CUDNN_VER="$(CUDNN_VERSION)") \