From a2f0ffabd98b046f65a69f19660eb9342efca731 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 20 Jan 2026 06:48:32 -0800 Subject: [PATCH 01/19] Add Linux CPU only CI build and tests Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- .../{libkineto_cuda.yml => linux_cpu.yml} | 0 .github/workflows/linux_cuda.yml | 135 ++++++++++++++++++ 2 files changed, 135 insertions(+) rename .github/workflows/{libkineto_cuda.yml => linux_cpu.yml} (100%) create mode 100644 .github/workflows/linux_cuda.yml diff --git a/.github/workflows/libkineto_cuda.yml b/.github/workflows/linux_cpu.yml similarity index 100% rename from .github/workflows/libkineto_cuda.yml rename to .github/workflows/linux_cpu.yml diff --git a/.github/workflows/linux_cuda.yml b/.github/workflows/linux_cuda.yml new file mode 100644 index 000000000..a42a56860 --- /dev/null +++ b/.github/workflows/linux_cuda.yml @@ -0,0 +1,135 @@ +name: libkineto PR Test on A10G + +on: + push: + branches: + - main + pull_request: + +# Use TorchBench's docker image which has all basic dependencies. +env: + DOCKER_IMAGE: "ghcr.io/pytorch/torchbench:latest" + +jobs: + pr-test: + # AWS A10G GPU instance label: linux.g5.4xlarge.nvidia.gpu + # OS version: Amazon Linux 2 + runs-on: linux.g5.4xlarge.nvidia.gpu + timeout-minutes: 180 # 3 hours + steps: + - name: Checkout Kineto + uses: actions/checkout@v5 + with: + path: kineto + submodules: recursive + + - name: Pull docker image + uses: pytorch/test-infra/.github/actions/pull-docker-image@main + with: + docker-image: ${{ env.DOCKER_IMAGE }} + + - name: Install NVIDIA Driver, docker runtime, set GPU_FLAG + uses: pytorch/test-infra/.github/actions/setup-nvidia@main + + - name: Get env vars + run: | + echo GITHUB_WORKFLOW = $GITHUB_WORKFLOW + echo HOME = $HOME + echo GITHUB_ACTION = $GITHUB_ACTION + echo GITHUB_ACTIONS = $GITHUB_ACTIONS + echo GITHUB_REPOSITORY = $GITHUB_REPOSITORY + echo GITHUB_EVENT_NAME = $GITHUB_EVENT_NAME + echo GITHUB_EVENT_PATH = $GITHUB_EVENT_PATH + echo GITHUB_WORKSPACE = $GITHUB_WORKSPACE + echo GITHUB_SHA = $GITHUB_SHA + echo GITHUB_REF = $GITHUB_REF + + - name: Build libkineto (static and shared library) and tests + run: | + container_name=$(docker run \ + ${GPU_FLAG:-} \ + --tty \ + --detach \ + --shm-size=32gb \ + -v "${PWD}/kineto:/kineto" \ + -w / \ + "${{ env.DOCKER_IMAGE }}" + ) + echo "Container name: ${container_name}" + + docker exec -t -w "/kineto" "${container_name}" bash -c " + set -eux + mkdir build_static + mkdir build_shared + pip install cmake + " + + docker exec -t -w "/kineto/build_static" "${container_name}" bash -c " + set -eux + cmake -DKINETO_LIBRARY_TYPE=static ../libkineto/ + make -j + " + + docker exec -t -w "/kineto/build_shared" "${container_name}" bash -c " + set -eux + cmake -DKINETO_LIBRARY_TYPE=shared ../libkineto/ + make -j + " + + docker exec -t -w "/kineto/build_static" "${container_name}" bash -c "make test" + + - name: Clone PyTorch + run: | + container_name=$(docker ps -lq) + docker exec -t -w "/" "${container_name}" bash -c " + set -eux + git clone --recursive --branch viable/strict https://github.com/pytorch/pytorch.git + " + + - name: Replace PyTorch's Kineto with PR version + run: | + container_name=$(docker ps -lq) + docker exec -t -w "/pytorch" "${container_name}" bash -c " + set -eux + rm -rf third_party/kineto + ln -s /kineto third_party/kineto + " + + - name: Build PyTorch from source + run: | + container_name=$(docker ps -lq) + docker exec -t -w "/pytorch" "${container_name}" bash -c " + set -eux + pip install -r requirements.txt + export USE_CUDA=1 + export BUILD_TEST=1 + python setup.py develop + " + + - name: Run PyTorch profiler tests + run: | + container_name=$(docker ps -lq) + docker exec -t -w "/pytorch" "${container_name}" bash -c " + set -eux + # TODO: Dynamically add/remove tests to the exclusion list based on their + # status on trunk instead of maintaining a hardcoded list of known failures. + # This will prevent the list from becoming stale as tests get fixed upstream. + python -m pytest test/profiler/ -v \ + --deselect=test/profiler/test_memory_profiler.py::TestDataFlow::test_data_flow_graph_complicated \ + --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_sequential_fwd_bwd \ + --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_simple_fwd_bwd \ + --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_simple_fwd_bwd_step \ + --deselect=test/profiler/test_profiler.py::TestProfiler::test_kineto \ + --deselect=test/profiler/test_profiler.py::TestProfiler::test_user_annotation \ + --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_fuzz_symbolize \ + --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_profiler_debug_autotuner \ + --deselect=test/profiler/test_torch_tidy.py::TestTorchTidyProfiler::test_tensorimpl_invalidation_scalar_args + " + + - name: Teardown Linux + uses: pytorch/test-infra/.github/actions/teardown-linux@main + if: always() + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true From ea7b04af61880043d8522459aab0f15a2e5f6028 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 20 Jan 2026 06:51:30 -0800 Subject: [PATCH 02/19] Fix CI names --- .github/workflows/linux_cpu.yml | 9 ++------- .github/workflows/linux_cuda.yml | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index a42a56860..d488212ff 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -1,4 +1,4 @@ -name: libkineto PR Test on A10G +name: Linux CPU build and test on: push: @@ -12,9 +12,7 @@ env: jobs: pr-test: - # AWS A10G GPU instance label: linux.g5.4xlarge.nvidia.gpu - # OS version: Amazon Linux 2 - runs-on: linux.g5.4xlarge.nvidia.gpu + runs-on: ubuntu-latest timeout-minutes: 180 # 3 hours steps: - name: Checkout Kineto @@ -28,9 +26,6 @@ jobs: with: docker-image: ${{ env.DOCKER_IMAGE }} - - name: Install NVIDIA Driver, docker runtime, set GPU_FLAG - uses: pytorch/test-infra/.github/actions/setup-nvidia@main - - name: Get env vars run: | echo GITHUB_WORKFLOW = $GITHUB_WORKFLOW diff --git a/.github/workflows/linux_cuda.yml b/.github/workflows/linux_cuda.yml index a42a56860..e6c19c083 100644 --- a/.github/workflows/linux_cuda.yml +++ b/.github/workflows/linux_cuda.yml @@ -1,4 +1,4 @@ -name: libkineto PR Test on A10G +name: Linux CUDA build and test on: push: From 20e37375807b5f1cda6ebc3342fd7a3fa8d4d8e2 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 20 Jan 2026 07:04:46 -0800 Subject: [PATCH 03/19] Use linux.c7i.2xlarge as CPU runner --- .github/workflows/linux_cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index d488212ff..ddd4f57a8 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -12,7 +12,7 @@ env: jobs: pr-test: - runs-on: ubuntu-latest + runs-on: linux.c7i.2xlarge timeout-minutes: 180 # 3 hours steps: - name: Checkout Kineto From c50f083434bc3a3e4f52b53b57858fbcc2481e1a Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 20 Jan 2026 07:26:39 -0800 Subject: [PATCH 04/19] Force CPU-only through cmake flags --- .github/workflows/linux_cpu.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index ddd4f57a8..5ae36f9e0 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -59,15 +59,16 @@ jobs: pip install cmake " + # Note that we are making these CPU-only builds through cmake flags. docker exec -t -w "/kineto/build_static" "${container_name}" bash -c " set -eux - cmake -DKINETO_LIBRARY_TYPE=static ../libkineto/ + cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=static ../libkineto/ make -j " docker exec -t -w "/kineto/build_shared" "${container_name}" bash -c " set -eux - cmake -DKINETO_LIBRARY_TYPE=shared ../libkineto/ + cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=shared ../libkineto/ make -j " From 67d542af068f0f63c23a57fec02d89c759d093fb Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 20 Jan 2026 08:10:54 -0800 Subject: [PATCH 05/19] Explicitly disable building CUDA support in PyTorch --- .github/workflows/linux_cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index 5ae36f9e0..2d361145c 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -97,7 +97,7 @@ jobs: docker exec -t -w "/pytorch" "${container_name}" bash -c " set -eux pip install -r requirements.txt - export USE_CUDA=1 + export USE_CUDA=0 export BUILD_TEST=1 python setup.py develop " From 8623b6013b229209f9857ca59edcfdf2d4c63d34 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 20 Jan 2026 08:58:30 -0800 Subject: [PATCH 06/19] OFF instead of 0? --- .github/workflows/linux_cpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index 2d361145c..7ec7ae311 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -91,13 +91,14 @@ jobs: ln -s /kineto third_party/kineto " + # Note that we are explicitly turning off CUDA support - name: Build PyTorch from source run: | container_name=$(docker ps -lq) docker exec -t -w "/pytorch" "${container_name}" bash -c " set -eux pip install -r requirements.txt - export USE_CUDA=0 + export USE_CUDA=OFF export BUILD_TEST=1 python setup.py develop " From 2e2582ce7acda9afc544d15e6ddf4e469ecefe7e Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 20 Jan 2026 11:00:21 -0800 Subject: [PATCH 07/19] USE_CUDA=0 --- .github/workflows/linux_cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index 7ec7ae311..75535d567 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -98,7 +98,7 @@ jobs: docker exec -t -w "/pytorch" "${container_name}" bash -c " set -eux pip install -r requirements.txt - export USE_CUDA=OFF + export USE_CUDA=0 export BUILD_TEST=1 python setup.py develop " From 658e88bf1b8a768eec7f2f88ebc10c4d02c9bc25 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 07:18:25 -0800 Subject: [PATCH 08/19] Separate build and test --- .github/workflows/linux_cpu.yml | 5 ++++- .github/workflows/linux_cuda.yml | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index 75535d567..9f0b8e0d8 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -39,7 +39,7 @@ jobs: echo GITHUB_SHA = $GITHUB_SHA echo GITHUB_REF = $GITHUB_REF - - name: Build libkineto (static and shared library) and tests + - name: Build libkineto (static and shared library) run: | container_name=$(docker run \ ${GPU_FLAG:-} \ @@ -72,6 +72,9 @@ jobs: make -j " + - name: Run libkineto tests + run: | + container_name=$(docker ps -lq) docker exec -t -w "/kineto/build_static" "${container_name}" bash -c "make test" - name: Clone PyTorch diff --git a/.github/workflows/linux_cuda.yml b/.github/workflows/linux_cuda.yml index e6c19c083..f3670de49 100644 --- a/.github/workflows/linux_cuda.yml +++ b/.github/workflows/linux_cuda.yml @@ -44,7 +44,7 @@ jobs: echo GITHUB_SHA = $GITHUB_SHA echo GITHUB_REF = $GITHUB_REF - - name: Build libkineto (static and shared library) and tests + - name: Build libkineto (static and shared library) run: | container_name=$(docker run \ ${GPU_FLAG:-} \ @@ -76,6 +76,9 @@ jobs: make -j " + - name: Run libkineto tests + run: | + container_name=$(docker ps -lq) docker exec -t -w "/kineto/build_static" "${container_name}" bash -c "make test" - name: Clone PyTorch From c10619e06ab084980a46bed1c6597d5779b4cd8b Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 07:32:20 -0800 Subject: [PATCH 09/19] Use bigger runner --- .github/workflows/linux_cpu.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index 9f0b8e0d8..8a430e8f9 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -94,7 +94,7 @@ jobs: ln -s /kineto third_party/kineto " - # Note that we are explicitly turning off CUDA support + # Note that we are explicitly turning off all GPU support. - name: Build PyTorch from source run: | container_name=$(docker ps -lq) @@ -102,6 +102,9 @@ jobs: set -eux pip install -r requirements.txt export USE_CUDA=0 + export USE_CUDNN=0 + export USE_NCCL=0 + export USE_ROCM=0 export BUILD_TEST=1 python setup.py develop " From 281f88cadb7958c696c66f1e70a0b1138b1261f9 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 08:10:14 -0800 Subject: [PATCH 10/19] Actually change to linux.c7i.4xlarge --- .github/workflows/linux_cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index 8a430e8f9..df497f2fa 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -12,7 +12,7 @@ env: jobs: pr-test: - runs-on: linux.c7i.2xlarge + runs-on: linux.c7i.4xlarge timeout-minutes: 180 # 3 hours steps: - name: Checkout Kineto From fbaef3528e008faceea2ff0fa34508a35bbb92ad Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 08:47:09 -0800 Subject: [PATCH 11/19] Limit max jobs; try non-docker version --- .github/workflows/linux_cpu.yml | 1 + .github/workflows/linux_cpu_no_docker.yml | 93 +++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 .github/workflows/linux_cpu_no_docker.yml diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index df497f2fa..decc892c8 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -106,6 +106,7 @@ jobs: export USE_NCCL=0 export USE_ROCM=0 export BUILD_TEST=1 + export MAX_JOBS=4 python setup.py develop " diff --git a/.github/workflows/linux_cpu_no_docker.yml b/.github/workflows/linux_cpu_no_docker.yml new file mode 100644 index 000000000..489a976dd --- /dev/null +++ b/.github/workflows/linux_cpu_no_docker.yml @@ -0,0 +1,93 @@ +name: Linux CPU build and test + +on: + push: + branches: + - main + pull_request: + +defaults: + run: + shell: bash -l -eo pipefail {0} + +jobs: + pr-test: + runs-on: linux.c7i.4xlarge + timeout-minutes: 180 # 3 hours + steps: + - name: Checkout Kineto + uses: actions/checkout@v5 + with: + path: kineto + submodules: recursive + + - name: Build libkineto (static and shared library) + run: | + + mkdir kineto/build_static + mkdir kineto/build_shared + pip install cmake + + # Note that we are making these CPU-only builds through cmake flags. + docker exec -t -w "/kineto/build_static" "${container_name}" bash -c " + cd kineto/build_static + cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=static ../libkineto/ + make -j + cd + + cd kineto/build_shared + cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=shared ../libkineto/ + make -j + cd + + - name: Run libkineto tests + run: | + cd kineto/build_static + make test + cd + + - name: Clone PyTorch + run: | + git clone --recursive --branch viable/strict https://github.com/pytorch/pytorch.git + + - name: Replace PyTorch's Kineto with PR version + run: | + cd pytorch + rm -rf third_party/kineto + ln -s /kineto third_party/kineto + + # Note that we are explicitly turning off all GPU support. + - name: Build PyTorch from source + run: | + pip install -r requirements.txt + export USE_CUDA=0 + export USE_CUDNN=0 + export USE_NCCL=0 + export USE_ROCM=0 + export BUILD_TEST=1 + export MAX_JOBS=4 + python setup.py develop + + - name: Run PyTorch profiler tests + run: | + # TODO: Dynamically add/remove tests to the exclusion list based on their + # status on trunk instead of maintaining a hardcoded list of known failures. + # This will prevent the list from becoming stale as tests get fixed upstream. + python -m pytest test/profiler/ -v \ + --deselect=test/profiler/test_memory_profiler.py::TestDataFlow::test_data_flow_graph_complicated \ + --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_sequential_fwd_bwd \ + --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_simple_fwd_bwd \ + --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_simple_fwd_bwd_step \ + --deselect=test/profiler/test_profiler.py::TestProfiler::test_kineto \ + --deselect=test/profiler/test_profiler.py::TestProfiler::test_user_annotation \ + --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_fuzz_symbolize \ + --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_profiler_debug_autotuner \ + --deselect=test/profiler/test_torch_tidy.py::TestTorchTidyProfiler::test_tensorimpl_invalidation_scalar_args + + - name: Teardown Linux + uses: pytorch/test-infra/.github/actions/teardown-linux@main + if: always() + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true From b4983ea918f19b5f025bab3a2513040810d13181 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 08:48:19 -0800 Subject: [PATCH 12/19] Different workflow names --- .github/workflows/linux_cpu_no_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_cpu_no_docker.yml b/.github/workflows/linux_cpu_no_docker.yml index 489a976dd..c5c9c199f 100644 --- a/.github/workflows/linux_cpu_no_docker.yml +++ b/.github/workflows/linux_cpu_no_docker.yml @@ -1,4 +1,4 @@ -name: Linux CPU build and test +name: Linux CPU build and test non-docker on: push: From c4437d06e5fa0d940de44ca9ea9f6176d3da27e9 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 08:50:39 -0800 Subject: [PATCH 13/19] Fix stray docker command --- .github/workflows/linux_cpu_no_docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/linux_cpu_no_docker.yml b/.github/workflows/linux_cpu_no_docker.yml index c5c9c199f..08b1c3da6 100644 --- a/.github/workflows/linux_cpu_no_docker.yml +++ b/.github/workflows/linux_cpu_no_docker.yml @@ -29,7 +29,6 @@ jobs: pip install cmake # Note that we are making these CPU-only builds through cmake flags. - docker exec -t -w "/kineto/build_static" "${container_name}" bash -c " cd kineto/build_static cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=static ../libkineto/ make -j From d35c493420fa8e9477699a974784a5c80030b65c Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 08:57:55 -0800 Subject: [PATCH 14/19] Use pushd/popd --- .github/workflows/linux_cpu_no_docker.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linux_cpu_no_docker.yml b/.github/workflows/linux_cpu_no_docker.yml index 08b1c3da6..c0b611b93 100644 --- a/.github/workflows/linux_cpu_no_docker.yml +++ b/.github/workflows/linux_cpu_no_docker.yml @@ -23,27 +23,26 @@ jobs: - name: Build libkineto (static and shared library) run: | - mkdir kineto/build_static mkdir kineto/build_shared pip install cmake # Note that we are making these CPU-only builds through cmake flags. - cd kineto/build_static + pushd kineto/build_static cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=static ../libkineto/ make -j - cd + popd - cd kineto/build_shared + pushd kineto/build_shared cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=shared ../libkineto/ make -j - cd + popd - name: Run libkineto tests run: | - cd kineto/build_static + pushd kineto/build_static make test - cd + popd - name: Clone PyTorch run: | @@ -51,13 +50,15 @@ jobs: - name: Replace PyTorch's Kineto with PR version run: | - cd pytorch + pushd pytorch rm -rf third_party/kineto ln -s /kineto third_party/kineto + popd # Note that we are explicitly turning off all GPU support. - name: Build PyTorch from source run: | + pushd pytorch pip install -r requirements.txt export USE_CUDA=0 export USE_CUDNN=0 @@ -66,9 +67,11 @@ jobs: export BUILD_TEST=1 export MAX_JOBS=4 python setup.py develop + popd - name: Run PyTorch profiler tests run: | + pushd pytorch # TODO: Dynamically add/remove tests to the exclusion list based on their # status on trunk instead of maintaining a hardcoded list of known failures. # This will prevent the list from becoming stale as tests get fixed upstream. @@ -82,6 +85,7 @@ jobs: --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_fuzz_symbolize \ --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_profiler_debug_autotuner \ --deselect=test/profiler/test_torch_tidy.py::TestTorchTidyProfiler::test_tensorimpl_invalidation_scalar_args + popd - name: Teardown Linux uses: pytorch/test-infra/.github/actions/teardown-linux@main From 9b4a76ca7ffacb079095f7d52047dc576c0d64fd Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 09:36:26 -0800 Subject: [PATCH 15/19] Improved non-docker CPU only tests --- .github/workflows/linux_cpu.yml | 3 +- .github/workflows/linux_cpu_no_docker.yml | 136 ++++++++++------------ 2 files changed, 62 insertions(+), 77 deletions(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index decc892c8..01f85c282 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -12,7 +12,7 @@ env: jobs: pr-test: - runs-on: linux.c7i.4xlarge + runs-on: linux.12xlarge timeout-minutes: 180 # 3 hours steps: - name: Checkout Kineto @@ -106,7 +106,6 @@ jobs: export USE_NCCL=0 export USE_ROCM=0 export BUILD_TEST=1 - export MAX_JOBS=4 python setup.py develop " diff --git a/.github/workflows/linux_cpu_no_docker.yml b/.github/workflows/linux_cpu_no_docker.yml index c0b611b93..6d920af2e 100644 --- a/.github/workflows/linux_cpu_no_docker.yml +++ b/.github/workflows/linux_cpu_no_docker.yml @@ -6,90 +6,76 @@ on: - main pull_request: -defaults: - run: - shell: bash -l -eo pipefail {0} - jobs: pr-test: - runs-on: linux.c7i.4xlarge - timeout-minutes: 180 # 3 hours - steps: - - name: Checkout Kineto - uses: actions/checkout@v5 - with: - path: kineto - submodules: recursive - - - name: Build libkineto (static and shared library) - run: | - mkdir kineto/build_static - mkdir kineto/build_shared - pip install cmake - - # Note that we are making these CPU-only builds through cmake flags. - pushd kineto/build_static - cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=static ../libkineto/ - make -j - popd + strategy: + matrix: + python-version: + - "3.12" + runner: ["linux.12xlarge"] + gpu-arch-type: ["cpu"] + fail-fast: true + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + with: + repository: pytorch/kineto + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + timeout: 120 + test-infra-ref: main + script: | + set -euo pipefail + mkdir kineto/build_static + mkdir kineto/build_shared + pip install cmake - pushd kineto/build_shared - cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=shared ../libkineto/ - make -j - popd + # Note that we are making these CPU-only builds through cmake flags. + pushd kineto/build_static + cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=static ../libkineto/ + make -j + popd - - name: Run libkineto tests - run: | - pushd kineto/build_static - make test - popd + pushd kineto/build_shared + cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=shared ../libkineto/ + make -j + popd - - name: Clone PyTorch - run: | - git clone --recursive --branch viable/strict https://github.com/pytorch/pytorch.git + pushd kineto/build_static + make test + popd - - name: Replace PyTorch's Kineto with PR version - run: | - pushd pytorch - rm -rf third_party/kineto - ln -s /kineto third_party/kineto - popd + git clone --recursive --branch viable/strict https://github.com/pytorch/pytorch.git - # Note that we are explicitly turning off all GPU support. - - name: Build PyTorch from source - run: | - pushd pytorch - pip install -r requirements.txt - export USE_CUDA=0 - export USE_CUDNN=0 - export USE_NCCL=0 - export USE_ROCM=0 - export BUILD_TEST=1 - export MAX_JOBS=4 - python setup.py develop - popd + pushd pytorch + rm -rf third_party/kineto + ln -s ../kineto third_party/kineto - - name: Run PyTorch profiler tests - run: | - pushd pytorch - # TODO: Dynamically add/remove tests to the exclusion list based on their - # status on trunk instead of maintaining a hardcoded list of known failures. - # This will prevent the list from becoming stale as tests get fixed upstream. - python -m pytest test/profiler/ -v \ - --deselect=test/profiler/test_memory_profiler.py::TestDataFlow::test_data_flow_graph_complicated \ - --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_sequential_fwd_bwd \ - --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_simple_fwd_bwd \ - --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_simple_fwd_bwd_step \ - --deselect=test/profiler/test_profiler.py::TestProfiler::test_kineto \ - --deselect=test/profiler/test_profiler.py::TestProfiler::test_user_annotation \ - --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_fuzz_symbolize \ - --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_profiler_debug_autotuner \ - --deselect=test/profiler/test_torch_tidy.py::TestTorchTidyProfiler::test_tensorimpl_invalidation_scalar_args - popd + # Note that we are explicitly turning off all GPU support. + pip install -r requirements.txt + export USE_CUDA=0 + export USE_CUDNN=0 + export USE_NCCL=0 + export USE_ROCM=0 + export BUILD_TEST=1 + export MAX_JOBS=4 + python setup.py develop - - name: Teardown Linux - uses: pytorch/test-infra/.github/actions/teardown-linux@main - if: always() + # TODO: Dynamically add/remove tests to the exclusion list based on their + # status on trunk instead of maintaining a hardcoded list of known failures. + # This will prevent the list from becoming stale as tests get fixed upstream. + python -m pytest test/profiler/ -v \ + --deselect=test/profiler/test_memory_profiler.py::TestDataFlow::test_data_flow_graph_complicated \ + --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_sequential_fwd_bwd \ + --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_simple_fwd_bwd \ + --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_simple_fwd_bwd_step \ + --deselect=test/profiler/test_profiler.py::TestProfiler::test_kineto \ + --deselect=test/profiler/test_profiler.py::TestProfiler::test_user_annotation \ + --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_fuzz_symbolize \ + --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_profiler_debug_autotuner \ + --deselect=test/profiler/test_torch_tidy.py::TestTorchTidyProfiler::test_tensorimpl_invalidation_scalar_args + popd concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} From 26ba8b52417c76ce4d70759ec6edbac42329c1ef Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 10:37:36 -0800 Subject: [PATCH 16/19] Remove non-docker attempt; deselect failing test --- .github/workflows/linux_cpu.yml | 5 +- .github/workflows/linux_cpu_no_docker.yml | 82 ----------------------- 2 files changed, 4 insertions(+), 83 deletions(-) delete mode 100644 .github/workflows/linux_cpu_no_docker.yml diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index 01f85c282..b0bf0ba80 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -126,7 +126,10 @@ jobs: --deselect=test/profiler/test_profiler.py::TestProfiler::test_user_annotation \ --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_fuzz_symbolize \ --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_profiler_debug_autotuner \ - --deselect=test/profiler/test_torch_tidy.py::TestTorchTidyProfiler::test_tensorimpl_invalidation_scalar_args + --deselect=test/profiler/test_torch_tidy.py::TestTorchTidyProfiler::test_tensorimpl_invalidation_scalar_args \ + + TODO: Investigate this test failure: https://github.com/pytorch/kineto/issues/1230 + --deselect=test/profiler/test_profiler_tree.py::TestProfilerTree::test_profiler_experimental_tree_with_stack_and_modules " - name: Teardown Linux diff --git a/.github/workflows/linux_cpu_no_docker.yml b/.github/workflows/linux_cpu_no_docker.yml deleted file mode 100644 index 6d920af2e..000000000 --- a/.github/workflows/linux_cpu_no_docker.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Linux CPU build and test non-docker - -on: - push: - branches: - - main - pull_request: - -jobs: - pr-test: - strategy: - matrix: - python-version: - - "3.12" - runner: ["linux.12xlarge"] - gpu-arch-type: ["cpu"] - fail-fast: true - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - permissions: - id-token: write - contents: read - with: - repository: pytorch/kineto - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - timeout: 120 - test-infra-ref: main - script: | - set -euo pipefail - mkdir kineto/build_static - mkdir kineto/build_shared - pip install cmake - - # Note that we are making these CPU-only builds through cmake flags. - pushd kineto/build_static - cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=static ../libkineto/ - make -j - popd - - pushd kineto/build_shared - cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=shared ../libkineto/ - make -j - popd - - pushd kineto/build_static - make test - popd - - git clone --recursive --branch viable/strict https://github.com/pytorch/pytorch.git - - pushd pytorch - rm -rf third_party/kineto - ln -s ../kineto third_party/kineto - - # Note that we are explicitly turning off all GPU support. - pip install -r requirements.txt - export USE_CUDA=0 - export USE_CUDNN=0 - export USE_NCCL=0 - export USE_ROCM=0 - export BUILD_TEST=1 - export MAX_JOBS=4 - python setup.py develop - - # TODO: Dynamically add/remove tests to the exclusion list based on their - # status on trunk instead of maintaining a hardcoded list of known failures. - # This will prevent the list from becoming stale as tests get fixed upstream. - python -m pytest test/profiler/ -v \ - --deselect=test/profiler/test_memory_profiler.py::TestDataFlow::test_data_flow_graph_complicated \ - --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_sequential_fwd_bwd \ - --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_simple_fwd_bwd \ - --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_simple_fwd_bwd_step \ - --deselect=test/profiler/test_profiler.py::TestProfiler::test_kineto \ - --deselect=test/profiler/test_profiler.py::TestProfiler::test_user_annotation \ - --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_fuzz_symbolize \ - --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_profiler_debug_autotuner \ - --deselect=test/profiler/test_torch_tidy.py::TestTorchTidyProfiler::test_tensorimpl_invalidation_scalar_args - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true From 6e722c13ebaf797a45b6cd7f6e69912252bf2cc8 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 10:40:07 -0800 Subject: [PATCH 17/19] Make it a proper comment --- .github/workflows/linux_cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index b0bf0ba80..b3a7ca622 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -128,7 +128,7 @@ jobs: --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_profiler_debug_autotuner \ --deselect=test/profiler/test_torch_tidy.py::TestTorchTidyProfiler::test_tensorimpl_invalidation_scalar_args \ - TODO: Investigate this test failure: https://github.com/pytorch/kineto/issues/1230 + # TODO: Investigate this test failure: https://github.com/pytorch/kineto/issues/1230 --deselect=test/profiler/test_profiler_tree.py::TestProfilerTree::test_profiler_experimental_tree_with_stack_and_modules " From ae04c7e6a6fd36947914f8d33f35be9190ea9b93 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 10:41:43 -0800 Subject: [PATCH 18/19] Block comment --- .github/workflows/linux_cpu.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index b3a7ca622..1d517034e 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -117,6 +117,9 @@ jobs: # TODO: Dynamically add/remove tests to the exclusion list based on their # status on trunk instead of maintaining a hardcoded list of known failures. # This will prevent the list from becoming stale as tests get fixed upstream. + # + # TODO: Investigate test_profiler_experimental_tree_with_stack_and_modules: + # https://github.com/pytorch/kineto/issues/1230 python -m pytest test/profiler/ -v \ --deselect=test/profiler/test_memory_profiler.py::TestDataFlow::test_data_flow_graph_complicated \ --deselect=test/profiler/test_memory_profiler.py::TestMemoryProfilerE2E::test_categories_e2e_sequential_fwd_bwd \ @@ -127,8 +130,6 @@ jobs: --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_fuzz_symbolize \ --deselect=test/profiler/test_profiler.py::TestExperimentalUtils::test_profiler_debug_autotuner \ --deselect=test/profiler/test_torch_tidy.py::TestTorchTidyProfiler::test_tensorimpl_invalidation_scalar_args \ - - # TODO: Investigate this test failure: https://github.com/pytorch/kineto/issues/1230 --deselect=test/profiler/test_profiler_tree.py::TestProfilerTree::test_profiler_experimental_tree_with_stack_and_modules " From 04617651bcf630e79e610c766aabb8abe093b810 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 21 Jan 2026 11:34:48 -0800 Subject: [PATCH 19/19] Defensively disable all known GPUs --- .github/workflows/linux_cpu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_cpu.yml b/.github/workflows/linux_cpu.yml index 1d517034e..be1ae2bcf 100644 --- a/.github/workflows/linux_cpu.yml +++ b/.github/workflows/linux_cpu.yml @@ -62,13 +62,13 @@ jobs: # Note that we are making these CPU-only builds through cmake flags. docker exec -t -w "/kineto/build_static" "${container_name}" bash -c " set -eux - cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=static ../libkineto/ + cmake -DLIBKINETO_NOCUPTI=1 -DLIBKINETO_NOROCTRACER=1 -DLIBKINETO_NOXPUPTI=1 -DLIBKINETO_NOAIUPTI=1 -DKINETO_LIBRARY_TYPE=static ../libkineto/ make -j " docker exec -t -w "/kineto/build_shared" "${container_name}" bash -c " set -eux - cmake -DLIBKINETO_NOCUPTI=1 -DKINETO_LIBRARY_TYPE=shared ../libkineto/ + cmake -DLIBKINETO_NOCUPTI=1 -DLIBKINETO_NOROCTRACER=1 -DLIBKINETO_NOXPUPTI=1 -DLIBKINETO_NOAIUPTI=1 -DKINETO_LIBRARY_TYPE=shared ../libkineto/ make -j "