From 261106db263ee7e6d7bc90112649fd261e0cdc6a Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 27 Jan 2026 15:20:09 -0500 Subject: [PATCH 01/10] Decouple configure/build/test tools from Docker Create a two-layer tool architecture: - Core tools (ck-configure, ck-build, ck-test): Environment-agnostic, work on any system with ROCm - no Docker dependency - Container tools (ck-docker): Manage Docker containers and delegate to core tools via docker exec Changes: - Add ck-configure: New CMake configuration tool with preset support, native GPU detection, and flexible options - Refactor ck-build: Remove Docker dependency, add --configure and --list options, call ninja directly - Refactor ck-test: Remove Docker dependency, add CTest integration with --smoke/--regression/--all options - Enhance common.sh: Add native GPU detection, build directory utils, and output helpers - Update ck-docker: Add configure/build/test/exec commands that delegate to core tools inside container This enables: - Native development on ROCm hosts without Docker - Simpler CI/CD integration - Consistent behavior inside and outside containers Co-Authored-By: Claude --- script/tools/ck-build | 138 +++++++++++++---------- script/tools/ck-configure | 187 +++++++++++++++++++++++++++++++ script/tools/ck-docker | 166 ++++++++-------------------- script/tools/ck-test | 225 ++++++++++++++++++++++++-------------- script/tools/common.sh | 92 +++++++++++++++- 5 files changed, 544 insertions(+), 264 deletions(-) create mode 100755 script/tools/ck-configure diff --git a/script/tools/ck-build b/script/tools/ck-build index 2c0bb24eda1..a2a02387eb7 100755 --- a/script/tools/ck-build +++ b/script/tools/ck-build @@ -2,7 +2,8 @@ # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -# CK Build - Build Composable Kernel targets in Docker +# CK Build - Build Composable Kernel targets +# Environment-agnostic: works natively on ROCm hosts or inside containers set -e set -o pipefail @@ -12,46 +13,51 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/common.sh" # Initialize configuration -PROJECT_ROOT=$(get_project_root "${SCRIPT_DIR}") -CONTAINER_NAME=$(get_container_name "${PROJECT_ROOT}") +PROJECT_ROOT=$(find_project_root "${SCRIPT_DIR}" || get_project_root "${SCRIPT_DIR}") +BUILD_DIR=$(get_build_dir "${PROJECT_ROOT}") # Help message show_help() { cat << EOF -CK Build - Build Composable Kernel targets in Docker +CK Build - Build Composable Kernel targets Usage: ck-build [options] [target...] Options: -h, --help Show this help message - --name Specify container name - --reconfigure Reconfigure CMake before building -j Parallel jobs (passed to ninja) + -v, --verbose Verbose output + --build-dir Build directory (default: ./build) --clean Clean before building + --configure Auto-configure if build.ninja missing + --list List available targets Arguments: target Target(s) to build (default: all) Environment: - CK_CONTAINER_NAME - Override default container name - GPU_TARGET - Override GPU target detection (e.g., gfx950, gfx942) + CK_BUILD_DIR - Override build directory + CK_GPU_TARGET - Override GPU target for auto-configure Examples: ck-build # Build all targets ck-build test_amdgcn_mma # Build specific target ck-build test_amdgcn_mma test_gemm # Build multiple targets - ck-build --reconfigure # Reconfigure CMake and build all + ck-build --configure # Auto-configure and build all ck-build --clean test_amdgcn_mma # Clean and build target ck-build -j 8 test_amdgcn_mma # Build with 8 parallel jobs + ck-build --list # List available targets EOF } # Parse arguments targets=() -reconfigure=false -clean=false parallel_jobs="" +verbose=false +clean=false +auto_configure=false +list_targets=false while [[ $# -gt 0 ]]; do case $1 in @@ -59,21 +65,35 @@ while [[ $# -gt 0 ]]; do show_help exit 0 ;; - --name) - CONTAINER_NAME="$2" + -j) + require_arg "$1" "${2:-}" + parallel_jobs="$2" shift 2 ;; - --reconfigure) - reconfigure=true + -j*) + parallel_jobs="${1#-j}" shift ;; + -v|--verbose) + verbose=true + shift + ;; + --build-dir) + require_arg "$1" "${2:-}" + BUILD_DIR="$2" + shift 2 + ;; --clean) clean=true shift ;; - -j) - parallel_jobs="-j $2" - shift 2 + --configure) + auto_configure=true + shift + ;; + --list) + list_targets=true + shift ;; *) targets+=("$1") @@ -82,62 +102,62 @@ while [[ $# -gt 0 ]]; do esac done -# Ensure container is running -if ! container_is_running "${CONTAINER_NAME}"; then - echo "Container '${CONTAINER_NAME}' not running. Starting..." - "${SCRIPT_DIR}/ck-start" "${CONTAINER_NAME}" +# Handle --list +if [ "$list_targets" = true ]; then + if ! is_build_configured "${BUILD_DIR}"; then + error "Build not configured. Run 'ck-configure' first or use --configure" + exit 1 + fi + info "Available targets:" + cd "${BUILD_DIR}" + ninja -t targets 2>/dev/null | grep -E '^[a-zA-Z_][a-zA-Z0-9_-]*:' | cut -d: -f1 | sort | head -100 echo "" + echo "(Showing first 100 targets. Use 'ninja -t targets' for full list)" + exit 0 fi -# Configure CMake if needed or requested -if [ "$reconfigure" = true ] || ! docker exec "${CONTAINER_NAME}" test -f /workspace/build/build.ninja 2>/dev/null; then - echo "Detecting GPU target..." - GPU_TARGET_DETECTED=$(detect_gpu_target "${CONTAINER_NAME}") - - if [ "$reconfigure" = true ]; then - echo "Reconfiguring CMake from scratch for GPU target: ${GPU_TARGET_DETECTED}" +# Auto-configure if needed +if ! is_build_configured "${BUILD_DIR}"; then + if [ "$auto_configure" = true ]; then + info "Build not configured. Running ck-configure..." + "${SCRIPT_DIR}/ck-configure" --build-dir "${BUILD_DIR}" + echo "" else - echo "Configuring build with CMake for GPU target: ${GPU_TARGET_DETECTED}" + error "Build not configured. Run 'ck-configure' first or use --configure" + exit 1 fi - - docker exec "${CONTAINER_NAME}" bash -c " - cd /workspace || exit 1 - rm -rf /workspace/build - mkdir /workspace/build - cd /workspace/build || exit 1 - cmake .. -GNinja \ - -DGPU_TARGETS=${GPU_TARGET_DETECTED} \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ - -DBUILD_TESTING=ON 2>&1 | tail -30 - " - echo "" fi # Clean if requested if [ "$clean" = true ]; then - echo "Cleaning build directory..." - docker exec "${CONTAINER_NAME}" bash -c " - cd /workspace/build || exit 1 - ninja clean - " + info "Cleaning build directory..." + cd "${BUILD_DIR}" + ninja clean echo "" fi +# Build ninja command +ninja_cmd=(ninja -C "${BUILD_DIR}") + +if [ -n "$parallel_jobs" ]; then + ninja_cmd+=("-j" "$parallel_jobs") +fi + +if [ "$verbose" = true ]; then + ninja_cmd+=(-v) +fi + +# Add targets +ninja_cmd+=("${targets[@]}") + # Build targets if [ ${#targets[@]} -eq 0 ]; then - echo "Building all configured targets..." - docker exec "${CONTAINER_NAME}" bash -c " - cd /workspace/build || exit 1 - ninja ${parallel_jobs} 2>&1 - " + info "Building all configured targets..." else - echo "Building targets: ${targets[*]}" - docker exec "${CONTAINER_NAME}" bash -c " - cd /workspace/build || exit 1 - ninja ${parallel_jobs} ${targets[*]} 2>&1 - " + info "Building targets: ${targets[*]}" fi +"${ninja_cmd[@]}" + echo "" -echo "Build complete ✓" +info "Build complete" diff --git a/script/tools/ck-configure b/script/tools/ck-configure new file mode 100755 index 00000000000..ffe5a4dacaa --- /dev/null +++ b/script/tools/ck-configure @@ -0,0 +1,187 @@ +#!/bin/bash +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +# CK Configure - Configure CMake build for Composable Kernel +# Environment-agnostic: works natively on ROCm hosts or inside containers + +set -e +set -o pipefail + +# Find script directory and load common utilities +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +# Initialize configuration +PROJECT_ROOT=$(find_project_root "${SCRIPT_DIR}" || get_project_root "${SCRIPT_DIR}") +BUILD_DIR=$(get_build_dir "${PROJECT_ROOT}") + +# Help message +show_help() { + cat << EOF +CK Configure - Configure CMake build for Composable Kernel + +Usage: ck-configure [options] + +Options: + -h, --help Show this help message + --preset Use CMake preset (dev, dev-gfx908, dev-gfx90a, dev-gfx942, dev-gfx950) + --gpu Override GPU_TARGETS (auto-detected if not specified) + --dtypes Set DTYPES (e.g., fp16,fp32,bf16) + --build-type CMAKE_BUILD_TYPE (default: Release) + --build-dir Build directory (default: ./build) + --clean Remove existing build directory before configuring + --list-presets List available CMake presets + -D = Pass additional CMake variable + +Environment: + CK_GPU_TARGET - Override GPU target detection (e.g., gfx950, gfx942) + CK_BUILD_DIR - Override build directory + +Examples: + ck-configure # Auto-detect GPU and configure + ck-configure --preset dev-gfx950 # Use CMake preset + ck-configure --gpu gfx942 # Configure for specific GPU + ck-configure --clean --preset dev # Clean and reconfigure + ck-configure -D BUILD_DEV=ON # Pass CMake variable + +EOF +} + +# Parse arguments +preset="" +gpu_target="" +dtypes="" +build_type="Release" +clean=false +list_presets=false +cmake_vars=() + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + show_help + exit 0 + ;; + --preset) + require_arg "$1" "${2:-}" + preset="$2" + shift 2 + ;; + --gpu) + require_arg "$1" "${2:-}" + gpu_target="$2" + shift 2 + ;; + --dtypes) + require_arg "$1" "${2:-}" + dtypes="$2" + shift 2 + ;; + --build-type) + require_arg "$1" "${2:-}" + build_type="$2" + shift 2 + ;; + --build-dir) + require_arg "$1" "${2:-}" + BUILD_DIR="$2" + shift 2 + ;; + --clean) + clean=true + shift + ;; + --list-presets) + list_presets=true + shift + ;; + -D) + require_arg "$1" "${2:-}" + cmake_vars+=("-D$2") + shift 2 + ;; + -D*) + cmake_vars+=("$1") + shift + ;; + *) + error "Unknown option: $1" + echo "" + show_help + exit 1 + ;; + esac +done + +# Handle --list-presets +if [ "$list_presets" = true ]; then + echo "Available CMake presets:" + presets=$(list_cmake_presets "${PROJECT_ROOT}" 2>/dev/null) + if [ -n "$presets" ]; then + echo "$presets" | sed 's/^/ /' + else + echo " (No CMakePresets.json found or jq not available)" + fi + exit 0 +fi + +# Clean build directory if requested +if [ "$clean" = true ]; then + if [ -d "${BUILD_DIR}" ]; then + info "Removing existing build directory: ${BUILD_DIR}" + rm -rf "${BUILD_DIR}" + fi +fi + +# Create build directory +mkdir -p "${BUILD_DIR}" + +# Change to project root for CMake +cd "${PROJECT_ROOT}" + +# Build CMake command +cmake_cmd=(cmake -S . -B "${BUILD_DIR}" -GNinja) + +# Use preset if specified +if [ -n "$preset" ]; then + cmake_cmd+=(--preset "${preset}") + info "Using CMake preset: ${preset}" +else + # Manual configuration + + # Detect GPU target if not specified + if [ -z "$gpu_target" ]; then + gpu_target=$(detect_gpu_native) + info "Auto-detected GPU target: ${gpu_target}" + else + info "Using specified GPU target: ${gpu_target}" + fi + + cmake_cmd+=(-DGPU_TARGETS="${gpu_target}") + cmake_cmd+=(-DCMAKE_BUILD_TYPE="${build_type}") + cmake_cmd+=(-DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++) + cmake_cmd+=(-DBUILD_TESTING=ON) + + # Add DTYPES if specified + if [ -n "$dtypes" ]; then + cmake_cmd+=(-DDTYPES="${dtypes}") + info "Using DTYPES: ${dtypes}" + fi +fi + +# Add any additional CMake variables +for var in "${cmake_vars[@]}"; do + cmake_cmd+=("$var") +done + +# Run CMake +info "Configuring build in: ${BUILD_DIR}" +echo "Running: ${cmake_cmd[*]}" +echo "" + +"${cmake_cmd[@]}" + +echo "" +info "Configuration complete. Build directory: ${BUILD_DIR}" +info "Next: run 'ck-build' to build targets" diff --git a/script/tools/ck-docker b/script/tools/ck-docker index 82bf770011f..6c118561b7f 100755 --- a/script/tools/ck-docker +++ b/script/tools/ck-docker @@ -22,25 +22,29 @@ CK Docker Tool - Build and test composable_kernel in Docker Usage: ck-docker [options] -Commands: - start [name] Start Docker container - build [target] [--reconfigure] Build target (optionally reconfigure CMake) - test [options] Run test - shell [name] Open shell in container - status [name] Check container status - stop [name] Stop and remove container +Container Management: + start [name] Start Docker container + stop [name] Stop and remove container + status [name] Check container status + shell [name] Open shell in container + +Build/Test (delegates to core tools inside container): + configure [opts] Run ck-configure in container + build [opts] Run ck-build in container + test [opts] Run ck-test in container + exec Run arbitrary command in container Examples: ck-docker start + ck-docker configure --preset dev-gfx950 ck-docker build test_amdgcn_mma - ck-docker build --reconfigure test_amdgcn_mma - ck-docker test test_amdgcn_mma --gtest_filter=*Fp16* + ck-docker test test_amdgcn_mma --filter '*Fp16*' ck-docker shell + ck-docker exec rocminfo Environment: CK_CONTAINER_NAME - Override default container name (default: ck__) CK_DOCKER_IMAGE - Override Docker image (default: rocm/composable_kernel:ck_ub24.04_rocm7.0.1) - GPU_TARGET - Override GPU target detection (e.g., gfx950, gfx942) EOF } @@ -77,126 +81,38 @@ cmd_start() { docker exec "${name}" bash -c "echo 'Working directory:' && pwd" } -# Build target -cmd_build() { - local target="" - local name="${CONTAINER_NAME}" - local reconfigure=false - - while [[ $# -gt 0 ]]; do - case $1 in - --name) - name="$2" - shift 2 - ;; - --reconfigure) - reconfigure=true - shift - ;; - *) - target="$1" - shift - ;; - esac - done - - # Check if container is running - if ! container_is_running "${name}"; then - echo "Container '${name}' not running. Starting..." - cmd_start "${name}" - fi - - # Reconfigure CMake if requested or if build.ninja doesn't exist - if [ "$reconfigure" = true ] || ! docker exec "${name}" test -f /workspace/build/build.ninja 2>/dev/null; then - echo "Detecting GPU target..." - local gpu_target=$(detect_gpu_target "${name}") - - if [ "$reconfigure" = true ]; then - echo "Reconfiguring CMake from scratch for GPU target: ${gpu_target}" - else - echo "Configuring build with CMake for GPU target: ${gpu_target}" - fi - - docker exec "${name}" bash -c " - cd /workspace || exit 1 - rm -rf /workspace/build - mkdir /workspace/build - cd /workspace/build || exit 1 - cmake .. -GNinja \ - -DGPU_TARGETS=${gpu_target} \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ - -DBUILD_TESTING=ON 2>&1 | tail -30 - " - fi - - if [ -z "$target" ]; then - echo "Building all configured targets..." - else - echo "Building target: ${target}" - fi - - docker exec "${name}" bash -c " - cd /workspace/build || exit 1 - ninja ${target} 2>&1 - " +# Configure (delegate to ck-configure in container) +cmd_configure() { + ensure_container_running "${CONTAINER_NAME}" "${SCRIPT_DIR}" + docker exec "${CONTAINER_NAME}" /workspace/script/tools/ck-configure "$@" +} - echo "Build complete" +# Build (delegate to ck-build in container) +cmd_build() { + ensure_container_running "${CONTAINER_NAME}" "${SCRIPT_DIR}" + docker exec "${CONTAINER_NAME}" /workspace/script/tools/ck-build "$@" } -# Run test +# Test (delegate to ck-test in container) cmd_test() { - local test_name="" - local name="${CONTAINER_NAME}" - local -a test_options=() - - while [[ $# -gt 0 ]]; do - case $1 in - --name) - name="$2" - shift 2 - ;; - --gtest_*|--help) - test_options+=("$1") - shift - ;; - *) - if [ -z "$test_name" ]; then - test_name="$1" - else - test_options+=("$1") - fi - shift - ;; - esac - done + ensure_container_running "${CONTAINER_NAME}" "${SCRIPT_DIR}" + docker exec "${CONTAINER_NAME}" /workspace/script/tools/ck-test "$@" +} - if [ -z "$test_name" ]; then - echo "Error: test_name required" - echo "Usage: ck-docker test [--name container_name] [gtest_options]" +# Execute arbitrary command in container +cmd_exec() { + if [ $# -eq 0 ]; then + error "command required" + echo "Usage: ck-docker exec " return 1 fi - # Check if container is running - if ! container_is_running "${name}"; then - echo "Error: Container '${name}' not running" - echo "Start it with: ck-docker start --name ${name}" - return 1 - fi + ensure_container_running "${CONTAINER_NAME}" "${SCRIPT_DIR}" - if ! docker exec "${name}" test -f "/workspace/build/bin/${test_name}" 2>/dev/null; then - echo "Test executable not found. Building ${test_name}..." - cmd_build "${test_name}" --name "${name}" - fi + local docker_flags=() + [ -t 0 ] && [ -t 1 ] && docker_flags+=("-it") - echo "Running: ${test_name} ${test_options[*]}" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - # Build the command with proper quoting - local cmd="cd /workspace/build && ./bin/${test_name}" - for opt in "${test_options[@]}"; do - cmd="${cmd} $(printf '%q' "$opt")" - done - docker exec "${name}" bash -c "${cmd}" + docker exec "${docker_flags[@]}" "${CONTAINER_NAME}" "$@" } # Shell @@ -220,7 +136,7 @@ cmd_status() { if [ -z "$name" ]; then echo "Composable Kernel Docker Containers:" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "---" docker ps -a --filter "ancestor=${docker_image}" \ --format "table {{.Names}}\t{{.Status}}\t{{.CreatedAt}}" || echo "No containers found" else @@ -262,6 +178,10 @@ case "${1:-}" in shift cmd_start "$@" ;; + configure) + shift + cmd_configure "$@" + ;; build) shift cmd_build "$@" @@ -270,6 +190,10 @@ case "${1:-}" in shift cmd_test "$@" ;; + exec) + shift + cmd_exec "$@" + ;; shell) shift cmd_shell "$@" diff --git a/script/tools/ck-test b/script/tools/ck-test index 712f904596d..1ee8d0defd7 100755 --- a/script/tools/ck-test +++ b/script/tools/ck-test @@ -2,7 +2,8 @@ # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -# CK Test - Build and test Composable Kernel in Docker +# CK Test - Run Composable Kernel tests +# Environment-agnostic: works natively on ROCm hosts or inside containers set -e set -o pipefail @@ -12,155 +13,219 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/common.sh" # Initialize configuration -PROJECT_ROOT=$(get_project_root "${SCRIPT_DIR}") -CONTAINER_NAME=$(get_container_name "${PROJECT_ROOT}") +PROJECT_ROOT=$(find_project_root "${SCRIPT_DIR}" || get_project_root "${SCRIPT_DIR}") +BUILD_DIR=$(get_build_dir "${PROJECT_ROOT}") # Help message show_help() { cat << EOF -CK Test - Build and test Composable Kernel in Docker +CK Test - Run Composable Kernel tests -Usage: ck-test [options] [test_options] +Usage: ck-test [options] [test_name] [-- gtest_options] Options: -h, --help Show this help message - --name Specify container name - --reconfigure Reconfigure CMake before building + --build-dir Build directory (default: ./build) --no-build Skip building, run test directly + --list List available tests + --smoke Run all smoke tests (via CTest -L SMOKE_TEST) + --regression Run all regression tests (via CTest -L REGRESSION_TEST) + --all Run all tests (via CTest) + --filter Shorthand for --gtest_filter= Arguments: - test_name Name of test executable (required) - test_options Additional options passed to test (e.g., --gtest_filter=*) + test_name Name of test executable (optional for --smoke/--regression/--all) + gtest_options Additional options passed to test (after --) Environment: - CK_CONTAINER_NAME - Override default container name - GPU_TARGET - Override GPU target detection (e.g., gfx950, gfx942) + CK_BUILD_DIR - Override build directory Examples: - ck-test test_amdgcn_mma - ck-test test_amdgcn_mma --gtest_filter=*Fp16* - ck-test --name my_container test_amdgcn_mma - ck-test --reconfigure test_amdgcn_mma + ck-test test_amdgcn_mma # Build and run specific test + ck-test test_amdgcn_mma --filter '*Fp16*' # Run with gtest filter + ck-test test_amdgcn_mma -- --gtest_filter=*Fp16* # Explicit gtest options + ck-test --no-build test_amdgcn_mma # Run without rebuilding + ck-test --list # List available tests + ck-test --smoke # Run all smoke tests + ck-test --regression # Run all regression tests + ck-test --all # Run all tests EOF } # Parse arguments test_name="" -reconfigure=false no_build=false -test_options=() +list_tests=false +run_smoke=false +run_regression=false +run_all=false +gtest_filter="" +gtest_options=() +parsing_gtest=false while [[ $# -gt 0 ]]; do + if [ "$parsing_gtest" = true ]; then + gtest_options+=("$1") + shift + continue + fi + case $1 in -h|--help) show_help exit 0 ;; - --name) - CONTAINER_NAME="$2" + --build-dir) + require_arg "$1" "${2:-}" + BUILD_DIR="$2" shift 2 ;; - --reconfigure) - reconfigure=true - shift - ;; --no-build) no_build=true shift ;; - --gtest_*|--help) - test_options+=("$1") + --list) + list_tests=true + shift + ;; + --smoke) + run_smoke=true + shift + ;; + --regression) + run_regression=true + shift + ;; + --all) + run_all=true + shift + ;; + --filter) + require_arg "$1" "${2:-}" + gtest_filter="$2" + shift 2 + ;; + --) + parsing_gtest=true + shift + ;; + --gtest_*) + gtest_options+=("$1") shift ;; *) if [ -z "$test_name" ]; then test_name="$1" else - test_options+=("$1") + gtest_options+=("$1") fi shift ;; esac done -# Validate test name -if [ -z "$test_name" ]; then - echo "Error: test_name required" - echo "" - show_help +# Add filter to gtest options if specified +if [ -n "$gtest_filter" ]; then + gtest_options+=("--gtest_filter=${gtest_filter}") +fi + +# Validate mutual exclusivity of test suite options +suite_count=0 +[ "$run_smoke" = true ] && suite_count=$((suite_count + 1)) +[ "$run_regression" = true ] && suite_count=$((suite_count + 1)) +[ "$run_all" = true ] && suite_count=$((suite_count + 1)) + +if [ "$suite_count" -gt 1 ]; then + error "Options --smoke, --regression, and --all are mutually exclusive" exit 1 fi -# Ensure container is running -if ! container_is_running "${CONTAINER_NAME}"; then - echo "Container '${CONTAINER_NAME}' not running. Starting..." - "${SCRIPT_DIR}/ck-start" "${CONTAINER_NAME}" +# Check build is configured +if ! is_build_configured "${BUILD_DIR}"; then + error "Build not configured. Run 'ck-configure' first" + exit 1 +fi + +# Handle --list +if [ "$list_tests" = true ]; then + info "Available tests:" + if [ -d "${BUILD_DIR}/bin" ]; then + ls -1 "${BUILD_DIR}/bin/" 2>/dev/null | grep -E '^test_' | sort || echo " (No test binaries found)" + else + echo " (No bin directory found)" + fi echo "" + echo "CTest labels:" + cd "${BUILD_DIR}" + ctest -N 2>/dev/null | head -20 || echo " (Run 'ctest -N' for full list)" + exit 0 fi -# Configure CMake if needed or requested -if [ "$reconfigure" = true ] || ! docker exec "${CONTAINER_NAME}" test -f /workspace/build/build.ninja 2>/dev/null; then - echo "Detecting GPU target..." - GPU_TARGET_DETECTED=$(detect_gpu_target "${CONTAINER_NAME}") +# Handle CTest-based test suites +if [ "$run_smoke" = true ] || [ "$run_regression" = true ] || [ "$run_all" = true ]; then + cd "${BUILD_DIR}" + + ctest_cmd=(ctest --output-on-failure) - if [ "$reconfigure" = true ]; then - echo "Reconfiguring CMake from scratch for GPU target: ${GPU_TARGET_DETECTED}" + if [ "$run_smoke" = true ]; then + ctest_cmd+=(-L SMOKE_TEST) + info "Running smoke tests..." + elif [ "$run_regression" = true ]; then + ctest_cmd+=(-L REGRESSION_TEST) + info "Running regression tests..." else - echo "Configuring build with CMake for GPU target: ${GPU_TARGET_DETECTED}" + info "Running all tests..." fi - docker exec "${CONTAINER_NAME}" bash -c " - cd /workspace || exit 1 - rm -rf /workspace/build - mkdir /workspace/build - cd /workspace/build || exit 1 - cmake .. -GNinja \ - -DGPU_TARGETS=${GPU_TARGET_DETECTED} \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ - -DBUILD_TESTING=ON 2>&1 | tail -30 - " + "${ctest_cmd[@]}" + exit_code=$? + + echo "" + if [ $exit_code -eq 0 ]; then + info "Tests completed successfully" + else + error "Tests failed with exit code: ${exit_code}" + fi + exit $exit_code +fi + +# Validate test name for individual test runs +if [ -z "$test_name" ]; then + error "test_name required (or use --smoke/--regression/--all for test suites)" echo "" + show_help + exit 1 fi # Build test if needed (unless --no-build is specified) if [ "$no_build" = false ]; then - if ! docker exec "${CONTAINER_NAME}" test -f "/workspace/build/bin/${test_name}" 2>/dev/null; then - echo "Building ${test_name}..." - docker exec "${CONTAINER_NAME}" bash -c " - cd /workspace/build || exit 1 - ninja ${test_name} 2>&1 - " - echo "" - else - echo "Test executable found, rebuilding to ensure latest version..." - docker exec "${CONTAINER_NAME}" bash -c " - cd /workspace/build || exit 1 - ninja ${test_name} 2>&1 - " - echo "" - fi + info "Building ${test_name}..." + "${SCRIPT_DIR}/ck-build" --build-dir "${BUILD_DIR}" "${test_name}" + echo "" fi -# Run test -echo "Running: ${test_name} ${test_options[*]}" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +# Verify test executable exists +test_binary="${BUILD_DIR}/bin/${test_name}" +if [ ! -f "$test_binary" ]; then + error "Test executable not found: ${test_binary}" + echo "Run 'ck-build ${test_name}' first" + exit 1 +fi -# Build the command with proper quoting -cmd="cd /workspace/build && ./bin/${test_name}" -for opt in "${test_options[@]}"; do - cmd="${cmd} $(printf '%q' "$opt")" -done +# Run test +echo "Running: ${test_name} ${gtest_options[*]}" +echo "---" -docker exec "${CONTAINER_NAME}" bash -c "${cmd}" +cd "${BUILD_DIR}" +"./bin/${test_name}" "${gtest_options[@]}" exit_code=$? -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "---" if [ $exit_code -eq 0 ]; then - echo "Test completed successfully" + info "Test completed successfully" else - echo "Test failed with exit code: ${exit_code}" + error "Test failed with exit code: ${exit_code}" fi exit $exit_code diff --git a/script/tools/common.sh b/script/tools/common.sh index 6683572c0f5..e5a39cea675 100644 --- a/script/tools/common.sh +++ b/script/tools/common.sh @@ -74,14 +74,14 @@ container_is_running() { detect_gpu_target() { local container="$1" - # Allow override via GPU_TARGET environment variable - if [ -n "${GPU_TARGET:-}" ]; then - echo "${GPU_TARGET}" + # Allow override via CK_GPU_TARGET environment variable + if [ -n "${CK_GPU_TARGET:-}" ]; then + echo "${CK_GPU_TARGET}" return 0 fi docker exec "${container}" bash -c " - rocminfo 2>/dev/null | grep -oP 'gfx[0-9a-z]+' | head -1 || echo 'gfx950' + rocminfo 2>/dev/null | grep -oE 'gfx[0-9a-z]+' | head -1 || echo 'gfx950' " | tr -d '\r\n' } @@ -95,3 +95,87 @@ ensure_container_running() { "${script_dir}/ck-docker" start "${container}" fi } + +# ============================================================================ +# Native (non-Docker) utilities +# ============================================================================ + +# Output utilities +info() { echo "[info] $*"; } +warn() { echo "[warn] $*" >&2; } +error() { echo "[error] $*" >&2; } + +# Require argument for option (validates $2 exists and is not another flag) +require_arg() { + local option="$1" + local value="$2" + if [ -z "$value" ] || [[ "$value" == -* ]]; then + error "Option $option requires an argument" + exit 1 + fi +} + +# Native GPU detection (no Docker required) +detect_gpu_native() { + # Allow override via CK_GPU_TARGET environment variable + if [ -n "${CK_GPU_TARGET:-}" ]; then + echo "${CK_GPU_TARGET}" + return 0 + fi + + # Try rocminfo if available + if command -v rocminfo &>/dev/null; then + local gpu + gpu=$(rocminfo 2>/dev/null | grep -oE 'gfx[0-9a-z]+' | head -1) + if [ -n "$gpu" ]; then + echo "$gpu" + return 0 + fi + fi + + # Fallback + echo "gfx950" +} + +# Get build directory (respects CK_BUILD_DIR env var) +get_build_dir() { + local project_root="${1:-$(get_project_root "$(dirname "${BASH_SOURCE[0]}")")}" + echo "${CK_BUILD_DIR:-${project_root}/build}" +} + +# Check if build is configured (build.ninja exists) +is_build_configured() { + local build_dir="${1:-$(get_build_dir)}" + [ -f "${build_dir}/build.ninja" ] +} + +# Find project root from any subdirectory (walks up to find .git) +find_project_root() { + local dir="${1:-$(pwd)}" + while [ "$dir" != "/" ]; do + if [ -d "$dir/.git" ]; then + echo "$dir" + return 0 + fi + dir=$(dirname "$dir") + done + return 1 +} + +# List available CMake presets +list_cmake_presets() { + local project_root="${1:-$(find_project_root)}" + local presets_file="${project_root}/CMakePresets.json" + + if [ ! -f "$presets_file" ]; then + return 1 + fi + + # Extract non-hidden preset names + if command -v jq &>/dev/null; then + jq -r '.configurePresets[] | select(.hidden != true) | .name' "$presets_file" 2>/dev/null + else + # Fallback: sed-based extraction (more portable than grep -P) + sed -n 's/.*"name"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$presets_file" | grep -v '^use-' + fi +} From 6385afe90c90c19488de11716db38ebc4d71563e Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 20 Jan 2026 15:36:13 -0600 Subject: [PATCH 02/10] Add ck-rocprof: GPU profiling tool for rocprof-compute Adds a command-line profiling tool to simplify GPU performance analysis workflow using AMD rocprof-compute. Features: - Easy setup with automatic Python venv configuration - Simple CLI: setup, run, analyze, compare, list - Automatic GPU architecture detection - Focus on LDS metrics (Block 12) for bank conflict analysis - Comprehensive documentation with examples and troubleshooting Usage: ck-rocprof setup # One-time environment setup ck-rocprof run # Profile executable ck-rocprof analyze [block] # Analyze metrics ck-rocprof compare # Compare two runs ck-rocprof list # List available runs --- script/tools/ck-rocprof | 414 ++++++++++++++++++++++++++++++++++++ script/tools/ck-rocprof.md | 415 +++++++++++++++++++++++++++++++++++++ 2 files changed, 829 insertions(+) create mode 100755 script/tools/ck-rocprof create mode 100644 script/tools/ck-rocprof.md diff --git a/script/tools/ck-rocprof b/script/tools/ck-rocprof new file mode 100755 index 00000000000..75375943011 --- /dev/null +++ b/script/tools/ck-rocprof @@ -0,0 +1,414 @@ +#!/bin/bash +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +# CK ROCProf Tool - Profile CK applications with rocprof-compute + +set -e +set -o pipefail + +# Find script directory and load common utilities +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +# Initialize configuration +PROJECT_ROOT=$(get_project_root "${SCRIPT_DIR}") +CONTAINER_NAME=$(get_container_name "${PROJECT_ROOT}") + +# Profiling configuration +VENV_PATH="${CK_PROFILE_VENV:-/opt/rocprof_venv}" +ROCPROF_BIN="${CK_ROCPROF_BIN:-/opt/rocm-7.0.1/bin/rocprof-compute}" +WORKLOAD_DIR="${CK_WORKLOAD_DIR:-/workspace/build/workloads}" +ROCM_REQUIREMENTS="/opt/rocm-7.0.1/libexec/rocprofiler-compute/requirements.txt" + +# Help message +show_help() { + cat << EOF +CK ROCProf Tool - Profile CK applications with rocprof-compute + +Usage: ck-rocprof [options] + +Commands: + setup One-time setup: create Python venv and install dependencies + run [args] Profile executable and save results as + analyze [block] Analyze profiling results (default: block 12 - LDS metrics) + compare Compare two profiling runs + list List available profiling runs + help Show this help message + +Examples: + ck-rocprof setup + ck-rocprof run baseline ./bin/tile_example_gemm_universal + ck-rocprof analyze baseline + ck-rocprof analyze baseline 12 + ck-rocprof compare baseline optimized + ck-rocprof list + +Environment Variables: + CK_PROFILE_VENV - Python venv path (default: /opt/rocprof_venv) + CK_ROCPROF_BIN - rocprof-compute binary path (default: /opt/rocm-7.0.1/bin/rocprof-compute) + CK_WORKLOAD_DIR - Workload storage directory (default: /workspace/build/workloads) + CK_CONTAINER_NAME - Docker container name (for remote profiling) + +Profiling Metrics: + Block 12 (LDS - Local Data Share): + - 12.1.3: Bank Conflict Rate (% of peak) + - 12.2.9: Bank Conflicts/Access (conflicts/access) + - 12.2.12: Bank Conflict (cycles per kernel) + - 12.2.17: LDS Data FIFO Full Rate (cycles) + +Notes: + - Profiling runs inside Docker container if CK_CONTAINER_NAME is set + - Results stored in ${WORKLOAD_DIR}/// + - Use 'analyze' to view detailed metrics for specific blocks + - Use 'compare' to see side-by-side metrics from two runs +EOF +} + +# Get rocprof-compute wrapper path +get_rocprof_wrapper() { + echo "${VENV_PATH}/bin/rocprof-compute" +} + +# Check if setup is complete +is_setup_complete() { + local wrapper=$(get_rocprof_wrapper) + [ -d "${VENV_PATH}" ] && [ -f "${wrapper}" ] +} + +# Setup: Create Python venv and install rocprof-compute dependencies +cmd_setup() { + echo "Setting up rocprof-compute profiling environment..." + echo "===========================================" + + # Check if rocprof-compute exists + if [ ! -f "${ROCPROF_BIN}" ]; then + echo "Error: rocprof-compute not found at ${ROCPROF_BIN}" + echo "Please ensure ROCm is installed and CK_ROCPROF_BIN is set correctly" + return 1 + fi + + # Check if requirements file exists + if [ ! -f "${ROCM_REQUIREMENTS}" ]; then + echo "Error: rocprofiler-compute requirements.txt not found at ${ROCM_REQUIREMENTS}" + echo "Please ensure ROCm is properly installed" + return 1 + fi + + # Create Python venv + if [ -d "${VENV_PATH}" ]; then + echo "Python venv already exists at ${VENV_PATH}" + else + echo "Creating Python virtual environment at ${VENV_PATH}..." + python3 -m venv "${VENV_PATH}" + echo "* Virtual environment created" + fi + + # Install dependencies + echo "Installing rocprofiler-compute dependencies..." + "${VENV_PATH}/bin/pip" install --quiet -r "${ROCM_REQUIREMENTS}" + echo "* Dependencies installed" + + # Create wrapper script + local wrapper=$(get_rocprof_wrapper) + cat > "${wrapper}" << 'WRAPPER_EOF' +#!/bin/bash +# rocprof-compute wrapper using venv Python +VENV_DIR="$(cd "$(dirname "$0")/.." && pwd)" +exec "${VENV_DIR}/bin/python" /opt/rocm-7.0.1/bin/rocprof-compute "$@" +WRAPPER_EOF + chmod +x "${wrapper}" + echo "* Wrapper script created at ${wrapper}" + + echo "" + echo "Setup complete! You can now use:" + echo " ck-rocprof run " +} + +# Detect GPU architecture +# rocprof-compute uses marketing names (MI350, MI300X, etc.) for directory naming +# This function attempts to match that convention +detect_gpu_arch() { + if [ -n "${GPU_TARGET:-}" ]; then + echo "${GPU_TARGET}" + return 0 + fi + + # Try to get marketing name from rocminfo + local marketing_name=$(rocminfo 2>/dev/null | grep "Marketing Name:" | grep -oP 'MI\d+[A-Z]*' | head -1 || echo "") + if [ -n "$marketing_name" ]; then + echo "$marketing_name" + return 0 + fi + + # Fallback: Try gfx name from rocminfo + local gfx_name=$(rocminfo 2>/dev/null | grep -oP 'gfx[0-9a-z]+' | head -1 || echo "") + if [ -n "$gfx_name" ]; then + echo "$gfx_name" + return 0 + fi + + # Fallback: try to detect from existing workload dirs (any architecture pattern) + if [ -d "${WORKLOAD_DIR}" ]; then + local first_dir=$(find "${WORKLOAD_DIR}" -maxdepth 2 -type d \( -name 'gfx*' -o -name 'MI*' \) 2>/dev/null | head -1) + if [ -n "$first_dir" ]; then + basename "$first_dir" + return 0 + fi + fi + + # Final fallback + echo "MI350" +} + +# Run profiling +cmd_run() { + local name="$1" + local executable="$2" + shift 2 + local -a exe_args=("$@") + + if [ -z "$name" ] || [ -z "$executable" ]; then + echo "Error: name and executable required" + echo "Usage: ck-rocprof run [args]" + return 1 + fi + + # Check setup + if ! is_setup_complete; then + echo "Error: Profiling environment not set up" + echo "Run: ck-rocprof setup" + return 1 + fi + + # Check if executable exists + if [ ! -f "$executable" ]; then + echo "Error: Executable not found: $executable" + return 1 + fi + + local wrapper=$(get_rocprof_wrapper) + local gpu_arch=$(detect_gpu_arch) + + echo "Profiling: $executable ${exe_args[*]}" + echo "Run name: $name" + echo "GPU arch: $gpu_arch" + echo "===========================================" + + # Build command + local cmd="${wrapper} profile --name ${name}" + cmd="${cmd} --" + cmd="${cmd} ${executable}" + for arg in "${exe_args[@]}"; do + cmd="${cmd} $(printf '%q' "$arg")" + done + + # Run profiling + eval "${cmd}" + + echo "" + echo "* Profiling complete" + echo "Results saved to: ${WORKLOAD_DIR}/${name}/${gpu_arch}/" + echo "" + echo "Analyze with: ck-rocprof analyze ${name}" +} + +# Find actual GPU architecture directory for a given run name +# rocprof-compute creates directories with various naming schemes (MI350, gfx950, etc.) +# This function finds whatever directory actually exists +find_workload_path() { + local name="$1" + local run_dir="${WORKLOAD_DIR}/${name}" + + if [ ! -d "$run_dir" ]; then + return 1 + fi + + # Find first subdirectory (should be GPU architecture) + local gpu_dir=$(find "$run_dir" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | head -1) + + if [ -n "$gpu_dir" ] && [ -d "$gpu_dir" ]; then + echo "$gpu_dir" + return 0 + fi + + return 1 +} + +# Analyze profiling results +cmd_analyze() { + local name="$1" + local block="${2:-12}" # Default to block 12 (LDS metrics) + + if [ -z "$name" ]; then + echo "Error: name required" + echo "Usage: ck-rocprof analyze [block]" + return 1 + fi + + # Check setup + if ! is_setup_complete; then + echo "Error: Profiling environment not set up" + echo "Run: ck-rocprof setup" + return 1 + fi + + local wrapper=$(get_rocprof_wrapper) + local workload_path=$(find_workload_path "${name}") + + if [ -z "$workload_path" ]; then + echo "Error: Profiling results not found for '${name}'" + echo "" + echo "Available runs:" + cmd_list + return 1 + fi + + local gpu_arch=$(basename "$workload_path") + echo "Analyzing: ${name} (GPU: ${gpu_arch}, Block ${block})" + echo "===========================================" + echo "" + + "${wrapper}" analyze --path "${workload_path}" --block "${block}" +} + +# Extract key LDS metrics from profiling results +extract_lds_metrics() { + local name="$1" + local gpu_arch=$(detect_gpu_arch) + local workload_path="${WORKLOAD_DIR}/${name}/${gpu_arch}" + local csv_file="${workload_path}/SQ_INST_LEVEL_LDS.csv" + + if [ ! -f "$csv_file" ]; then + echo "N/A (CSV not found)" + return 1 + fi + + # Extract key metrics (simplified - would need proper CSV parsing) + # For now, just indicate that raw data is available + echo "Raw data available at: ${csv_file}" +} + +# Compare two profiling runs +cmd_compare() { + local name1="$1" + local name2="$2" + + if [ -z "$name1" ] || [ -z "$name2" ]; then + echo "Error: two run names required" + echo "Usage: ck-rocprof compare " + return 1 + fi + + # Check setup + if ! is_setup_complete; then + echo "Error: Profiling environment not set up" + echo "Run: ck-rocprof setup" + return 1 + fi + + # Verify both runs exist + local path1=$(find_workload_path "${name1}") + local path2=$(find_workload_path "${name2}") + + if [ -z "$path1" ]; then + echo "Error: Profiling results not found for '${name1}'" + return 1 + fi + + if [ -z "$path2" ]; then + echo "Error: Profiling results not found for '${name2}'" + return 1 + fi + + local gpu1=$(basename "$path1") + local gpu2=$(basename "$path2") + + echo "Comparing profiling runs:" + echo " Baseline: ${name1} (${gpu1})" + echo " Optimized: ${name2} (${gpu2})" + echo "===========================================" + echo "" + + echo "=== ${name1} - Block 12 (LDS) ===" + cmd_analyze "${name1}" 12 2>/dev/null | head -40 + + echo "" + echo "=== ${name2} - Block 12 (LDS) ===" + cmd_analyze "${name2}" 12 2>/dev/null | head -40 + + echo "" + echo "===========================================" + echo "For detailed analysis, run:" + echo " ck-rocprof analyze ${name1} 12" + echo " ck-rocprof analyze ${name2} 12" +} + +# List available profiling runs +cmd_list() { + if [ ! -d "${WORKLOAD_DIR}" ]; then + echo "No profiling runs found (workload directory doesn't exist)" + return 0 + fi + + local runs=$(find "${WORKLOAD_DIR}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort) + + if [ -z "$runs" ]; then + echo "No profiling runs found in ${WORKLOAD_DIR}" + return 0 + fi + + echo "Available profiling runs:" + echo "===========================================" + + while IFS= read -r run; do + local path=$(find_workload_path "$run") + + if [ -n "$path" ]; then + local gpu_arch=$(basename "$path") + local size=$(du -sh "$path" 2>/dev/null | cut -f1) + local date=$(stat -c %y "$path" 2>/dev/null | cut -d' ' -f1) + printf " %-25s [%-8s, %s, %s]\n" "$run" "$gpu_arch" "$size" "$date" + else + printf " %-25s [no data]\n" "$run" + fi + done <<< "$runs" + + echo "" + echo "Analyze with: ck-rocprof analyze " +} + +# Main command dispatcher +case "${1:-}" in + setup) + cmd_setup + ;; + run) + shift + cmd_run "$@" + ;; + analyze) + shift + cmd_analyze "$@" + ;; + compare) + shift + cmd_compare "$@" + ;; + list) + cmd_list + ;; + help|--help|-h) + show_help + ;; + *) + if [ -z "${1:-}" ]; then + show_help + else + echo "Unknown command: ${1}" + echo "" + show_help + exit 1 + fi + ;; +esac diff --git a/script/tools/ck-rocprof.md b/script/tools/ck-rocprof.md new file mode 100644 index 00000000000..e4642ef20ca --- /dev/null +++ b/script/tools/ck-rocprof.md @@ -0,0 +1,415 @@ +# CK ROCProf Tool + +Performance profiling tool for Composable Kernel applications using AMD rocprof-compute. + +## Overview + +`ck-rocprof` simplifies GPU performance profiling by wrapping rocprofiler-compute with an easy-to-use CLI. It handles Python environment setup, manages profiling runs, and provides convenient comparison tools for optimization workflows. + +## Quick Start + +```bash +# One-time setup +./script/tools/ck-rocprof setup + +# Profile an executable +cd build +../script/tools/ck-rocprof run baseline ./bin/tile_example_gemm_universal + +# Analyze results +../script/tools/ck-rocprof analyze baseline + +# Compare two runs +../script/tools/ck-rocprof run optimized ./bin/tile_example_gemm_universal +../script/tools/ck-rocprof compare baseline optimized +``` + +## Commands + +### `setup` + +One-time setup of the profiling environment. + +```bash +ck-rocprof setup +``` + +**What it does:** +- Creates Python virtual environment at `/opt/rocprof_venv` +- Installs rocprofiler-compute dependencies +- Creates wrapper script for easy invocation + +**Requirements:** +- ROCm installed (tested with ROCm 7.0.1) +- Python 3 with venv support +- Write access to `/opt/` directory + +--- + +### `run [args]` + +Profile an executable and save results with a given name. + +```bash +ck-rocprof run [args...] +``` + +**Arguments:** +- `name` - Unique identifier for this profiling run +- `executable` - Path to executable to profile +- `args` - Optional arguments to pass to the executable + +**Examples:** +```bash +# Profile with default arguments +ck-rocprof run baseline ./bin/tile_example_gemm_universal + +# Profile with custom arguments +ck-rocprof run custom_size ./bin/tile_example_gemm_universal -m 8192 -n 8192 -k 4096 + +# Profile test with gtest filter +ck-rocprof run unit_test ./bin/test_ck_tile_gemm --gtest_filter="*Fp16*" +``` + +**Output:** +- Results stored in `/workspace/build/workloads///` +- GPU architecture auto-detected (e.g., `gfx950`, `MI350`) + +--- + +### `analyze [block]` + +Analyze profiling results for a specific run. + +```bash +ck-rocprof analyze [block] +``` + +**Arguments:** +- `name` - Name of profiling run to analyze +- `block` - Optional block number to display (default: 12 for LDS metrics) + +**Examples:** +```bash +# Analyze LDS metrics (Block 12) +ck-rocprof analyze baseline + +# Analyze specific block +ck-rocprof analyze baseline 2 # L2 Cache +ck-rocprof analyze baseline 7 # Compute Unit - Instruction Mix +ck-rocprof analyze baseline 12 # Local Data Share (LDS) +``` + +**Key Metric Blocks:** +- **Block 2**: L2 Cache +- **Block 7**: Compute Unit - Instruction Mix +- **Block 12**: Local Data Share (LDS) + - 12.1.3: Bank Conflict Rate (% of peak) + - 12.2.9: Bank Conflicts/Access (conflicts/access) + - 12.2.12: Bank Conflict (cycles per kernel) + - 12.2.17: LDS Data FIFO Full Rate + +--- + +### `compare ` + +Compare LDS metrics from two profiling runs side-by-side. + +```bash +ck-rocprof compare +``` + +**Arguments:** +- `name1` - First profiling run (typically baseline) +- `name2` - Second profiling run (typically optimized) + +**Example:** +```bash +ck-rocprof compare baseline optimized +``` + +**Output:** +- Side-by-side Block 12 (LDS) metrics +- Useful for validating optimization improvements + +--- + +### `list` + +List all available profiling runs. + +```bash +ck-rocprof list +``` + +**Output:** +- Run name +- Disk usage +- Creation date + +**Example:** +```bash +$ ck-rocprof list +Available profiling runs: +=========================================== + baseline [156K, 2026-01-20] + optimized [148K, 2026-01-20] + custom_size [162K, 2026-01-20] +``` + +--- + +## Environment Variables + +- `CK_PROFILE_VENV` - Python venv path (default: `/opt/rocprof_venv`) +- `CK_ROCPROF_BIN` - rocprof-compute binary (default: `/opt/rocm-7.0.1/bin/rocprof-compute`) +- `CK_WORKLOAD_DIR` - Workload storage directory (default: `/workspace/build/workloads`) +- `GPU_TARGET` - Override GPU architecture detection (e.g., `gfx950`, `gfx942`) + +## Common Workflows + +### Optimizing LDS Bank Conflicts + +```bash +# 1. Capture baseline metrics +cd build +../script/tools/ck-rocprof run baseline ./bin/tile_example_gemm_universal + +# 2. Check baseline bank conflict rate +../script/tools/ck-rocprof analyze baseline +# Look for: +# - Section 12.2.9: Bank Conflicts/Access (target: <0.01) +# - Section 12.2.12: Bank Conflict cycles (target: minimize) + +# 3. Apply optimization (e.g., XOR transform) +# ... modify code ... + +# 4. Rebuild and profile +ninja tile_example_gemm_universal +../script/tools/ck-rocprof run optimized ./bin/tile_example_gemm_universal + +# 5. Compare results +../script/tools/ck-rocprof compare baseline optimized + +# 6. Verify improvement +# Expected: Bank Conflicts/Access reduced by 8-10x +``` + +### Profiling Multiple Configurations + +```bash +# Profile different tile sizes +ck-rocprof run tile_64x64 ./bin/example -tile 64 +ck-rocprof run tile_128x128 ./bin/example -tile 128 +ck-rocprof run tile_256x256 ./bin/example -tile 256 + +# Compare all runs +ck-rocprof list +ck-rocprof compare tile_64x64 tile_128x128 +ck-rocprof compare tile_128x128 tile_256x256 +``` + +## Understanding LDS Metrics (Block 12) + +### Key Metrics + +**12.1.3 - Bank Conflict Rate** +- Percentage of theoretical peak LDS bandwidth achieved +- Lower is worse (conflicts reduce bandwidth) +- Target: >90% of peak + +**12.2.9 - Bank Conflicts/Access** +- Average number of bank conflicts per LDS access +- Direct measure of conflict severity +- Target: <0.01 (1% conflict rate) +- Baseline (naive layout): ~0.04 (4% conflicts) +- Optimized (XOR/padding): <0.005 (<0.5% conflicts) + +**12.2.12 - Bank Conflict (cycles)** +- Total cycles lost to bank conflicts per kernel +- Shows MAX, MIN, AVG across compute units +- Look at MAX column for worst-case bottleneck +- Target: Minimize (reduce by 8-10x with optimization) + +**12.2.17 - LDS Data FIFO Full Rate** +- Cycles stalled due to LDS data FIFO full +- Indicates memory system pressure +- Reduces with bank conflict mitigation + +### Interpreting Results + +**Good LDS Performance:** +``` +Section 12.1 - Speed-of-Light: + Bank Conflict Rate: Low % of peak + +Section 12.2 - LDS Stats: + Bank Conflicts/Access: Low value (ideally <0.01) + Bank Conflict (MAX): Minimal cycles + LDS Data FIFO Full: Low cycles +``` + +**Poor LDS Performance (needs optimization):** +``` +Section 12.1 - Speed-of-Light: + Bank Conflict Rate: Higher % of peak + +Section 12.2 - LDS Stats: + Bank Conflicts/Access: High value (>0.02 indicates issues) + Bank Conflict (MAX): Significant cycles lost + LDS Data FIFO Full: High memory pressure +``` + +**Typical Improvement After Optimization:** +- Bank Conflicts/Access: Reduced by 10-20x +- Bank Conflict cycles: Significantly reduced +- Overall kernel speedup: Variable depending on workload + +## Troubleshooting + +### "Error: Profiling environment not set up" + +**Solution:** Run `ck-rocprof setup` first + +### "Error: rocprof-compute not found" + +**Cause:** ROCm not installed or in non-standard location + +**Solution:** Set `CK_ROCPROF_BIN` environment variable: +```bash +export CK_ROCPROF_BIN=/path/to/rocm/bin/rocprof-compute +ck-rocprof setup +``` + +### "Error: Profiling results not found" + +**Cause:** Profiling run doesn't exist or wrong GPU architecture + +**Solution:** +```bash +# List available runs +ck-rocprof list + +# Check GPU architecture +rocminfo | grep gfx + +# Override if needed +export GPU_TARGET=gfx950 +``` + +### Python Dependency Conflicts + +**Cause:** Ubuntu 24.04 has externally-managed Python environment + +**Solution:** The tool automatically creates a venv to avoid conflicts. If setup fails: +```bash +# Manual venv creation +sudo python3 -m venv /opt/rocprof_venv +sudo /opt/rocprof_venv/bin/pip install -r /opt/rocm-7.0.1/libexec/rocprofiler-compute/requirements.txt +``` + +## Technical Details + +### Profiling Workflow + +1. **Setup Phase** (`ck-rocprof setup`): + - Creates isolated Python venv + - Installs rocprofiler-compute dependencies + - Creates wrapper script that uses venv Python + +2. **Profiling Phase** (`ck-rocprof run`): + - Detects GPU architecture + - Runs `rocprof-compute profile --name -- ` + - Stores results in `workloads///` + +3. **Analysis Phase** (`ck-rocprof analyze`): + - Runs `rocprof-compute analyze --path --block ` + - Displays formatted metrics from specified block + - Default: Block 12 (LDS metrics) + +### Storage Layout + +``` +/workspace/build/workloads/ ++-- baseline/ +| +-- gfx950/ +| +-- SQ_INST_LEVEL_LDS.csv # LDS metrics (Block 12) +| +-- pmc_perf.csv # Performance counters +| +-- counter_collection.csv # All counters ++-- optimized/ +| +-- gfx950/ +| +-- ... ++-- custom_run/ + +-- MI350/ + +-- ... +``` + +### Supported GPU Architectures + +- AMD Instinct MI300 series (gfx940, gfx941, gfx942) +- AMD Instinct MI350 series (gfx950, gfx951) +- Auto-detection via `rocminfo` +- Manual override via `GPU_TARGET` environment variable + +## Related Tools + +- `ck-docker` - Docker container management for CK builds +- `rocprof-compute` - AMD's GPU profiling tool (v2) +- `rocm-smi` - ROCm System Management Interface + +## References + +- [ROCm Profiler Documentation](https://rocm.docs.amd.com/projects/rocprofiler/en/latest/) +- [Composable Kernel GitHub](https://github.com/ROCmSoftwarePlatform/composable_kernel) +- [LDS Bank Conflict Optimization Guide](../../../github_formatting_bot_ideas.md) + +## Examples + +### Example 1: Basic Profiling Session + +```bash +# Setup (one-time) +$ ./script/tools/ck-rocprof setup +Setting up rocprof-compute profiling environment... +* Virtual environment created +* Dependencies installed +* Wrapper script created +Setup complete! + +# Profile an executable +$ cd build +$ ../script/tools/ck-rocprof run my_profile ./bin/tile_example_gemm_universal +Profiling: ./bin/tile_example_gemm_universal +* Profiling complete +Results saved to: /workspace/build/workloads/my_profile// + +# Analyze results +$ ../script/tools/ck-rocprof analyze my_profile +Analyzing: my_profile (Block 12) +[Shows Section 12: Local Data Share (LDS) metrics] +``` + +### Example 2: Optimization Workflow + +```bash +# Profile baseline +$ ck-rocprof run baseline ./bin/my_kernel + +# Make code changes +# ... modify source files ... + +# Rebuild +$ ninja my_kernel + +# Profile optimized version +$ ck-rocprof run optimized ./bin/my_kernel + +# Compare results +$ ck-rocprof compare baseline optimized +[Shows side-by-side Block 12 (LDS) metrics for comparison] +``` + +## License + +Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +SPDX-License-Identifier: MIT From de8c04fe86c4c53a1709d60a1e78f5b685b6f2a2 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 22 Jan 2026 13:50:58 -0600 Subject: [PATCH 03/10] Make ck-rocprof documentation concise and improve Docker integration - Streamlined documentation from 416 to 157 lines (62% reduction) - Focused on essential commands, metrics, and workflows - Enhanced script to run all operations inside Docker containers - Fixed workload directory path and improved container management - Added automatic rocprofiler-compute installation and dependency handling --- script/tools/ck-rocprof | 142 +++++++------ script/tools/ck-rocprof.md | 414 +++++++------------------------------ 2 files changed, 162 insertions(+), 394 deletions(-) diff --git a/script/tools/ck-rocprof b/script/tools/ck-rocprof index 75375943011..88aeee67e3c 100755 --- a/script/tools/ck-rocprof +++ b/script/tools/ck-rocprof @@ -18,7 +18,7 @@ CONTAINER_NAME=$(get_container_name "${PROJECT_ROOT}") # Profiling configuration VENV_PATH="${CK_PROFILE_VENV:-/opt/rocprof_venv}" ROCPROF_BIN="${CK_ROCPROF_BIN:-/opt/rocm-7.0.1/bin/rocprof-compute}" -WORKLOAD_DIR="${CK_WORKLOAD_DIR:-/workspace/build/workloads}" +WORKLOAD_DIR="${CK_WORKLOAD_DIR:-/workspace/workloads}" ROCM_REQUIREMENTS="/opt/rocm-7.0.1/libexec/rocprofiler-compute/requirements.txt" # Help message @@ -58,8 +58,8 @@ Profiling Metrics: - 12.2.17: LDS Data FIFO Full Rate (cycles) Notes: - - Profiling runs inside Docker container if CK_CONTAINER_NAME is set - - Results stored in ${WORKLOAD_DIR}/// + - All commands run inside Docker container (auto-started if needed) + - Results stored in /workspace/workloads/// - Use 'analyze' to view detailed metrics for specific blocks - Use 'compare' to see side-by-side metrics from two runs EOF @@ -70,54 +70,83 @@ get_rocprof_wrapper() { echo "${VENV_PATH}/bin/rocprof-compute" } -# Check if setup is complete +# Check if setup is complete (inside container) is_setup_complete() { local wrapper=$(get_rocprof_wrapper) - [ -d "${VENV_PATH}" ] && [ -f "${wrapper}" ] + # Check if container is running and setup is complete inside it + container_is_running "${CONTAINER_NAME}" && \ + docker exec "${CONTAINER_NAME}" test -d "${VENV_PATH}" 2>/dev/null && \ + docker exec "${CONTAINER_NAME}" test -f "${wrapper}" 2>/dev/null } -# Setup: Create Python venv and install rocprof-compute dependencies +# Setup: Create Python venv and install rocprof-compute dependencies in Docker container cmd_setup() { echo "Setting up rocprof-compute profiling environment..." echo "===========================================" - # Check if rocprof-compute exists - if [ ! -f "${ROCPROF_BIN}" ]; then - echo "Error: rocprof-compute not found at ${ROCPROF_BIN}" - echo "Please ensure ROCm is installed and CK_ROCPROF_BIN is set correctly" + # Ensure Docker container is running + echo "Ensuring Docker container '${CONTAINER_NAME}' is running..." + ensure_container_running "${CONTAINER_NAME}" "${SCRIPT_DIR}" + + # Check if rocprofiler-compute is installed, install if not + echo "Checking for rocprofiler-compute package..." + if ! docker exec "${CONTAINER_NAME}" test -f "${ROCPROF_BIN}" 2>/dev/null; then + echo "Installing rocprofiler-compute package..." + docker exec "${CONTAINER_NAME}" bash -c "apt update && apt install -y rocprofiler-compute" 2>&1 | tail -5 + echo "* rocprofiler-compute installed" + else + echo "* rocprofiler-compute already installed" + fi + + # Verify rocprof-compute exists after installation + if ! docker exec "${CONTAINER_NAME}" test -f "${ROCPROF_BIN}" 2>/dev/null; then + echo "Error: rocprof-compute not found at ${ROCPROF_BIN} after installation" + echo "Please check ROCm installation in container" return 1 fi # Check if requirements file exists - if [ ! -f "${ROCM_REQUIREMENTS}" ]; then + if ! docker exec "${CONTAINER_NAME}" test -f "${ROCM_REQUIREMENTS}" 2>/dev/null; then echo "Error: rocprofiler-compute requirements.txt not found at ${ROCM_REQUIREMENTS}" - echo "Please ensure ROCm is properly installed" + echo "Please ensure ROCm is properly installed in container" return 1 fi - # Create Python venv - if [ -d "${VENV_PATH}" ]; then + # Install uv if not present + echo "Checking for uv package manager..." + if ! docker exec "${CONTAINER_NAME}" command -v uv &>/dev/null; then + echo "Installing uv package manager..." + docker exec "${CONTAINER_NAME}" bash -c "curl -LsSf https://astral.sh/uv/install.sh | sh" 2>&1 | tail -3 + echo "* uv installed" + else + echo "* uv already installed" + fi + + # Create Python venv in container using uv + if docker exec "${CONTAINER_NAME}" test -d "${VENV_PATH}" 2>/dev/null; then echo "Python venv already exists at ${VENV_PATH}" else echo "Creating Python virtual environment at ${VENV_PATH}..." - python3 -m venv "${VENV_PATH}" + docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv venv '${VENV_PATH}'" echo "* Virtual environment created" fi - # Install dependencies + # Install dependencies in container using uv echo "Installing rocprofiler-compute dependencies..." - "${VENV_PATH}/bin/pip" install --quiet -r "${ROCM_REQUIREMENTS}" + docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv pip install --python '${VENV_PATH}/bin/python' -r '${ROCM_REQUIREMENTS}'" + # Pin pandas to <3.0 to avoid CSV conversion bug in rocprof-compute + docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv pip install --python '${VENV_PATH}/bin/python' 'pandas<3.0'" echo "* Dependencies installed" - # Create wrapper script + # Create wrapper script in container local wrapper=$(get_rocprof_wrapper) - cat > "${wrapper}" << 'WRAPPER_EOF' + docker exec "${CONTAINER_NAME}" bash -c "cat > '${wrapper}' << 'WRAPPER_EOF' #!/bin/bash # rocprof-compute wrapper using venv Python -VENV_DIR="$(cd "$(dirname "$0")/.." && pwd)" -exec "${VENV_DIR}/bin/python" /opt/rocm-7.0.1/bin/rocprof-compute "$@" -WRAPPER_EOF - chmod +x "${wrapper}" +VENV_DIR=\"\$(cd \"\$(dirname \"\$0\")/..\" && pwd)\" +exec \"\${VENV_DIR}/bin/python\" /opt/rocm-7.0.1/bin/rocprof-compute \"\$@\" +WRAPPER_EOF" + docker exec "${CONTAINER_NAME}" chmod +x "${wrapper}" echo "* Wrapper script created at ${wrapper}" echo "" @@ -125,7 +154,7 @@ WRAPPER_EOF echo " ck-rocprof run " } -# Detect GPU architecture +# Detect GPU architecture (inside container) # rocprof-compute uses marketing names (MI350, MI300X, etc.) for directory naming # This function attempts to match that convention detect_gpu_arch() { @@ -134,34 +163,32 @@ detect_gpu_arch() { return 0 fi - # Try to get marketing name from rocminfo - local marketing_name=$(rocminfo 2>/dev/null | grep "Marketing Name:" | grep -oP 'MI\d+[A-Z]*' | head -1 || echo "") + # Try to get marketing name from rocminfo in container + local marketing_name=$(docker exec "${CONTAINER_NAME}" bash -c "rocminfo 2>/dev/null | grep 'Marketing Name:' | grep -oP 'MI\d+[A-Z]*' | head -1" 2>/dev/null || echo "") if [ -n "$marketing_name" ]; then echo "$marketing_name" return 0 fi - # Fallback: Try gfx name from rocminfo - local gfx_name=$(rocminfo 2>/dev/null | grep -oP 'gfx[0-9a-z]+' | head -1 || echo "") + # Fallback: Try gfx name from rocminfo in container + local gfx_name=$(docker exec "${CONTAINER_NAME}" bash -c "rocminfo 2>/dev/null | grep -oP 'gfx[0-9a-z]+' | head -1" 2>/dev/null || echo "") if [ -n "$gfx_name" ]; then echo "$gfx_name" return 0 fi - # Fallback: try to detect from existing workload dirs (any architecture pattern) - if [ -d "${WORKLOAD_DIR}" ]; then - local first_dir=$(find "${WORKLOAD_DIR}" -maxdepth 2 -type d \( -name 'gfx*' -o -name 'MI*' \) 2>/dev/null | head -1) - if [ -n "$first_dir" ]; then - basename "$first_dir" - return 0 - fi + # Fallback: try to detect from existing workload dirs in container + local first_dir=$(docker exec "${CONTAINER_NAME}" bash -c "find '${WORKLOAD_DIR}' -maxdepth 2 -type d \( -name 'gfx*' -o -name 'MI*' \) 2>/dev/null | head -1" 2>/dev/null || echo "") + if [ -n "$first_dir" ]; then + basename "$first_dir" + return 0 fi # Final fallback echo "MI350" } -# Run profiling +# Run profiling (inside container) cmd_run() { local name="$1" local executable="$2" @@ -181,9 +208,9 @@ cmd_run() { return 1 fi - # Check if executable exists - if [ ! -f "$executable" ]; then - echo "Error: Executable not found: $executable" + # Check if executable exists in container + if ! docker exec "${CONTAINER_NAME}" test -f "$executable" 2>/dev/null; then + echo "Error: Executable not found in container: $executable" return 1 fi @@ -193,18 +220,17 @@ cmd_run() { echo "Profiling: $executable ${exe_args[*]}" echo "Run name: $name" echo "GPU arch: $gpu_arch" + echo "Container: $CONTAINER_NAME" echo "===========================================" - # Build command - local cmd="${wrapper} profile --name ${name}" - cmd="${cmd} --" - cmd="${cmd} ${executable}" + # Build command for container execution + local cmd="${wrapper} profile --name ${name} -- ${executable}" for arg in "${exe_args[@]}"; do cmd="${cmd} $(printf '%q' "$arg")" done - # Run profiling - eval "${cmd}" + # Run profiling in container + docker exec "${CONTAINER_NAME}" bash -c "${cmd}" echo "" echo "* Profiling complete" @@ -213,21 +239,21 @@ cmd_run() { echo "Analyze with: ck-rocprof analyze ${name}" } -# Find actual GPU architecture directory for a given run name +# Find actual GPU architecture directory for a given run name (inside container) # rocprof-compute creates directories with various naming schemes (MI350, gfx950, etc.) # This function finds whatever directory actually exists find_workload_path() { local name="$1" local run_dir="${WORKLOAD_DIR}/${name}" - if [ ! -d "$run_dir" ]; then + if ! docker exec "${CONTAINER_NAME}" test -d "$run_dir" 2>/dev/null; then return 1 fi # Find first subdirectory (should be GPU architecture) - local gpu_dir=$(find "$run_dir" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | head -1) + local gpu_dir=$(docker exec "${CONTAINER_NAME}" bash -c "find '$run_dir' -maxdepth 1 -mindepth 1 -type d 2>/dev/null | head -1" 2>/dev/null) - if [ -n "$gpu_dir" ] && [ -d "$gpu_dir" ]; then + if [ -n "$gpu_dir" ]; then echo "$gpu_dir" return 0 fi @@ -235,7 +261,7 @@ find_workload_path() { return 1 } -# Analyze profiling results +# Analyze profiling results (inside container) cmd_analyze() { local name="$1" local block="${2:-12}" # Default to block 12 (LDS metrics) @@ -269,17 +295,17 @@ cmd_analyze() { echo "===========================================" echo "" - "${wrapper}" analyze --path "${workload_path}" --block "${block}" + docker exec "${CONTAINER_NAME}" "${wrapper}" analyze --path "${workload_path}" --block "${block}" } -# Extract key LDS metrics from profiling results +# Extract key LDS metrics from profiling results (inside container) extract_lds_metrics() { local name="$1" local gpu_arch=$(detect_gpu_arch) local workload_path="${WORKLOAD_DIR}/${name}/${gpu_arch}" local csv_file="${workload_path}/SQ_INST_LEVEL_LDS.csv" - if [ ! -f "$csv_file" ]; then + if ! docker exec "${CONTAINER_NAME}" test -f "$csv_file" 2>/dev/null; then echo "N/A (CSV not found)" return 1 fi @@ -344,14 +370,14 @@ cmd_compare() { echo " ck-rocprof analyze ${name2} 12" } -# List available profiling runs +# List available profiling runs (inside container) cmd_list() { - if [ ! -d "${WORKLOAD_DIR}" ]; then + if ! docker exec "${CONTAINER_NAME}" test -d "${WORKLOAD_DIR}" 2>/dev/null; then echo "No profiling runs found (workload directory doesn't exist)" return 0 fi - local runs=$(find "${WORKLOAD_DIR}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort) + local runs=$(docker exec "${CONTAINER_NAME}" bash -c "find '${WORKLOAD_DIR}' -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort" 2>/dev/null) if [ -z "$runs" ]; then echo "No profiling runs found in ${WORKLOAD_DIR}" @@ -366,8 +392,8 @@ cmd_list() { if [ -n "$path" ]; then local gpu_arch=$(basename "$path") - local size=$(du -sh "$path" 2>/dev/null | cut -f1) - local date=$(stat -c %y "$path" 2>/dev/null | cut -d' ' -f1) + local size=$(docker exec "${CONTAINER_NAME}" bash -c "du -sh '$path' 2>/dev/null | cut -f1" 2>/dev/null) + local date=$(docker exec "${CONTAINER_NAME}" bash -c "stat -c %y '$path' 2>/dev/null | cut -d' ' -f1" 2>/dev/null) printf " %-25s [%-8s, %s, %s]\n" "$run" "$gpu_arch" "$size" "$date" else printf " %-25s [no data]\n" "$run" diff --git a/script/tools/ck-rocprof.md b/script/tools/ck-rocprof.md index e4642ef20ca..b9dc858dfaf 100644 --- a/script/tools/ck-rocprof.md +++ b/script/tools/ck-rocprof.md @@ -1,10 +1,6 @@ # CK ROCProf Tool -Performance profiling tool for Composable Kernel applications using AMD rocprof-compute. - -## Overview - -`ck-rocprof` simplifies GPU performance profiling by wrapping rocprofiler-compute with an easy-to-use CLI. It handles Python environment setup, manages profiling runs, and provides convenient comparison tools for optimization workflows. +GPU performance profiling for Composable Kernel applications using AMD rocprof-compute. ## Quick Start @@ -12,14 +8,14 @@ Performance profiling tool for Composable Kernel applications using AMD rocprof- # One-time setup ./script/tools/ck-rocprof setup -# Profile an executable +# Profile executable cd build ../script/tools/ck-rocprof run baseline ./bin/tile_example_gemm_universal -# Analyze results +# Analyze LDS metrics ../script/tools/ck-rocprof analyze baseline -# Compare two runs +# Compare optimizations ../script/tools/ck-rocprof run optimized ./bin/tile_example_gemm_universal ../script/tools/ck-rocprof compare baseline optimized ``` @@ -27,389 +23,135 @@ cd build ## Commands ### `setup` - -One-time setup of the profiling environment. - -```bash -ck-rocprof setup -``` - -**What it does:** -- Creates Python virtual environment at `/opt/rocprof_venv` -- Installs rocprofiler-compute dependencies -- Creates wrapper script for easy invocation - -**Requirements:** -- ROCm installed (tested with ROCm 7.0.1) -- Python 3 with venv support -- Write access to `/opt/` directory - ---- +One-time setup: creates Python venv, installs dependencies, configures rocprof-compute. ### `run [args]` +Profile executable and save results. -Profile an executable and save results with a given name. - -```bash -ck-rocprof run [args...] -``` - -**Arguments:** -- `name` - Unique identifier for this profiling run -- `executable` - Path to executable to profile -- `args` - Optional arguments to pass to the executable - -**Examples:** ```bash -# Profile with default arguments -ck-rocprof run baseline ./bin/tile_example_gemm_universal +# Basic profiling +ck-rocprof run baseline ./bin/gemm_example -# Profile with custom arguments -ck-rocprof run custom_size ./bin/tile_example_gemm_universal -m 8192 -n 8192 -k 4096 +# With arguments +ck-rocprof run large_matrix ./bin/gemm_example -m 8192 -n 8192 -k 4096 -# Profile test with gtest filter -ck-rocprof run unit_test ./bin/test_ck_tile_gemm --gtest_filter="*Fp16*" +# Test filtering +ck-rocprof run unit_test ./bin/test_gemm --gtest_filter="*Fp16*" ``` -**Output:** -- Results stored in `/workspace/build/workloads///` -- GPU architecture auto-detected (e.g., `gfx950`, `MI350`) - ---- - ### `analyze [block]` - -Analyze profiling results for a specific run. +Display profiling metrics (default: Block 12 - LDS). ```bash -ck-rocprof analyze [block] +ck-rocprof analyze baseline # LDS metrics +ck-rocprof analyze baseline 2 # L2 Cache +ck-rocprof analyze baseline 7 # Instruction Mix ``` -**Arguments:** -- `name` - Name of profiling run to analyze -- `block` - Optional block number to display (default: 12 for LDS metrics) - -**Examples:** -```bash -# Analyze LDS metrics (Block 12) -ck-rocprof analyze baseline - -# Analyze specific block -ck-rocprof analyze baseline 2 # L2 Cache -ck-rocprof analyze baseline 7 # Compute Unit - Instruction Mix -ck-rocprof analyze baseline 12 # Local Data Share (LDS) -``` - -**Key Metric Blocks:** -- **Block 2**: L2 Cache -- **Block 7**: Compute Unit - Instruction Mix -- **Block 12**: Local Data Share (LDS) - - 12.1.3: Bank Conflict Rate (% of peak) - - 12.2.9: Bank Conflicts/Access (conflicts/access) - - 12.2.12: Bank Conflict (cycles per kernel) - - 12.2.17: LDS Data FIFO Full Rate - ---- - ### `compare ` - -Compare LDS metrics from two profiling runs side-by-side. - -```bash -ck-rocprof compare -``` - -**Arguments:** -- `name1` - First profiling run (typically baseline) -- `name2` - Second profiling run (typically optimized) - -**Example:** -```bash -ck-rocprof compare baseline optimized -``` - -**Output:** -- Side-by-side Block 12 (LDS) metrics -- Useful for validating optimization improvements - ---- +Side-by-side comparison of two runs. ### `list` +List all profiling runs with size and date. -List all available profiling runs. +## Key LDS Metrics (Block 12) -```bash -ck-rocprof list -``` +**Target Values:** +- Bank Conflicts/Access: <0.01 (1% conflict rate) +- Bank Conflict Rate: >90% of peak bandwidth -**Output:** -- Run name -- Disk usage -- Creation date +**Critical Metrics:** +- **12.2.9 Bank Conflicts/Access**: Direct conflict measure + - Baseline (naive): ~0.04 (4% conflicts) + - Optimized: <0.005 (<0.5% conflicts) +- **12.2.12 Bank Conflict Cycles**: Wasted cycles per kernel +- **12.2.17 LDS Data FIFO Full**: Memory system pressure -**Example:** -```bash -$ ck-rocprof list -Available profiling runs: -=========================================== - baseline [156K, 2026-01-20] - optimized [148K, 2026-01-20] - custom_size [162K, 2026-01-20] -``` - ---- - -## Environment Variables - -- `CK_PROFILE_VENV` - Python venv path (default: `/opt/rocprof_venv`) -- `CK_ROCPROF_BIN` - rocprof-compute binary (default: `/opt/rocm-7.0.1/bin/rocprof-compute`) -- `CK_WORKLOAD_DIR` - Workload storage directory (default: `/workspace/build/workloads`) -- `GPU_TARGET` - Override GPU architecture detection (e.g., `gfx950`, `gfx942`) - -## Common Workflows - -### Optimizing LDS Bank Conflicts +## Optimization Workflow ```bash -# 1. Capture baseline metrics -cd build -../script/tools/ck-rocprof run baseline ./bin/tile_example_gemm_universal - -# 2. Check baseline bank conflict rate -../script/tools/ck-rocprof analyze baseline -# Look for: -# - Section 12.2.9: Bank Conflicts/Access (target: <0.01) -# - Section 12.2.12: Bank Conflict cycles (target: minimize) +# 1. Baseline +ck-rocprof run baseline ./bin/my_kernel -# 3. Apply optimization (e.g., XOR transform) -# ... modify code ... - -# 4. Rebuild and profile -ninja tile_example_gemm_universal -../script/tools/ck-rocprof run optimized ./bin/tile_example_gemm_universal - -# 5. Compare results -../script/tools/ck-rocprof compare baseline optimized +# 2. Check conflicts +ck-rocprof analyze baseline +# Look for Bank Conflicts/Access > 0.02 -# 6. Verify improvement -# Expected: Bank Conflicts/Access reduced by 8-10x -``` +# 3. Optimize code (XOR transforms, padding, etc.) +# ... edit source ... -### Profiling Multiple Configurations +# 4. Test optimization +ninja my_kernel +ck-rocprof run optimized ./bin/my_kernel -```bash -# Profile different tile sizes -ck-rocprof run tile_64x64 ./bin/example -tile 64 -ck-rocprof run tile_128x128 ./bin/example -tile 128 -ck-rocprof run tile_256x256 ./bin/example -tile 256 - -# Compare all runs -ck-rocprof list -ck-rocprof compare tile_64x64 tile_128x128 -ck-rocprof compare tile_128x128 tile_256x256 +# 5. Verify improvement +ck-rocprof compare baseline optimized +# Target: 8-10x reduction in conflicts ``` -## Understanding LDS Metrics (Block 12) - -### Key Metrics - -**12.1.3 - Bank Conflict Rate** -- Percentage of theoretical peak LDS bandwidth achieved -- Lower is worse (conflicts reduce bandwidth) -- Target: >90% of peak - -**12.2.9 - Bank Conflicts/Access** -- Average number of bank conflicts per LDS access -- Direct measure of conflict severity -- Target: <0.01 (1% conflict rate) -- Baseline (naive layout): ~0.04 (4% conflicts) -- Optimized (XOR/padding): <0.005 (<0.5% conflicts) - -**12.2.12 - Bank Conflict (cycles)** -- Total cycles lost to bank conflicts per kernel -- Shows MAX, MIN, AVG across compute units -- Look at MAX column for worst-case bottleneck -- Target: Minimize (reduce by 8-10x with optimization) +## Environment Variables -**12.2.17 - LDS Data FIFO Full Rate** -- Cycles stalled due to LDS data FIFO full -- Indicates memory system pressure -- Reduces with bank conflict mitigation +- `CK_PROFILE_VENV`: Python venv path (default: `/opt/rocprof_venv`) +- `CK_ROCPROF_BIN`: rocprof-compute binary path +- `CK_WORKLOAD_DIR`: Results directory (default: `/workspace/build/workloads`) +- `GPU_TARGET`: Override GPU detection (e.g., `gfx950`, `gfx942`) -### Interpreting Results +## Interpreting Results -**Good LDS Performance:** +**Good Performance:** ``` -Section 12.1 - Speed-of-Light: - Bank Conflict Rate: Low % of peak - -Section 12.2 - LDS Stats: - Bank Conflicts/Access: Low value (ideally <0.01) - Bank Conflict (MAX): Minimal cycles - LDS Data FIFO Full: Low cycles +Bank Conflicts/Access: <0.01 +Bank Conflict Rate: >90% of peak +LDS Data FIFO Full: Minimal cycles ``` -**Poor LDS Performance (needs optimization):** +**Needs Optimization:** ``` -Section 12.1 - Speed-of-Light: - Bank Conflict Rate: Higher % of peak - -Section 12.2 - LDS Stats: - Bank Conflicts/Access: High value (>0.02 indicates issues) - Bank Conflict (MAX): Significant cycles lost - LDS Data FIFO Full: High memory pressure +Bank Conflicts/Access: >0.02 +Bank Conflict Cycles: High MAX values +LDS Data FIFO Full: High memory pressure ``` -**Typical Improvement After Optimization:** -- Bank Conflicts/Access: Reduced by 10-20x -- Bank Conflict cycles: Significantly reduced -- Overall kernel speedup: Variable depending on workload - ## Troubleshooting -### "Error: Profiling environment not set up" - -**Solution:** Run `ck-rocprof setup` first - -### "Error: rocprof-compute not found" - -**Cause:** ROCm not installed or in non-standard location - -**Solution:** Set `CK_ROCPROF_BIN` environment variable: +**"Profiling environment not set up"** ```bash -export CK_ROCPROF_BIN=/path/to/rocm/bin/rocprof-compute ck-rocprof setup ``` -### "Error: Profiling results not found" - -**Cause:** Profiling run doesn't exist or wrong GPU architecture - -**Solution:** +**"rocprof-compute not found"** ```bash -# List available runs -ck-rocprof list - -# Check GPU architecture -rocminfo | grep gfx - -# Override if needed -export GPU_TARGET=gfx950 +export CK_ROCPROF_BIN=/custom/path/rocprof-compute +ck-rocprof setup ``` -### Python Dependency Conflicts - -**Cause:** Ubuntu 24.04 has externally-managed Python environment - -**Solution:** The tool automatically creates a venv to avoid conflicts. If setup fails: +**"Profiling results not found"** ```bash -# Manual venv creation -sudo python3 -m venv /opt/rocprof_venv -sudo /opt/rocprof_venv/bin/pip install -r /opt/rocm-7.0.1/libexec/rocprofiler-compute/requirements.txt +ck-rocprof list # Check available runs +rocminfo | grep gfx # Verify GPU arch +export GPU_TARGET=gfx950 # Override if needed ``` -## Technical Details - -### Profiling Workflow - -1. **Setup Phase** (`ck-rocprof setup`): - - Creates isolated Python venv - - Installs rocprofiler-compute dependencies - - Creates wrapper script that uses venv Python +## Storage Layout -2. **Profiling Phase** (`ck-rocprof run`): - - Detects GPU architecture - - Runs `rocprof-compute profile --name -- ` - - Stores results in `workloads///` +Results stored in `/workspace/build/workloads///`: +- `SQ_INST_LEVEL_LDS.csv`: LDS metrics +- `pmc_perf.csv`: Performance counters +- `counter_collection.csv`: All metrics -3. **Analysis Phase** (`ck-rocprof analyze`): - - Runs `rocprof-compute analyze --path --block ` - - Displays formatted metrics from specified block - - Default: Block 12 (LDS metrics) - -### Storage Layout - -``` -/workspace/build/workloads/ -+-- baseline/ -| +-- gfx950/ -| +-- SQ_INST_LEVEL_LDS.csv # LDS metrics (Block 12) -| +-- pmc_perf.csv # Performance counters -| +-- counter_collection.csv # All counters -+-- optimized/ -| +-- gfx950/ -| +-- ... -+-- custom_run/ - +-- MI350/ - +-- ... -``` - -### Supported GPU Architectures +## Technical Details -- AMD Instinct MI300 series (gfx940, gfx941, gfx942) -- AMD Instinct MI350 series (gfx950, gfx951) -- Auto-detection via `rocminfo` -- Manual override via `GPU_TARGET` environment variable +- **Setup**: Creates isolated Python venv, installs dependencies +- **Profiling**: Runs `rocprof-compute profile --name -- ` +- **Analysis**: Runs `rocprof-compute analyze --path --block ` +- **GPU Support**: MI300/MI350 series, auto-detects architecture ## Related Tools -- `ck-docker` - Docker container management for CK builds -- `rocprof-compute` - AMD's GPU profiling tool (v2) -- `rocm-smi` - ROCm System Management Interface - -## References - -- [ROCm Profiler Documentation](https://rocm.docs.amd.com/projects/rocprofiler/en/latest/) -- [Composable Kernel GitHub](https://github.com/ROCmSoftwarePlatform/composable_kernel) -- [LDS Bank Conflict Optimization Guide](../../../github_formatting_bot_ideas.md) - -## Examples - -### Example 1: Basic Profiling Session - -```bash -# Setup (one-time) -$ ./script/tools/ck-rocprof setup -Setting up rocprof-compute profiling environment... -* Virtual environment created -* Dependencies installed -* Wrapper script created -Setup complete! - -# Profile an executable -$ cd build -$ ../script/tools/ck-rocprof run my_profile ./bin/tile_example_gemm_universal -Profiling: ./bin/tile_example_gemm_universal -* Profiling complete -Results saved to: /workspace/build/workloads/my_profile// - -# Analyze results -$ ../script/tools/ck-rocprof analyze my_profile -Analyzing: my_profile (Block 12) -[Shows Section 12: Local Data Share (LDS) metrics] -``` - -### Example 2: Optimization Workflow - -```bash -# Profile baseline -$ ck-rocprof run baseline ./bin/my_kernel - -# Make code changes -# ... modify source files ... - -# Rebuild -$ ninja my_kernel - -# Profile optimized version -$ ck-rocprof run optimized ./bin/my_kernel - -# Compare results -$ ck-rocprof compare baseline optimized -[Shows side-by-side Block 12 (LDS) metrics for comparison] -``` +- `ck-docker`: Container management +- `rocprof-compute`: AMD GPU profiler v2 +- `rocm-smi`: System monitoring ## License -Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -SPDX-License-Identifier: MIT +Copyright (c) Advanced Micro Devices, Inc. SPDX-License-Identifier: MIT From c13f22f2b093492efdd05bad00fe4df573671070 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 23 Jan 2026 20:30:37 -0600 Subject: [PATCH 04/10] Add --no-roof flag to ck-rocprof profile command Skip roofline analysis by default to speed up profiling. Roofline analysis can add significant time to profiling runs but is not needed for most LDS bank conflict analysis workflows. --- script/tools/ck-rocprof | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/tools/ck-rocprof b/script/tools/ck-rocprof index 88aeee67e3c..cf72e3cf9a0 100755 --- a/script/tools/ck-rocprof +++ b/script/tools/ck-rocprof @@ -224,7 +224,8 @@ cmd_run() { echo "===========================================" # Build command for container execution - local cmd="${wrapper} profile --name ${name} -- ${executable}" + # --no-roof skips roofline analysis to speed up profiling + local cmd="${wrapper} profile --no-roof --name ${name} -- ${executable}" for arg in "${exe_args[@]}"; do cmd="${cmd} $(printf '%q' "$arg")" done From 8f428dba6228efb3dd48c41f285d0911e06c40d3 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Wed, 28 Jan 2026 14:50:06 -0500 Subject: [PATCH 05/10] Make ck-rocprof work independently of Docker Add native execution mode that runs rocprof-compute directly on the host system when available, falling back to Docker mode when not. Key changes: - Auto-detect native mode when rocprof-compute is in PATH or common locations - Add execution mode wrappers (exec_cmd, file_exists, dir_exists, etc.) - Native mode stores venv at .ck-rocprof-venv in project root - Native mode stores workloads at build/workloads/ - Support user-installed rocprofiler-compute (e.g., ~/.local/rocprofiler-compute) - Add CK_FORCE_DOCKER env var to force Docker mode - Update help message to show current execution mode - Maintain full backward compatibility with existing Docker workflow Tested successfully with rocprofiler-compute 3.4.0 installed from source on MI300X GPU in native mode. Co-Authored-By: Claude --- script/tools/ck-rocprof | 552 +++++++++++++++++++++++++++++++--------- 1 file changed, 430 insertions(+), 122 deletions(-) diff --git a/script/tools/ck-rocprof b/script/tools/ck-rocprof index cf72e3cf9a0..cdae7778a4d 100755 --- a/script/tools/ck-rocprof +++ b/script/tools/ck-rocprof @@ -15,17 +15,193 @@ source "${SCRIPT_DIR}/common.sh" PROJECT_ROOT=$(get_project_root "${SCRIPT_DIR}") CONTAINER_NAME=$(get_container_name "${PROJECT_ROOT}") -# Profiling configuration -VENV_PATH="${CK_PROFILE_VENV:-/opt/rocprof_venv}" -ROCPROF_BIN="${CK_ROCPROF_BIN:-/opt/rocm-7.0.1/bin/rocprof-compute}" -WORKLOAD_DIR="${CK_WORKLOAD_DIR:-/workspace/workloads}" -ROCM_REQUIREMENTS="/opt/rocm-7.0.1/libexec/rocprofiler-compute/requirements.txt" +# ============================================================================ +# Execution mode detection +# ============================================================================ + +# Determine if running in native mode (no Docker) +# Native mode is used when: +# 1. CK_FORCE_DOCKER is not set to 1 +# 2. rocprof-compute is available locally +is_native_mode() { + # Force Docker mode if explicitly requested + if [ "${CK_FORCE_DOCKER:-0}" = "1" ]; then + return 1 + fi + + # Use native mode if rocprof-compute exists locally + if command -v rocprof-compute &>/dev/null; then + return 0 + fi + + # Check common ROCm locations and user installations + local candidates=( + "${HOME}/.local/rocprofiler-compute/3.4.0/bin/rocprof-compute" + "/opt/rocm/bin/rocprof-compute" + "/opt/rocm-7.2.0/bin/rocprof-compute" + "/opt/rocm-7.0.1/bin/rocprof-compute" + "/opt/rocm-6.2.0/bin/rocprof-compute" + "/opt/rocm-6.1.0/bin/rocprof-compute" + ) + + for bin in "${candidates[@]}"; do + if [ -f "$bin" ]; then + return 0 + fi + done + + # Default to Docker mode + return 1 +} + +# Find rocprof-compute binary (native mode) +find_rocprof_bin() { + # Check CK_ROCPROF_BIN first + if [ -n "${CK_ROCPROF_BIN:-}" ] && [ -f "${CK_ROCPROF_BIN}" ]; then + echo "${CK_ROCPROF_BIN}" + return 0 + fi + + # Check PATH + if command -v rocprof-compute &>/dev/null; then + command -v rocprof-compute + return 0 + fi + + # Check common ROCm locations and user installations + local candidates=( + "${HOME}/.local/rocprofiler-compute/3.4.0/bin/rocprof-compute" + "/opt/rocm/bin/rocprof-compute" + "/opt/rocm-7.2.0/bin/rocprof-compute" + "/opt/rocm-7.0.1/bin/rocprof-compute" + "/opt/rocm-6.2.0/bin/rocprof-compute" + "/opt/rocm-6.1.0/bin/rocprof-compute" + ) + + for bin in "${candidates[@]}"; do + if [ -f "$bin" ]; then + echo "$bin" + return 0 + fi + done + + return 1 +} + +# Find ROCm requirements file (native mode) +find_rocm_requirements() { + local rocprof_bin="${1:-$(find_rocprof_bin)}" + if [ -z "$rocprof_bin" ]; then + return 1 + fi + + # Requirements file is typically at ../libexec/rocprofiler-compute/requirements.txt + local rocm_dir=$(dirname $(dirname "$rocprof_bin")) + local req_file="${rocm_dir}/libexec/rocprofiler-compute/requirements.txt" + + if [ -f "$req_file" ]; then + echo "$req_file" + return 0 + fi + + return 1 +} + +# ============================================================================ +# Configuration (mode-dependent) +# ============================================================================ + +if is_native_mode; then + # Native mode configuration + ROCPROF_BIN="${CK_ROCPROF_BIN:-$(find_rocprof_bin)}" + VENV_PATH="${CK_PROFILE_VENV:-${PROJECT_ROOT}/.ck-rocprof-venv}" + WORKLOAD_DIR="${CK_WORKLOAD_DIR:-$(get_build_dir "${PROJECT_ROOT}")/workloads}" + ROCM_REQUIREMENTS="${CK_ROCM_REQUIREMENTS:-$(find_rocm_requirements "${ROCPROF_BIN}")}" +else + # Docker mode configuration + ROCPROF_BIN="${CK_ROCPROF_BIN:-/opt/rocm-7.0.1/bin/rocprof-compute}" + VENV_PATH="${CK_PROFILE_VENV:-/opt/rocprof_venv}" + WORKLOAD_DIR="${CK_WORKLOAD_DIR:-/workspace/workloads}" + ROCM_REQUIREMENTS="/opt/rocm-7.0.1/libexec/rocprofiler-compute/requirements.txt" +fi + +# ============================================================================ +# Command execution wrappers (abstract native vs Docker execution) +# ============================================================================ + +# Execute command in appropriate environment +exec_cmd() { + if is_native_mode; then + bash -c "$*" + else + docker exec "${CONTAINER_NAME}" bash -c "$*" + fi +} + +# Check if file exists +file_exists() { + local filepath="$1" + if is_native_mode; then + [ -f "$filepath" ] + else + docker exec "${CONTAINER_NAME}" test -f "$filepath" 2>/dev/null + fi +} + +# Check if directory exists +dir_exists() { + local dirpath="$1" + if is_native_mode; then + [ -d "$dirpath" ] + else + docker exec "${CONTAINER_NAME}" test -d "$dirpath" 2>/dev/null + fi +} + +# Check if command exists in execution environment +cmd_exists() { + local cmd="$1" + if is_native_mode; then + command -v "$cmd" &>/dev/null + else + docker exec "${CONTAINER_NAME}" command -v "$cmd" &>/dev/null + fi +} + +# Get file/directory size +get_size() { + local path="$1" + if is_native_mode; then + du -sh "$path" 2>/dev/null | cut -f1 + else + docker exec "${CONTAINER_NAME}" bash -c "du -sh '$path' 2>/dev/null | cut -f1" 2>/dev/null + fi +} + +# Get file modification date +get_date() { + local path="$1" + if is_native_mode; then + stat -c %y "$path" 2>/dev/null | cut -d' ' -f1 + else + docker exec "${CONTAINER_NAME}" bash -c "stat -c %y '$path' 2>/dev/null | cut -d' ' -f1" 2>/dev/null + fi +} # Help message show_help() { + local mode_info + if is_native_mode; then + mode_info="NATIVE (rocprof-compute found locally)" + else + mode_info="DOCKER (container: ${CONTAINER_NAME})" + fi + cat << EOF CK ROCProf Tool - Profile CK applications with rocprof-compute +Current mode: ${mode_info} + Usage: ck-rocprof [options] Commands: @@ -44,11 +220,18 @@ Examples: ck-rocprof compare baseline optimized ck-rocprof list +Execution Modes: + NATIVE: Runs directly on ROCm host (auto-detected if rocprof-compute in PATH) + DOCKER: Runs in container (auto-started if needed) + Environment Variables: - CK_PROFILE_VENV - Python venv path (default: /opt/rocprof_venv) - CK_ROCPROF_BIN - rocprof-compute binary path (default: /opt/rocm-7.0.1/bin/rocprof-compute) - CK_WORKLOAD_DIR - Workload storage directory (default: /workspace/build/workloads) - CK_CONTAINER_NAME - Docker container name (for remote profiling) + CK_FORCE_DOCKER - Set to 1 to force Docker mode + CK_GPU_TARGET - Override GPU detection (e.g., gfx950, MI300X) + CK_PROFILE_VENV - Python venv path + CK_ROCPROF_BIN - rocprof-compute binary path + CK_ROCM_REQUIREMENTS - Path to rocprofiler-compute requirements.txt + CK_WORKLOAD_DIR - Workload storage directory + CK_CONTAINER_NAME - Docker container name (Docker mode only) Profiling Metrics: Block 12 (LDS - Local Data Share): @@ -58,8 +241,7 @@ Profiling Metrics: - 12.2.17: LDS Data FIFO Full Rate (cycles) Notes: - - All commands run inside Docker container (auto-started if needed) - - Results stored in /workspace/workloads/// + - Results stored in workloads/// - Use 'analyze' to view detailed metrics for specific blocks - Use 'compare' to see side-by-side metrics from two runs EOF @@ -70,125 +252,232 @@ get_rocprof_wrapper() { echo "${VENV_PATH}/bin/rocprof-compute" } -# Check if setup is complete (inside container) +# Check if setup is complete is_setup_complete() { local wrapper=$(get_rocprof_wrapper) - # Check if container is running and setup is complete inside it - container_is_running "${CONTAINER_NAME}" && \ - docker exec "${CONTAINER_NAME}" test -d "${VENV_PATH}" 2>/dev/null && \ - docker exec "${CONTAINER_NAME}" test -f "${wrapper}" 2>/dev/null + + if is_native_mode; then + # Native: check local venv and wrapper + [ -d "${VENV_PATH}" ] && [ -f "${wrapper}" ] + else + # Docker: check container is running and setup is complete inside it + container_is_running "${CONTAINER_NAME}" && \ + docker exec "${CONTAINER_NAME}" test -d "${VENV_PATH}" 2>/dev/null && \ + docker exec "${CONTAINER_NAME}" test -f "${wrapper}" 2>/dev/null + fi } -# Setup: Create Python venv and install rocprof-compute dependencies in Docker container +# Setup: Create Python venv and install rocprof-compute dependencies cmd_setup() { echo "Setting up rocprof-compute profiling environment..." echo "===========================================" - # Ensure Docker container is running - echo "Ensuring Docker container '${CONTAINER_NAME}' is running..." - ensure_container_running "${CONTAINER_NAME}" "${SCRIPT_DIR}" + if is_native_mode; then + # ====== NATIVE MODE SETUP ====== + info "Running in NATIVE mode" - # Check if rocprofiler-compute is installed, install if not - echo "Checking for rocprofiler-compute package..." - if ! docker exec "${CONTAINER_NAME}" test -f "${ROCPROF_BIN}" 2>/dev/null; then - echo "Installing rocprofiler-compute package..." - docker exec "${CONTAINER_NAME}" bash -c "apt update && apt install -y rocprofiler-compute" 2>&1 | tail -5 - echo "* rocprofiler-compute installed" - else - echo "* rocprofiler-compute already installed" - fi + # Check rocprof-compute exists + if [ -z "${ROCPROF_BIN}" ] || [ ! -f "${ROCPROF_BIN}" ]; then + error "rocprof-compute not found" + echo "Install with: sudo apt install rocprofiler-compute" + return 1 + fi + info "Found rocprof-compute: ${ROCPROF_BIN}" - # Verify rocprof-compute exists after installation - if ! docker exec "${CONTAINER_NAME}" test -f "${ROCPROF_BIN}" 2>/dev/null; then - echo "Error: rocprof-compute not found at ${ROCPROF_BIN} after installation" - echo "Please check ROCm installation in container" - return 1 - fi + # Check requirements file + if [ -z "${ROCM_REQUIREMENTS}" ] || [ ! -f "${ROCM_REQUIREMENTS}" ]; then + error "ROCm requirements file not found" + echo "Expected at: $(dirname $(dirname ${ROCPROF_BIN}))/libexec/rocprofiler-compute/requirements.txt" + return 1 + fi + info "Found requirements: ${ROCM_REQUIREMENTS}" - # Check if requirements file exists - if ! docker exec "${CONTAINER_NAME}" test -f "${ROCM_REQUIREMENTS}" 2>/dev/null; then - echo "Error: rocprofiler-compute requirements.txt not found at ${ROCM_REQUIREMENTS}" - echo "Please ensure ROCm is properly installed in container" - return 1 - fi + # Check GPU access + if [ ! -r /dev/kfd ]; then + warn "No read access to /dev/kfd - GPU profiling may fail" + warn "Add user to video/render group: sudo usermod -a -G video,render \$USER" + fi - # Install uv if not present - echo "Checking for uv package manager..." - if ! docker exec "${CONTAINER_NAME}" command -v uv &>/dev/null; then - echo "Installing uv package manager..." - docker exec "${CONTAINER_NAME}" bash -c "curl -LsSf https://astral.sh/uv/install.sh | sh" 2>&1 | tail -3 - echo "* uv installed" - else - echo "* uv already installed" - fi + # Install uv if needed + if ! command -v uv &>/dev/null; then + info "Installing uv package manager..." + curl -LsSf https://astral.sh/uv/install.sh | sh + # shellcheck disable=SC1090 + source ~/.local/bin/env 2>/dev/null || true + fi - # Create Python venv in container using uv - if docker exec "${CONTAINER_NAME}" test -d "${VENV_PATH}" 2>/dev/null; then - echo "Python venv already exists at ${VENV_PATH}" + # Create venv + if [ -d "${VENV_PATH}" ]; then + info "Python venv already exists at ${VENV_PATH}" + else + info "Creating Python venv at ${VENV_PATH}..." + uv venv "${VENV_PATH}" + fi + + # Install dependencies + info "Installing dependencies..." + uv pip install --python "${VENV_PATH}/bin/python" -r "${ROCM_REQUIREMENTS}" + uv pip install --python "${VENV_PATH}/bin/python" 'pandas<3.0' + + # Create wrapper script + local wrapper=$(get_rocprof_wrapper) + mkdir -p "$(dirname "${wrapper}")" + cat > "${wrapper}" << WRAPPER_EOF +#!/bin/bash +# rocprof-compute wrapper using venv Python +VENV_DIR="\$(cd "\$(dirname "\$0")/.." && pwd)" +exec "\${VENV_DIR}/bin/python" "${ROCPROF_BIN}" "\$@" +WRAPPER_EOF + chmod +x "${wrapper}" + info "Wrapper created at ${wrapper}" + + # Create workload directory + mkdir -p "${WORKLOAD_DIR}" + info "Workload directory: ${WORKLOAD_DIR}" + + echo "" + info "Setup complete! You can now use:" + echo " ck-rocprof run " else - echo "Creating Python virtual environment at ${VENV_PATH}..." - docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv venv '${VENV_PATH}'" - echo "* Virtual environment created" - fi + # ====== DOCKER MODE SETUP ====== + info "Running in DOCKER mode" + + # Ensure Docker container is running + echo "Ensuring Docker container '${CONTAINER_NAME}' is running..." + ensure_container_running "${CONTAINER_NAME}" "${SCRIPT_DIR}" + + # Check if rocprofiler-compute is installed, install if not + echo "Checking for rocprofiler-compute package..." + if ! docker exec "${CONTAINER_NAME}" test -f "${ROCPROF_BIN}" 2>/dev/null; then + echo "Installing rocprofiler-compute package..." + docker exec "${CONTAINER_NAME}" bash -c "apt update && apt install -y rocprofiler-compute" 2>&1 | tail -5 + echo "* rocprofiler-compute installed" + else + echo "* rocprofiler-compute already installed" + fi - # Install dependencies in container using uv - echo "Installing rocprofiler-compute dependencies..." - docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv pip install --python '${VENV_PATH}/bin/python' -r '${ROCM_REQUIREMENTS}'" - # Pin pandas to <3.0 to avoid CSV conversion bug in rocprof-compute - docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv pip install --python '${VENV_PATH}/bin/python' 'pandas<3.0'" - echo "* Dependencies installed" + # Verify rocprof-compute exists after installation + if ! docker exec "${CONTAINER_NAME}" test -f "${ROCPROF_BIN}" 2>/dev/null; then + error "rocprof-compute not found at ${ROCPROF_BIN} after installation" + echo "Please check ROCm installation in container" + return 1 + fi - # Create wrapper script in container - local wrapper=$(get_rocprof_wrapper) - docker exec "${CONTAINER_NAME}" bash -c "cat > '${wrapper}' << 'WRAPPER_EOF' + # Check if requirements file exists + if ! docker exec "${CONTAINER_NAME}" test -f "${ROCM_REQUIREMENTS}" 2>/dev/null; then + error "rocprofiler-compute requirements.txt not found at ${ROCM_REQUIREMENTS}" + echo "Please ensure ROCm is properly installed in container" + return 1 + fi + + # Install uv if not present + echo "Checking for uv package manager..." + if ! docker exec "${CONTAINER_NAME}" command -v uv &>/dev/null; then + echo "Installing uv package manager..." + docker exec "${CONTAINER_NAME}" bash -c "curl -LsSf https://astral.sh/uv/install.sh | sh" 2>&1 | tail -3 + echo "* uv installed" + else + echo "* uv already installed" + fi + + # Create Python venv in container using uv + if docker exec "${CONTAINER_NAME}" test -d "${VENV_PATH}" 2>/dev/null; then + echo "Python venv already exists at ${VENV_PATH}" + else + echo "Creating Python virtual environment at ${VENV_PATH}..." + docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv venv '${VENV_PATH}'" + echo "* Virtual environment created" + fi + + # Install dependencies in container using uv + echo "Installing rocprofiler-compute dependencies..." + docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv pip install --python '${VENV_PATH}/bin/python' -r '${ROCM_REQUIREMENTS}'" + # Pin pandas to <3.0 to avoid CSV conversion bug in rocprof-compute + docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv pip install --python '${VENV_PATH}/bin/python' 'pandas<3.0'" + echo "* Dependencies installed" + + # Create wrapper script in container + local wrapper=$(get_rocprof_wrapper) + docker exec "${CONTAINER_NAME}" bash -c "cat > '${wrapper}' << 'WRAPPER_EOF' #!/bin/bash # rocprof-compute wrapper using venv Python VENV_DIR=\"\$(cd \"\$(dirname \"\$0\")/..\" && pwd)\" exec \"\${VENV_DIR}/bin/python\" /opt/rocm-7.0.1/bin/rocprof-compute \"\$@\" WRAPPER_EOF" - docker exec "${CONTAINER_NAME}" chmod +x "${wrapper}" - echo "* Wrapper script created at ${wrapper}" + docker exec "${CONTAINER_NAME}" chmod +x "${wrapper}" + echo "* Wrapper script created at ${wrapper}" - echo "" - echo "Setup complete! You can now use:" - echo " ck-rocprof run " + echo "" + echo "Setup complete! You can now use:" + echo " ck-rocprof run " + fi } -# Detect GPU architecture (inside container) +# Detect GPU architecture # rocprof-compute uses marketing names (MI350, MI300X, etc.) for directory naming # This function attempts to match that convention detect_gpu_arch() { - if [ -n "${GPU_TARGET:-}" ]; then - echo "${GPU_TARGET}" + # Allow override via environment variable + if [ -n "${CK_GPU_TARGET:-}" ]; then + echo "${CK_GPU_TARGET}" return 0 fi - # Try to get marketing name from rocminfo in container - local marketing_name=$(docker exec "${CONTAINER_NAME}" bash -c "rocminfo 2>/dev/null | grep 'Marketing Name:' | grep -oP 'MI\d+[A-Z]*' | head -1" 2>/dev/null || echo "") - if [ -n "$marketing_name" ]; then - echo "$marketing_name" - return 0 - fi + if is_native_mode; then + # ====== NATIVE MODE ====== + if command -v rocminfo &>/dev/null; then + # Try marketing name first (MI350, MI300X) + local marketing_name=$(rocminfo 2>/dev/null | grep 'Marketing Name:' | grep -oE 'MI[0-9]+[A-Z]*' | head -1) + if [ -n "$marketing_name" ]; then + echo "$marketing_name" + return 0 + fi + + # Fallback to gfx name + local gfx_name=$(rocminfo 2>/dev/null | grep -oE 'gfx[0-9a-z]+' | head -1) + if [ -n "$gfx_name" ]; then + echo "$gfx_name" + return 0 + fi + fi - # Fallback: Try gfx name from rocminfo in container - local gfx_name=$(docker exec "${CONTAINER_NAME}" bash -c "rocminfo 2>/dev/null | grep -oP 'gfx[0-9a-z]+' | head -1" 2>/dev/null || echo "") - if [ -n "$gfx_name" ]; then - echo "$gfx_name" - return 0 - fi + # Try existing workload directories + if [ -d "${WORKLOAD_DIR}" ]; then + local first_dir=$(find "${WORKLOAD_DIR}" -maxdepth 2 -type d \( -name 'gfx*' -o -name 'MI*' \) 2>/dev/null | head -1) + if [ -n "$first_dir" ]; then + basename "$first_dir" + return 0 + fi + fi + else + # ====== DOCKER MODE ====== + # Try to get marketing name from rocminfo in container + local marketing_name=$(docker exec "${CONTAINER_NAME}" bash -c "rocminfo 2>/dev/null | grep 'Marketing Name:' | grep -oE 'MI[0-9]+[A-Z]*' | head -1" 2>/dev/null || echo "") + if [ -n "$marketing_name" ]; then + echo "$marketing_name" + return 0 + fi - # Fallback: try to detect from existing workload dirs in container - local first_dir=$(docker exec "${CONTAINER_NAME}" bash -c "find '${WORKLOAD_DIR}' -maxdepth 2 -type d \( -name 'gfx*' -o -name 'MI*' \) 2>/dev/null | head -1" 2>/dev/null || echo "") - if [ -n "$first_dir" ]; then - basename "$first_dir" - return 0 + # Fallback: Try gfx name from rocminfo in container + local gfx_name=$(docker exec "${CONTAINER_NAME}" bash -c "rocminfo 2>/dev/null | grep -oE 'gfx[0-9a-z]+' | head -1" 2>/dev/null || echo "") + if [ -n "$gfx_name" ]; then + echo "$gfx_name" + return 0 + fi + + # Fallback: try to detect from existing workload dirs in container + local first_dir=$(docker exec "${CONTAINER_NAME}" bash -c "find '${WORKLOAD_DIR}' -maxdepth 2 -type d \( -name 'gfx*' -o -name 'MI*' \) 2>/dev/null | head -1" 2>/dev/null || echo "") + if [ -n "$first_dir" ]; then + basename "$first_dir" + return 0 + fi fi # Final fallback echo "MI350" } -# Run profiling (inside container) +# Run profiling cmd_run() { local name="$1" local executable="$2" @@ -196,21 +485,21 @@ cmd_run() { local -a exe_args=("$@") if [ -z "$name" ] || [ -z "$executable" ]; then - echo "Error: name and executable required" + error "name and executable required" echo "Usage: ck-rocprof run [args]" return 1 fi # Check setup if ! is_setup_complete; then - echo "Error: Profiling environment not set up" + error "Profiling environment not set up" echo "Run: ck-rocprof setup" return 1 fi - # Check if executable exists in container - if ! docker exec "${CONTAINER_NAME}" test -f "$executable" 2>/dev/null; then - echo "Error: Executable not found in container: $executable" + # Check if executable exists + if ! file_exists "$executable"; then + error "Executable not found: $executable" return 1 fi @@ -220,39 +509,49 @@ cmd_run() { echo "Profiling: $executable ${exe_args[*]}" echo "Run name: $name" echo "GPU arch: $gpu_arch" - echo "Container: $CONTAINER_NAME" + if is_native_mode; then + echo "Mode: NATIVE" + else + echo "Container: $CONTAINER_NAME" + fi echo "===========================================" - # Build command for container execution + # Build command # --no-roof skips roofline analysis to speed up profiling - local cmd="${wrapper} profile --no-roof --name ${name} -- ${executable}" + # --path sets the full output path for profiling results (includes name) + local cmd="${wrapper} profile --no-roof --path ${WORKLOAD_DIR}/${name} --name ${name} -- ${executable}" for arg in "${exe_args[@]}"; do cmd="${cmd} $(printf '%q' "$arg")" done - # Run profiling in container - docker exec "${CONTAINER_NAME}" bash -c "${cmd}" + # Run profiling + exec_cmd "${cmd}" echo "" - echo "* Profiling complete" + info "Profiling complete" echo "Results saved to: ${WORKLOAD_DIR}/${name}/${gpu_arch}/" echo "" echo "Analyze with: ck-rocprof analyze ${name}" } -# Find actual GPU architecture directory for a given run name (inside container) +# Find actual GPU architecture directory for a given run name # rocprof-compute creates directories with various naming schemes (MI350, gfx950, etc.) # This function finds whatever directory actually exists find_workload_path() { local name="$1" local run_dir="${WORKLOAD_DIR}/${name}" - if ! docker exec "${CONTAINER_NAME}" test -d "$run_dir" 2>/dev/null; then + if ! dir_exists "$run_dir"; then return 1 fi # Find first subdirectory (should be GPU architecture) - local gpu_dir=$(docker exec "${CONTAINER_NAME}" bash -c "find '$run_dir' -maxdepth 1 -mindepth 1 -type d 2>/dev/null | head -1" 2>/dev/null) + local gpu_dir + if is_native_mode; then + gpu_dir=$(find "$run_dir" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | head -1) + else + gpu_dir=$(docker exec "${CONTAINER_NAME}" bash -c "find '$run_dir' -maxdepth 1 -mindepth 1 -type d 2>/dev/null | head -1" 2>/dev/null) + fi if [ -n "$gpu_dir" ]; then echo "$gpu_dir" @@ -262,20 +561,20 @@ find_workload_path() { return 1 } -# Analyze profiling results (inside container) +# Analyze profiling results cmd_analyze() { local name="$1" local block="${2:-12}" # Default to block 12 (LDS metrics) if [ -z "$name" ]; then - echo "Error: name required" + error "name required" echo "Usage: ck-rocprof analyze [block]" return 1 fi # Check setup if ! is_setup_complete; then - echo "Error: Profiling environment not set up" + error "Profiling environment not set up" echo "Run: ck-rocprof setup" return 1 fi @@ -284,7 +583,7 @@ cmd_analyze() { local workload_path=$(find_workload_path "${name}") if [ -z "$workload_path" ]; then - echo "Error: Profiling results not found for '${name}'" + error "Profiling results not found for '${name}'" echo "" echo "Available runs:" cmd_list @@ -296,17 +595,21 @@ cmd_analyze() { echo "===========================================" echo "" - docker exec "${CONTAINER_NAME}" "${wrapper}" analyze --path "${workload_path}" --block "${block}" + if is_native_mode; then + "${wrapper}" analyze --path "${workload_path}" --block "${block}" + else + docker exec "${CONTAINER_NAME}" "${wrapper}" analyze --path "${workload_path}" --block "${block}" + fi } -# Extract key LDS metrics from profiling results (inside container) +# Extract key LDS metrics from profiling results extract_lds_metrics() { local name="$1" local gpu_arch=$(detect_gpu_arch) local workload_path="${WORKLOAD_DIR}/${name}/${gpu_arch}" local csv_file="${workload_path}/SQ_INST_LEVEL_LDS.csv" - if ! docker exec "${CONTAINER_NAME}" test -f "$csv_file" 2>/dev/null; then + if ! file_exists "$csv_file"; then echo "N/A (CSV not found)" return 1 fi @@ -322,14 +625,14 @@ cmd_compare() { local name2="$2" if [ -z "$name1" ] || [ -z "$name2" ]; then - echo "Error: two run names required" + error "two run names required" echo "Usage: ck-rocprof compare " return 1 fi # Check setup if ! is_setup_complete; then - echo "Error: Profiling environment not set up" + error "Profiling environment not set up" echo "Run: ck-rocprof setup" return 1 fi @@ -339,12 +642,12 @@ cmd_compare() { local path2=$(find_workload_path "${name2}") if [ -z "$path1" ]; then - echo "Error: Profiling results not found for '${name1}'" + error "Profiling results not found for '${name1}'" return 1 fi if [ -z "$path2" ]; then - echo "Error: Profiling results not found for '${name2}'" + error "Profiling results not found for '${name2}'" return 1 fi @@ -371,14 +674,19 @@ cmd_compare() { echo " ck-rocprof analyze ${name2} 12" } -# List available profiling runs (inside container) +# List available profiling runs cmd_list() { - if ! docker exec "${CONTAINER_NAME}" test -d "${WORKLOAD_DIR}" 2>/dev/null; then + if ! dir_exists "${WORKLOAD_DIR}"; then echo "No profiling runs found (workload directory doesn't exist)" return 0 fi - local runs=$(docker exec "${CONTAINER_NAME}" bash -c "find '${WORKLOAD_DIR}' -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort" 2>/dev/null) + local runs + if is_native_mode; then + runs=$(find "${WORKLOAD_DIR}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort) + else + runs=$(docker exec "${CONTAINER_NAME}" bash -c "find '${WORKLOAD_DIR}' -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort" 2>/dev/null) + fi if [ -z "$runs" ]; then echo "No profiling runs found in ${WORKLOAD_DIR}" @@ -393,8 +701,8 @@ cmd_list() { if [ -n "$path" ]; then local gpu_arch=$(basename "$path") - local size=$(docker exec "${CONTAINER_NAME}" bash -c "du -sh '$path' 2>/dev/null | cut -f1" 2>/dev/null) - local date=$(docker exec "${CONTAINER_NAME}" bash -c "stat -c %y '$path' 2>/dev/null | cut -d' ' -f1" 2>/dev/null) + local size=$(get_size "$path") + local date=$(get_date "$path") printf " %-25s [%-8s, %s, %s]\n" "$run" "$gpu_arch" "$size" "$date" else printf " %-25s [no data]\n" "$run" From 5f2a1b733e6d7a8eb856980b42d44d3e51f7912c Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Wed, 28 Jan 2026 17:10:18 -0500 Subject: [PATCH 06/10] Add clean/status commands and improve ck-rocprof robustness - Add 'clean' command to remove profiling runs (supports --all) - Add 'status' command to show configuration and environment info - Add workload name validation to prevent path traversal attacks - Fix uv installation to use pip instead of curl for reliability - Add cross-platform stat support for macOS compatibility - Consolidate ROCPROF_CANDIDATES to avoid code duplication - Expand help documentation with all profiling block descriptions - Fix Docker wrapper script escaping issues Co-Authored-By: Claude --- script/tools/ck-rocprof | 262 +++++++++++++++++++++++++++++----------- 1 file changed, 190 insertions(+), 72 deletions(-) diff --git a/script/tools/ck-rocprof b/script/tools/ck-rocprof index cdae7778a4d..f8f7d48ba4e 100755 --- a/script/tools/ck-rocprof +++ b/script/tools/ck-rocprof @@ -19,6 +19,17 @@ CONTAINER_NAME=$(get_container_name "${PROJECT_ROOT}") # Execution mode detection # ============================================================================ +# Common rocprof-compute binary locations (shared between detection functions) +# Order: user installs first, then system ROCm versions (newest first) +ROCPROF_CANDIDATES=( + "${HOME}/.local/rocprofiler-compute/3.4.0/bin/rocprof-compute" + "/opt/rocm/bin/rocprof-compute" + "/opt/rocm-7.2.0/bin/rocprof-compute" + "/opt/rocm-7.0.1/bin/rocprof-compute" + "/opt/rocm-6.2.0/bin/rocprof-compute" + "/opt/rocm-6.1.0/bin/rocprof-compute" +) + # Determine if running in native mode (no Docker) # Native mode is used when: # 1. CK_FORCE_DOCKER is not set to 1 @@ -35,16 +46,7 @@ is_native_mode() { fi # Check common ROCm locations and user installations - local candidates=( - "${HOME}/.local/rocprofiler-compute/3.4.0/bin/rocprof-compute" - "/opt/rocm/bin/rocprof-compute" - "/opt/rocm-7.2.0/bin/rocprof-compute" - "/opt/rocm-7.0.1/bin/rocprof-compute" - "/opt/rocm-6.2.0/bin/rocprof-compute" - "/opt/rocm-6.1.0/bin/rocprof-compute" - ) - - for bin in "${candidates[@]}"; do + for bin in "${ROCPROF_CANDIDATES[@]}"; do if [ -f "$bin" ]; then return 0 fi @@ -69,16 +71,7 @@ find_rocprof_bin() { fi # Check common ROCm locations and user installations - local candidates=( - "${HOME}/.local/rocprofiler-compute/3.4.0/bin/rocprof-compute" - "/opt/rocm/bin/rocprof-compute" - "/opt/rocm-7.2.0/bin/rocprof-compute" - "/opt/rocm-7.0.1/bin/rocprof-compute" - "/opt/rocm-6.2.0/bin/rocprof-compute" - "/opt/rocm-6.1.0/bin/rocprof-compute" - ) - - for bin in "${candidates[@]}"; do + for bin in "${ROCPROF_CANDIDATES[@]}"; do if [ -f "$bin" ]; then echo "$bin" return 0 @@ -158,16 +151,6 @@ dir_exists() { fi } -# Check if command exists in execution environment -cmd_exists() { - local cmd="$1" - if is_native_mode; then - command -v "$cmd" &>/dev/null - else - docker exec "${CONTAINER_NAME}" command -v "$cmd" &>/dev/null - fi -} - # Get file/directory size get_size() { local path="$1" @@ -178,12 +161,18 @@ get_size() { fi } -# Get file modification date +# Get file modification date (cross-platform: Linux and macOS) get_date() { local path="$1" if is_native_mode; then - stat -c %y "$path" 2>/dev/null | cut -d' ' -f1 + # Try GNU stat first (Linux), fall back to BSD stat (macOS) + if stat --version &>/dev/null 2>&1; then + stat -c %y "$path" 2>/dev/null | cut -d' ' -f1 + else + stat -f %Sm -t %Y-%m-%d "$path" 2>/dev/null + fi else + # Docker containers typically run Linux docker exec "${CONTAINER_NAME}" bash -c "stat -c %y '$path' 2>/dev/null | cut -d' ' -f1" 2>/dev/null fi } @@ -210,6 +199,8 @@ Commands: analyze [block] Analyze profiling results (default: block 12 - LDS metrics) compare Compare two profiling runs list List available profiling runs + clean Remove a profiling run (use --all for all runs) + status Show current configuration and status help Show this help message Examples: @@ -219,6 +210,8 @@ Examples: ck-rocprof analyze baseline 12 ck-rocprof compare baseline optimized ck-rocprof list + ck-rocprof clean baseline + ck-rocprof status Execution Modes: NATIVE: Runs directly on ROCm host (auto-detected if rocprof-compute in PATH) @@ -233,15 +226,25 @@ Environment Variables: CK_WORKLOAD_DIR - Workload storage directory CK_CONTAINER_NAME - Docker container name (Docker mode only) -Profiling Metrics: - Block 12 (LDS - Local Data Share): - - 12.1.3: Bank Conflict Rate (% of peak) - - 12.2.9: Bank Conflicts/Access (conflicts/access) - - 12.2.12: Bank Conflict (cycles per kernel) - - 12.2.17: LDS Data FIFO Full Rate (cycles) +Profiling Blocks (use with 'analyze '): + Block 2: System Speed-of-Light (SOL) + Block 6: Shader Engine (SE) utilization + Block 7: L2 Cache metrics + Block 11: Vector L1D Cache metrics + Block 12: LDS (Local Data Share) - DEFAULT + Block 16: Instruction mix statistics + Block 17: Compute Unit (CU) metrics + +LDS Metrics (Block 12): + - 12.1.3: Bank Conflict Rate (% of peak) + - 12.2.9: Bank Conflicts/Access (conflicts/access) + - 12.2.12: Bank Conflict (cycles per kernel) + - 12.2.17: LDS Data FIFO Full Rate (cycles) Notes: - - Results stored in workloads/// + - Workload names must be alphanumeric with hyphens/underscores only + - Profiling skips roofline analysis (--no-roof) for faster execution + - Results stored in workloads// - Use 'analyze' to view detailed metrics for specific blocks - Use 'compare' to see side-by-side metrics from two runs EOF @@ -252,6 +255,23 @@ get_rocprof_wrapper() { echo "${VENV_PATH}/bin/rocprof-compute" } +# Validate workload name to prevent path traversal and shell injection +# Allowed: alphanumeric, hyphens, underscores +validate_workload_name() { + local name="$1" + if [[ ! "$name" =~ ^[a-zA-Z0-9_-]+$ ]]; then + error "Invalid workload name: '$name'" + echo "Names must contain only letters, numbers, hyphens, and underscores" + return 1 + fi + # Prevent reserved names + if [[ "$name" == "." || "$name" == ".." ]]; then + error "Invalid workload name: '$name'" + return 1 + fi + return 0 +} + # Check if setup is complete is_setup_complete() { local wrapper=$(get_rocprof_wrapper) @@ -300,10 +320,10 @@ cmd_setup() { # Install uv if needed if ! command -v uv &>/dev/null; then - info "Installing uv package manager..." - curl -LsSf https://astral.sh/uv/install.sh | sh - # shellcheck disable=SC1090 - source ~/.local/bin/env 2>/dev/null || true + info "Installing uv package manager via pip..." + python3 -m pip install --user uv + # Add user bin to PATH if not already there + export PATH="${HOME}/.local/bin:${PATH}" fi # Create venv @@ -373,8 +393,8 @@ WRAPPER_EOF # Install uv if not present echo "Checking for uv package manager..." if ! docker exec "${CONTAINER_NAME}" command -v uv &>/dev/null; then - echo "Installing uv package manager..." - docker exec "${CONTAINER_NAME}" bash -c "curl -LsSf https://astral.sh/uv/install.sh | sh" 2>&1 | tail -3 + echo "Installing uv package manager via pip..." + docker exec "${CONTAINER_NAME}" bash -c "python3 -m pip install --user uv && export PATH=\${HOME}/.local/bin:\${PATH}" 2>&1 | tail -3 echo "* uv installed" else echo "* uv already installed" @@ -385,24 +405,24 @@ WRAPPER_EOF echo "Python venv already exists at ${VENV_PATH}" else echo "Creating Python virtual environment at ${VENV_PATH}..." - docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv venv '${VENV_PATH}'" + docker exec "${CONTAINER_NAME}" bash -c "export PATH=\${HOME}/.local/bin:\${PATH} && uv venv '${VENV_PATH}'" echo "* Virtual environment created" fi # Install dependencies in container using uv echo "Installing rocprofiler-compute dependencies..." - docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv pip install --python '${VENV_PATH}/bin/python' -r '${ROCM_REQUIREMENTS}'" + docker exec "${CONTAINER_NAME}" bash -c "export PATH=\${HOME}/.local/bin:\${PATH} && uv pip install --python '${VENV_PATH}/bin/python' -r '${ROCM_REQUIREMENTS}'" # Pin pandas to <3.0 to avoid CSV conversion bug in rocprof-compute - docker exec "${CONTAINER_NAME}" bash -c "source ~/.local/bin/env && uv pip install --python '${VENV_PATH}/bin/python' 'pandas<3.0'" + docker exec "${CONTAINER_NAME}" bash -c "export PATH=\${HOME}/.local/bin:\${PATH} && uv pip install --python '${VENV_PATH}/bin/python' 'pandas<3.0'" echo "* Dependencies installed" # Create wrapper script in container local wrapper=$(get_rocprof_wrapper) - docker exec "${CONTAINER_NAME}" bash -c "cat > '${wrapper}' << 'WRAPPER_EOF' + docker exec "${CONTAINER_NAME}" bash -c "cat > '${wrapper}' << WRAPPER_EOF #!/bin/bash # rocprof-compute wrapper using venv Python -VENV_DIR=\"\$(cd \"\$(dirname \"\$0\")/..\" && pwd)\" -exec \"\${VENV_DIR}/bin/python\" /opt/rocm-7.0.1/bin/rocprof-compute \"\$@\" +VENV_DIR=\"\\\$(cd \"\\\$(dirname \"\\\$0\")/..\" && pwd)\" +exec \"\\\${VENV_DIR}/bin/python\" \"${ROCPROF_BIN}\" \"\\\$@\" WRAPPER_EOF" docker exec "${CONTAINER_NAME}" chmod +x "${wrapper}" echo "* Wrapper script created at ${wrapper}" @@ -479,14 +499,20 @@ detect_gpu_arch() { # Run profiling cmd_run() { + # Validate argument count before shifting + if [ $# -lt 2 ]; then + error "name and executable required" + echo "Usage: ck-rocprof run [args]" + return 1 + fi + local name="$1" local executable="$2" shift 2 local -a exe_args=("$@") - if [ -z "$name" ] || [ -z "$executable" ]; then - error "name and executable required" - echo "Usage: ck-rocprof run [args]" + # Validate workload name (prevents path traversal) + if ! validate_workload_name "$name"; then return 1 fi @@ -529,7 +555,7 @@ cmd_run() { echo "" info "Profiling complete" - echo "Results saved to: ${WORKLOAD_DIR}/${name}/${gpu_arch}/" + echo "Results saved to: ${WORKLOAD_DIR}/${name}/" echo "" echo "Analyze with: ck-rocprof analyze ${name}" } @@ -602,23 +628,6 @@ cmd_analyze() { fi } -# Extract key LDS metrics from profiling results -extract_lds_metrics() { - local name="$1" - local gpu_arch=$(detect_gpu_arch) - local workload_path="${WORKLOAD_DIR}/${name}/${gpu_arch}" - local csv_file="${workload_path}/SQ_INST_LEVEL_LDS.csv" - - if ! file_exists "$csv_file"; then - echo "N/A (CSV not found)" - return 1 - fi - - # Extract key metrics (simplified - would need proper CSV parsing) - # For now, just indicate that raw data is available - echo "Raw data available at: ${csv_file}" -} - # Compare two profiling runs cmd_compare() { local name1="$1" @@ -713,6 +722,108 @@ cmd_list() { echo "Analyze with: ck-rocprof analyze " } +# Clean (remove) profiling runs +cmd_clean() { + local name="${1:-}" + + if [ -z "$name" ]; then + error "name required (or use --all to remove all runs)" + echo "Usage: ck-rocprof clean " + echo " ck-rocprof clean --all" + return 1 + fi + + if [ "$name" = "--all" ]; then + # Remove all profiling runs + if ! dir_exists "${WORKLOAD_DIR}"; then + echo "No profiling runs to clean" + return 0 + fi + + echo "This will remove ALL profiling runs in ${WORKLOAD_DIR}" + read -r -p "Are you sure? [y/N] " confirm + if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Cancelled" + return 0 + fi + + if is_native_mode; then + rm -rf "${WORKLOAD_DIR:?}"/* + else + docker exec "${CONTAINER_NAME}" bash -c "rm -rf '${WORKLOAD_DIR:?}'/*" + fi + info "All profiling runs removed" + else + # Validate name + if ! validate_workload_name "$name"; then + return 1 + fi + + local run_dir="${WORKLOAD_DIR}/${name}" + if ! dir_exists "$run_dir"; then + error "Profiling run not found: ${name}" + return 1 + fi + + if is_native_mode; then + rm -rf "${run_dir}" + else + docker exec "${CONTAINER_NAME}" bash -c "rm -rf '${run_dir}'" + fi + info "Removed profiling run: ${name}" + fi +} + +# Show status information +cmd_status() { + echo "CK ROCProf Status" + echo "===========================================" + echo "" + + # Mode + if is_native_mode; then + echo "Mode: NATIVE" + echo "rocprof-compute: ${ROCPROF_BIN:-not found}" + else + echo "Mode: DOCKER" + echo "Container: ${CONTAINER_NAME}" + if container_is_running "${CONTAINER_NAME}"; then + echo "Container status: running" + else + echo "Container status: not running" + fi + fi + echo "" + + # Paths + echo "Paths:" + echo " Venv: ${VENV_PATH}" + echo " Workloads: ${WORKLOAD_DIR}" + echo "" + + # Setup status + echo "Setup status:" + if is_setup_complete; then + echo " Profiling environment: ready" + else + echo " Profiling environment: not configured (run 'ck-rocprof setup')" + fi + echo "" + + # Workload count + if dir_exists "${WORKLOAD_DIR}"; then + local count + if is_native_mode; then + count=$(find "${WORKLOAD_DIR}" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l) + else + count=$(docker exec "${CONTAINER_NAME}" bash -c "find '${WORKLOAD_DIR}' -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l" 2>/dev/null) + fi + echo "Profiling runs: ${count}" + else + echo "Profiling runs: 0" + fi +} + # Main command dispatcher case "${1:-}" in setup) @@ -733,6 +844,13 @@ case "${1:-}" in list) cmd_list ;; + clean) + shift + cmd_clean "$@" + ;; + status) + cmd_status + ;; help|--help|-h) show_help ;; From 7fbb96f27f9a916757feb2e8310359a200a8b306 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Wed, 28 Jan 2026 17:16:27 -0500 Subject: [PATCH 07/10] Fix analyze command to use correct workload path rocprof-compute stores results directly in the workload directory (pmc_perf.csv) rather than in a GPU architecture subdirectory. Updated find_workload_path to detect this correctly. Co-Authored-By: Claude --- script/tools/ck-rocprof | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/script/tools/ck-rocprof b/script/tools/ck-rocprof index f8f7d48ba4e..da2382ae878 100755 --- a/script/tools/ck-rocprof +++ b/script/tools/ck-rocprof @@ -560,9 +560,8 @@ cmd_run() { echo "Analyze with: ck-rocprof analyze ${name}" } -# Find actual GPU architecture directory for a given run name -# rocprof-compute creates directories with various naming schemes (MI350, gfx950, etc.) -# This function finds whatever directory actually exists +# Find workload path for a given run name +# rocprof-compute stores results directly in the workload directory (pmc_perf.csv) find_workload_path() { local name="$1" local run_dir="${WORKLOAD_DIR}/${name}" @@ -571,16 +570,9 @@ find_workload_path() { return 1 fi - # Find first subdirectory (should be GPU architecture) - local gpu_dir - if is_native_mode; then - gpu_dir=$(find "$run_dir" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | head -1) - else - gpu_dir=$(docker exec "${CONTAINER_NAME}" bash -c "find '$run_dir' -maxdepth 1 -mindepth 1 -type d 2>/dev/null | head -1" 2>/dev/null) - fi - - if [ -n "$gpu_dir" ]; then - echo "$gpu_dir" + # Check if profiling data exists directly in run directory (current rocprof-compute behavior) + if file_exists "${run_dir}/pmc_perf.csv"; then + echo "$run_dir" return 0 fi @@ -616,8 +608,7 @@ cmd_analyze() { return 1 fi - local gpu_arch=$(basename "$workload_path") - echo "Analyzing: ${name} (GPU: ${gpu_arch}, Block ${block})" + echo "Analyzing: ${name} (Block ${block})" echo "===========================================" echo "" @@ -660,12 +651,9 @@ cmd_compare() { return 1 fi - local gpu1=$(basename "$path1") - local gpu2=$(basename "$path2") - echo "Comparing profiling runs:" - echo " Baseline: ${name1} (${gpu1})" - echo " Optimized: ${name2} (${gpu2})" + echo " Baseline: ${name1}" + echo " Optimized: ${name2}" echo "===========================================" echo "" @@ -709,10 +697,9 @@ cmd_list() { local path=$(find_workload_path "$run") if [ -n "$path" ]; then - local gpu_arch=$(basename "$path") local size=$(get_size "$path") local date=$(get_date "$path") - printf " %-25s [%-8s, %s, %s]\n" "$run" "$gpu_arch" "$size" "$date" + printf " %-25s [%s, %s]\n" "$run" "$size" "$date" else printf " %-25s [no data]\n" "$run" fi From 409c5920fd6f73bcb040949cadac461e2ad21024 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Wed, 28 Jan 2026 17:27:38 -0500 Subject: [PATCH 08/10] Address PR review security and robustness issues Security fixes: - Escape executable path in cmd_run to prevent shell injection - Add workload name validation to cmd_analyze and cmd_compare Robustness improvements: - Add error checking for uv package manager installation - Use consistent project root detection (find_project_root || get_project_root) - Use /opt/rocm instead of hardcoded /opt/rocm-7.0.1 in Docker mode - Derive ROCM_REQUIREMENTS path from ROCPROF_BIN for flexibility - Use gfx950 as fallback GPU consistent with common.sh Documentation updates: - Fix env var name GPU_TARGET -> CK_GPU_TARGET - Update storage layout to reflect current structure (workloads//) - Document clean and status commands - Clarify native vs Docker default paths Co-Authored-By: Claude --- script/tools/ck-rocprof | 55 +++++++++++++++++++++++++++++++------- script/tools/ck-rocprof.md | 25 ++++++++++------- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/script/tools/ck-rocprof b/script/tools/ck-rocprof index da2382ae878..d20676cb682 100755 --- a/script/tools/ck-rocprof +++ b/script/tools/ck-rocprof @@ -12,7 +12,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/common.sh" # Initialize configuration -PROJECT_ROOT=$(get_project_root "${SCRIPT_DIR}") +PROJECT_ROOT=$(find_project_root "${SCRIPT_DIR}" || get_project_root "${SCRIPT_DIR}") CONTAINER_NAME=$(get_container_name "${PROJECT_ROOT}") # ============================================================================ @@ -112,10 +112,12 @@ if is_native_mode; then ROCM_REQUIREMENTS="${CK_ROCM_REQUIREMENTS:-$(find_rocm_requirements "${ROCPROF_BIN}")}" else # Docker mode configuration - ROCPROF_BIN="${CK_ROCPROF_BIN:-/opt/rocm-7.0.1/bin/rocprof-compute}" + # Use /opt/rocm (symlink to current version) as default, can be overridden + ROCPROF_BIN="${CK_ROCPROF_BIN:-/opt/rocm/bin/rocprof-compute}" VENV_PATH="${CK_PROFILE_VENV:-/opt/rocprof_venv}" WORKLOAD_DIR="${CK_WORKLOAD_DIR:-/workspace/workloads}" - ROCM_REQUIREMENTS="/opt/rocm-7.0.1/libexec/rocprofiler-compute/requirements.txt" + # Requirements path derived from ROCPROF_BIN location + ROCM_REQUIREMENTS="${CK_ROCM_REQUIREMENTS:-$(dirname "$(dirname "${ROCPROF_BIN}")")/libexec/rocprofiler-compute/requirements.txt}" fi # ============================================================================ @@ -321,9 +323,18 @@ cmd_setup() { # Install uv if needed if ! command -v uv &>/dev/null; then info "Installing uv package manager via pip..." - python3 -m pip install --user uv + if ! python3 -m pip install --user uv; then + error "Failed to install uv package manager" + return 1 + fi # Add user bin to PATH if not already there export PATH="${HOME}/.local/bin:${PATH}" + # Verify installation + if ! command -v uv &>/dev/null; then + error "uv installed but not found in PATH" + echo "Try adding ~/.local/bin to your PATH" + return 1 + fi fi # Create venv @@ -394,7 +405,15 @@ WRAPPER_EOF echo "Checking for uv package manager..." if ! docker exec "${CONTAINER_NAME}" command -v uv &>/dev/null; then echo "Installing uv package manager via pip..." - docker exec "${CONTAINER_NAME}" bash -c "python3 -m pip install --user uv && export PATH=\${HOME}/.local/bin:\${PATH}" 2>&1 | tail -3 + if ! docker exec "${CONTAINER_NAME}" bash -c "python3 -m pip install --user uv" 2>&1 | tail -3; then + error "Failed to install uv package manager in container" + return 1 + fi + # Verify installation + if ! docker exec "${CONTAINER_NAME}" bash -c "export PATH=\${HOME}/.local/bin:\${PATH} && command -v uv" &>/dev/null; then + error "uv installed but not found in container PATH" + return 1 + fi echo "* uv installed" else echo "* uv already installed" @@ -493,8 +512,8 @@ detect_gpu_arch() { fi fi - # Final fallback - echo "MI350" + # Final fallback - use gfx950 consistent with common.sh + echo "gfx950" } # Run profiling @@ -542,10 +561,15 @@ cmd_run() { fi echo "===========================================" - # Build command + # Build command with proper escaping to prevent shell injection # --no-roof skips roofline analysis to speed up profiling # --path sets the full output path for profiling results (includes name) - local cmd="${wrapper} profile --no-roof --path ${WORKLOAD_DIR}/${name} --name ${name} -- ${executable}" + local escaped_executable + escaped_executable=$(printf '%q' "$executable") + local escaped_workload_dir + escaped_workload_dir=$(printf '%q' "${WORKLOAD_DIR}/${name}") + + local cmd="${wrapper} profile --no-roof --path ${escaped_workload_dir} --name ${name} -- ${escaped_executable}" for arg in "${exe_args[@]}"; do cmd="${cmd} $(printf '%q' "$arg")" done @@ -590,6 +614,11 @@ cmd_analyze() { return 1 fi + # Validate workload name (prevents path traversal) + if ! validate_workload_name "$name"; then + return 1 + fi + # Check setup if ! is_setup_complete; then error "Profiling environment not set up" @@ -630,6 +659,14 @@ cmd_compare() { return 1 fi + # Validate workload names (prevents path traversal) + if ! validate_workload_name "$name1"; then + return 1 + fi + if ! validate_workload_name "$name2"; then + return 1 + fi + # Check setup if ! is_setup_complete; then error "Profiling environment not set up" diff --git a/script/tools/ck-rocprof.md b/script/tools/ck-rocprof.md index b9dc858dfaf..e834c6758ca 100644 --- a/script/tools/ck-rocprof.md +++ b/script/tools/ck-rocprof.md @@ -54,6 +54,12 @@ Side-by-side comparison of two runs. ### `list` List all profiling runs with size and date. +### `clean ` / `clean --all` +Remove profiling runs. Use `--all` to remove all runs. + +### `status` +Show current configuration: mode (native/Docker), paths, setup status. + ## Key LDS Metrics (Block 12) **Target Values:** @@ -91,10 +97,10 @@ ck-rocprof compare baseline optimized ## Environment Variables -- `CK_PROFILE_VENV`: Python venv path (default: `/opt/rocprof_venv`) -- `CK_ROCPROF_BIN`: rocprof-compute binary path -- `CK_WORKLOAD_DIR`: Results directory (default: `/workspace/build/workloads`) -- `GPU_TARGET`: Override GPU detection (e.g., `gfx950`, `gfx942`) +- `CK_PROFILE_VENV`: Python venv path (default: `$PROJECT/.ck-rocprof-venv` native, `/opt/rocprof_venv` Docker) +- `CK_ROCPROF_BIN`: rocprof-compute binary path (auto-detected from PATH or /opt/rocm) +- `CK_WORKLOAD_DIR`: Results directory (default: `$PROJECT/build/workloads` native, `/workspace/workloads` Docker) +- `CK_GPU_TARGET`: Override GPU detection (e.g., `gfx950`, `MI300X`) ## Interpreting Results @@ -129,15 +135,16 @@ ck-rocprof setup ```bash ck-rocprof list # Check available runs rocminfo | grep gfx # Verify GPU arch -export GPU_TARGET=gfx950 # Override if needed +export CK_GPU_TARGET=gfx950 # Override if needed ``` ## Storage Layout -Results stored in `/workspace/build/workloads///`: -- `SQ_INST_LEVEL_LDS.csv`: LDS metrics -- `pmc_perf.csv`: Performance counters -- `counter_collection.csv`: All metrics +Results stored in `workloads//`: +- `pmc_perf.csv`: Performance counters (primary data file) +- `perfmon/`: Input metric files +- `out/`: Raw output data from profiler runs +- `log.txt`: Profiling log ## Technical Details From a12e1868173eed007874580346c22457af2cdfbd Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Wed, 28 Jan 2026 17:38:00 -0500 Subject: [PATCH 09/10] Simplify ck-rocprof to native-only mode Remove Docker mode from ck-rocprof. Docker users should run the tool via `ck-docker exec ck-rocprof ...` instead. This simplification: - Removes ~210 lines of Docker-specific code - Eliminates mode detection complexity - Makes the script easier to maintain - Provides clearer error messages when rocprof-compute is not found The setup command now lists all searched locations when rocprof-compute is not found, helping users understand how to install it. Co-Authored-By: Claude --- script/tools/ck-rocprof | 503 +++++++++++-------------------------- script/tools/ck-rocprof.md | 9 +- 2 files changed, 151 insertions(+), 361 deletions(-) diff --git a/script/tools/ck-rocprof b/script/tools/ck-rocprof index d20676cb682..bb1076a35ad 100755 --- a/script/tools/ck-rocprof +++ b/script/tools/ck-rocprof @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT # CK ROCProf Tool - Profile CK applications with rocprof-compute +# Native-only tool. For Docker usage, run via: ck-docker exec ck-rocprof ... set -e set -o pipefail @@ -13,13 +14,12 @@ source "${SCRIPT_DIR}/common.sh" # Initialize configuration PROJECT_ROOT=$(find_project_root "${SCRIPT_DIR}" || get_project_root "${SCRIPT_DIR}") -CONTAINER_NAME=$(get_container_name "${PROJECT_ROOT}") # ============================================================================ -# Execution mode detection +# rocprof-compute detection # ============================================================================ -# Common rocprof-compute binary locations (shared between detection functions) +# Common rocprof-compute binary locations # Order: user installs first, then system ROCm versions (newest first) ROCPROF_CANDIDATES=( "${HOME}/.local/rocprofiler-compute/3.4.0/bin/rocprof-compute" @@ -30,33 +30,7 @@ ROCPROF_CANDIDATES=( "/opt/rocm-6.1.0/bin/rocprof-compute" ) -# Determine if running in native mode (no Docker) -# Native mode is used when: -# 1. CK_FORCE_DOCKER is not set to 1 -# 2. rocprof-compute is available locally -is_native_mode() { - # Force Docker mode if explicitly requested - if [ "${CK_FORCE_DOCKER:-0}" = "1" ]; then - return 1 - fi - - # Use native mode if rocprof-compute exists locally - if command -v rocprof-compute &>/dev/null; then - return 0 - fi - - # Check common ROCm locations and user installations - for bin in "${ROCPROF_CANDIDATES[@]}"; do - if [ -f "$bin" ]; then - return 0 - fi - done - - # Default to Docker mode - return 1 -} - -# Find rocprof-compute binary (native mode) +# Find rocprof-compute binary find_rocprof_bin() { # Check CK_ROCPROF_BIN first if [ -n "${CK_ROCPROF_BIN:-}" ] && [ -f "${CK_ROCPROF_BIN}" ]; then @@ -81,7 +55,7 @@ find_rocprof_bin() { return 1 } -# Find ROCm requirements file (native mode) +# Find ROCm requirements file find_rocm_requirements() { local rocprof_bin="${1:-$(find_rocprof_bin)}" if [ -z "$rocprof_bin" ]; then @@ -89,7 +63,8 @@ find_rocm_requirements() { fi # Requirements file is typically at ../libexec/rocprofiler-compute/requirements.txt - local rocm_dir=$(dirname $(dirname "$rocprof_bin")) + local rocm_dir + rocm_dir=$(dirname "$(dirname "$rocprof_bin")") local req_file="${rocm_dir}/libexec/rocprofiler-compute/requirements.txt" if [ -f "$req_file" ]; then @@ -101,98 +76,40 @@ find_rocm_requirements() { } # ============================================================================ -# Configuration (mode-dependent) +# Configuration # ============================================================================ -if is_native_mode; then - # Native mode configuration - ROCPROF_BIN="${CK_ROCPROF_BIN:-$(find_rocprof_bin)}" - VENV_PATH="${CK_PROFILE_VENV:-${PROJECT_ROOT}/.ck-rocprof-venv}" - WORKLOAD_DIR="${CK_WORKLOAD_DIR:-$(get_build_dir "${PROJECT_ROOT}")/workloads}" - ROCM_REQUIREMENTS="${CK_ROCM_REQUIREMENTS:-$(find_rocm_requirements "${ROCPROF_BIN}")}" -else - # Docker mode configuration - # Use /opt/rocm (symlink to current version) as default, can be overridden - ROCPROF_BIN="${CK_ROCPROF_BIN:-/opt/rocm/bin/rocprof-compute}" - VENV_PATH="${CK_PROFILE_VENV:-/opt/rocprof_venv}" - WORKLOAD_DIR="${CK_WORKLOAD_DIR:-/workspace/workloads}" - # Requirements path derived from ROCPROF_BIN location - ROCM_REQUIREMENTS="${CK_ROCM_REQUIREMENTS:-$(dirname "$(dirname "${ROCPROF_BIN}")")/libexec/rocprofiler-compute/requirements.txt}" -fi +ROCPROF_BIN="${CK_ROCPROF_BIN:-$(find_rocprof_bin || echo "")}" +VENV_PATH="${CK_PROFILE_VENV:-${PROJECT_ROOT}/.ck-rocprof-venv}" +WORKLOAD_DIR="${CK_WORKLOAD_DIR:-$(get_build_dir "${PROJECT_ROOT}")/workloads}" +ROCM_REQUIREMENTS="${CK_ROCM_REQUIREMENTS:-$(find_rocm_requirements "${ROCPROF_BIN}" || echo "")}" # ============================================================================ -# Command execution wrappers (abstract native vs Docker execution) +# Helper functions # ============================================================================ -# Execute command in appropriate environment -exec_cmd() { - if is_native_mode; then - bash -c "$*" - else - docker exec "${CONTAINER_NAME}" bash -c "$*" - fi -} - -# Check if file exists -file_exists() { - local filepath="$1" - if is_native_mode; then - [ -f "$filepath" ] - else - docker exec "${CONTAINER_NAME}" test -f "$filepath" 2>/dev/null - fi -} - -# Check if directory exists -dir_exists() { - local dirpath="$1" - if is_native_mode; then - [ -d "$dirpath" ] - else - docker exec "${CONTAINER_NAME}" test -d "$dirpath" 2>/dev/null - fi -} - # Get file/directory size get_size() { local path="$1" - if is_native_mode; then - du -sh "$path" 2>/dev/null | cut -f1 - else - docker exec "${CONTAINER_NAME}" bash -c "du -sh '$path' 2>/dev/null | cut -f1" 2>/dev/null - fi + du -sh "$path" 2>/dev/null | cut -f1 } # Get file modification date (cross-platform: Linux and macOS) get_date() { local path="$1" - if is_native_mode; then - # Try GNU stat first (Linux), fall back to BSD stat (macOS) - if stat --version &>/dev/null 2>&1; then - stat -c %y "$path" 2>/dev/null | cut -d' ' -f1 - else - stat -f %Sm -t %Y-%m-%d "$path" 2>/dev/null - fi + # Try GNU stat first (Linux), fall back to BSD stat (macOS) + if stat --version &>/dev/null 2>&1; then + stat -c %y "$path" 2>/dev/null | cut -d' ' -f1 else - # Docker containers typically run Linux - docker exec "${CONTAINER_NAME}" bash -c "stat -c %y '$path' 2>/dev/null | cut -d' ' -f1" 2>/dev/null + stat -f %Sm -t %Y-%m-%d "$path" 2>/dev/null fi } # Help message show_help() { - local mode_info - if is_native_mode; then - mode_info="NATIVE (rocprof-compute found locally)" - else - mode_info="DOCKER (container: ${CONTAINER_NAME})" - fi - cat << EOF CK ROCProf Tool - Profile CK applications with rocprof-compute -Current mode: ${mode_info} - Usage: ck-rocprof [options] Commands: @@ -215,18 +132,12 @@ Examples: ck-rocprof clean baseline ck-rocprof status -Execution Modes: - NATIVE: Runs directly on ROCm host (auto-detected if rocprof-compute in PATH) - DOCKER: Runs in container (auto-started if needed) - Environment Variables: - CK_FORCE_DOCKER - Set to 1 to force Docker mode CK_GPU_TARGET - Override GPU detection (e.g., gfx950, MI300X) - CK_PROFILE_VENV - Python venv path + CK_PROFILE_VENV - Python venv path (default: \$PROJECT/.ck-rocprof-venv) CK_ROCPROF_BIN - rocprof-compute binary path CK_ROCM_REQUIREMENTS - Path to rocprofiler-compute requirements.txt CK_WORKLOAD_DIR - Workload storage directory - CK_CONTAINER_NAME - Docker container name (Docker mode only) Profiling Blocks (use with 'analyze '): Block 2: System Speed-of-Light (SOL) @@ -247,8 +158,7 @@ Notes: - Workload names must be alphanumeric with hyphens/underscores only - Profiling skips roofline analysis (--no-roof) for faster execution - Results stored in workloads// - - Use 'analyze' to view detailed metrics for specific blocks - - Use 'compare' to see side-by-side metrics from two runs + - For Docker usage, run via: ck-docker exec ck-rocprof ... EOF } @@ -276,185 +186,106 @@ validate_workload_name() { # Check if setup is complete is_setup_complete() { - local wrapper=$(get_rocprof_wrapper) - - if is_native_mode; then - # Native: check local venv and wrapper - [ -d "${VENV_PATH}" ] && [ -f "${wrapper}" ] - else - # Docker: check container is running and setup is complete inside it - container_is_running "${CONTAINER_NAME}" && \ - docker exec "${CONTAINER_NAME}" test -d "${VENV_PATH}" 2>/dev/null && \ - docker exec "${CONTAINER_NAME}" test -f "${wrapper}" 2>/dev/null - fi + local wrapper + wrapper=$(get_rocprof_wrapper) + [ -d "${VENV_PATH}" ] && [ -f "${wrapper}" ] } +# ============================================================================ +# Commands +# ============================================================================ + # Setup: Create Python venv and install rocprof-compute dependencies cmd_setup() { echo "Setting up rocprof-compute profiling environment..." echo "===========================================" - if is_native_mode; then - # ====== NATIVE MODE SETUP ====== - info "Running in NATIVE mode" + # Check rocprof-compute exists + if [ -z "${ROCPROF_BIN}" ] || [ ! -f "${ROCPROF_BIN}" ]; then + error "rocprof-compute not found" + echo "" + echo "Install rocprofiler-compute using one of these methods:" + echo " 1. System package: sudo apt install rocprofiler-compute" + echo " 2. Set CK_ROCPROF_BIN to point to an existing installation" + echo "" + echo "Searched locations:" + for bin in "${ROCPROF_CANDIDATES[@]}"; do + echo " - $bin" + done + return 1 + fi + info "Found rocprof-compute: ${ROCPROF_BIN}" - # Check rocprof-compute exists - if [ -z "${ROCPROF_BIN}" ] || [ ! -f "${ROCPROF_BIN}" ]; then - error "rocprof-compute not found" - echo "Install with: sudo apt install rocprofiler-compute" - return 1 - fi - info "Found rocprof-compute: ${ROCPROF_BIN}" + # Check requirements file + if [ -z "${ROCM_REQUIREMENTS}" ] || [ ! -f "${ROCM_REQUIREMENTS}" ]; then + error "ROCm requirements file not found" + local expected_path + expected_path="$(dirname "$(dirname "${ROCPROF_BIN}")")/libexec/rocprofiler-compute/requirements.txt" + echo "Expected at: ${expected_path}" + echo "Set CK_ROCM_REQUIREMENTS to override" + return 1 + fi + info "Found requirements: ${ROCM_REQUIREMENTS}" - # Check requirements file - if [ -z "${ROCM_REQUIREMENTS}" ] || [ ! -f "${ROCM_REQUIREMENTS}" ]; then - error "ROCm requirements file not found" - echo "Expected at: $(dirname $(dirname ${ROCPROF_BIN}))/libexec/rocprofiler-compute/requirements.txt" - return 1 - fi - info "Found requirements: ${ROCM_REQUIREMENTS}" + # Check GPU access + if [ ! -r /dev/kfd ]; then + warn "No read access to /dev/kfd - GPU profiling may fail" + warn "Add user to video/render group: sudo usermod -a -G video,render \$USER" + fi - # Check GPU access - if [ ! -r /dev/kfd ]; then - warn "No read access to /dev/kfd - GPU profiling may fail" - warn "Add user to video/render group: sudo usermod -a -G video,render \$USER" + # Install uv if needed + if ! command -v uv &>/dev/null; then + info "Installing uv package manager via pip..." + if ! python3 -m pip install --user uv; then + error "Failed to install uv package manager" + return 1 fi - - # Install uv if needed + # Add user bin to PATH if not already there + export PATH="${HOME}/.local/bin:${PATH}" + # Verify installation if ! command -v uv &>/dev/null; then - info "Installing uv package manager via pip..." - if ! python3 -m pip install --user uv; then - error "Failed to install uv package manager" - return 1 - fi - # Add user bin to PATH if not already there - export PATH="${HOME}/.local/bin:${PATH}" - # Verify installation - if ! command -v uv &>/dev/null; then - error "uv installed but not found in PATH" - echo "Try adding ~/.local/bin to your PATH" - return 1 - fi + error "uv installed but not found in PATH" + echo "Try adding ~/.local/bin to your PATH" + return 1 fi + fi - # Create venv - if [ -d "${VENV_PATH}" ]; then - info "Python venv already exists at ${VENV_PATH}" - else - info "Creating Python venv at ${VENV_PATH}..." - uv venv "${VENV_PATH}" - fi + # Create venv + if [ -d "${VENV_PATH}" ]; then + info "Python venv already exists at ${VENV_PATH}" + else + info "Creating Python venv at ${VENV_PATH}..." + uv venv "${VENV_PATH}" + fi - # Install dependencies - info "Installing dependencies..." - uv pip install --python "${VENV_PATH}/bin/python" -r "${ROCM_REQUIREMENTS}" - uv pip install --python "${VENV_PATH}/bin/python" 'pandas<3.0' + # Install dependencies + info "Installing dependencies..." + uv pip install --python "${VENV_PATH}/bin/python" -r "${ROCM_REQUIREMENTS}" + uv pip install --python "${VENV_PATH}/bin/python" 'pandas<3.0' - # Create wrapper script - local wrapper=$(get_rocprof_wrapper) - mkdir -p "$(dirname "${wrapper}")" - cat > "${wrapper}" << WRAPPER_EOF + # Create wrapper script + local wrapper + wrapper=$(get_rocprof_wrapper) + mkdir -p "$(dirname "${wrapper}")" + cat > "${wrapper}" << WRAPPER_EOF #!/bin/bash # rocprof-compute wrapper using venv Python VENV_DIR="\$(cd "\$(dirname "\$0")/.." && pwd)" exec "\${VENV_DIR}/bin/python" "${ROCPROF_BIN}" "\$@" WRAPPER_EOF - chmod +x "${wrapper}" - info "Wrapper created at ${wrapper}" - - # Create workload directory - mkdir -p "${WORKLOAD_DIR}" - info "Workload directory: ${WORKLOAD_DIR}" - - echo "" - info "Setup complete! You can now use:" - echo " ck-rocprof run " - else - # ====== DOCKER MODE SETUP ====== - info "Running in DOCKER mode" - - # Ensure Docker container is running - echo "Ensuring Docker container '${CONTAINER_NAME}' is running..." - ensure_container_running "${CONTAINER_NAME}" "${SCRIPT_DIR}" - - # Check if rocprofiler-compute is installed, install if not - echo "Checking for rocprofiler-compute package..." - if ! docker exec "${CONTAINER_NAME}" test -f "${ROCPROF_BIN}" 2>/dev/null; then - echo "Installing rocprofiler-compute package..." - docker exec "${CONTAINER_NAME}" bash -c "apt update && apt install -y rocprofiler-compute" 2>&1 | tail -5 - echo "* rocprofiler-compute installed" - else - echo "* rocprofiler-compute already installed" - fi - - # Verify rocprof-compute exists after installation - if ! docker exec "${CONTAINER_NAME}" test -f "${ROCPROF_BIN}" 2>/dev/null; then - error "rocprof-compute not found at ${ROCPROF_BIN} after installation" - echo "Please check ROCm installation in container" - return 1 - fi - - # Check if requirements file exists - if ! docker exec "${CONTAINER_NAME}" test -f "${ROCM_REQUIREMENTS}" 2>/dev/null; then - error "rocprofiler-compute requirements.txt not found at ${ROCM_REQUIREMENTS}" - echo "Please ensure ROCm is properly installed in container" - return 1 - fi - - # Install uv if not present - echo "Checking for uv package manager..." - if ! docker exec "${CONTAINER_NAME}" command -v uv &>/dev/null; then - echo "Installing uv package manager via pip..." - if ! docker exec "${CONTAINER_NAME}" bash -c "python3 -m pip install --user uv" 2>&1 | tail -3; then - error "Failed to install uv package manager in container" - return 1 - fi - # Verify installation - if ! docker exec "${CONTAINER_NAME}" bash -c "export PATH=\${HOME}/.local/bin:\${PATH} && command -v uv" &>/dev/null; then - error "uv installed but not found in container PATH" - return 1 - fi - echo "* uv installed" - else - echo "* uv already installed" - fi - - # Create Python venv in container using uv - if docker exec "${CONTAINER_NAME}" test -d "${VENV_PATH}" 2>/dev/null; then - echo "Python venv already exists at ${VENV_PATH}" - else - echo "Creating Python virtual environment at ${VENV_PATH}..." - docker exec "${CONTAINER_NAME}" bash -c "export PATH=\${HOME}/.local/bin:\${PATH} && uv venv '${VENV_PATH}'" - echo "* Virtual environment created" - fi - - # Install dependencies in container using uv - echo "Installing rocprofiler-compute dependencies..." - docker exec "${CONTAINER_NAME}" bash -c "export PATH=\${HOME}/.local/bin:\${PATH} && uv pip install --python '${VENV_PATH}/bin/python' -r '${ROCM_REQUIREMENTS}'" - # Pin pandas to <3.0 to avoid CSV conversion bug in rocprof-compute - docker exec "${CONTAINER_NAME}" bash -c "export PATH=\${HOME}/.local/bin:\${PATH} && uv pip install --python '${VENV_PATH}/bin/python' 'pandas<3.0'" - echo "* Dependencies installed" + chmod +x "${wrapper}" + info "Wrapper created at ${wrapper}" - # Create wrapper script in container - local wrapper=$(get_rocprof_wrapper) - docker exec "${CONTAINER_NAME}" bash -c "cat > '${wrapper}' << WRAPPER_EOF -#!/bin/bash -# rocprof-compute wrapper using venv Python -VENV_DIR=\"\\\$(cd \"\\\$(dirname \"\\\$0\")/..\" && pwd)\" -exec \"\\\${VENV_DIR}/bin/python\" \"${ROCPROF_BIN}\" \"\\\$@\" -WRAPPER_EOF" - docker exec "${CONTAINER_NAME}" chmod +x "${wrapper}" - echo "* Wrapper script created at ${wrapper}" + # Create workload directory + mkdir -p "${WORKLOAD_DIR}" + info "Workload directory: ${WORKLOAD_DIR}" - echo "" - echo "Setup complete! You can now use:" - echo " ck-rocprof run " - fi + echo "" + info "Setup complete! You can now use:" + echo " ck-rocprof run " } # Detect GPU architecture -# rocprof-compute uses marketing names (MI350, MI300X, etc.) for directory naming -# This function attempts to match that convention detect_gpu_arch() { # Allow override via environment variable if [ -n "${CK_GPU_TARGET:-}" ]; then @@ -462,50 +293,28 @@ detect_gpu_arch() { return 0 fi - if is_native_mode; then - # ====== NATIVE MODE ====== - if command -v rocminfo &>/dev/null; then - # Try marketing name first (MI350, MI300X) - local marketing_name=$(rocminfo 2>/dev/null | grep 'Marketing Name:' | grep -oE 'MI[0-9]+[A-Z]*' | head -1) - if [ -n "$marketing_name" ]; then - echo "$marketing_name" - return 0 - fi - - # Fallback to gfx name - local gfx_name=$(rocminfo 2>/dev/null | grep -oE 'gfx[0-9a-z]+' | head -1) - if [ -n "$gfx_name" ]; then - echo "$gfx_name" - return 0 - fi - fi - - # Try existing workload directories - if [ -d "${WORKLOAD_DIR}" ]; then - local first_dir=$(find "${WORKLOAD_DIR}" -maxdepth 2 -type d \( -name 'gfx*' -o -name 'MI*' \) 2>/dev/null | head -1) - if [ -n "$first_dir" ]; then - basename "$first_dir" - return 0 - fi - fi - else - # ====== DOCKER MODE ====== - # Try to get marketing name from rocminfo in container - local marketing_name=$(docker exec "${CONTAINER_NAME}" bash -c "rocminfo 2>/dev/null | grep 'Marketing Name:' | grep -oE 'MI[0-9]+[A-Z]*' | head -1" 2>/dev/null || echo "") + if command -v rocminfo &>/dev/null; then + # Try marketing name first (MI350, MI300X) + local marketing_name + marketing_name=$(rocminfo 2>/dev/null | grep 'Marketing Name:' | grep -oE 'MI[0-9]+[A-Z]*' | head -1) if [ -n "$marketing_name" ]; then echo "$marketing_name" return 0 fi - # Fallback: Try gfx name from rocminfo in container - local gfx_name=$(docker exec "${CONTAINER_NAME}" bash -c "rocminfo 2>/dev/null | grep -oE 'gfx[0-9a-z]+' | head -1" 2>/dev/null || echo "") + # Fallback to gfx name + local gfx_name + gfx_name=$(rocminfo 2>/dev/null | grep -oE 'gfx[0-9a-z]+' | head -1) if [ -n "$gfx_name" ]; then echo "$gfx_name" return 0 fi + fi - # Fallback: try to detect from existing workload dirs in container - local first_dir=$(docker exec "${CONTAINER_NAME}" bash -c "find '${WORKLOAD_DIR}' -maxdepth 2 -type d \( -name 'gfx*' -o -name 'MI*' \) 2>/dev/null | head -1" 2>/dev/null || echo "") + # Try existing workload directories + if [ -d "${WORKLOAD_DIR}" ]; then + local first_dir + first_dir=$(find "${WORKLOAD_DIR}" -maxdepth 2 -type d \( -name 'gfx*' -o -name 'MI*' \) 2>/dev/null | head -1) if [ -n "$first_dir" ]; then basename "$first_dir" return 0 @@ -543,27 +352,23 @@ cmd_run() { fi # Check if executable exists - if ! file_exists "$executable"; then + if [ ! -f "$executable" ]; then error "Executable not found: $executable" return 1 fi - local wrapper=$(get_rocprof_wrapper) - local gpu_arch=$(detect_gpu_arch) + local wrapper + wrapper=$(get_rocprof_wrapper) + local gpu_arch + gpu_arch=$(detect_gpu_arch) echo "Profiling: $executable ${exe_args[*]}" echo "Run name: $name" echo "GPU arch: $gpu_arch" - if is_native_mode; then - echo "Mode: NATIVE" - else - echo "Container: $CONTAINER_NAME" - fi echo "===========================================" # Build command with proper escaping to prevent shell injection # --no-roof skips roofline analysis to speed up profiling - # --path sets the full output path for profiling results (includes name) local escaped_executable escaped_executable=$(printf '%q' "$executable") local escaped_workload_dir @@ -575,7 +380,7 @@ cmd_run() { done # Run profiling - exec_cmd "${cmd}" + bash -c "${cmd}" echo "" info "Profiling complete" @@ -585,17 +390,16 @@ cmd_run() { } # Find workload path for a given run name -# rocprof-compute stores results directly in the workload directory (pmc_perf.csv) find_workload_path() { local name="$1" local run_dir="${WORKLOAD_DIR}/${name}" - if ! dir_exists "$run_dir"; then + if [ ! -d "$run_dir" ]; then return 1 fi - # Check if profiling data exists directly in run directory (current rocprof-compute behavior) - if file_exists "${run_dir}/pmc_perf.csv"; then + # Check if profiling data exists + if [ -f "${run_dir}/pmc_perf.csv" ]; then echo "$run_dir" return 0 fi @@ -626,8 +430,10 @@ cmd_analyze() { return 1 fi - local wrapper=$(get_rocprof_wrapper) - local workload_path=$(find_workload_path "${name}") + local wrapper + wrapper=$(get_rocprof_wrapper) + local workload_path + workload_path=$(find_workload_path "${name}") if [ -z "$workload_path" ]; then error "Profiling results not found for '${name}'" @@ -641,11 +447,7 @@ cmd_analyze() { echo "===========================================" echo "" - if is_native_mode; then - "${wrapper}" analyze --path "${workload_path}" --block "${block}" - else - docker exec "${CONTAINER_NAME}" "${wrapper}" analyze --path "${workload_path}" --block "${block}" - fi + "${wrapper}" analyze --path "${workload_path}" --block "${block}" } # Compare two profiling runs @@ -675,8 +477,10 @@ cmd_compare() { fi # Verify both runs exist - local path1=$(find_workload_path "${name1}") - local path2=$(find_workload_path "${name2}") + local path1 + path1=$(find_workload_path "${name1}") + local path2 + path2=$(find_workload_path "${name2}") if [ -z "$path1" ]; then error "Profiling results not found for '${name1}'" @@ -710,17 +514,13 @@ cmd_compare() { # List available profiling runs cmd_list() { - if ! dir_exists "${WORKLOAD_DIR}"; then + if [ ! -d "${WORKLOAD_DIR}" ]; then echo "No profiling runs found (workload directory doesn't exist)" return 0 fi local runs - if is_native_mode; then - runs=$(find "${WORKLOAD_DIR}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort) - else - runs=$(docker exec "${CONTAINER_NAME}" bash -c "find '${WORKLOAD_DIR}' -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort" 2>/dev/null) - fi + runs=$(find "${WORKLOAD_DIR}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort) if [ -z "$runs" ]; then echo "No profiling runs found in ${WORKLOAD_DIR}" @@ -731,11 +531,14 @@ cmd_list() { echo "===========================================" while IFS= read -r run; do - local path=$(find_workload_path "$run") + local path + path=$(find_workload_path "$run") if [ -n "$path" ]; then - local size=$(get_size "$path") - local date=$(get_date "$path") + local size + size=$(get_size "$path") + local date + date=$(get_date "$path") printf " %-25s [%s, %s]\n" "$run" "$size" "$date" else printf " %-25s [no data]\n" "$run" @@ -759,7 +562,7 @@ cmd_clean() { if [ "$name" = "--all" ]; then # Remove all profiling runs - if ! dir_exists "${WORKLOAD_DIR}"; then + if [ ! -d "${WORKLOAD_DIR}" ]; then echo "No profiling runs to clean" return 0 fi @@ -771,11 +574,7 @@ cmd_clean() { return 0 fi - if is_native_mode; then - rm -rf "${WORKLOAD_DIR:?}"/* - else - docker exec "${CONTAINER_NAME}" bash -c "rm -rf '${WORKLOAD_DIR:?}'/*" - fi + rm -rf "${WORKLOAD_DIR:?}"/* info "All profiling runs removed" else # Validate name @@ -784,16 +583,12 @@ cmd_clean() { fi local run_dir="${WORKLOAD_DIR}/${name}" - if ! dir_exists "$run_dir"; then + if [ ! -d "$run_dir" ]; then error "Profiling run not found: ${name}" return 1 fi - if is_native_mode; then - rm -rf "${run_dir}" - else - docker exec "${CONTAINER_NAME}" bash -c "rm -rf '${run_dir}'" - fi + rm -rf "${run_dir}" info "Removed profiling run: ${name}" fi } @@ -804,18 +599,11 @@ cmd_status() { echo "===========================================" echo "" - # Mode - if is_native_mode; then - echo "Mode: NATIVE" - echo "rocprof-compute: ${ROCPROF_BIN:-not found}" + # rocprof-compute binary + if [ -n "${ROCPROF_BIN}" ] && [ -f "${ROCPROF_BIN}" ]; then + echo "rocprof-compute: ${ROCPROF_BIN}" else - echo "Mode: DOCKER" - echo "Container: ${CONTAINER_NAME}" - if container_is_running "${CONTAINER_NAME}"; then - echo "Container status: running" - else - echo "Container status: not running" - fi + echo "rocprof-compute: not found" fi echo "" @@ -835,20 +623,19 @@ cmd_status() { echo "" # Workload count - if dir_exists "${WORKLOAD_DIR}"; then + if [ -d "${WORKLOAD_DIR}" ]; then local count - if is_native_mode; then - count=$(find "${WORKLOAD_DIR}" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l) - else - count=$(docker exec "${CONTAINER_NAME}" bash -c "find '${WORKLOAD_DIR}' -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l" 2>/dev/null) - fi + count=$(find "${WORKLOAD_DIR}" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l) echo "Profiling runs: ${count}" else echo "Profiling runs: 0" fi } +# ============================================================================ # Main command dispatcher +# ============================================================================ + case "${1:-}" in setup) cmd_setup diff --git a/script/tools/ck-rocprof.md b/script/tools/ck-rocprof.md index e834c6758ca..0588846097f 100644 --- a/script/tools/ck-rocprof.md +++ b/script/tools/ck-rocprof.md @@ -2,10 +2,12 @@ GPU performance profiling for Composable Kernel applications using AMD rocprof-compute. +**Note:** This is a native-only tool. For Docker usage, run via `ck-docker exec ck-rocprof ...` + ## Quick Start ```bash -# One-time setup +# One-time setup (requires rocprofiler-compute installed) ./script/tools/ck-rocprof setup # Profile executable @@ -97,9 +99,10 @@ ck-rocprof compare baseline optimized ## Environment Variables -- `CK_PROFILE_VENV`: Python venv path (default: `$PROJECT/.ck-rocprof-venv` native, `/opt/rocprof_venv` Docker) +- `CK_PROFILE_VENV`: Python venv path (default: `$PROJECT/.ck-rocprof-venv`) - `CK_ROCPROF_BIN`: rocprof-compute binary path (auto-detected from PATH or /opt/rocm) -- `CK_WORKLOAD_DIR`: Results directory (default: `$PROJECT/build/workloads` native, `/workspace/workloads` Docker) +- `CK_ROCM_REQUIREMENTS`: Path to rocprofiler-compute requirements.txt (auto-detected) +- `CK_WORKLOAD_DIR`: Results directory (default: `$PROJECT/build/workloads`) - `CK_GPU_TARGET`: Override GPU detection (e.g., `gfx950`, `MI300X`) ## Interpreting Results From 76beb8a7cc84516991580846fbb2809571169db3 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Wed, 28 Jan 2026 18:18:57 -0500 Subject: [PATCH 10/10] Add rocprofiler-compute source installation fallback When rocprof-compute is not found in system locations, automatically install rocprofiler-compute 3.4.0 from source as a fallback. This eliminates the hard dependency on system ROCm packages. Implementation details: - Clone rocprofiler-compute from GitHub to ~/.local/ - Install dependencies via requirements.txt (not editable install) - Create wrapper that sets PYTHONPATH to source directory - Execute source script directly rather than importing as module This approach matches the project's development workflow and works around the incomplete pyproject.toml that prevents editable installs. Co-Authored-By: Claude --- script/tools/ck-rocprof | 224 +++++++++++++++++++++++++++++++--------- 1 file changed, 176 insertions(+), 48 deletions(-) diff --git a/script/tools/ck-rocprof b/script/tools/ck-rocprof index bb1076a35ad..2b41a7403c2 100755 --- a/script/tools/ck-rocprof +++ b/script/tools/ck-rocprof @@ -191,6 +191,93 @@ is_setup_complete() { [ -d "${VENV_PATH}" ] && [ -f "${wrapper}" ] } +# ============================================================================ +# Source installation +# ============================================================================ + +# rocprofiler-compute source installation location +ROCPROF_SOURCE_VERSION="3.4.0" +ROCPROF_SOURCE_DIR="${HOME}/.local/rocprofiler-compute/${ROCPROF_SOURCE_VERSION}" +ROCPROF_SOURCE_BIN="${ROCPROF_SOURCE_DIR}/bin/rocprof-compute" +ROCPROF_REPO_URL="https://github.com/ROCm/rocprofiler-compute.git" +ROCPROF_REPO_BRANCH="release/rocprofiler-compute-v${ROCPROF_SOURCE_VERSION}" + +# Install rocprofiler-compute from source +install_from_source() { + local install_dir="${ROCPROF_SOURCE_DIR}" + local src_dir="${install_dir}/src" + + info "Installing rocprofiler-compute ${ROCPROF_SOURCE_VERSION} from source..." + echo "Install location: ${install_dir}" + echo "" + + # Ensure uv is available + if ! command -v uv &>/dev/null; then + info "Installing uv package manager via pip..." + if ! python3 -m pip install --user uv; then + error "Failed to install uv package manager" + return 1 + fi + export PATH="${HOME}/.local/bin:${PATH}" + if ! command -v uv &>/dev/null; then + error "uv installed but not found in PATH" + return 1 + fi + fi + + # Create installation directory + mkdir -p "${install_dir}" + + # Clone repository + if [ -d "${src_dir}" ]; then + info "Source already exists, updating..." + git -C "${src_dir}" fetch --quiet + git -C "${src_dir}" checkout --quiet "${ROCPROF_REPO_BRANCH}" 2>/dev/null || \ + git -C "${src_dir}" checkout --quiet "amd-mainline" + else + info "Cloning rocprofiler-compute repository..." + if ! git clone --quiet --branch "${ROCPROF_REPO_BRANCH}" --depth 1 "${ROCPROF_REPO_URL}" "${src_dir}" 2>/dev/null; then + # Fall back to amd-mainline if release branch doesn't exist + info "Release branch not found, using amd-mainline..." + git clone --quiet --branch "amd-mainline" --depth 1 "${ROCPROF_REPO_URL}" "${src_dir}" + fi + fi + + # Create venv for source installation + local venv_dir="${install_dir}/venv" + if [ ! -d "${venv_dir}" ]; then + info "Creating Python virtual environment..." + uv venv "${venv_dir}" + fi + + # Install dependencies from requirements.txt + info "Installing dependencies (this may take a minute)..." + uv pip install --python "${venv_dir}/bin/python" -r "${src_dir}/requirements.txt" --quiet + # Pin pandas to avoid CSV conversion bug + uv pip install --python "${venv_dir}/bin/python" 'pandas<3.0' --quiet + + # Create bin directory and wrapper script + mkdir -p "${install_dir}/bin" + cat > "${ROCPROF_SOURCE_BIN}" << 'WRAPPER_EOF' +#!/bin/bash +# rocprof-compute wrapper for source installation +INSTALL_DIR="$(cd "$(dirname "$0")/.." && pwd)" +SRC_DIR="${INSTALL_DIR}/src/src" +VENV_DIR="${INSTALL_DIR}/venv" + +# Set PYTHONPATH to source directory for module imports +export PYTHONPATH="${SRC_DIR}:${PYTHONPATH}" + +# Execute rocprof-compute script with venv Python +exec "${VENV_DIR}/bin/python3" "${SRC_DIR}/rocprof-compute" "$@" +WRAPPER_EOF + chmod +x "${ROCPROF_SOURCE_BIN}" + + info "rocprofiler-compute installed successfully!" + echo " Binary: ${ROCPROF_SOURCE_BIN}" + echo "" +} + # ============================================================================ # Commands # ============================================================================ @@ -200,32 +287,53 @@ cmd_setup() { echo "Setting up rocprof-compute profiling environment..." echo "===========================================" - # Check rocprof-compute exists + # Check if rocprof-compute exists, install from source if not if [ -z "${ROCPROF_BIN}" ] || [ ! -f "${ROCPROF_BIN}" ]; then - error "rocprof-compute not found" - echo "" - echo "Install rocprofiler-compute using one of these methods:" - echo " 1. System package: sudo apt install rocprofiler-compute" - echo " 2. Set CK_ROCPROF_BIN to point to an existing installation" + warn "rocprof-compute not found in standard locations" echo "" echo "Searched locations:" for bin in "${ROCPROF_CANDIDATES[@]}"; do echo " - $bin" done - return 1 + echo "" + + # Check if we can install from source + if ! command -v git &>/dev/null; then + error "git is required to install from source" + return 1 + fi + if ! command -v python3 &>/dev/null; then + error "python3 is required to install from source" + return 1 + fi + + echo "Installing rocprofiler-compute from source..." + echo "" + if ! install_from_source; then + error "Failed to install rocprofiler-compute from source" + return 1 + fi + + # Update configuration with source installation + ROCPROF_BIN="${ROCPROF_SOURCE_BIN}" + ROCM_REQUIREMENTS="${ROCPROF_SOURCE_DIR}/libexec/rocprofiler-compute/requirements.txt" fi - info "Found rocprof-compute: ${ROCPROF_BIN}" + info "Using rocprof-compute: ${ROCPROF_BIN}" - # Check requirements file + # Check requirements file (only needed for non-source installs that use separate venv) if [ -z "${ROCM_REQUIREMENTS}" ] || [ ! -f "${ROCM_REQUIREMENTS}" ]; then - error "ROCm requirements file not found" - local expected_path - expected_path="$(dirname "$(dirname "${ROCPROF_BIN}")")/libexec/rocprofiler-compute/requirements.txt" - echo "Expected at: ${expected_path}" - echo "Set CK_ROCM_REQUIREMENTS to override" - return 1 + # For source installs, requirements are bundled + if [[ "${ROCPROF_BIN}" == "${ROCPROF_SOURCE_BIN}" ]]; then + ROCM_REQUIREMENTS="${ROCPROF_SOURCE_DIR}/libexec/rocprofiler-compute/requirements.txt" + else + error "ROCm requirements file not found" + local expected_path + expected_path="$(dirname "$(dirname "${ROCPROF_BIN}")")/libexec/rocprofiler-compute/requirements.txt" + echo "Expected at: ${expected_path}" + echo "Set CK_ROCM_REQUIREMENTS to override" + return 1 + fi fi - info "Found requirements: ${ROCM_REQUIREMENTS}" # Check GPU access if [ ! -r /dev/kfd ]; then @@ -233,48 +341,68 @@ cmd_setup() { warn "Add user to video/render group: sudo usermod -a -G video,render \$USER" fi - # Install uv if needed - if ! command -v uv &>/dev/null; then - info "Installing uv package manager via pip..." - if ! python3 -m pip install --user uv; then - error "Failed to install uv package manager" - return 1 - fi - # Add user bin to PATH if not already there - export PATH="${HOME}/.local/bin:${PATH}" - # Verify installation + # For source installations, the venv is already set up - just create wrapper + if [[ "${ROCPROF_BIN}" == "${ROCPROF_SOURCE_BIN}" ]]; then + # Source install already has everything set up + local wrapper + wrapper=$(get_rocprof_wrapper) + mkdir -p "$(dirname "${wrapper}")" + + # For source install, wrapper just calls the source binary + cat > "${wrapper}" << WRAPPER_EOF +#!/bin/bash +# rocprof-compute wrapper (using source installation) +exec "${ROCPROF_BIN}" "\$@" +WRAPPER_EOF + chmod +x "${wrapper}" + info "Wrapper created at ${wrapper}" + + # Create marker file for venv directory + mkdir -p "${VENV_PATH}/bin" + touch "${VENV_PATH}/.source-install" + else + # System install - need to set up venv with dependencies + # Install uv if needed if ! command -v uv &>/dev/null; then - error "uv installed but not found in PATH" - echo "Try adding ~/.local/bin to your PATH" - return 1 + info "Installing uv package manager via pip..." + if ! python3 -m pip install --user uv; then + error "Failed to install uv package manager" + return 1 + fi + export PATH="${HOME}/.local/bin:${PATH}" + if ! command -v uv &>/dev/null; then + error "uv installed but not found in PATH" + echo "Try adding ~/.local/bin to your PATH" + return 1 + fi fi - fi - # Create venv - if [ -d "${VENV_PATH}" ]; then - info "Python venv already exists at ${VENV_PATH}" - else - info "Creating Python venv at ${VENV_PATH}..." - uv venv "${VENV_PATH}" - fi + # Create venv + if [ -d "${VENV_PATH}" ]; then + info "Python venv already exists at ${VENV_PATH}" + else + info "Creating Python venv at ${VENV_PATH}..." + uv venv "${VENV_PATH}" + fi - # Install dependencies - info "Installing dependencies..." - uv pip install --python "${VENV_PATH}/bin/python" -r "${ROCM_REQUIREMENTS}" - uv pip install --python "${VENV_PATH}/bin/python" 'pandas<3.0' + # Install dependencies + info "Installing dependencies..." + uv pip install --python "${VENV_PATH}/bin/python" -r "${ROCM_REQUIREMENTS}" + uv pip install --python "${VENV_PATH}/bin/python" 'pandas<3.0' - # Create wrapper script - local wrapper - wrapper=$(get_rocprof_wrapper) - mkdir -p "$(dirname "${wrapper}")" - cat > "${wrapper}" << WRAPPER_EOF + # Create wrapper script + local wrapper + wrapper=$(get_rocprof_wrapper) + mkdir -p "$(dirname "${wrapper}")" + cat > "${wrapper}" << WRAPPER_EOF #!/bin/bash # rocprof-compute wrapper using venv Python VENV_DIR="\$(cd "\$(dirname "\$0")/.." && pwd)" exec "\${VENV_DIR}/bin/python" "${ROCPROF_BIN}" "\$@" WRAPPER_EOF - chmod +x "${wrapper}" - info "Wrapper created at ${wrapper}" + chmod +x "${wrapper}" + info "Wrapper created at ${wrapper}" + fi # Create workload directory mkdir -p "${WORKLOAD_DIR}"