From 0a8437dd164059d650580fd69066db2596b4f97d Mon Sep 17 00:00:00 2001 From: ryanzhang22 Date: Wed, 9 Sep 2026 16:56:55 +0000 Subject: [PATCH 01/10] Add new config options for PM sampling --- libkineto/include/Config.h | 14 +++++ libkineto/src/Config.cpp | 38 +++++++++++ libkineto/src/CuptiPMSamplingApi.cpp | 31 ++++++--- libkineto/src/CuptiPMSamplingApi.h | 11 +++- libkineto/src/CuptiPMSamplingProfiler.cpp | 8 +-- libkineto/test/ConfigTest.cpp | 23 ++++++- libkineto/test/CuptiPMSamplingApiTest.cpp | 63 +++++++++++++++++-- .../test/CuptiPMSamplingProfilerTest.cpp | 18 +----- 8 files changed, 171 insertions(+), 35 deletions(-) diff --git a/libkineto/include/Config.h b/libkineto/include/Config.h index 9835ce0a9..dc409b1c1 100644 --- a/libkineto/include/Config.h +++ b/libkineto/include/Config.h @@ -236,6 +236,16 @@ class Config : public AbstractConfig { return performanceMetricsDeviceId_; } + [[nodiscard]] std::chrono::nanoseconds performanceMetricsSamplingInterval() + const { + return performanceMetricsSamplingInterval_; + } + + [[nodiscard]] std::chrono::nanoseconds performanceMetricsLookbackWindow() + const { + return performanceMetricsLookbackWindow_; + } + [[nodiscard]] bool memoryProfilerEnabled() const { return memoryProfilerEnabled_; } @@ -375,6 +385,10 @@ class Config : public AbstractConfig { // Performance metrics std::vector performanceMetricNames_; int32_t performanceMetricsDeviceId_{-1}; + std::chrono::nanoseconds performanceMetricsSamplingInterval_{ + std::chrono::milliseconds{1}}; + std::chrono::nanoseconds performanceMetricsLookbackWindow_{ + std::chrono::seconds{10}}; // CUPTI Timestamp Format bool useTSCTimestamp_{true}; diff --git a/libkineto/src/Config.cpp b/libkineto/src/Config.cpp index b1644458e..78a05d473 100644 --- a/libkineto/src/Config.cpp +++ b/libkineto/src/Config.cpp @@ -9,7 +9,10 @@ #include "Config.h" #include "ThrowUtil.h" +#include +#include #include +#include #include #include @@ -55,6 +58,10 @@ constexpr char kCuptiPerThreadBufferEnabledKey[] = constexpr char kPerformanceMetricsKey[] = "PERFORMANCE_METRICS"; constexpr char kPerformanceMetricsDeviceIdKey[] = "PERFORMANCE_METRICS_DEVICE_ID"; +constexpr char kPerformanceMetricsSamplingIntervalMsecsKey[] = + "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS"; +constexpr char kPerformanceMetricsLookbackWindowMsecsKey[] = + "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS"; constexpr char kActivityTypesKey[] = "ACTIVITY_TYPES"; constexpr char kActivitiesLogFileKey[] = "ACTIVITIES_LOG_FILE"; constexpr char kActivitiesDurationKey[] = "ACTIVITIES_DURATION_SECS"; @@ -228,6 +235,31 @@ bool isAllowedOnDemandTraceFile(const string& path) { return path.starts_with(dir) && path.find("..") == string::npos; } +nanoseconds parsePositiveMilliseconds( + const string& value, + const char* optionName) { + errno = 0; + char* end = nullptr; + const long double milliseconds = std::strtold(value.c_str(), &end); + constexpr long double kNanosecondsPerMillisecond = 1'000'000.0L; + const long double nanosecondCount = milliseconds * kNanosecondsPerMillisecond; + if (value.empty() || end != value.c_str() + value.size() || errno == ERANGE || + !std::isfinite(milliseconds) || nanosecondCount < 1.0L || + nanosecondCount > + static_cast( + std::numeric_limits::max())) { + KINETO_THROW( + std::invalid_argument, + fmt::format( + "Invalid {}: {} - expected positive milliseconds representable " + "as nanoseconds", + optionName, + value)); + } + return std::chrono::nanoseconds{ + static_cast(nanosecondCount)}; +} + } // namespace Config::Config() @@ -351,6 +383,12 @@ bool Config::handleOption(const std::string& name, std::string& val) { performanceMetricNames_ = splitAndTrim(val, ','); } else if (!name.compare(kPerformanceMetricsDeviceIdKey)) { performanceMetricsDeviceId_ = toInt32(val); + } else if (!name.compare(kPerformanceMetricsSamplingIntervalMsecsKey)) { + performanceMetricsSamplingInterval_ = parsePositiveMilliseconds( + val, kPerformanceMetricsSamplingIntervalMsecsKey); + } else if (!name.compare(kPerformanceMetricsLookbackWindowMsecsKey)) { + performanceMetricsLookbackWindow_ = parsePositiveMilliseconds( + val, kPerformanceMetricsLookbackWindowMsecsKey); } else if (!name.compare(kProfileMemory)) { memoryProfilerEnabled_ = toBool(val); if (memoryProfilerEnabled_) { diff --git a/libkineto/src/CuptiPMSamplingApi.cpp b/libkineto/src/CuptiPMSamplingApi.cpp index dbe29e3c4..9869738f0 100644 --- a/libkineto/src/CuptiPMSamplingApi.cpp +++ b/libkineto/src/CuptiPMSamplingApi.cpp @@ -8,6 +8,7 @@ #include "CuptiPMSamplingApi.h" +#include #include #include @@ -47,7 +48,6 @@ namespace KINETO_NAMESPACE { namespace { constexpr size_t kHardwareBufferSizeBytes = 64 * 1024 * 1024; -constexpr uint32_t kMaxSamplesPerDecode = 1024; // GA100 only supports variable-frequency SYSCLK sampling. One option is to // measure hardware clock frequency and estimate the number of cycles needed to @@ -55,6 +55,22 @@ constexpr uint32_t kMaxSamplesPerDecode = 1024; // fixed. This is almost certainly something we'll need to revisit/tune. constexpr uint64_t kGa100SamplingIntervalCycles = 1'000'000; +uint32_t validatedSampleCapacity(const CuptiPMSamplingConfig& config) { + const auto capacity = config.sampleCapacity(); + if (capacity == 0) { + KINETO_THROW( + std::runtime_error, + "CUPTI PM sampling interval and lookback window must be positive"); + } + if (capacity > std::numeric_limits::max()) { + KINETO_THROW( + std::runtime_error, + "CUPTI PM sampling lookback window requires more samples than CUPTI " + "supports"); + } + return static_cast(capacity); +} + /* * ========================== CUPTI PM SAMPLING API ========================== * @@ -110,6 +126,7 @@ void CuptiPMSamplingApi::configure(const CuptiPMSamplingConfig& config) { "Cannot configure CUPTI PM sampling before the previous " "configuration is fully disabled"); } + static_cast(validatedSampleCapacity(config)); config_ = config; // CUPTI expects metric names as std::vector @@ -171,10 +188,6 @@ void CuptiPMSamplingApi::configureCupti() { } else if ( deviceProperties.major > 8 || (deviceProperties.major == 8 && deviceProperties.minor >= 6)) { - if (config_.samplingInterval.count() <= 0) { - KINETO_THROW( - std::runtime_error, "CUPTI PM sampling interval must be positive"); - } triggerMode = CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_TIME_INTERVAL; samplingInterval = static_cast(config_.samplingInterval.count()); } else { @@ -272,15 +285,14 @@ void CuptiPMSamplingApi::configureCupti() { CUPTI_PM_SAMPLING_HARDWARE_BUFFER_APPEND_MODE_KEEP_LATEST; CUPTI_CALL_THROW(cuptiPmSamplingSetConfig(&setConfig)); - // Asking CUPTI how large the (counter) data image should be. - // Since the data image has an opage CUPTI-defined layout, the size - // depends on sampling config, metrics, etc. + // Size the counter data image to hold the requested lookback window. CUPTI + // 12.x requires SetConfig to run before GetCounterDataSize. CUpti_PmSampling_GetCounterDataSize_Params counterDataSize{ CUpti_PmSampling_GetCounterDataSize_Params_STRUCT_SIZE}; counterDataSize.pPmSamplingObject = samplingObject_; counterDataSize.pMetricNames = metricNamePtrs_.data(); counterDataSize.numMetrics = metricNamePtrs_.size(); - counterDataSize.maxSamples = kMaxSamplesPerDecode; + counterDataSize.maxSamples = validatedSampleCapacity(config_); CUPTI_CALL_THROW(cuptiPmSamplingGetCounterDataSize(&counterDataSize)); counterDataImage_.resize(counterDataSize.counterDataSize); @@ -425,6 +437,7 @@ void CuptiPMSamplingApi::disable() { metricNamePtrs_.clear(); config_.metricNames.clear(); config_.samplingInterval = std::chrono::nanoseconds::zero(); + config_.lookbackWindow = std::chrono::nanoseconds::zero(); } } // namespace KINETO_NAMESPACE diff --git a/libkineto/src/CuptiPMSamplingApi.h b/libkineto/src/CuptiPMSamplingApi.h index a38c3b1f2..b6fe84e41 100644 --- a/libkineto/src/CuptiPMSamplingApi.h +++ b/libkineto/src/CuptiPMSamplingApi.h @@ -30,7 +30,16 @@ struct CuptiPMSample { struct CuptiPMSamplingConfig { int32_t deviceId{-1}; std::vector metricNames; - std::chrono::nanoseconds samplingInterval{0}; + std::chrono::nanoseconds samplingInterval{std::chrono::milliseconds{1}}; + std::chrono::nanoseconds lookbackWindow{std::chrono::seconds{10}}; + + [[nodiscard]] uint64_t sampleCapacity() const { + if (samplingInterval.count() <= 0 || lookbackWindow.count() <= 0) { + return 0; + } + const auto samples = lookbackWindow.count() / samplingInterval.count(); + return static_cast(samples > 0 ? samples : 1); + } }; class CuptiPMSamplingApi { diff --git a/libkineto/src/CuptiPMSamplingProfiler.cpp b/libkineto/src/CuptiPMSamplingProfiler.cpp index 50e1e91a4..3f31847eb 100644 --- a/libkineto/src/CuptiPMSamplingProfiler.cpp +++ b/libkineto/src/CuptiPMSamplingProfiler.cpp @@ -26,9 +26,6 @@ namespace { const std::string kProfilerName{"CUPTI PM Sampling"}; const std::set kSupportedActivities{ libkineto::ActivityType::HARDWARE_COUNTERS}; -// TODO: This is a temporary constant -- we need to tune the interval by -// hardware type to prevent hardware buffer overflow and sample loss. -constexpr std::chrono::milliseconds kSamplingInterval{1}; } // namespace @@ -201,7 +198,10 @@ std::unique_ptr CuptiPMSamplingProfiler:: } const CuptiPMSamplingConfig pmConfig{ - deviceId, metricNames, kSamplingInterval}; + deviceId, + metricNames, + config.performanceMetricsSamplingInterval(), + config.performanceMetricsLookbackWindow()}; auto session = std::make_unique(pmConfig); if (!session->prepare()) { return nullptr; diff --git a/libkineto/test/ConfigTest.cpp b/libkineto/test/ConfigTest.cpp index ef4529cae..06a1dcdec 100644 --- a/libkineto/test/ConfigTest.cpp +++ b/libkineto/test/ConfigTest.cpp @@ -143,21 +143,42 @@ TEST(ParseTest, PerformanceMetrics) { Config cfg; EXPECT_TRUE(cfg.performanceMetricNames().empty()); EXPECT_EQ(cfg.performanceMetricsDeviceId(), -1); + EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), milliseconds(1)); + EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(10)); EXPECT_TRUE( cfg.parse("PERFORMANCE_METRICS_DEVICE_ID=2\n" "PERFORMANCE_METRICS=" - "sm__cycles_active.avg,dram__bytes_read.sum")); + "sm__cycles_active.avg,dram__bytes_read.sum\n" + "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=0.5\n" + "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS=2000")); EXPECT_EQ( cfg.performanceMetricNames(), std::vector( {"sm__cycles_active.avg", "dram__bytes_read.sum"})); EXPECT_EQ(cfg.performanceMetricsDeviceId(), 2); + EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), microseconds(500)); + EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(2)); auto clone = cfg.clone(); EXPECT_EQ(clone->performanceMetricNames(), cfg.performanceMetricNames()); EXPECT_EQ(clone->performanceMetricsDeviceId(), 2); + EXPECT_EQ(clone->performanceMetricsSamplingInterval(), microseconds(500)); + EXPECT_EQ(clone->performanceMetricsLookbackWindow(), seconds(2)); +} + +TEST(ParseTest, PerformanceMetricsDurationsRequirePositiveFiniteValues) { + for (const auto* key : + {"PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS", + "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS"}) { + for (const auto* value : + {"", "0", "-1", "nan", "inf", "1ms", "0.0000001", "1e10000"}) { + Config cfg; + EXPECT_FALSE(cfg.parse(fmt::format("{}={}", key, value))) + << key << "=" << value; + } + } } TEST(ParseTest, ProfileStartTime) { diff --git a/libkineto/test/CuptiPMSamplingApiTest.cpp b/libkineto/test/CuptiPMSamplingApiTest.cpp index e23e6ca03..a9fedda4b 100644 --- a/libkineto/test/CuptiPMSamplingApiTest.cpp +++ b/libkineto/test/CuptiPMSamplingApiTest.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,7 @@ struct FakeCuptiState { CUpti_PmSampling_HardwareBuffer_AppendMode appendMode{}; std::vector counterDataMetricNames; uint32_t maxSamples{0}; + size_t counterDataSize{8 * 1024}; std::byte hostObject{}; std::byte samplingObject{}; @@ -103,9 +105,10 @@ std::vector copyMetricNames( CuptiPMSamplingConfig makeConfig( std::chrono::nanoseconds samplingInterval = 500us, int32_t deviceId = 0, - std::vector metricNames = {"sm__cycles_active.avg"}) { + std::vector metricNames = {"sm__cycles_active.avg"}, + std::chrono::nanoseconds lookbackWindow = 10s) { return CuptiPMSamplingConfig{ - deviceId, std::move(metricNames), samplingInterval}; + deviceId, std::move(metricNames), samplingInterval, lookbackWindow}; } void configureForDevice( @@ -227,7 +230,7 @@ CUptiResult CUPTIAPI cuptiPmSamplingGetCounterDataSize( fakeCupti().counterDataMetricNames = copyMetricNames(params->pMetricNames, params->numMetrics); fakeCupti().maxSamples = params->maxSamples; - params->counterDataSize = 1; + params->counterDataSize = fakeCupti().counterDataSize; return CUPTI_SUCCESS; } @@ -334,12 +337,38 @@ TEST_F(CuptiPMSamplingApiTest, ConfiguresDeviceMetricsAndBuffers) { fakeCupti().appendMode, CUPTI_PM_SAMPLING_HARDWARE_BUFFER_APPEND_MODE_KEEP_LATEST); EXPECT_EQ(fakeCupti().counterDataMetricNames, config.metricNames); - EXPECT_EQ(fakeCupti().maxSamples, 1024); + EXPECT_EQ(fakeCupti().maxSamples, 40'000); + api.disable(); +} + +TEST_F(CuptiPMSamplingApiTest, SizesCounterDataFromLookbackWindow) { + CuptiPMSamplingApi api; + + api.configure(makeConfig( + 250us, + /*deviceId=*/0, + {"sm__cycles_active.avg"}, + 2s)); + + EXPECT_EQ(fakeCupti().maxSamples, 8'000); + api.disable(); +} + +TEST_F(CuptiPMSamplingApiTest, KeepsAtLeastOneSample) { + CuptiPMSamplingApi api; + + api.configure(makeConfig( + 2ms, + /*deviceId=*/0, + {"sm__cycles_active.avg"}, + 1ms)); + + EXPECT_EQ(fakeCupti().maxSamples, 1); api.disable(); } TEST_F(CuptiPMSamplingApiTest, UsesFixedSysclkIntervalOnGa100) { - configureForDevice(8, 0, 0ns); + configureForDevice(8, 0, 500us); EXPECT_EQ( fakeCupti().triggerMode, @@ -370,6 +399,30 @@ TEST_F(CuptiPMSamplingApiTest, RejectsNonpositiveTimeIntervals) { EXPECT_THROW(configureForDevice(8, 6, 0ns), std::runtime_error); } +TEST_F(CuptiPMSamplingApiTest, RejectsNonpositiveLookbackWindows) { + CuptiPMSamplingApi api; + EXPECT_THROW( + api.configure(makeConfig( + 500us, + /*deviceId=*/0, + {"sm__cycles_active.avg"}, + 0ns)), + std::runtime_error); +} + +TEST_F(CuptiPMSamplingApiTest, RejectsSampleCapacityAboveCuptiLimit) { + CuptiPMSamplingApi api; + constexpr auto kTooManySamples = + static_cast(std::numeric_limits::max()) + 1; + EXPECT_THROW( + api.configure(makeConfig( + 1ns, + /*deviceId=*/0, + {"sm__cycles_active.avg"}, + std::chrono::nanoseconds{kTooManySamples})), + std::runtime_error); +} + TEST_F(CuptiPMSamplingApiTest, RejectsMultipassConfigurationBeforeEnabling) { fakeCupti().numPasses = 2; CuptiPMSamplingApi api; diff --git a/libkineto/test/CuptiPMSamplingProfilerTest.cpp b/libkineto/test/CuptiPMSamplingProfilerTest.cpp index 9a038181e..7717bd953 100644 --- a/libkineto/test/CuptiPMSamplingProfilerTest.cpp +++ b/libkineto/test/CuptiPMSamplingProfilerTest.cpp @@ -86,29 +86,17 @@ TEST(CuptiPMSamplingProfilerTest, ReportsNameAndSupportedActivity) { EXPECT_EQ(profiler.availableActivities(), kHardwareCounterActivities); } -TEST(CuptiPMSamplingProfilerTest, RequiresActivityMetricsAndDevice) { +TEST(CuptiPMSamplingProfilerTest, RequiresMetricsAndDevice) { CuptiPMSamplingProfiler profiler; - Config validConfig; - ASSERT_TRUE( - validConfig.parse("CUPTI_PM_SAMPLING_METRICS = sm__cycles_active.avg\n" - "CUPTI_PM_SAMPLING_DEVICE_ID = 0")); - EXPECT_EQ( - profiler.configure( - /*startTimeMs=*/123, - /*durationMs=*/456, - /*activityTypes=*/{}, - validConfig), - nullptr); - Config missingMetrics; - ASSERT_TRUE(missingMetrics.parse("CUPTI_PM_SAMPLING_DEVICE_ID = 0")); + ASSERT_TRUE(missingMetrics.parse("PERFORMANCE_METRICS_DEVICE_ID = 0")); EXPECT_EQ( profiler.configure(kHardwareCounterActivities, missingMetrics), nullptr); Config missingDevice; ASSERT_TRUE( - missingDevice.parse("CUPTI_PM_SAMPLING_METRICS = sm__cycles_active.avg")); + missingDevice.parse("PERFORMANCE_METRICS = sm__cycles_active.avg")); EXPECT_EQ( profiler.configure(kHardwareCounterActivities, missingDevice), nullptr); } From 3472b339a7e581d37595e9f5f5e04453b3c9ecf7 Mon Sep 17 00:00:00 2001 From: ryanzhang22 Date: Wed, 9 Sep 2026 17:18:19 +0000 Subject: [PATCH 02/10] fixup --- libkineto/src/Config.cpp | 39 ++++++++++++----------------------- libkineto/test/ConfigTest.cpp | 21 ++++++++++--------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/libkineto/src/Config.cpp b/libkineto/src/Config.cpp index 78a05d473..8ddea0dbc 100644 --- a/libkineto/src/Config.cpp +++ b/libkineto/src/Config.cpp @@ -9,10 +9,8 @@ #include "Config.h" #include "ThrowUtil.h" -#include #include #include -#include #include #include @@ -235,29 +233,18 @@ bool isAllowedOnDemandTraceFile(const string& path) { return path.starts_with(dir) && path.find("..") == string::npos; } -nanoseconds parsePositiveMilliseconds( +nanoseconds parseMilliseconds( const string& value, - const char* optionName) { - errno = 0; + nanoseconds fallback) noexcept { char* end = nullptr; - const long double milliseconds = std::strtold(value.c_str(), &end); - constexpr long double kNanosecondsPerMillisecond = 1'000'000.0L; - const long double nanosecondCount = milliseconds * kNanosecondsPerMillisecond; - if (value.empty() || end != value.c_str() + value.size() || errno == ERANGE || - !std::isfinite(milliseconds) || nanosecondCount < 1.0L || - nanosecondCount > - static_cast( - std::numeric_limits::max())) { - KINETO_THROW( - std::invalid_argument, - fmt::format( - "Invalid {}: {} - expected positive milliseconds representable " - "as nanoseconds", - optionName, - value)); + const duration milliseconds{ + std::strtod(value.c_str(), &end)}; + if (end != value.c_str() + value.size() || + !std::isfinite(milliseconds.count()) || milliseconds < nanoseconds{1} || + milliseconds > nanoseconds::max()) { + return fallback; } - return std::chrono::nanoseconds{ - static_cast(nanosecondCount)}; + return duration_cast(milliseconds); } } // namespace @@ -384,11 +371,11 @@ bool Config::handleOption(const std::string& name, std::string& val) { } else if (!name.compare(kPerformanceMetricsDeviceIdKey)) { performanceMetricsDeviceId_ = toInt32(val); } else if (!name.compare(kPerformanceMetricsSamplingIntervalMsecsKey)) { - performanceMetricsSamplingInterval_ = parsePositiveMilliseconds( - val, kPerformanceMetricsSamplingIntervalMsecsKey); + performanceMetricsSamplingInterval_ = + parseMilliseconds(val, performanceMetricsSamplingInterval_); } else if (!name.compare(kPerformanceMetricsLookbackWindowMsecsKey)) { - performanceMetricsLookbackWindow_ = parsePositiveMilliseconds( - val, kPerformanceMetricsLookbackWindowMsecsKey); + performanceMetricsLookbackWindow_ = + parseMilliseconds(val, performanceMetricsLookbackWindow_); } else if (!name.compare(kProfileMemory)) { memoryProfilerEnabled_ = toBool(val); if (memoryProfilerEnabled_) { diff --git a/libkineto/test/ConfigTest.cpp b/libkineto/test/ConfigTest.cpp index 06a1dcdec..db1123ece 100644 --- a/libkineto/test/ConfigTest.cpp +++ b/libkineto/test/ConfigTest.cpp @@ -168,16 +168,17 @@ TEST(ParseTest, PerformanceMetrics) { EXPECT_EQ(clone->performanceMetricsLookbackWindow(), seconds(2)); } -TEST(ParseTest, PerformanceMetricsDurationsRequirePositiveFiniteValues) { - for (const auto* key : - {"PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS", - "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS"}) { - for (const auto* value : - {"", "0", "-1", "nan", "inf", "1ms", "0.0000001", "1e10000"}) { - Config cfg; - EXPECT_FALSE(cfg.parse(fmt::format("{}={}", key, value))) - << key << "=" << value; - } +TEST(ParseTest, PerformanceMetricsInvalidDurationsUseDefaults) { + for (const auto* value : + {"", "0", "-1", "nan", "inf", "1ms", "0.0000001", "1e10000"}) { + Config cfg; + EXPECT_TRUE(cfg.parse(fmt::format( + "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS={}\n" + "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS={}", + value, + value))); + EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), milliseconds(1)); + EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(10)); } } From 2a55075a57c28fb531912b5e4f6ffc42a5c0614a Mon Sep 17 00:00:00 2001 From: ryanzhang22 Date: Wed, 9 Sep 2026 17:34:04 +0000 Subject: [PATCH 03/10] fixup --- libkineto/src/Config.cpp | 35 ++++++++++-------- libkineto/src/CuptiPMSamplingApi.cpp | 26 ++++---------- libkineto/src/CuptiPMSamplingApi.h | 8 ----- libkineto/test/ConfigTest.cpp | 17 ++++----- libkineto/test/CuptiPMSamplingApiTest.cpp | 44 ++--------------------- 5 files changed, 36 insertions(+), 94 deletions(-) diff --git a/libkineto/src/Config.cpp b/libkineto/src/Config.cpp index 8ddea0dbc..8b40f4457 100644 --- a/libkineto/src/Config.cpp +++ b/libkineto/src/Config.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -233,18 +234,20 @@ bool isAllowedOnDemandTraceFile(const string& path) { return path.starts_with(dir) && path.find("..") == string::npos; } -nanoseconds parseMilliseconds( - const string& value, - nanoseconds fallback) noexcept { - char* end = nullptr; - const duration milliseconds{ - std::strtod(value.c_str(), &end)}; - if (end != value.c_str() + value.size() || - !std::isfinite(milliseconds.count()) || milliseconds < nanoseconds{1} || - milliseconds > nanoseconds::max()) { - return fallback; +std::optional parseMilliseconds(const string& text) noexcept { + char* parseEnd = nullptr; + const double count = std::strtod(text.c_str(), &parseEnd); + if (parseEnd != text.c_str() + text.size()) { + return std::nullopt; } - return duration_cast(milliseconds); + + const duration value{count}; + if (!std::isfinite(count) || value < nanoseconds{1} || + value > nanoseconds::max()) { + return std::nullopt; + } + + return duration_cast(value); } } // namespace @@ -371,11 +374,13 @@ bool Config::handleOption(const std::string& name, std::string& val) { } else if (!name.compare(kPerformanceMetricsDeviceIdKey)) { performanceMetricsDeviceId_ = toInt32(val); } else if (!name.compare(kPerformanceMetricsSamplingIntervalMsecsKey)) { - performanceMetricsSamplingInterval_ = - parseMilliseconds(val, performanceMetricsSamplingInterval_); + if (const auto value = parseMilliseconds(val)) { + performanceMetricsSamplingInterval_ = *value; + } } else if (!name.compare(kPerformanceMetricsLookbackWindowMsecsKey)) { - performanceMetricsLookbackWindow_ = - parseMilliseconds(val, performanceMetricsLookbackWindow_); + if (const auto value = parseMilliseconds(val)) { + performanceMetricsLookbackWindow_ = *value; + } } else if (!name.compare(kProfileMemory)) { memoryProfilerEnabled_ = toBool(val); if (memoryProfilerEnabled_) { diff --git a/libkineto/src/CuptiPMSamplingApi.cpp b/libkineto/src/CuptiPMSamplingApi.cpp index 9869738f0..9f09d1ebd 100644 --- a/libkineto/src/CuptiPMSamplingApi.cpp +++ b/libkineto/src/CuptiPMSamplingApi.cpp @@ -8,7 +8,7 @@ #include "CuptiPMSamplingApi.h" -#include +#include #include #include @@ -55,22 +55,6 @@ constexpr size_t kHardwareBufferSizeBytes = 64 * 1024 * 1024; // fixed. This is almost certainly something we'll need to revisit/tune. constexpr uint64_t kGa100SamplingIntervalCycles = 1'000'000; -uint32_t validatedSampleCapacity(const CuptiPMSamplingConfig& config) { - const auto capacity = config.sampleCapacity(); - if (capacity == 0) { - KINETO_THROW( - std::runtime_error, - "CUPTI PM sampling interval and lookback window must be positive"); - } - if (capacity > std::numeric_limits::max()) { - KINETO_THROW( - std::runtime_error, - "CUPTI PM sampling lookback window requires more samples than CUPTI " - "supports"); - } - return static_cast(capacity); -} - /* * ========================== CUPTI PM SAMPLING API ========================== * @@ -126,7 +110,6 @@ void CuptiPMSamplingApi::configure(const CuptiPMSamplingConfig& config) { "Cannot configure CUPTI PM sampling before the previous " "configuration is fully disabled"); } - static_cast(validatedSampleCapacity(config)); config_ = config; // CUPTI expects metric names as std::vector @@ -188,6 +171,10 @@ void CuptiPMSamplingApi::configureCupti() { } else if ( deviceProperties.major > 8 || (deviceProperties.major == 8 && deviceProperties.minor >= 6)) { + if (config_.samplingInterval.count() <= 0) { + KINETO_THROW( + std::runtime_error, "CUPTI PM sampling interval must be positive"); + } triggerMode = CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_TIME_INTERVAL; samplingInterval = static_cast(config_.samplingInterval.count()); } else { @@ -292,7 +279,8 @@ void CuptiPMSamplingApi::configureCupti() { counterDataSize.pPmSamplingObject = samplingObject_; counterDataSize.pMetricNames = metricNamePtrs_.data(); counterDataSize.numMetrics = metricNamePtrs_.size(); - counterDataSize.maxSamples = validatedSampleCapacity(config_); + counterDataSize.maxSamples = static_cast( + std::max(1, config_.lookbackWindow / config_.samplingInterval)); CUPTI_CALL_THROW(cuptiPmSamplingGetCounterDataSize(&counterDataSize)); counterDataImage_.resize(counterDataSize.counterDataSize); diff --git a/libkineto/src/CuptiPMSamplingApi.h b/libkineto/src/CuptiPMSamplingApi.h index b6fe84e41..d91cb659f 100644 --- a/libkineto/src/CuptiPMSamplingApi.h +++ b/libkineto/src/CuptiPMSamplingApi.h @@ -32,14 +32,6 @@ struct CuptiPMSamplingConfig { std::vector metricNames; std::chrono::nanoseconds samplingInterval{std::chrono::milliseconds{1}}; std::chrono::nanoseconds lookbackWindow{std::chrono::seconds{10}}; - - [[nodiscard]] uint64_t sampleCapacity() const { - if (samplingInterval.count() <= 0 || lookbackWindow.count() <= 0) { - return 0; - } - const auto samples = lookbackWindow.count() / samplingInterval.count(); - return static_cast(samples > 0 ? samples : 1); - } }; class CuptiPMSamplingApi { diff --git a/libkineto/test/ConfigTest.cpp b/libkineto/test/ConfigTest.cpp index db1123ece..7b84807df 100644 --- a/libkineto/test/ConfigTest.cpp +++ b/libkineto/test/ConfigTest.cpp @@ -169,17 +169,12 @@ TEST(ParseTest, PerformanceMetrics) { } TEST(ParseTest, PerformanceMetricsInvalidDurationsUseDefaults) { - for (const auto* value : - {"", "0", "-1", "nan", "inf", "1ms", "0.0000001", "1e10000"}) { - Config cfg; - EXPECT_TRUE(cfg.parse(fmt::format( - "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS={}\n" - "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS={}", - value, - value))); - EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), milliseconds(1)); - EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(10)); - } + Config cfg; + EXPECT_TRUE( + cfg.parse("PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=invalid\n" + "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS=0")); + EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), milliseconds(1)); + EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(10)); } TEST(ParseTest, ProfileStartTime) { diff --git a/libkineto/test/CuptiPMSamplingApiTest.cpp b/libkineto/test/CuptiPMSamplingApiTest.cpp index a9fedda4b..2d0f548e5 100644 --- a/libkineto/test/CuptiPMSamplingApiTest.cpp +++ b/libkineto/test/CuptiPMSamplingApiTest.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -59,7 +58,6 @@ struct FakeCuptiState { CUpti_PmSampling_HardwareBuffer_AppendMode appendMode{}; std::vector counterDataMetricNames; uint32_t maxSamples{0}; - size_t counterDataSize{8 * 1024}; std::byte hostObject{}; std::byte samplingObject{}; @@ -230,7 +228,7 @@ CUptiResult CUPTIAPI cuptiPmSamplingGetCounterDataSize( fakeCupti().counterDataMetricNames = copyMetricNames(params->pMetricNames, params->numMetrics); fakeCupti().maxSamples = params->maxSamples; - params->counterDataSize = fakeCupti().counterDataSize; + params->counterDataSize = 1; return CUPTI_SUCCESS; } @@ -321,7 +319,8 @@ TEST_F(CuptiPMSamplingApiTest, ConfiguresDeviceMetricsAndBuffers) { const auto config = makeConfig( 250us, /*deviceId=*/2, - {"sm__cycles_active.avg", "dram__bytes_read.sum"}); + {"sm__cycles_active.avg", "dram__bytes_read.sum"}, + 2s); CuptiPMSamplingApi api; api.configure(config); @@ -337,19 +336,6 @@ TEST_F(CuptiPMSamplingApiTest, ConfiguresDeviceMetricsAndBuffers) { fakeCupti().appendMode, CUPTI_PM_SAMPLING_HARDWARE_BUFFER_APPEND_MODE_KEEP_LATEST); EXPECT_EQ(fakeCupti().counterDataMetricNames, config.metricNames); - EXPECT_EQ(fakeCupti().maxSamples, 40'000); - api.disable(); -} - -TEST_F(CuptiPMSamplingApiTest, SizesCounterDataFromLookbackWindow) { - CuptiPMSamplingApi api; - - api.configure(makeConfig( - 250us, - /*deviceId=*/0, - {"sm__cycles_active.avg"}, - 2s)); - EXPECT_EQ(fakeCupti().maxSamples, 8'000); api.disable(); } @@ -399,30 +385,6 @@ TEST_F(CuptiPMSamplingApiTest, RejectsNonpositiveTimeIntervals) { EXPECT_THROW(configureForDevice(8, 6, 0ns), std::runtime_error); } -TEST_F(CuptiPMSamplingApiTest, RejectsNonpositiveLookbackWindows) { - CuptiPMSamplingApi api; - EXPECT_THROW( - api.configure(makeConfig( - 500us, - /*deviceId=*/0, - {"sm__cycles_active.avg"}, - 0ns)), - std::runtime_error); -} - -TEST_F(CuptiPMSamplingApiTest, RejectsSampleCapacityAboveCuptiLimit) { - CuptiPMSamplingApi api; - constexpr auto kTooManySamples = - static_cast(std::numeric_limits::max()) + 1; - EXPECT_THROW( - api.configure(makeConfig( - 1ns, - /*deviceId=*/0, - {"sm__cycles_active.avg"}, - std::chrono::nanoseconds{kTooManySamples})), - std::runtime_error); -} - TEST_F(CuptiPMSamplingApiTest, RejectsMultipassConfigurationBeforeEnabling) { fakeCupti().numPasses = 2; CuptiPMSamplingApi api; From 091b58427393efc4e9d249134384da959f873d11 Mon Sep 17 00:00:00 2001 From: ryanzhang22 Date: Wed, 9 Sep 2026 17:59:58 +0000 Subject: [PATCH 04/10] fixup --- libkineto/src/Config.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libkineto/src/Config.cpp b/libkineto/src/Config.cpp index 8b40f4457..86bd1e815 100644 --- a/libkineto/src/Config.cpp +++ b/libkineto/src/Config.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -234,10 +235,11 @@ bool isAllowedOnDemandTraceFile(const string& path) { return path.starts_with(dir) && path.find("..") == string::npos; } -std::optional parseMilliseconds(const string& text) noexcept { - char* parseEnd = nullptr; - const double count = std::strtod(text.c_str(), &parseEnd); - if (parseEnd != text.c_str() + text.size()) { +std::optional parseMilliseconds(std::string_view text) noexcept { + double count = 0; + const auto [parseEnd, error] = + std::from_chars(text.begin(), text.end(), count); + if (error != std::errc{} || parseEnd != text.end()) { return std::nullopt; } From 80ffbf991b41e09d55d15dae681ab7116b4c96ef Mon Sep 17 00:00:00 2001 From: ryanzhang22 Date: Tue, 15 Sep 2026 16:07:07 +0000 Subject: [PATCH 05/10] address comments --- libkineto/src/Config.cpp | 4 +-- libkineto/src/CuptiPMSamplingApi.cpp | 34 +++++++++++++++++++++-- libkineto/test/CuptiPMSamplingApiTest.cpp | 33 ++++++++++++++++++++-- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/libkineto/src/Config.cpp b/libkineto/src/Config.cpp index 86bd1e815..0429d3350 100644 --- a/libkineto/src/Config.cpp +++ b/libkineto/src/Config.cpp @@ -238,8 +238,8 @@ bool isAllowedOnDemandTraceFile(const string& path) { std::optional parseMilliseconds(std::string_view text) noexcept { double count = 0; const auto [parseEnd, error] = - std::from_chars(text.begin(), text.end(), count); - if (error != std::errc{} || parseEnd != text.end()) { + std::from_chars(text.data(), text.data() + text.size(), count); + if (error != std::errc{} || parseEnd != text.data() + text.size()) { return std::nullopt; } diff --git a/libkineto/src/CuptiPMSamplingApi.cpp b/libkineto/src/CuptiPMSamplingApi.cpp index 9f09d1ebd..490d28fd8 100644 --- a/libkineto/src/CuptiPMSamplingApi.cpp +++ b/libkineto/src/CuptiPMSamplingApi.cpp @@ -9,6 +9,7 @@ #include "CuptiPMSamplingApi.h" #include +#include #include #include @@ -55,6 +56,23 @@ constexpr size_t kHardwareBufferSizeBytes = 64 * 1024 * 1024; // fixed. This is almost certainly something we'll need to revisit/tune. constexpr uint64_t kGa100SamplingIntervalCycles = 1'000'000; +uint32_t maxSamplesForDecode( + std::chrono::nanoseconds lookbackWindow, + std::chrono::nanoseconds samplingInterval) { + if (lookbackWindow.count() <= 0 || samplingInterval.count() <= 0) { + KINETO_THROW( + std::runtime_error, + "CUPTI PM sampling interval and lookback window must be positive"); + } + const auto samples = std::max(1, lookbackWindow / samplingInterval); + if (samples > std::numeric_limits::max()) { + KINETO_THROW( + std::runtime_error, + "CUPTI PM sampling lookback window exceeds CUPTI sample capacity"); + } + return static_cast(samples); +} + /* * ========================== CUPTI PM SAMPLING API ========================== * @@ -163,11 +181,22 @@ void CuptiPMSamplingApi::configureCupti() { } CUpti_PmSampling_TriggerMode triggerMode; uint64_t samplingInterval; + auto capacityInterval = config_.samplingInterval; // GPU_TIME_INTERVAL is unavailable on GA100. The wall-time duration of this // fixed SYSCLK interval varies with the GPU's clock frequency. if (deviceProperties.major == 8 && deviceProperties.minor == 0) { + if (deviceProperties.clockRate <= 0) { + KINETO_THROW( + std::runtime_error, "GA100 reported an invalid GPU clock rate"); + } triggerMode = CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_SYSCLK_INTERVAL; samplingInterval = kGa100SamplingIntervalCycles; + // clockRate is the maximum clock in kHz. It gives the shortest possible + // period and therefore enough capacity at GA100's fastest sampling rate. + capacityInterval = std::chrono::duration_cast( + std::chrono::duration{ + static_cast(kGa100SamplingIntervalCycles) / + (static_cast(deviceProperties.clockRate) * 1000.0)}); } else if ( deviceProperties.major > 8 || (deviceProperties.major == 8 && deviceProperties.minor >= 6)) { @@ -183,6 +212,8 @@ void CuptiPMSamplingApi::configureCupti() { "Minimum supported GPU for CUPTI PM sampling is GA100 (compute " "capability 8.0); other GPUs require compute capability 8.6 or newer"); } + const auto maxSamples = + maxSamplesForDecode(config_.lookbackWindow, capacityInterval); // Building the availability image. This is a CUPTI byte buffer describing // which raw hardware counters are available on this machine. The first call @@ -279,8 +310,7 @@ void CuptiPMSamplingApi::configureCupti() { counterDataSize.pPmSamplingObject = samplingObject_; counterDataSize.pMetricNames = metricNamePtrs_.data(); counterDataSize.numMetrics = metricNamePtrs_.size(); - counterDataSize.maxSamples = static_cast( - std::max(1, config_.lookbackWindow / config_.samplingInterval)); + counterDataSize.maxSamples = maxSamples; CUPTI_CALL_THROW(cuptiPmSamplingGetCounterDataSize(&counterDataSize)); counterDataImage_.resize(counterDataSize.counterDataSize); diff --git a/libkineto/test/CuptiPMSamplingApiTest.cpp b/libkineto/test/CuptiPMSamplingApiTest.cpp index 2d0f548e5..dda329985 100644 --- a/libkineto/test/CuptiPMSamplingApiTest.cpp +++ b/libkineto/test/CuptiPMSamplingApiTest.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,7 @@ struct FakeSample { struct FakeCuptiState { int major{9}; int minor{0}; + int clockRate{1'500'000}; size_t numPasses{1}; CUpti_PmSampling_DecodeStopReason decodeStopReason{ CUPTI_PM_SAMPLING_DECODE_STOP_REASON_END_OF_RECORDS}; @@ -157,6 +159,7 @@ cudaGetDeviceProperties(cudaDeviceProp* properties, int device) { *properties = cudaDeviceProp{}; properties->major = fakeCupti().major; properties->minor = fakeCupti().minor; + properties->clockRate = fakeCupti().clockRate; return cudaSuccess; } @@ -353,13 +356,15 @@ TEST_F(CuptiPMSamplingApiTest, KeepsAtLeastOneSample) { api.disable(); } -TEST_F(CuptiPMSamplingApiTest, UsesFixedSysclkIntervalOnGa100) { - configureForDevice(8, 0, 500us); +TEST_F(CuptiPMSamplingApiTest, UsesEffectiveSysclkCadenceOnGa100) { + configureForDevice(8, 0, 0ns); EXPECT_EQ( fakeCupti().triggerMode, CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_SYSCLK_INTERVAL); EXPECT_EQ(fakeCupti().samplingInterval, 1'000'000); + // At the fake 1.5 GHz maximum clock, 1M cycles yields 15K samples in 10s. + EXPECT_EQ(fakeCupti().maxSamples, 15'000); } TEST_F(CuptiPMSamplingApiTest, UsesRequestedTimeIntervalOnGa10xAndNewer) { @@ -385,6 +390,30 @@ TEST_F(CuptiPMSamplingApiTest, RejectsNonpositiveTimeIntervals) { EXPECT_THROW(configureForDevice(8, 6, 0ns), std::runtime_error); } +TEST_F(CuptiPMSamplingApiTest, RejectsNonpositiveLookbackWindows) { + CuptiPMSamplingApi api; + EXPECT_THROW( + api.configure(makeConfig( + 500us, + /*deviceId=*/0, + {"sm__cycles_active.avg"}, + 0ns)), + std::runtime_error); +} + +TEST_F(CuptiPMSamplingApiTest, RejectsSampleCapacityAboveCuptiLimit) { + CuptiPMSamplingApi api; + constexpr auto kTooManySamples = + static_cast(std::numeric_limits::max()) + 1; + EXPECT_THROW( + api.configure(makeConfig( + 1ns, + /*deviceId=*/0, + {"sm__cycles_active.avg"}, + std::chrono::nanoseconds{kTooManySamples})), + std::runtime_error); +} + TEST_F(CuptiPMSamplingApiTest, RejectsMultipassConfigurationBeforeEnabling) { fakeCupti().numPasses = 2; CuptiPMSamplingApi api; From db302bb98889f7f123c01fd8741ac3fe90051582 Mon Sep 17 00:00:00 2001 From: ryanzhang22 Date: Tue, 15 Sep 2026 17:59:22 +0000 Subject: [PATCH 06/10] address comments --- libkineto/include/Config.h | 2 +- libkineto/src/CuptiPMSamplingApi.h | 2 +- libkineto/test/ConfigTest.cpp | 4 ++-- libkineto/test/CuptiPMSamplingApiTest.cpp | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libkineto/include/Config.h b/libkineto/include/Config.h index dc409b1c1..bf35bc85d 100644 --- a/libkineto/include/Config.h +++ b/libkineto/include/Config.h @@ -388,7 +388,7 @@ class Config : public AbstractConfig { std::chrono::nanoseconds performanceMetricsSamplingInterval_{ std::chrono::milliseconds{1}}; std::chrono::nanoseconds performanceMetricsLookbackWindow_{ - std::chrono::seconds{10}}; + std::chrono::seconds{1}}; // CUPTI Timestamp Format bool useTSCTimestamp_{true}; diff --git a/libkineto/src/CuptiPMSamplingApi.h b/libkineto/src/CuptiPMSamplingApi.h index d91cb659f..f153de553 100644 --- a/libkineto/src/CuptiPMSamplingApi.h +++ b/libkineto/src/CuptiPMSamplingApi.h @@ -31,7 +31,7 @@ struct CuptiPMSamplingConfig { int32_t deviceId{-1}; std::vector metricNames; std::chrono::nanoseconds samplingInterval{std::chrono::milliseconds{1}}; - std::chrono::nanoseconds lookbackWindow{std::chrono::seconds{10}}; + std::chrono::nanoseconds lookbackWindow{std::chrono::seconds{1}}; }; class CuptiPMSamplingApi { diff --git a/libkineto/test/ConfigTest.cpp b/libkineto/test/ConfigTest.cpp index 7b84807df..e90161ddf 100644 --- a/libkineto/test/ConfigTest.cpp +++ b/libkineto/test/ConfigTest.cpp @@ -144,7 +144,7 @@ TEST(ParseTest, PerformanceMetrics) { EXPECT_TRUE(cfg.performanceMetricNames().empty()); EXPECT_EQ(cfg.performanceMetricsDeviceId(), -1); EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), milliseconds(1)); - EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(10)); + EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(1)); EXPECT_TRUE( cfg.parse("PERFORMANCE_METRICS_DEVICE_ID=2\n" @@ -174,7 +174,7 @@ TEST(ParseTest, PerformanceMetricsInvalidDurationsUseDefaults) { cfg.parse("PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=invalid\n" "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS=0")); EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), milliseconds(1)); - EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(10)); + EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(1)); } TEST(ParseTest, ProfileStartTime) { diff --git a/libkineto/test/CuptiPMSamplingApiTest.cpp b/libkineto/test/CuptiPMSamplingApiTest.cpp index dda329985..1426e4708 100644 --- a/libkineto/test/CuptiPMSamplingApiTest.cpp +++ b/libkineto/test/CuptiPMSamplingApiTest.cpp @@ -106,7 +106,7 @@ CuptiPMSamplingConfig makeConfig( std::chrono::nanoseconds samplingInterval = 500us, int32_t deviceId = 0, std::vector metricNames = {"sm__cycles_active.avg"}, - std::chrono::nanoseconds lookbackWindow = 10s) { + std::chrono::nanoseconds lookbackWindow = 1s) { return CuptiPMSamplingConfig{ deviceId, std::move(metricNames), samplingInterval, lookbackWindow}; } @@ -363,8 +363,8 @@ TEST_F(CuptiPMSamplingApiTest, UsesEffectiveSysclkCadenceOnGa100) { fakeCupti().triggerMode, CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_SYSCLK_INTERVAL); EXPECT_EQ(fakeCupti().samplingInterval, 1'000'000); - // At the fake 1.5 GHz maximum clock, 1M cycles yields 15K samples in 10s. - EXPECT_EQ(fakeCupti().maxSamples, 15'000); + // At the fake 1.5 GHz maximum clock, 1M cycles yields 1.5K samples in 1s. + EXPECT_EQ(fakeCupti().maxSamples, 1'500); } TEST_F(CuptiPMSamplingApiTest, UsesRequestedTimeIntervalOnGa10xAndNewer) { From 38521f1cf6902a0115c2a54126ff19872a20fb81 Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Wed, 16 Sep 2026 16:23:04 -0700 Subject: [PATCH 07/10] update --- libkineto/include/Config.h | 8 +- libkineto/src/Config.cpp | 4 +- libkineto/src/CuptiPMSamplingApi.cpp | 32 ++++-- libkineto/src/CuptiPMSamplingApi.h | 4 +- libkineto/test/ConfigTest.cpp | 22 +++- libkineto/test/CuptiPMSamplingApiTest.cpp | 121 ++++++++++++++++++++-- 6 files changed, 168 insertions(+), 23 deletions(-) diff --git a/libkineto/include/Config.h b/libkineto/include/Config.h index bf35bc85d..80fbec58c 100644 --- a/libkineto/include/Config.h +++ b/libkineto/include/Config.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -236,8 +237,8 @@ class Config : public AbstractConfig { return performanceMetricsDeviceId_; } - [[nodiscard]] std::chrono::nanoseconds performanceMetricsSamplingInterval() - const { + [[nodiscard]] std::optional + performanceMetricsSamplingInterval() const { return performanceMetricsSamplingInterval_; } @@ -385,8 +386,7 @@ class Config : public AbstractConfig { // Performance metrics std::vector performanceMetricNames_; int32_t performanceMetricsDeviceId_{-1}; - std::chrono::nanoseconds performanceMetricsSamplingInterval_{ - std::chrono::milliseconds{1}}; + std::optional performanceMetricsSamplingInterval_; std::chrono::nanoseconds performanceMetricsLookbackWindow_{ std::chrono::seconds{1}}; diff --git a/libkineto/src/Config.cpp b/libkineto/src/Config.cpp index 0429d3350..54906cf0b 100644 --- a/libkineto/src/Config.cpp +++ b/libkineto/src/Config.cpp @@ -244,8 +244,10 @@ std::optional parseMilliseconds(std::string_view text) noexcept { } const duration value{count}; + // nanoseconds::max() rounds up to 2^63 in double precision, so exclude + // equality before converting back to an integer. if (!std::isfinite(count) || value < nanoseconds{1} || - value > nanoseconds::max()) { + value >= nanoseconds::max()) { return std::nullopt; } diff --git a/libkineto/src/CuptiPMSamplingApi.cpp b/libkineto/src/CuptiPMSamplingApi.cpp index 490d28fd8..08eec82ec 100644 --- a/libkineto/src/CuptiPMSamplingApi.cpp +++ b/libkineto/src/CuptiPMSamplingApi.cpp @@ -181,31 +181,47 @@ void CuptiPMSamplingApi::configureCupti() { } CUpti_PmSampling_TriggerMode triggerMode; uint64_t samplingInterval; - auto capacityInterval = config_.samplingInterval; + std::chrono::nanoseconds capacityInterval = + config_.samplingInterval.value_or(std::chrono::milliseconds{1}); // GPU_TIME_INTERVAL is unavailable on GA100. The wall-time duration of this // fixed SYSCLK interval varies with the GPU's clock frequency. if (deviceProperties.major == 8 && deviceProperties.minor == 0) { - if (deviceProperties.clockRate <= 0) { + if (config_.samplingInterval.has_value()) { + LOG(WARNING) + << "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS is ignored on GA100; " + "continuing with the fixed 1,000,000-cycle SYSCLK sampling " + "interval. The time between samples varies with the GPU clock."; + } + int peakClockRateKHz = 0; + const cudaError_t clockRateStatus = cudaDeviceGetAttribute( + &peakClockRateKHz, cudaDevAttrClockRate, config_.deviceId); + if (clockRateStatus != cudaSuccess) { + KINETO_THROW( + std::runtime_error, + std::string{"cudaDeviceGetAttribute(cudaDevAttrClockRate) failed: "} + + cudaGetErrorString(clockRateStatus)); + } + if (peakClockRateKHz <= 0) { KINETO_THROW( std::runtime_error, "GA100 reported an invalid GPU clock rate"); } triggerMode = CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_SYSCLK_INTERVAL; samplingInterval = kGa100SamplingIntervalCycles; - // clockRate is the maximum clock in kHz. It gives the shortest possible - // period and therefore enough capacity at GA100's fastest sampling rate. + // Use the peak clock only for conservative decode capacity sizing; + // GA100's actual SYSCLK sampling period varies with the GPU clock. capacityInterval = std::chrono::duration_cast( std::chrono::duration{ static_cast(kGa100SamplingIntervalCycles) / - (static_cast(deviceProperties.clockRate) * 1000.0)}); + (static_cast(peakClockRateKHz) * 1000.0)}); } else if ( deviceProperties.major > 8 || (deviceProperties.major == 8 && deviceProperties.minor >= 6)) { - if (config_.samplingInterval.count() <= 0) { + if (capacityInterval.count() <= 0) { KINETO_THROW( std::runtime_error, "CUPTI PM sampling interval must be positive"); } triggerMode = CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_TIME_INTERVAL; - samplingInterval = static_cast(config_.samplingInterval.count()); + samplingInterval = static_cast(capacityInterval.count()); } else { KINETO_THROW( std::runtime_error, @@ -454,7 +470,7 @@ void CuptiPMSamplingApi::disable() { config_.deviceId = -1; metricNamePtrs_.clear(); config_.metricNames.clear(); - config_.samplingInterval = std::chrono::nanoseconds::zero(); + config_.samplingInterval.reset(); config_.lookbackWindow = std::chrono::nanoseconds::zero(); } diff --git a/libkineto/src/CuptiPMSamplingApi.h b/libkineto/src/CuptiPMSamplingApi.h index f153de553..64a708df2 100644 --- a/libkineto/src/CuptiPMSamplingApi.h +++ b/libkineto/src/CuptiPMSamplingApi.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -30,7 +31,8 @@ struct CuptiPMSample { struct CuptiPMSamplingConfig { int32_t deviceId{-1}; std::vector metricNames; - std::chrono::nanoseconds samplingInterval{std::chrono::milliseconds{1}}; + // Unset selects 1 ms on GA10x and newer, or 1M SYSCLK cycles on GA100. + std::optional samplingInterval; std::chrono::nanoseconds lookbackWindow{std::chrono::seconds{1}}; }; diff --git a/libkineto/test/ConfigTest.cpp b/libkineto/test/ConfigTest.cpp index e90161ddf..06d470933 100644 --- a/libkineto/test/ConfigTest.cpp +++ b/libkineto/test/ConfigTest.cpp @@ -143,7 +143,7 @@ TEST(ParseTest, PerformanceMetrics) { Config cfg; EXPECT_TRUE(cfg.performanceMetricNames().empty()); EXPECT_EQ(cfg.performanceMetricsDeviceId(), -1); - EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), milliseconds(1)); + EXPECT_FALSE(cfg.performanceMetricsSamplingInterval().has_value()); EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(1)); EXPECT_TRUE( @@ -168,12 +168,30 @@ TEST(ParseTest, PerformanceMetrics) { EXPECT_EQ(clone->performanceMetricsLookbackWindow(), seconds(2)); } +TEST(ParseTest, PerformanceMetricsPreservesExplicitDefaultInterval) { + Config cfg; + EXPECT_FALSE(cfg.clone()->performanceMetricsSamplingInterval().has_value()); + + ASSERT_TRUE(cfg.parse("PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=1")); + EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), milliseconds(1)); + EXPECT_EQ(cfg.clone()->performanceMetricsSamplingInterval(), milliseconds(1)); +} + TEST(ParseTest, PerformanceMetricsInvalidDurationsUseDefaults) { Config cfg; EXPECT_TRUE( cfg.parse("PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=invalid\n" "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS=0")); - EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), milliseconds(1)); + EXPECT_FALSE(cfg.performanceMetricsSamplingInterval().has_value()); + EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(1)); +} + +TEST(ParseTest, PerformanceMetricsRejectDurationsAtNanosecondLimit) { + Config cfg; + EXPECT_TRUE( + cfg.parse("PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=9223372036854.776\n" + "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS=9223372036854.776")); + EXPECT_FALSE(cfg.performanceMetricsSamplingInterval().has_value()); EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(1)); } diff --git a/libkineto/test/CuptiPMSamplingApiTest.cpp b/libkineto/test/CuptiPMSamplingApiTest.cpp index 1426e4708..38a990d0e 100644 --- a/libkineto/test/CuptiPMSamplingApiTest.cpp +++ b/libkineto/test/CuptiPMSamplingApiTest.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,12 @@ #include #include +#include "include/Config.h" #include "src/CuptiPMSamplingApi.h" +#include "src/CuptiPMSamplingProfiler.h" +#include "src/CuptiTimestamp.h" +#include "src/Logger.h" +#include "src/LoggerCollector.h" using namespace KINETO_NAMESPACE; using namespace std::chrono_literals; @@ -44,6 +50,8 @@ struct FakeCuptiState { int major{9}; int minor{0}; int clockRate{1'500'000}; + int clockRateDevice{-1}; + cudaError_t clockRateResult{cudaSuccess}; size_t numPasses{1}; CUpti_PmSampling_DecodeStopReason decodeStopReason{ CUPTI_PM_SAMPLING_DECODE_STOP_REASON_END_OF_RECORDS}; @@ -103,7 +111,7 @@ std::vector copyMetricNames( } CuptiPMSamplingConfig makeConfig( - std::chrono::nanoseconds samplingInterval = 500us, + std::optional samplingInterval = 500us, int32_t deviceId = 0, std::vector metricNames = {"sm__cycles_active.avg"}, std::chrono::nanoseconds lookbackWindow = 1s) { @@ -114,7 +122,7 @@ CuptiPMSamplingConfig makeConfig( void configureForDevice( int major, int minor, - std::chrono::nanoseconds samplingInterval) { + std::optional samplingInterval) { fakeCupti().major = major; fakeCupti().minor = minor; CuptiPMSamplingApi api; @@ -159,10 +167,17 @@ cudaGetDeviceProperties(cudaDeviceProp* properties, int device) { *properties = cudaDeviceProp{}; properties->major = fakeCupti().major; properties->minor = fakeCupti().minor; - properties->clockRate = fakeCupti().clockRate; return cudaSuccess; } +cudaError_t CUDARTAPI +cudaDeviceGetAttribute(int* value, cudaDeviceAttr attribute, int device) { + EXPECT_EQ(attribute, cudaDevAttrClockRate); + fakeCupti().clockRateDevice = device; + *value = fakeCupti().clockRate; + return fakeCupti().clockRateResult; +} + const char* CUDARTAPI cudaGetErrorString(cudaError_t error) { static_cast(error); return "mock CUDA error"; @@ -356,16 +371,107 @@ TEST_F(CuptiPMSamplingApiTest, KeepsAtLeastOneSample) { api.disable(); } -TEST_F(CuptiPMSamplingApiTest, UsesEffectiveSysclkCadenceOnGa100) { - configureForDevice(8, 0, 0ns); - +TEST_F(CuptiPMSamplingApiTest, SizesGa100CapacityFromPeakClock) { + fakeCupti().major = 8; + fakeCupti().minor = 0; + Config config; + ASSERT_TRUE( + config.parse("PERFORMANCE_METRICS_DEVICE_ID=2\n" + "PERFORMANCE_METRICS=sm__cycles_active.avg\n" + "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS=2000")); + configureCuptiTimestampSource(false); + CuptiPMSamplingProfiler profiler; + auto session = profiler.configure({ActivityType::HARDWARE_COUNTERS}, config); + ASSERT_NE(session, nullptr); + + EXPECT_EQ(fakeCupti().clockRateDevice, 2); EXPECT_EQ( fakeCupti().triggerMode, CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_SYSCLK_INTERVAL); EXPECT_EQ(fakeCupti().samplingInterval, 1'000'000); - // At the fake 1.5 GHz maximum clock, 1M cycles yields 1.5K samples in 1s. + // At the fake 1.5 GHz maximum clock, 1M cycles yields 3K samples in 2s. + EXPECT_EQ(fakeCupti().maxSamples, 3'000); +} + +TEST_F(CuptiPMSamplingApiTest, IgnoresExplicitTimeIntervalsOnGa100) { + for (const std::chrono::nanoseconds interval : + {0ns, 1ns, 500'000ns, 1'000'000ns}) { + SCOPED_TRACE(interval.count()); + EXPECT_NO_THROW(configureForDevice(8, 0, interval)); + EXPECT_EQ( + fakeCupti().triggerMode, + CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_SYSCLK_INTERVAL); + EXPECT_EQ(fakeCupti().samplingInterval, 1'000'000); + EXPECT_EQ(fakeCupti().maxSamples, 1'500); + } +} + +#if !USE_GOOGLE_LOG +TEST_F(CuptiPMSamplingApiTest, WarnsAndContinuesWithExplicitGa100Interval) { + fakeCupti().major = 8; + fakeCupti().minor = 0; + Config config; + ASSERT_TRUE( + config.parse("PERFORMANCE_METRICS_DEVICE_ID=0\n" + "PERFORMANCE_METRICS=sm__cycles_active.avg\n" + "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=1")); + configureCuptiTimestampSource(false); + CuptiPMSamplingProfiler profiler; + LoggerCollector collector; + const int previousSeverity = Logger::severityLevel(); + Logger::setSeverityLevel(LoggerOutputType::WARNING); + Logger::addLoggerObserver(&collector); + + auto session = profiler.configure({ActivityType::HARDWARE_COUNTERS}, config); + + Logger::removeLoggerObserver(&collector); + Logger::setSeverityLevel(previousSeverity); + EXPECT_NE(session, nullptr); + auto messages = collector.extractCollectorMetadata(); + EXPECT_TRUE(messages[LoggerOutputType::ERROR].empty()); + const auto& warnings = messages[LoggerOutputType::WARNING]; + ASSERT_EQ(warnings.size(), 1); + EXPECT_NE( + warnings.front().find("PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS"), + std::string::npos); + EXPECT_NE(warnings.front().find("ignored"), std::string::npos); + EXPECT_NE(warnings.front().find("1,000,000-cycle"), std::string::npos); + EXPECT_EQ(fakeCupti().clockRateDevice, 0); + EXPECT_EQ(fakeCupti().samplingInterval, 1'000'000); EXPECT_EQ(fakeCupti().maxSamples, 1'500); } +#endif + +TEST_F(CuptiPMSamplingApiTest, CleansUpAfterGa100ClockQueryFailure) { + fakeCupti().major = 8; + fakeCupti().minor = 0; + fakeCupti().clockRateResult = cudaErrorInvalidValue; + CuptiPMSamplingApi api; + + EXPECT_THROW(api.configure(makeConfig(std::nullopt)), std::runtime_error); + + clearCalls(); + api.disable(); + expectCalls({"profilerDeInitialize"}); +} + +TEST_F(CuptiPMSamplingApiTest, RejectsNonpositiveGa100ClockRates) { + for (const int clockRate : {0, -1}) { + SCOPED_TRACE(clockRate); + fakeCupti().clockRate = clockRate; + EXPECT_THROW(configureForDevice(8, 0, std::nullopt), std::runtime_error); + } +} + +TEST_F(CuptiPMSamplingApiTest, UsesDefaultTimeIntervalOnGa10xAndNewer) { + for (const auto& [major, minor] : {std::pair{8, 6}, std::pair{9, 0}}) { + SCOPED_TRACE(testing::Message() << major << "." << minor); + configureForDevice(major, minor, std::nullopt); + + EXPECT_EQ(fakeCupti().samplingInterval, 1'000'000); + EXPECT_EQ(fakeCupti().maxSamples, 1'000); + } +} TEST_F(CuptiPMSamplingApiTest, UsesRequestedTimeIntervalOnGa10xAndNewer) { for (const auto& [major, minor] : {std::pair{8, 6}, std::pair{9, 0}}) { @@ -376,6 +482,7 @@ TEST_F(CuptiPMSamplingApiTest, UsesRequestedTimeIntervalOnGa10xAndNewer) { fakeCupti().triggerMode, CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_TIME_INTERVAL); EXPECT_EQ(fakeCupti().samplingInterval, 500'000); + EXPECT_EQ(fakeCupti().clockRateDevice, -1); } } From 3274e6c2a6faeec5551245f100c1da5c33f12ace Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Wed, 16 Sep 2026 16:28:55 -0700 Subject: [PATCH 08/10] update --- libkineto/src/Config.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libkineto/src/Config.cpp b/libkineto/src/Config.cpp index 54906cf0b..dfe9e3e83 100644 --- a/libkineto/src/Config.cpp +++ b/libkineto/src/Config.cpp @@ -244,8 +244,6 @@ std::optional parseMilliseconds(std::string_view text) noexcept { } const duration value{count}; - // nanoseconds::max() rounds up to 2^63 in double precision, so exclude - // equality before converting back to an integer. if (!std::isfinite(count) || value < nanoseconds{1} || value >= nanoseconds::max()) { return std::nullopt; From 8e2d188f5a03de497d263f2048f9370be1e4bcde Mon Sep 17 00:00:00 2001 From: ryanzhang22 Date: Mon, 21 Sep 2026 18:46:13 +0000 Subject: [PATCH 09/10] update --- libkineto/src/Config.cpp | 13 +++++++------ libkineto/src/CuptiPMSamplingApi.cpp | 8 ++++++++ libkineto/test/CuptiPMSamplingApiTest.cpp | 14 +++++++++++++- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/libkineto/src/Config.cpp b/libkineto/src/Config.cpp index dfe9e3e83..2dc3e47d7 100644 --- a/libkineto/src/Config.cpp +++ b/libkineto/src/Config.cpp @@ -9,6 +9,7 @@ #include "Config.h" #include "ThrowUtil.h" +#include #include #include @@ -17,7 +18,6 @@ #include #include -#include #include #include #include @@ -235,11 +235,12 @@ bool isAllowedOnDemandTraceFile(const string& path) { return path.starts_with(dir) && path.find("..") == string::npos; } -std::optional parseMilliseconds(std::string_view text) noexcept { - double count = 0; - const auto [parseEnd, error] = - std::from_chars(text.data(), text.data() + text.size(), count); - if (error != std::errc{} || parseEnd != text.data() + text.size()) { +std::optional parseMilliseconds(const string& text) noexcept { + const char* const begin = text.c_str(); + char* parseEnd = nullptr; + errno = 0; + const double count = std::strtod(begin, &parseEnd); + if (errno == ERANGE || parseEnd == begin || parseEnd != begin + text.size()) { return std::nullopt; } diff --git a/libkineto/src/CuptiPMSamplingApi.cpp b/libkineto/src/CuptiPMSamplingApi.cpp index 08eec82ec..f6a036457 100644 --- a/libkineto/src/CuptiPMSamplingApi.cpp +++ b/libkineto/src/CuptiPMSamplingApi.cpp @@ -49,6 +49,9 @@ namespace KINETO_NAMESPACE { namespace { constexpr size_t kHardwareBufferSizeBytes = 64 * 1024 * 1024; +// Counter-data images are fully allocated in host memory. Bound their size +// independently of CUPTI's uint32_t sample-count limit. +constexpr size_t kMaxCounterDataImageSizeBytes = 256 * 1024 * 1024; // GA100 only supports variable-frequency SYSCLK sampling. One option is to // measure hardware clock frequency and estimate the number of cycles needed to @@ -328,6 +331,11 @@ void CuptiPMSamplingApi::configureCupti() { counterDataSize.numMetrics = metricNamePtrs_.size(); counterDataSize.maxSamples = maxSamples; CUPTI_CALL_THROW(cuptiPmSamplingGetCounterDataSize(&counterDataSize)); + if (counterDataSize.counterDataSize > kMaxCounterDataImageSizeBytes) { + KINETO_THROW( + std::runtime_error, + "CUPTI PM sampling counter data image exceeds the 256 MiB limit"); + } counterDataImage_.resize(counterDataSize.counterDataSize); resetImage(); diff --git a/libkineto/test/CuptiPMSamplingApiTest.cpp b/libkineto/test/CuptiPMSamplingApiTest.cpp index 38a990d0e..3f9784be4 100644 --- a/libkineto/test/CuptiPMSamplingApiTest.cpp +++ b/libkineto/test/CuptiPMSamplingApiTest.cpp @@ -68,6 +68,7 @@ struct FakeCuptiState { CUpti_PmSampling_HardwareBuffer_AppendMode appendMode{}; std::vector counterDataMetricNames; uint32_t maxSamples{0}; + size_t counterDataSize{1}; std::byte hostObject{}; std::byte samplingObject{}; @@ -246,7 +247,7 @@ CUptiResult CUPTIAPI cuptiPmSamplingGetCounterDataSize( fakeCupti().counterDataMetricNames = copyMetricNames(params->pMetricNames, params->numMetrics); fakeCupti().maxSamples = params->maxSamples; - params->counterDataSize = 1; + params->counterDataSize = fakeCupti().counterDataSize; return CUPTI_SUCCESS; } @@ -521,6 +522,17 @@ TEST_F(CuptiPMSamplingApiTest, RejectsSampleCapacityAboveCuptiLimit) { std::runtime_error); } +TEST_F(CuptiPMSamplingApiTest, RejectsCounterDataImageAboveMemoryLimit) { + fakeCupti().counterDataSize = 256 * 1024 * 1024 + 1; + CuptiPMSamplingApi api; + + EXPECT_THROW(api.configure(makeConfig()), std::runtime_error); + + clearCalls(); + api.disable(); + expectCalls({"samplingDisable", "hostDeinitialize", "profilerDeInitialize"}); +} + TEST_F(CuptiPMSamplingApiTest, RejectsMultipassConfigurationBeforeEnabling) { fakeCupti().numPasses = 2; CuptiPMSamplingApi api; From d6b6801f516e597266cc4cf3295e58b7822ff8dd Mon Sep 17 00:00:00 2001 From: ryanzhang22 Date: Mon, 21 Sep 2026 19:03:06 +0000 Subject: [PATCH 10/10] update --- libkineto/src/CuptiPMSamplingApi.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libkineto/src/CuptiPMSamplingApi.cpp b/libkineto/src/CuptiPMSamplingApi.cpp index f6a036457..e77ab286f 100644 --- a/libkineto/src/CuptiPMSamplingApi.cpp +++ b/libkineto/src/CuptiPMSamplingApi.cpp @@ -334,7 +334,12 @@ void CuptiPMSamplingApi::configureCupti() { if (counterDataSize.counterDataSize > kMaxCounterDataImageSizeBytes) { KINETO_THROW( std::runtime_error, - "CUPTI PM sampling counter data image exceeds the 256 MiB limit"); + "CUPTI PM sampling counter data image requires " + + std::to_string(counterDataSize.counterDataSize) + + " bytes, exceeding the 256 MiB limit. Reduce " + "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS, increase " + "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS on non-GA100 GPUs, or " + "request fewer PERFORMANCE_METRICS."); } counterDataImage_.resize(counterDataSize.counterDataSize);