[Kineto] Add lookback window and sampling interval as config options

libkineto/include/Config.h

@@ -15,6 +15,7 @@
 #include <chrono>
 #include <cstdint>
 #include <functional>
+#include <optional>
 #include <set>
 #include <string>
 #include <vector>
@@ -236,6 +237,16 @@ class Config : public AbstractConfig {
     return performanceMetricsDeviceId_;
   }
 
+  [[nodiscard]] std::optional<std::chrono::nanoseconds>
+  performanceMetricsSamplingInterval() const {
+    return performanceMetricsSamplingInterval_;
+  }
+
+  [[nodiscard]] std::chrono::nanoseconds performanceMetricsLookbackWindow()
+      const {
+    return performanceMetricsLookbackWindow_;
+  }
+
   [[nodiscard]] bool memoryProfilerEnabled() const {
     return memoryProfilerEnabled_;
   }
@@ -375,6 +386,9 @@ class Config : public AbstractConfig {
   // Performance metrics
   std::vector<std::string> performanceMetricNames_;
   int32_t performanceMetricsDeviceId_{-1};
+  std::optional<std::chrono::nanoseconds> performanceMetricsSamplingInterval_;
+  std::chrono::nanoseconds performanceMetricsLookbackWindow_{
+      std::chrono::seconds{1}};
 
   // CUPTI Timestamp Format
   bool useTSCTimestamp_{true};

libkineto/src/Config.cpp

@@ -9,6 +9,8 @@
 #include "Config.h"
 #include "ThrowUtil.h"
 
+#include <cerrno>
+#include <cmath>
 #include <cstdlib>
 
 #include <fmt/chrono.h>
@@ -20,6 +22,7 @@
 #include <ctime>
 #include <functional>
 #include <mutex>
+#include <optional>
 #include <ostream>
 #include <string_view>
 #include <utility>
@@ -55,6 +58,10 @@ constexpr char kCuptiPerThreadBufferEnabledKey[] =
 constexpr char kPerformanceMetricsKey[] = "PERFORMANCE_METRICS";
 constexpr char kPerformanceMetricsDeviceIdKey[] =
     "PERFORMANCE_METRICS_DEVICE_ID";
+constexpr char kPerformanceMetricsSamplingIntervalMsecsKey[] =
+    "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS";
+constexpr char kPerformanceMetricsLookbackWindowMsecsKey[] =
+    "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS";
 constexpr char kActivityTypesKey[] = "ACTIVITY_TYPES";
 constexpr char kActivitiesLogFileKey[] = "ACTIVITIES_LOG_FILE";
 constexpr char kActivitiesDurationKey[] = "ACTIVITIES_DURATION_SECS";
@@ -228,6 +235,24 @@ bool isAllowedOnDemandTraceFile(const string& path) {
   return path.starts_with(dir) && path.find("..") == string::npos;
 }
 
+std::optional<nanoseconds> parseMilliseconds(const string& text) noexcept {
+  const char* const begin = text.c_str();
+  char* parseEnd = nullptr;
+  errno = 0;
+  const double count = std::strtod(begin, &parseEnd);
+  if (errno == ERANGE || parseEnd == begin || parseEnd != begin + text.size()) {
+    return std::nullopt;
+  }
+
+  const duration<double, std::milli> value{count};
+  if (!std::isfinite(count) || value < nanoseconds{1} ||
+      value >= nanoseconds::max()) {
+    return std::nullopt;
+  }
+
+  return duration_cast<nanoseconds>(value);
+}
+
 } // namespace
 
 Config::Config()
@@ -351,6 +376,14 @@ bool Config::handleOption(const std::string& name, std::string& val) {
     performanceMetricNames_ = splitAndTrim(val, ',');
   } else if (!name.compare(kPerformanceMetricsDeviceIdKey)) {
     performanceMetricsDeviceId_ = toInt32(val);
+  } else if (!name.compare(kPerformanceMetricsSamplingIntervalMsecsKey)) {
+    if (const auto value = parseMilliseconds(val)) {
+      performanceMetricsSamplingInterval_ = *value;
+    }
+  } else if (!name.compare(kPerformanceMetricsLookbackWindowMsecsKey)) {
+    if (const auto value = parseMilliseconds(val)) {
+      performanceMetricsLookbackWindow_ = *value;
+    }
   } else if (!name.compare(kProfileMemory)) {
     memoryProfilerEnabled_ = toBool(val);
     if (memoryProfilerEnabled_) {

libkineto/src/CuptiPMSamplingApi.cpp

@@ -8,6 +8,8 @@
 
 #include "CuptiPMSamplingApi.h"
 
+#include <algorithm>
+#include <limits>
 #include <stdexcept>
 
 #include <cuda_runtime_api.h>
@@ -47,14 +49,33 @@ namespace KINETO_NAMESPACE {
 namespace {
 
 constexpr size_t kHardwareBufferSizeBytes = 64 * 1024 * 1024;
-constexpr uint32_t kMaxSamplesPerDecode = 1024;
+// Counter-data images are fully allocated in host memory. Bound their size
+// independently of CUPTI's uint32_t sample-count limit.
+constexpr size_t kMaxCounterDataImageSizeBytes = 256 * 1024 * 1024;
 
 // GA100 only supports variable-frequency SYSCLK sampling. One option is to
 // measure hardware clock frequency and estimate the number of cycles needed to
 // fill a certain sampling duration, but clock frequency is not necessarily
 // fixed. This is almost certainly something we'll need to revisit/tune.
 constexpr uint64_t kGa100SamplingIntervalCycles = 1'000'000;
 
+uint32_t maxSamplesForDecode(
+    std::chrono::nanoseconds lookbackWindow,
+    std::chrono::nanoseconds samplingInterval) {
+  if (lookbackWindow.count() <= 0 || samplingInterval.count() <= 0) {
+    KINETO_THROW(
+        std::runtime_error,
+        "CUPTI PM sampling interval and lookback window must be positive");
+  }
+  const auto samples = std::max<int64_t>(1, lookbackWindow / samplingInterval);
+  if (samples > std::numeric_limits<uint32_t>::max()) {
+    KINETO_THROW(
+        std::runtime_error,
+        "CUPTI PM sampling lookback window exceeds CUPTI sample capacity");
+  }
+  return static_cast<uint32_t>(samples);
+}
+
 /*
  * ========================== CUPTI PM SAMPLING API ==========================
  *
@@ -163,26 +184,55 @@ void CuptiPMSamplingApi::configureCupti() {
   }
   CUpti_PmSampling_TriggerMode triggerMode;
   uint64_t samplingInterval;
+  std::chrono::nanoseconds capacityInterval =
+      config_.samplingInterval.value_or(std::chrono::milliseconds{1});
   // GPU_TIME_INTERVAL is unavailable on GA100. The wall-time duration of this
   // fixed SYSCLK interval varies with the GPU's clock frequency.
   if (deviceProperties.major == 8 && deviceProperties.minor == 0) {
+    if (config_.samplingInterval.has_value()) {
+      LOG(WARNING)
+          << "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS is ignored on GA100; "
+             "continuing with the fixed 1,000,000-cycle SYSCLK sampling "
+             "interval. The time between samples varies with the GPU clock.";
+    }
+    int peakClockRateKHz = 0;
+    const cudaError_t clockRateStatus = cudaDeviceGetAttribute(
+        &peakClockRateKHz, cudaDevAttrClockRate, config_.deviceId);
+    if (clockRateStatus != cudaSuccess) {
+      KINETO_THROW(
+          std::runtime_error,
+          std::string{"cudaDeviceGetAttribute(cudaDevAttrClockRate) failed: "} +
+              cudaGetErrorString(clockRateStatus));
+    }
+    if (peakClockRateKHz <= 0) {
+      KINETO_THROW(
+          std::runtime_error, "GA100 reported an invalid GPU clock rate");
+    }
     triggerMode = CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_SYSCLK_INTERVAL;
     samplingInterval = kGa100SamplingIntervalCycles;
+    // Use the peak clock only for conservative decode capacity sizing;
+    // GA100's actual SYSCLK sampling period varies with the GPU clock.
+    capacityInterval = std::chrono::duration_cast<std::chrono::nanoseconds>(
+        std::chrono::duration<double>{
+            static_cast<double>(kGa100SamplingIntervalCycles) /
+            (static_cast<double>(peakClockRateKHz) * 1000.0)});
   } else if (
       deviceProperties.major > 8 ||
       (deviceProperties.major == 8 && deviceProperties.minor >= 6)) {
-    if (config_.samplingInterval.count() <= 0) {
+    if (capacityInterval.count() <= 0) {
       KINETO_THROW(
           std::runtime_error, "CUPTI PM sampling interval must be positive");
     }
     triggerMode = CUPTI_PM_SAMPLING_TRIGGER_MODE_GPU_TIME_INTERVAL;
-    samplingInterval = static_cast<uint64_t>(config_.samplingInterval.count());
+    samplingInterval = static_cast<uint64_t>(capacityInterval.count());
   } else {
     KINETO_THROW(
         std::runtime_error,
         "Minimum supported GPU for CUPTI PM sampling is GA100 (compute "
         "capability 8.0); other GPUs require compute capability 8.6 or newer");
   }
+  const auto maxSamples =
+      maxSamplesForDecode(config_.lookbackWindow, capacityInterval);
 
   // Building the availability image. This is a CUPTI byte buffer describing
   // which raw hardware counters are available on this machine. The first call
@@ -272,16 +322,25 @@ void CuptiPMSamplingApi::configureCupti() {
       CUPTI_PM_SAMPLING_HARDWARE_BUFFER_APPEND_MODE_KEEP_LATEST;
   CUPTI_CALL_THROW(cuptiPmSamplingSetConfig(&setConfig));
 
-  // Asking CUPTI how large the (counter) data image should be.
-  // Since the data image has an opage CUPTI-defined layout, the size
-  // depends on sampling config, metrics, etc.
+  // Size the counter data image to hold the requested lookback window. CUPTI
+  // 12.x requires SetConfig to run before GetCounterDataSize.
   CUpti_PmSampling_GetCounterDataSize_Params counterDataSize{
       CUpti_PmSampling_GetCounterDataSize_Params_STRUCT_SIZE};
   counterDataSize.pPmSamplingObject = samplingObject_;
   counterDataSize.pMetricNames = metricNamePtrs_.data();
   counterDataSize.numMetrics = metricNamePtrs_.size();
-  counterDataSize.maxSamples = kMaxSamplesPerDecode;
+  counterDataSize.maxSamples = maxSamples;
   CUPTI_CALL_THROW(cuptiPmSamplingGetCounterDataSize(&counterDataSize));
+  if (counterDataSize.counterDataSize > kMaxCounterDataImageSizeBytes) {
+    KINETO_THROW(
+        std::runtime_error,
+        "CUPTI PM sampling counter data image requires " +
+            std::to_string(counterDataSize.counterDataSize) +
+            " bytes, exceeding the 256 MiB limit. Reduce "
+            "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS, increase "
+            "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS on non-GA100 GPUs, or "
+            "request fewer PERFORMANCE_METRICS.");
+  }
 
   counterDataImage_.resize(counterDataSize.counterDataSize);
   resetImage();
@@ -424,7 +483,8 @@ void CuptiPMSamplingApi::disable() {
   config_.deviceId = -1;
   metricNamePtrs_.clear();
   config_.metricNames.clear();
-  config_.samplingInterval = std::chrono::nanoseconds::zero();
+  config_.samplingInterval.reset();
+  config_.lookbackWindow = std::chrono::nanoseconds::zero();
 }
 
 } // namespace KINETO_NAMESPACE

libkineto/src/CuptiPMSamplingApi.h

@@ -11,6 +11,7 @@
 #include <chrono>
 #include <cstddef>
 #include <cstdint>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -30,7 +31,9 @@ struct CuptiPMSample {
 struct CuptiPMSamplingConfig {
   int32_t deviceId{-1};
   std::vector<std::string> metricNames;
-  std::chrono::nanoseconds samplingInterval{0};
+  // Unset selects 1 ms on GA10x and newer, or 1M SYSCLK cycles on GA100.
+  std::optional<std::chrono::nanoseconds> samplingInterval;
+  std::chrono::nanoseconds lookbackWindow{std::chrono::seconds{1}};
 };
 
 class CuptiPMSamplingApi {

libkineto/src/CuptiPMSamplingProfiler.cpp

@@ -26,9 +26,6 @@ namespace {
 const std::string kProfilerName{"CUPTI PM Sampling"};
 const std::set<libkineto::ActivityType> kSupportedActivities{
     libkineto::ActivityType::HARDWARE_COUNTERS};
-// TODO: This is a temporary constant -- we need to tune the interval by
-// hardware type to prevent hardware buffer overflow and sample loss.
-constexpr std::chrono::milliseconds kSamplingInterval{1};
 
 } // namespace
 
@@ -201,7 +198,10 @@ std::unique_ptr<libkineto::IActivityProfilerSession> CuptiPMSamplingProfiler::
   }
 
   const CuptiPMSamplingConfig pmConfig{
-      deviceId, metricNames, kSamplingInterval};
+      deviceId,
+      metricNames,
+      config.performanceMetricsSamplingInterval(),
+      config.performanceMetricsLookbackWindow()};
   auto session = std::make_unique<CuptiPMSamplingSession>(pmConfig);
   if (!session->prepare()) {
     return nullptr;

libkineto/test/ConfigTest.cpp

@@ -143,21 +143,56 @@ TEST(ParseTest, PerformanceMetrics) {
   Config cfg;
   EXPECT_TRUE(cfg.performanceMetricNames().empty());
   EXPECT_EQ(cfg.performanceMetricsDeviceId(), -1);
+  EXPECT_FALSE(cfg.performanceMetricsSamplingInterval().has_value());
+  EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(1));
 
   EXPECT_TRUE(
       cfg.parse("PERFORMANCE_METRICS_DEVICE_ID=2\n"
                 "PERFORMANCE_METRICS="
-                "sm__cycles_active.avg,dram__bytes_read.sum"));
+                "sm__cycles_active.avg,dram__bytes_read.sum\n"
+                "PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=0.5\n"
+                "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS=2000"));
 
   EXPECT_EQ(
       cfg.performanceMetricNames(),
       std::vector<std::string>(
           {"sm__cycles_active.avg", "dram__bytes_read.sum"}));
   EXPECT_EQ(cfg.performanceMetricsDeviceId(), 2);
+  EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), microseconds(500));
+  EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(2));
 
   auto clone = cfg.clone();
   EXPECT_EQ(clone->performanceMetricNames(), cfg.performanceMetricNames());
   EXPECT_EQ(clone->performanceMetricsDeviceId(), 2);
+  EXPECT_EQ(clone->performanceMetricsSamplingInterval(), microseconds(500));
+  EXPECT_EQ(clone->performanceMetricsLookbackWindow(), seconds(2));
+}
+
+TEST(ParseTest, PerformanceMetricsPreservesExplicitDefaultInterval) {
+  Config cfg;
+  EXPECT_FALSE(cfg.clone()->performanceMetricsSamplingInterval().has_value());
+
+  ASSERT_TRUE(cfg.parse("PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=1"));
+  EXPECT_EQ(cfg.performanceMetricsSamplingInterval(), milliseconds(1));
+  EXPECT_EQ(cfg.clone()->performanceMetricsSamplingInterval(), milliseconds(1));
+}
+
+TEST(ParseTest, PerformanceMetricsInvalidDurationsUseDefaults) {
+  Config cfg;
+  EXPECT_TRUE(
+      cfg.parse("PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=invalid\n"
+                "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS=0"));
+  EXPECT_FALSE(cfg.performanceMetricsSamplingInterval().has_value());
+  EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(1));
+}
+
+TEST(ParseTest, PerformanceMetricsRejectDurationsAtNanosecondLimit) {
+  Config cfg;
+  EXPECT_TRUE(
+      cfg.parse("PERFORMANCE_METRICS_SAMPLING_INTERVAL_MS=9223372036854.776\n"
+                "PERFORMANCE_METRICS_LOOKBACK_WINDOW_MS=9223372036854.776"));
+  EXPECT_FALSE(cfg.performanceMetricsSamplingInterval().has_value());
+  EXPECT_EQ(cfg.performanceMetricsLookbackWindow(), seconds(1));
 }
 
 TEST(ParseTest, ProfileStartTime) {