[CAS] Implemented llvm-cas --show-stat

clang/include/clang/Basic/DiagnosticCASKinds.td

@@ -44,6 +44,8 @@ def err_cas_missing_input_cache_entry : Error<
   "CAS missing expected input cache key '%0'">, DefaultFatal;
 def err_cas_cannot_load_api_notes_include_tree : Error<
   "cannot load APINotes from include-tree-id '%0'">, DefaultFatal;
+def err_cas_stats_cannot_be_initialized : Error<
+  "CAS stats cannot be initialized: %0">, DefaultFatal;
 
 def warn_clang_cache_disabled_caching: Warning<
   "caching disabled because %0">,

clang/include/clang/Frontend/CompilerInstance.h

@@ -27,6 +27,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CAS/CASID.h"
 #include "llvm/CAS/CASOutputBackend.h"
+#include "llvm/CAS/Stats.h"
 #include "llvm/Support/BuryPointer.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/PrefixMapper.h"
@@ -1085,6 +1086,22 @@ class CompilerInstance : public ModuleLoader {
 
   void setCASDatabases(std::shared_ptr<llvm::cas::ObjectStore> CAS,
                        std::shared_ptr<llvm::cas::ActionCache> Cache);
+
+  static bool setupCacheStatsDiagnostics(DiagnosticsEngine &Diags);
+};
+
+class CacheStatsDiagnosticConsumer : public DiagnosticConsumer {
+public:
+  CacheStatsDiagnosticConsumer() = default;
+  CacheStatsDiagnosticConsumer(CacheStatsDiagnosticConsumer &&Other)
+      : Stats(std::move(Other.Stats)) {};
+  CacheStatsDiagnosticConsumer(llvm::cas::Stats &Stats)
+      : Stats(std::move(Stats)) {};
+  void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                        const Diagnostic &Info) override;
+
+private:
+  llvm::cas::Stats Stats;
 };
 
 } // end namespace clang

clang/lib/Frontend/CompilerInstance.cpp

@@ -67,6 +67,7 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/SmallVectorMemoryBuffer.h"
 #include "llvm/Support/Threading.h"
@@ -381,6 +382,27 @@ static void SetupSerializedDiagnostics(DiagnosticOptions &DiagOpts,
   }
 }
 
+bool CompilerInstance::setupCacheStatsDiagnostics(DiagnosticsEngine &Diags) {
+  auto CASPath = llvm::sys::Process::GetEnv("LLVM_CACHE_CAS_PATH");
+  if (!CASPath)
+    return true;
+  llvm::cas::Stats Stats;
+  if (auto Err = Stats.open(*CASPath)) {
+    Diags.Report(diag::err_cas_stats_cannot_be_initialized)
+        << llvm::toString(std::move(Err));
+    return false;
+  }
+  auto SC = std::make_unique<CacheStatsDiagnosticConsumer>(Stats);
+  if (Diags.ownsClient()) {
+    Diags.setClient(
+        new ChainedDiagnosticConsumer(Diags.takeClient(), std::move(SC)));
+  } else {
+    Diags.setClient(
+        new ChainedDiagnosticConsumer(Diags.getClient(), std::move(SC)));
+  }
+  return true;
+}
+
 void CompilerInstance::createDiagnostics(DiagnosticConsumer *Client,
                                          bool ShouldOwnClient) {
   Diagnostics = createDiagnostics(getVirtualFileSystem(), getDiagnosticOpts(),
@@ -403,6 +425,8 @@ IntrusiveRefCntPtr<DiagnosticsEngine> CompilerInstance::createDiagnostics(
   } else
     Diags->setClient(new TextDiagnosticPrinter(llvm::errs(), Opts));
 
+  setupCacheStatsDiagnostics(*Diags);
+
   // Chain in -verify checker, if requested.
   if (Opts.VerifyDiagnostics)
     Diags->setClient(new VerifyDiagnosticConsumer(*Diags));
@@ -2915,3 +2939,67 @@ bool CompileCacheASTReaderHelper::readIncludeTreeID(StringRef ID,
   return checkCASID(Complain, ID, diag::err_cas_cannot_parse_include_tree_id,
                     diag::err_cas_missing_include_tree_id);
 }
+
+void CacheStatsDiagnosticConsumer::HandleDiagnostic(
+    DiagnosticsEngine::Level Level, const Diagnostic &Info) {
+  if (Level == DiagnosticsEngine::Remark) {
+    // StringRef remark =
+    // Info.getDiags()->getDiagnosticIDs()->getWarningOptionForDiag(Info.getID());
+    // "compile-job-cache-miss"
+
+    //SmallString<100> MsgBuf;
+    //Info.FormatDiagnostic(MsgBuf);
+    // std::string Message(MsgBuf);
+
+    double T = 0.0;
+    if (Info.getNumArgs() > 0 &&
+        Info.getArgKind(0) == DiagnosticsEngine::ak_std_string) {
+      llvm::StringRef S = StringRef(Info.getArgStdStr(0)).rtrim(" s\r\n\t");
+      llvm::to_float(S, T);
+    }
+
+    switch (Info.getID()) {
+    case diag::remark_compile_job_cache_hit:
+      Stats.inc(llvm::cas::Stats::Key::hit);
+      break;
+    case diag::remark_compile_job_cache_miss:
+      Stats.inc(llvm::cas::Stats::Key::miss);
+      break;
+    case diag::remark_compile_job_cache_miss_result_not_found:
+      Stats.inc(llvm::cas::Stats::Key::miss_result_not_found);
+      break;
+    case diag::remark_compile_job_cache_backend_output_not_found:
+      Stats.inc(llvm::cas::Stats::Key::backend_output_not_found);
+      break;
+    case diag::remark_compile_job_cache_skipped:
+      Stats.inc(llvm::cas::Stats::Key::skipped);
+      break;
+
+    case diag::remark_compile_job_cache_timing_depscan:
+      Stats.inc(llvm::cas::Stats::Key::depscan_count);
+      Stats.inc_f(llvm::cas::Stats::Key::depscan_time, T);
+      break;
+    case diag::remark_compile_job_cache_timing_backend_key_query:
+      Stats.inc(llvm::cas::Stats::Key::backend_key_query_count);
+      Stats.inc_f(llvm::cas::Stats::Key::backend_key_query_time, T);
+      break;
+    case diag::remark_compile_job_cache_timing_backend_key_update:
+      Stats.inc(llvm::cas::Stats::Key::backend_key_update_count);
+      Stats.inc_f(llvm::cas::Stats::Key::backend_key_update_time, T);
+      break;
+    case diag::remark_compile_job_cache_timing_backend_load:
+      Stats.inc(llvm::cas::Stats::Key::backend_load_count);
+      Stats.inc_f(llvm::cas::Stats::Key::backend_load_time, T);
+      break;
+    case diag::remark_compile_job_cache_timing_backend_store:
+      Stats.inc(llvm::cas::Stats::Key::backend_store_count);
+      Stats.inc_f(llvm::cas::Stats::Key::backend_store_time, T);
+      break;
+
+    default:
+      break;
+    }
+  }
+
+  // DiagnosticConsumer::HandleDiagnostic(Level, Info);
+}

clang/test/CAS/stats.c

@@ -0,0 +1,32 @@
+// REQUIRES: ondisk_cas
+// REQUIRES: llvm-cas
+
+// RUN: rm -rf %t && mkdir -p %t
+
+// RUN: %llvm-cas --show-stats -cas %t/cas | FileCheck %s -check-prefix=NO-CACHE
+
+// RUN: env LLVM_CACHE_CAS_PATH=%t/cas %clang-cache %clang -c %s -o %t/test.o -Xclang -Rcompile-job-cache
+// RUN: %llvm-cas --show-stats -cas %t/cas | FileCheck %s -check-prefix=CACHE-MISS
+
+// RUN: env LLVM_CACHE_CAS_PATH=%t/cas %clang-cache %clang -c %s -o %t/test.o -Xclang -Rcompile-job-cache
+// RUN: %llvm-cas --show-stats -cas %t/cas | FileCheck %s -check-prefix=CACHE-HIT
+
+// RUN: %llvm-cas --zero-stats -cas %t/cas | FileCheck %s -check-prefix=ZERO
+
+// RUN: %llvm-cas --show-stats -cas %t/cas | FileCheck %s -check-prefix=AFTER-ZERO
+
+// NO-CACHE: Cache hits:                      0
+// NO-CACHE: Cache misses:                    0
+
+// CACHE-MISS: Cache hits:                      0
+// CACHE-MISS: Cache misses:                    1
+
+// CACHE-HIT: Cache hits:                      1
+// CACHE-HIT: Cache misses:                    1
+
+// ZERO: zero-stats: successfull
+
+// AFTER-ZERO: Cache hits:                      0
+// AFTER-ZERO: Cache misses:                    0
+
+int main() { return 0; }

clang/tools/driver/cc1depscan_main.cpp

@@ -736,6 +736,10 @@ int cc1depscan_main(ArrayRef<const char *> Argv, const char *Argv0,
       std::make_unique<TextDiagnosticPrinter>(llvm::errs(), *DiagOpts, false);
   DiagnosticsEngine Diags(new DiagnosticIDs(), *DiagOpts);
   Diags.setClient(DiagsConsumer.get(), /*ShouldOwnClient=*/false);
+
+  if (!CompilerInstance::setupCacheStatsDiagnostics(Diags))
+    return 1;
+
   auto VFS = [] {
     auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
     return llvm::vfs::getRealFileSystem();

llvm/include/llvm/CAS/Stats.h

@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// 
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_STATS_H
+#define LLVM_CAS_STATS_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+
+namespace llvm::cas {
+
+class Stats {
+public:
+  static const uint64_t Version = 1;
+
+  enum class Key : uint16_t {
+    version = 0,
+    hit = 1,
+    miss = 2,
+    miss_result_not_found = 3,
+    backend_output_not_found = 4,
+    skipped = 5,
+    depscan_time = 6,
+    depscan_count = 7,
+    backend_key_query_time = 8,
+    backend_key_query_count = 9,
+    backend_key_update_time = 10,
+    backend_key_update_count = 11,
+    backend_load_time = 12,
+    backend_load_count = 13,
+    backend_store_time = 14,
+    backend_store_count = 15,
+    last = 16,
+  };
+
+  Stats() : FD(-1) {};
+  Stats(Stats &&Other) : Map(std::move(Other.Map)), FD(Other.FD) {
+    Other.FD = -1;
+  };
+  ~Stats() { close(); }
+
+  Error open(StringRef Path);
+  void close();
+
+  uint64_t get(Key K);
+  void set(Key K, uint64_t V);
+  void inc(Key K, uint64_t V = 1);
+
+  double get_f(Key K);
+  void set_f(Key K, double V);
+  void inc_f(Key K, double V);
+
+private:
+  bool key_valid(Key K) { return (Map && K < Key::last); }
+  void *key2addr(Key K) {
+    return reinterpret_cast<void *>(Map.data() + (uint64_t)K * sizeof(uint64_t));
+  }
+  int FD;
+  sys::fs::mapped_file_region Map;
+};
+
+} // namespace llvm::cas
+
+#endif // LLVM_CAS_STATS_H

llvm/lib/CAS/CMakeLists.txt

@@ -30,6 +30,7 @@ add_llvm_component_library(LLVMCAS
   OnDiskTrieRawHashMap.cpp
   OnDiskKeyValueDB.cpp
   PluginCAS.cpp
+  Stats.cpp
   TreeSchema.cpp
   UnifiedOnDiskCache.cpp
   Utils.cpp