From 5c5688af955f9f2d2bfaadd3fcc968fc99fea5fa Mon Sep 17 00:00:00 2001 From: kweronsx Date: Fri, 16 Oct 2026 12:59:26 +0200 Subject: [PATCH 1/3] [UR] Delay recycling of events created during recording until graph destruction --- .../level_zero/v2/command_list_manager.cpp | 15 +++++++++++++++ .../level_zero/v2/command_list_manager.hpp | 2 ++ .../source/adapters/level_zero/v2/graph.cpp | 10 ++++++++++ .../source/adapters/level_zero/v2/graph.hpp | 6 ++++++ 4 files changed, 33 insertions(+) diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp index 8c97ade034533..4e5e877c29dbe 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp @@ -220,11 +220,25 @@ ur_result_t ur_command_list_manager::appendRegionCopyUnlocked( return UR_RESULT_SUCCESS; } +void ur_command_list_manager::retainEventIfCapturing(ur_event_handle_t hEvent) { + bool captureActive = false; + if (!hEvent || queryGraphCaptureActive(&captureActive) != UR_RESULT_SUCCESS || + !captureActive) { + return; + } + + ur_exp_graph_handle_t hGraph = nullptr; + if (getGraph(&hGraph) == UR_RESULT_SUCCESS && hGraph) { + hGraph->retainCapturedEvent(hEvent); + } +} + ze_event_handle_t ur_command_list_manager::getSignalEvent(ur_event_handle_t hUserEvent, ur_command_t commandType) { if (hUserEvent) { hUserEvent->setCommandType(commandType); + retainEventIfCapturing(hUserEvent); return hUserEvent->getZeEvent(); } else { return nullptr; @@ -603,6 +617,7 @@ ur_result_t ur_command_list_manager::appendTimestampRecordingExp( auto [pWaitEvents, numWaitEvents, _] = waitListView; phEvent->initTimestampRecording(); + retainEventIfCapturing(phEvent); auto [timestampPtr, zeSignalEvent] = phEvent->getEventEndTimestampAndHandle(); diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp index 35cd8e16a39ea..a12427c6f0c18 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp @@ -274,6 +274,8 @@ struct ur_command_list_manager { ze_event_handle_t getSignalEvent(ur_event_handle_t hUserEvent, ur_command_t commandType); + void retainEventIfCapturing(ur_event_handle_t hEvent); + ur_result_t appendKernelLaunchLocked( ur_kernel_handle_t hKernel, ze_kernel_handle_t hZeKernel, uint32_t workDim, const size_t *pGlobalWorkOffset, diff --git a/unified-runtime/source/adapters/level_zero/v2/graph.cpp b/unified-runtime/source/adapters/level_zero/v2/graph.cpp index a93fc167207a8..47b13140cab0f 100644 --- a/unified-runtime/source/adapters/level_zero/v2/graph.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/graph.cpp @@ -10,6 +10,7 @@ #include "graph.hpp" #include "common.hpp" #include "context.hpp" +#include "event.hpp" #include "ur_interface_loader.hpp" #include @@ -34,6 +35,15 @@ ur_exp_graph_handle_t_::~ur_exp_graph_handle_t_() { UR_LOG_SAFE(WARN, "Failed to destroy graph handle: {}", ZeResult); } } + for (auto hEvent : capturedEvents) { + hEvent->release(); + } +} + +void ur_exp_graph_handle_t_::retainCapturedEvent(ur_event_handle_t hEvent) { + hEvent->retain(); + std::scoped_lock lock(capturedEventsMutex); + capturedEvents.push_back(hEvent); } ur_exp_executable_graph_handle_t_::ur_exp_executable_graph_handle_t_( diff --git a/unified-runtime/source/adapters/level_zero/v2/graph.hpp b/unified-runtime/source/adapters/level_zero/v2/graph.hpp index 869b0ffc17afd..8a054cc7d7313 100644 --- a/unified-runtime/source/adapters/level_zero/v2/graph.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/graph.hpp @@ -28,9 +28,15 @@ struct ur_exp_graph_handle_t_ : v2::ur_object_t { ze_graph_handle_t getZeHandle() { return zeGraph; } ur_context_handle_t getContext() { return hContext; } + // Keeps the event out of its pool until the graph is destroyed. + void retainCapturedEvent(ur_event_handle_t hEvent); + private: ur_context_handle_t hContext = nullptr; ze_graph_handle_t zeGraph = nullptr; + + ur_mutex capturedEventsMutex; + std::vector capturedEvents; }; struct ur_exp_executable_graph_handle_t_ : v2::ur_object_t { From 496ab3a86bcfacfd36eaf6c02c9628705054c1ae Mon Sep 17 00:00:00 2001 From: kweronsx Date: Mon, 19 Oct 2026 13:27:06 +0200 Subject: [PATCH 2/3] [UR] Only recycle events from pool if they are not in use --- .../adapters/level_zero/v2/event_pool.cpp | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp index 7d8926aceee04..9939ed25845c4 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp @@ -12,6 +12,8 @@ #include "queue_api.hpp" #include "unified-runtime/ur_api.h" +#include + namespace ur::level_zero::v2 { ur_event_handle_t event_pool::allocate() { @@ -19,14 +21,23 @@ ur_event_handle_t event_pool::allocate() { std::unique_lock lock(mutex); - if (freelist.empty()) { + auto it = std::find_if( + freelist.rbegin(), freelist.rend(), [](ur_event_handle_t event) { + return !event->isCounter() || + ZE_CALL_NOCHECK(zeEventQueryStatus, (event->getZeEvent())) == + ZE_RESULT_SUCCESS; + }); + + ur_event_handle_t event; + if (it == freelist.rend()) { events.emplace_back(hContext, provider->allocate(), this); - freelist.push_back(&events.back()); + event = &events.back(); + } else { + event = *it; + *it = freelist.back(); + freelist.pop_back(); } - auto event = freelist.back(); - freelist.pop_back(); - #ifndef NDEBUG // Set the command type to an invalid value to catch any misuses in tests event->setQueue(nullptr); From 332a863dd674695603e57308d96c8ad77e0da509 Mon Sep 17 00:00:00 2001 From: kweronsx Date: Tue, 20 Oct 2026 15:59:17 +0200 Subject: [PATCH 3/3] [UR] Add UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION flag --- .../include/unified-runtime/ur_api.h | 6 +++++- .../include/unified-runtime/ur_print.hpp | 14 ++++++++++++++ .../scripts/core/exp-reusable-events.yml | 6 ++++++ .../source/adapters/level_zero/v2/event.cpp | 6 ++++-- .../adapters/level_zero/v2/event_pool.cpp | 18 +++++++++++------- .../adapters/level_zero/v2/event_pool.hpp | 5 +++-- 6 files changed, 43 insertions(+), 12 deletions(-) diff --git a/unified-runtime/include/unified-runtime/ur_api.h b/unified-runtime/include/unified-runtime/ur_api.h index 3f6f786b78459..9cfdab305910b 100644 --- a/unified-runtime/include/unified-runtime/ur_api.h +++ b/unified-runtime/include/unified-runtime/ur_api.h @@ -14084,13 +14084,17 @@ typedef enum ur_exp_event_flag_t { /// will later be obtained; it cannot be enabled after creation. Must not /// be combined with ::UR_EXP_EVENT_FLAG_ENABLE_PROFILING. UR_EXP_EVENT_FLAG_IPC_EXP = UR_BIT(1), + /// A new native event is created instead of reusing one previously + /// released to the adapter's event pool. Once released, the event will + /// still be returned to the pool. + UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION = UR_BIT(2), /// @cond UR_EXP_EVENT_FLAG_FORCE_UINT32 = 0x7fffffff /// @endcond } ur_exp_event_flag_t; /// @brief Bit Mask for validating ur_exp_event_flags_t -#define UR_EXP_EVENT_FLAGS_MASK 0xfffffffc +#define UR_EXP_EVENT_FLAGS_MASK 0xfffffff8 /////////////////////////////////////////////////////////////////////////////// /// @brief Descriptor type for creating reusable events. diff --git a/unified-runtime/include/unified-runtime/ur_print.hpp b/unified-runtime/include/unified-runtime/ur_print.hpp index 27b96da2967ce..ef968c8fbfc67 100644 --- a/unified-runtime/include/unified-runtime/ur_print.hpp +++ b/unified-runtime/include/unified-runtime/ur_print.hpp @@ -13753,6 +13753,9 @@ inline std::ostream &operator<<(std::ostream &os, case UR_EXP_EVENT_FLAG_IPC_EXP: os << "UR_EXP_EVENT_FLAG_IPC_EXP"; break; + case UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION: + os << "UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION"; + break; default: os << "unknown enumerator"; break; @@ -13790,6 +13793,17 @@ inline ur_result_t printFlag(std::ostream &os, } os << UR_EXP_EVENT_FLAG_IPC_EXP; } + + if ((val & UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION) == + (uint32_t)UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION) { + val ^= (uint32_t)UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION; + if (!first) { + os << " | "; + } else { + first = false; + } + os << UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION; + } if (val != 0) { std::bitset<32> bits(val); if (!first) { diff --git a/unified-runtime/scripts/core/exp-reusable-events.yml b/unified-runtime/scripts/core/exp-reusable-events.yml index 7b9b10d735508..f64af882d4f4c 100644 --- a/unified-runtime/scripts/core/exp-reusable-events.yml +++ b/unified-runtime/scripts/core/exp-reusable-events.yml @@ -35,6 +35,12 @@ etors: desc: > Event captures UR_PROFILING_INFO_COMMAND_START and UR_PROFILING_INFO_COMMAND_END timestamps when signalled. + - name: FORCE_EVENT_CREATION + value: "$X_BIT(2)" + desc: > + A new native event is created instead of reusing one previously + released to the adapter's event pool. Once released, the event will + still be returned to the pool. --- #-------------------------------------------------------------------------- type: struct diff --git a/unified-runtime/source/adapters/level_zero/v2/event.cpp b/unified-runtime/source/adapters/level_zero/v2/event.cpp index 87d1b40c5745b..b9a13df86450c 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.cpp @@ -454,8 +454,10 @@ ur_result_t urEventCreateExp(::ur_context_handle_t hContextOpque, // IPC events must not be recycled (their native handle may outlive this // process's reference), so they get a detached, self-owning event. - *phEvent = (flags & v2::EVENT_FLAGS_IPC) ? eventPool->allocateDetached() - : eventPool->allocate(); + *phEvent = (flags & v2::EVENT_FLAGS_IPC) + ? eventPool->allocateDetached() + : eventPool->allocate(pEventDesc->flags & + UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp index 9939ed25845c4..bb736c805cb7e 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp @@ -16,17 +16,21 @@ namespace ur::level_zero::v2 { -ur_event_handle_t event_pool::allocate() { +ur_event_handle_t event_pool::allocate(bool forceCreate) { TRACK_SCOPE_LATENCY("event_pool::allocate"); std::unique_lock lock(mutex); - auto it = std::find_if( - freelist.rbegin(), freelist.rend(), [](ur_event_handle_t event) { - return !event->isCounter() || - ZE_CALL_NOCHECK(zeEventQueryStatus, (event->getZeEvent())) == - ZE_RESULT_SUCCESS; - }); + auto it = + forceCreate + ? freelist.rend() + : std::find_if(freelist.rbegin(), freelist.rend(), + [](ur_event_handle_t event) { + return !event->isCounter() || + ZE_CALL_NOCHECK(zeEventQueryStatus, + (event->getZeEvent())) == + ZE_RESULT_SUCCESS; + }); ur_event_handle_t event; if (it == freelist.rend()) { diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool.hpp b/unified-runtime/source/adapters/level_zero/v2/event_pool.hpp index 5887c6810b3b7..d5cf884d8c760 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool.hpp @@ -40,8 +40,9 @@ class event_pool { ~event_pool() = default; - // Allocate an event from the pool. Thread safe. - ur_event_handle_t allocate(); + // Allocate an event from the pool. If forceCreate is set, a new event is + // created even if a reusable one is available. Thread safe. + ur_event_handle_t allocate(bool forceCreate = false); // Allocate a detached event that owns its underlying ze_event_handle_t and is // never returned to the pool for recycling, it is destroyed when released.