Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion unified-runtime/include/unified-runtime/ur_api.h
Original file line number Diff line number Diff line change
Expand Up @@ -14084,13 +14084,17 @@ typedef enum ur_exp_event_flag_t {
/// will later be obtained; it cannot be enabled after creation. Must not
/// be combined with ::UR_EXP_EVENT_FLAG_ENABLE_PROFILING.
UR_EXP_EVENT_FLAG_IPC_EXP = UR_BIT(1),
/// A new native event is created instead of reusing one previously
/// released to the adapter's event pool. Once released, the event will
/// still be returned to the pool.
UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION = UR_BIT(2),
/// @cond
UR_EXP_EVENT_FLAG_FORCE_UINT32 = 0x7fffffff
/// @endcond

} ur_exp_event_flag_t;
/// @brief Bit Mask for validating ur_exp_event_flags_t
#define UR_EXP_EVENT_FLAGS_MASK 0xfffffffc
#define UR_EXP_EVENT_FLAGS_MASK 0xfffffff8

///////////////////////////////////////////////////////////////////////////////
/// @brief Descriptor type for creating reusable events.
Expand Down
14 changes: 14 additions & 0 deletions unified-runtime/include/unified-runtime/ur_print.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -13753,6 +13753,9 @@ inline std::ostream &operator<<(std::ostream &os,
case UR_EXP_EVENT_FLAG_IPC_EXP:
os << "UR_EXP_EVENT_FLAG_IPC_EXP";
break;
case UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION:
os << "UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION";
break;
default:
os << "unknown enumerator";
break;
Expand Down Expand Up @@ -13790,6 +13793,17 @@ inline ur_result_t printFlag<ur_exp_event_flag_t>(std::ostream &os,
}
os << UR_EXP_EVENT_FLAG_IPC_EXP;
}

if ((val & UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION) ==
(uint32_t)UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION) {
val ^= (uint32_t)UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION;
if (!first) {
os << " | ";
} else {
first = false;
}
os << UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION;
}
if (val != 0) {
std::bitset<32> bits(val);
if (!first) {
Expand Down
6 changes: 6 additions & 0 deletions unified-runtime/scripts/core/exp-reusable-events.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ etors:
desc: >
Event captures UR_PROFILING_INFO_COMMAND_START and
UR_PROFILING_INFO_COMMAND_END timestamps when signalled.
- name: FORCE_EVENT_CREATION
value: "$X_BIT(2)"
desc: >
A new native event is created instead of reusing one previously
released to the adapter's event pool. Once released, the event will
still be returned to the pool.

--- #--------------------------------------------------------------------------
type: struct
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -220,11 +220,25 @@ ur_result_t ur_command_list_manager::appendRegionCopyUnlocked(
return UR_RESULT_SUCCESS;
}

void ur_command_list_manager::retainEventIfCapturing(ur_event_handle_t hEvent) {
bool captureActive = false;
if (!hEvent || queryGraphCaptureActive(&captureActive) != UR_RESULT_SUCCESS ||
!captureActive) {
return;
}

ur_exp_graph_handle_t hGraph = nullptr;
if (getGraph(&hGraph) == UR_RESULT_SUCCESS && hGraph) {
hGraph->retainCapturedEvent(hEvent);
}
}

ze_event_handle_t
ur_command_list_manager::getSignalEvent(ur_event_handle_t hUserEvent,
ur_command_t commandType) {
if (hUserEvent) {
hUserEvent->setCommandType(commandType);
retainEventIfCapturing(hUserEvent);
return hUserEvent->getZeEvent();
} else {
return nullptr;
Expand Down Expand Up @@ -603,6 +617,7 @@ ur_result_t ur_command_list_manager::appendTimestampRecordingExp(
auto [pWaitEvents, numWaitEvents, _] = waitListView;

phEvent->initTimestampRecording();
retainEventIfCapturing(phEvent);

auto [timestampPtr, zeSignalEvent] = phEvent->getEventEndTimestampAndHandle();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,8 @@ struct ur_command_list_manager {
ze_event_handle_t getSignalEvent(ur_event_handle_t hUserEvent,
ur_command_t commandType);

void retainEventIfCapturing(ur_event_handle_t hEvent);

ur_result_t appendKernelLaunchLocked(
ur_kernel_handle_t hKernel, ze_kernel_handle_t hZeKernel,
uint32_t workDim, const size_t *pGlobalWorkOffset,
Expand Down
6 changes: 4 additions & 2 deletions unified-runtime/source/adapters/level_zero/v2/event.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -454,8 +454,10 @@ ur_result_t urEventCreateExp(::ur_context_handle_t hContextOpque,

// IPC events must not be recycled (their native handle may outlive this
// process's reference), so they get a detached, self-owning event.
*phEvent = (flags & v2::EVENT_FLAGS_IPC) ? eventPool->allocateDetached()
: eventPool->allocate();
*phEvent = (flags & v2::EVENT_FLAGS_IPC)
? eventPool->allocateDetached()
: eventPool->allocate(pEventDesc->flags &
UR_EXP_EVENT_FLAG_FORCE_EVENT_CREATION);
return UR_RESULT_SUCCESS;
} catch (...) {
return exceptionToResult(std::current_exception());
Expand Down
27 changes: 21 additions & 6 deletions unified-runtime/source/adapters/level_zero/v2/event_pool.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,36 @@
#include "queue_api.hpp"
#include "unified-runtime/ur_api.h"

#include <algorithm>

namespace ur::level_zero::v2 {

ur_event_handle_t event_pool::allocate() {
ur_event_handle_t event_pool::allocate(bool forceCreate) {
TRACK_SCOPE_LATENCY("event_pool::allocate");

std::unique_lock<ur_mutex> lock(mutex);

if (freelist.empty()) {
auto it =
forceCreate
? freelist.rend()
: std::find_if(freelist.rbegin(), freelist.rend(),
[](ur_event_handle_t event) {
return !event->isCounter() ||
ZE_CALL_NOCHECK(zeEventQueryStatus,
(event->getZeEvent())) ==
ZE_RESULT_SUCCESS;
});

ur_event_handle_t event;
if (it == freelist.rend()) {
events.emplace_back(hContext, provider->allocate(), this);
freelist.push_back(&events.back());
event = &events.back();
} else {
event = *it;
*it = freelist.back();
freelist.pop_back();
}

auto event = freelist.back();
freelist.pop_back();

#ifndef NDEBUG
// Set the command type to an invalid value to catch any misuses in tests
event->setQueue(nullptr);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,9 @@ class event_pool {

~event_pool() = default;

// Allocate an event from the pool. Thread safe.
ur_event_handle_t allocate();
// Allocate an event from the pool. If forceCreate is set, a new event is
// created even if a reusable one is available. Thread safe.
ur_event_handle_t allocate(bool forceCreate = false);

// Allocate a detached event that owns its underlying ze_event_handle_t and is
// never returned to the pool for recycling, it is destroyed when released.
Expand Down
10 changes: 10 additions & 0 deletions unified-runtime/source/adapters/level_zero/v2/graph.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include "graph.hpp"
#include "common.hpp"
#include "context.hpp"
#include "event.hpp"
#include "ur_interface_loader.hpp"

#include <memory>
Expand All @@ -34,6 +35,15 @@ ur_exp_graph_handle_t_::~ur_exp_graph_handle_t_() {
UR_LOG_SAFE(WARN, "Failed to destroy graph handle: {}", ZeResult);
}
}
for (auto hEvent : capturedEvents) {
hEvent->release();
}
}

void ur_exp_graph_handle_t_::retainCapturedEvent(ur_event_handle_t hEvent) {
hEvent->retain();
std::scoped_lock<ur_mutex> lock(capturedEventsMutex);
capturedEvents.push_back(hEvent);
}

ur_exp_executable_graph_handle_t_::ur_exp_executable_graph_handle_t_(
Expand Down
6 changes: 6 additions & 0 deletions unified-runtime/source/adapters/level_zero/v2/graph.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,15 @@ struct ur_exp_graph_handle_t_ : v2::ur_object_t {
ze_graph_handle_t getZeHandle() { return zeGraph; }
ur_context_handle_t getContext() { return hContext; }

// Keeps the event out of its pool until the graph is destroyed.
void retainCapturedEvent(ur_event_handle_t hEvent);

private:
ur_context_handle_t hContext = nullptr;
ze_graph_handle_t zeGraph = nullptr;

ur_mutex capturedEventsMutex;
std::vector<ur_event_handle_t> capturedEvents;
};

struct ur_exp_executable_graph_handle_t_ : v2::ur_object_t {
Expand Down
Loading