Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 12 additions & 18 deletions unified-runtime/source/adapters/level_zero/common/device.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2177,32 +2177,26 @@ ur_result_t ur_device_handle_t_::initialize(int SubSubDeviceOrdinal,

ZeQueuePriorityRange.Compute = [this,
ZeDevice](ze_queue_priority_range &Range) {
std::optional<std::pair<int32_t, int32_t>> DriverLevels;
int32_t Lowest = 0, Highest = 0;
auto GetPriorityLevels =
Platform->ZeQueuePriorityExt.zeDeviceGetPriorityLevels;
if (!GetPriorityLevels)
return;
// The function is loaded directly from the driver, so it expects a driver
// handle rather than a loader handle.
ze_device_handle_t DriverDevice = nullptr;
if (ZE_CALL_NOCHECK(zelLoaderTranslateHandle,
if (GetPriorityLevels &&
ZE_CALL_NOCHECK(zelLoaderTranslateHandle,
(ZEL_HANDLE_DEVICE, ZeDevice,
reinterpret_cast<void **>(&DriverDevice))) !=
ZE_RESULT_SUCCESS ||
ZE_CALL_NOCHECK(GetPriorityLevels,
(DriverDevice, &Range.Lowest, &Range.Highest)) !=
reinterpret_cast<void **>(&DriverDevice))) ==
ZE_RESULT_SUCCESS &&
ZE_CALL_NOCHECK(GetPriorityLevels, (DriverDevice, &Lowest, &Highest)) ==
ZE_RESULT_SUCCESS)
return;
// A single level or an empty range means there are no levels to choose
// from.
if (Range.Highest >= Range.Lowest)
return;
DriverLevels = {Lowest, Highest};
// The UR API guarantees that the range contains the default priority 0.
if (Range.Highest > 0 || Range.Lowest < 0) {
UR_LOG(WARN, "Ignoring queue priority range [{}, {}] without 0.",
Range.Highest, Range.Lowest);
return;
}
Range.Supported = true;
if (DriverLevels && (Highest > 0 || Lowest < 0))
UR_LOG(WARN, "Ignoring queue priority range [{}, {}] without 0.", Highest,
Lowest);
Range = makeQueuePriorityRange(DriverLevels);
};

#ifdef ZE_INTEL_DEVICE_BLOCK_ARRAY_EXP_NAME
Expand Down
62 changes: 60 additions & 2 deletions unified-runtime/source/adapters/level_zero/common/device.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
//===----------------------------------------------------------------------===//
#pragma once

#include <algorithm>
#include <cassert>
#include <list>
#include <map>
Expand All @@ -17,6 +18,7 @@
#include <unified-runtime/ur_api.h>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>

#include "api.hpp"
Expand Down Expand Up @@ -69,13 +71,69 @@ struct ze_global_memsize {
uint64_t value;
};

// Queue priority range reported by zeDeviceGetPriorityLevels.
// Queue priority range, see makeQueuePriorityRange.
struct ze_queue_priority_range {
bool Supported;
int32_t Highest;
// Maps to ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_LOW instead of a priority level.
int32_t Lowest;
// Whether the levels above Lowest are the levels of
// zeDeviceGetPriorityLevels. Otherwise the range is {1, -1}, and -1 maps to
// ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_HIGH instead of a priority level.
bool HasLevels;
};

// Makes the queue priority range from the {lowest, highest} levels reported by
// zeDeviceGetPriorityLevels, if any. The driver levels only cover the context
// group, while the low priority class uses a separate context below all of
// them, so it's exposed as an extra lowest level. Without at least two levels
// around the default priority 0, the range is made of the low, normal and high
// priority classes instead.
inline ze_queue_priority_range makeQueuePriorityRange(
std::optional<std::pair<int32_t, int32_t>> DriverLevels) {
if (!DriverLevels)
return {-1, 1, false};
const auto [Lowest, Highest] = *DriverLevels;
if (Highest >= Lowest || Highest > 0 || Lowest < 0)
return {-1, 1, false};
return {Highest, Lowest + 1, true};
}

// The Level Zero priority class and level of a queue.
struct ze_queue_priority {
ze_command_queue_priority_t Class;
std::optional<int32_t> Level;
};

// Returns the Level Zero priority of a queue created with the given flags or,
// if set, the given numeric priority.
inline ze_queue_priority
getZeQueuePriority(const ze_queue_priority_range &Range, ur_queue_flags_t Flags,
std::optional<int32_t> Priority) {
if (!Priority) {
if (Flags & UR_QUEUE_FLAG_PRIORITY_LOW)
return {ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_LOW, std::nullopt};
if (!(Flags & UR_QUEUE_FLAG_PRIORITY_HIGH))
return {ZE_COMMAND_QUEUE_PRIORITY_NORMAL, std::nullopt};
// The high priority class alone gets the driver's default priority of high
// priority queues, which may be below the highest priority level.
if (Range.HasLevels)
return {ZE_COMMAND_QUEUE_PRIORITY_NORMAL, Range.Highest};
return {ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_HIGH, std::nullopt};
}
// Clamp here rather than relying on the driver, so that values mapping to
// the same priority also share cached command lists.
const int32_t Level = std::clamp(*Priority, Range.Highest, Range.Lowest);
// The driver derives the priority class from a priority level, so the
// lowest level can only be expressed by the low class without a level.
if (Level == Range.Lowest)
return {ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_LOW, std::nullopt};
if (Range.HasLevels)
return {ZE_COMMAND_QUEUE_PRIORITY_NORMAL, Level};
return {Level < 0 ? ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_HIGH
: ZE_COMMAND_QUEUE_PRIORITY_NORMAL,
std::nullopt};
}

enum ur_ze_external_memory_desc_type {
UR_ZE_EXTERNAL_OPAQUE_FD,
UR_ZE_EXTERNAL_WIN32,
Expand Down
7 changes: 5 additions & 2 deletions unified-runtime/source/adapters/level_zero/device.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,14 @@ ur_result_t urDeviceGetInfo(::ur_device_handle_t hDeviceOpque,
return ReturnValue(false);
case UR_DEVICE_INFO_IPC_EVENT_SUPPORT_EXP:
return ReturnValue(false);
// The low, normal and high priority classes are the priority levels 1, 0
// and -1, see urQueueCreate.
case UR_DEVICE_INFO_QUEUE_PRIORITY_SUPPORT_EXP:
return ReturnValue(static_cast<ur_bool_t>(false));
return ReturnValue(static_cast<ur_bool_t>(true));
case UR_DEVICE_INFO_QUEUE_PRIORITY_HIGHEST_EXP:
return ReturnValue(int32_t{-1});
case UR_DEVICE_INFO_QUEUE_PRIORITY_LOWEST_EXP:
return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION;
return ReturnValue(int32_t{1});
default:
return ur::level_zero::urDeviceGetInfo(hDeviceOpque, propName, propSize,
propValue, propSizeRet);
Expand Down
24 changes: 18 additions & 6 deletions unified-runtime/source/adapters/level_zero/queue.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,7 @@ ur_result_t urQueueCreate(
}

int ForceComputeIndex = -1; // Use default/round-robin.
std::optional<int32_t> Priority;
if (Props) {
if (Props->pNext) {
const ur_base_properties_t *extendedDesc =
Expand All @@ -511,8 +512,12 @@ ur_result_t urQueueCreate(
ForceComputeIndex = IndexProperties->computeIndex;
}
}
if (find_stype_node<ur_exp_queue_priority_properties_t>(Props))
return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
if (const auto *PriorityProperties =
find_stype_node<ur_exp_queue_priority_properties_t>(Props)) {
if (Flags & (UR_QUEUE_FLAG_PRIORITY_LOW | UR_QUEUE_FLAG_PRIORITY_HIGH))
return UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES;
Priority = PriorityProperties->priority;
}
}

UR_ASSERT(Context->isValidDevice(Device), UR_RESULT_ERROR_INVALID_DEVICE);
Expand Down Expand Up @@ -550,7 +555,7 @@ ur_result_t urQueueCreate(
try {
*Queue = new ur::level_zero::v1::ur_queue_handle_t_(
ZeComputeCommandQueues, ZeCopyCommandQueues, Context, Device, true,
Flags, ForceComputeIndex);
Flags, ForceComputeIndex, Priority);
} catch (const std::bad_alloc &) {
return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY;
} catch (...) {
Expand Down Expand Up @@ -1179,9 +1184,16 @@ ur::level_zero::v1::ur_queue_handle_t_::ur_queue_handle_t_(
std::vector<ze_command_queue_handle_t> &ComputeQueues,
std::vector<ze_command_queue_handle_t> &CopyQueues,
ur_context_handle_t Context, ur_device_handle_t Device,
bool OwnZeCommandQueue, ur_queue_flags_t Properties, int ForceComputeIndex)
bool OwnZeCommandQueue, ur_queue_flags_t Properties, int ForceComputeIndex,
std::optional<int32_t> Priority)
: Context{Context}, Device{Device}, OwnZeCommandQueue{OwnZeCommandQueue},
Properties(Properties) {
if (Priority)
PriorityClass = std::clamp(*Priority, int32_t{-1}, int32_t{1});
else if (Properties & UR_QUEUE_FLAG_PRIORITY_LOW)
PriorityClass = 1;
else if (Properties & UR_QUEUE_FLAG_PRIORITY_HIGH)
PriorityClass = -1;
// Set the type of commandlists the queue will use when user-selected
// submission mode. Otherwise use env var setting and if unset, use default.
if (isBatchedSubmission())
Expand Down Expand Up @@ -1730,11 +1742,11 @@ bool ur::level_zero::v1::ur_queue_handle_t_::isDiscardEvents() const {
}

bool ur::level_zero::v1::ur_queue_handle_t_::isPriorityLow() const {
return ((this->Properties & UR_QUEUE_FLAG_PRIORITY_LOW) != 0);
return PriorityClass > 0;
}

bool ur::level_zero::v1::ur_queue_handle_t_::isPriorityHigh() const {
return ((this->Properties & UR_QUEUE_FLAG_PRIORITY_HIGH) != 0);
return PriorityClass < 0;
}

bool ur::level_zero::v1::ur_queue_handle_t_::isBatchedSubmission() const {
Expand Down
7 changes: 6 additions & 1 deletion unified-runtime/source/adapters/level_zero/queue.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ struct ur_queue_handle_t_ : ur_object_t {
std::vector<ze_command_queue_handle_t> &CopyQueues,
ur_context_handle_t Context, ur_device_handle_t Device,
bool OwnZeCommandQueue, ur_queue_flags_t Properties = 0,
int ForceComputeIndex = -1);
int ForceComputeIndex = -1,
std::optional<int32_t> Priority = std::nullopt);

using queue_type = ur_device_handle_t_::queue_group_info_t::type;
// PI queue is in general a one to many mapping to L0 native queues.
Expand Down Expand Up @@ -372,6 +373,10 @@ struct ur_queue_handle_t_ : ur_object_t {
// Keeps the properties of this queue.
ur_queue_flags_t Properties;

// The priority levels 1, 0 and -1 for the low, normal and high priority
// classes. Set by a numeric priority or by the priority flags.
int32_t PriorityClass = 0;

// Keeps track of whether we are using Counter-based Events
bool CounterBasedEventsEnabled = false;

Expand Down
5 changes: 1 addition & 4 deletions unified-runtime/source/adapters/level_zero/v2/device.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,10 @@ ur_result_t urDeviceGetInfo(::ur_device_handle_t hDeviceOpque,
#endif
}
case UR_DEVICE_INFO_QUEUE_PRIORITY_SUPPORT_EXP:
return ReturnValue(
static_cast<ur_bool_t>(hDevice->ZeQueuePriorityRange->Supported));
return ReturnValue(static_cast<ur_bool_t>(true));
case UR_DEVICE_INFO_QUEUE_PRIORITY_HIGHEST_EXP:
case UR_DEVICE_INFO_QUEUE_PRIORITY_LOWEST_EXP: {
const auto &Range = hDevice->ZeQueuePriorityRange.get();
if (!Range.Supported)
return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION;
return ReturnValue(propName == UR_DEVICE_INFO_QUEUE_PRIORITY_HIGHEST_EXP
? Range.Highest
: Range.Lowest);
Expand Down
42 changes: 12 additions & 30 deletions unified-runtime/source/adapters/level_zero/v2/queue_create.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@
#include "queue_immediate_in_order.hpp"
#include "unified-runtime/ur_api.h"

#include <algorithm>

static const bool ForceBatched = getenv_tobool("UR_L0_V2_FORCE_BATCHED");

namespace ur::level_zero::v2 {
Expand All @@ -43,14 +41,6 @@ getPriorityLevel(const ur_queue_properties_t *pProps) {
return std::nullopt;
}

static ze_command_queue_priority_t getZePriority(ur_queue_flags_t flags) {
if ((flags & UR_QUEUE_FLAG_PRIORITY_LOW) != 0)
return ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_LOW;
if ((flags & UR_QUEUE_FLAG_PRIORITY_HIGH) != 0)
return ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_HIGH;
return ZE_COMMAND_QUEUE_PRIORITY_NORMAL;
}

static event_flags_t eventFlagsFromQueueFlags(ur_queue_flags_t flags) {
event_flags_t eventFlags = EVENT_FLAGS_COUNTER;
if (flags & UR_QUEUE_FLAG_PROFILING_ENABLE)
Expand Down Expand Up @@ -104,37 +94,29 @@ ur_result_t urQueueCreate(::ur_context_handle_t hContextOpque,
}

auto zeIndex = v2::getZeIndex(pProperties);
auto priorityLevel = v2::getPriorityLevel(pProperties);

if (priorityLevel) {
if (flags & (UR_QUEUE_FLAG_PRIORITY_LOW | UR_QUEUE_FLAG_PRIORITY_HIGH))
return UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES;
const auto &Range = hDevice->ZeQueuePriorityRange.get();
if (!Range.Supported)
return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
// Clamp here rather than relying on the driver, so that values mapping to
// the same priority also share cached command lists.
priorityLevel = std::clamp(*priorityLevel, Range.Highest, Range.Lowest);
}
auto priority = v2::getPriorityLevel(pProperties);
if (priority &&
(flags & (UR_QUEUE_FLAG_PRIORITY_LOW | UR_QUEUE_FLAG_PRIORITY_HIGH)))
return UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES;
const auto [zePriority, priorityLevel] =
getZeQueuePriority(hDevice->ZeQueuePriorityRange.get(), flags, priority);

if (isBatched) {
// out of order not supported
*phQueue = ur_queue_handle_t_::create<v2::ur_queue_batched_t>(
hContext, hDevice, v2::getZeOrdinal(hDevice), v2::getZePriority(flags),
zeIndex, priorityLevel, v2::eventFlagsFromQueueFlags(flags), flags);
hContext, hDevice, v2::getZeOrdinal(hDevice), zePriority, zeIndex,
priorityLevel, v2::eventFlagsFromQueueFlags(flags), flags);
} else {
if (isOutOfOrder) {
*phQueue =
ur_queue_handle_t_::create<v2::ur_queue_immediate_out_of_order_t>(
hContext, hDevice, v2::getZeOrdinal(hDevice),
v2::getZePriority(flags), zeIndex, priorityLevel,
v2::eventFlagsFromQueueFlags(flags), flags);
hContext, hDevice, v2::getZeOrdinal(hDevice), zePriority, zeIndex,
priorityLevel, v2::eventFlagsFromQueueFlags(flags), flags);
} else {
// immediate
*phQueue = ur_queue_handle_t_::create<v2::ur_queue_immediate_in_order_t>(
hContext, hDevice, v2::getZeOrdinal(hDevice),
v2::getZePriority(flags), zeIndex, priorityLevel,
v2::eventFlagsFromQueueFlags(flags), flags);
hContext, hDevice, v2::getZeOrdinal(hDevice), zePriority, zeIndex,
priorityLevel, v2::eventFlagsFromQueueFlags(flags), flags);
}
}

Expand Down
4 changes: 4 additions & 0 deletions unified-runtime/test/adapters/level_zero/v2/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ add_l0_v2_devices_test(batched_queue
batched_queue_test.cpp
)

add_l0_v2_devices_test(queue_priority
queue_priority_test.cpp
)

if(NOT UR_FOUND_DPCXX)
# Tests that require kernels can't be used if we aren't generating
# device binaries
Expand Down
Loading
Loading