Adding flag placeholders to semaphores/events. (#18122)

Events also gain a queue affinity that can be used to indicate which
queues a particular event may be set/waited on. The flags are currently
unused but will allow us to specify behavior modes in the future (such
as making exportable semaphores opt-in).

Since semaphores and events aren't yet exposed in the compiler there
were no changes needed there. The compiler will not (in common usage)
ever return new semaphores so will not need exportable flags and other
things we'll potentially add - applications will be setting those via
the C API.

Progress on #18121 (event queue affinity needed for
[VK_KHR_device_group](https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_device_group.html)).

experimental/web/sample_webgpu/main.c

@@ -782,7 +782,8 @@ static iree_status_t process_call_outputs(
   }
   iree_hal_semaphore_t* signal_semaphore = NULL;
   if (iree_status_is_ok(status)) {
-    status = iree_hal_semaphore_create(device, 0ull, &signal_semaphore);
+    status = iree_hal_semaphore_create(
+        device, 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &signal_semaphore);
   }
   uint64_t signal_value = 1ull;
   if (iree_status_is_ok(status)) {

experimental/webgpu/nop_event.c

@@ -24,8 +24,9 @@ static iree_hal_webgpu_nop_event_t* iree_hal_webgpu_nop_event_cast(
   return (iree_hal_webgpu_nop_event_t*)base_value;
 }
 
-iree_status_t iree_hal_webgpu_nop_event_create(iree_allocator_t host_allocator,
-                                               iree_hal_event_t** out_event) {
+iree_status_t iree_hal_webgpu_nop_event_create(
+    iree_hal_queue_affinity_t queue_affinity, iree_hal_event_flags_t flags,
+    iree_allocator_t host_allocator, iree_hal_event_t** out_event) {
   IREE_ASSERT_ARGUMENT(out_event);
   *out_event = NULL;
   IREE_TRACE_ZONE_BEGIN(z0);

experimental/webgpu/nop_event.h

@@ -14,8 +14,9 @@
 extern "C" {
 #endif  // __cplusplus
 
-iree_status_t iree_hal_webgpu_nop_event_create(iree_allocator_t host_allocator,
-                                               iree_hal_event_t** out_event);
+iree_status_t iree_hal_webgpu_nop_event_create(
+    iree_hal_queue_affinity_t queue_affinity, iree_hal_event_flags_t flags,
+    iree_allocator_t host_allocator, iree_hal_event_t** out_event);
 
 #ifdef __cplusplus
 }  // extern "C"

experimental/webgpu/webgpu_device.c

@@ -263,9 +263,11 @@ static iree_status_t iree_hal_webgpu_device_create_descriptor_set_layout(
 }
 
 static iree_status_t iree_hal_webgpu_device_create_event(
-    iree_hal_device_t* base_device, iree_hal_event_t** out_event) {
+    iree_hal_device_t* base_device, iree_hal_queue_affinity_t queue_affinity,
+    iree_hal_event_flags_t flags, iree_hal_event_t** out_event) {
   iree_hal_webgpu_device_t* device = iree_hal_webgpu_device_cast(base_device);
-  return iree_hal_webgpu_nop_event_create(device->host_allocator, out_event);
+  return iree_hal_webgpu_nop_event_create(queue_affinity, flags,
+                                          device->host_allocator, out_event);
 }
 
 static iree_status_t iree_hal_webgpu_device_create_executable_cache(
@@ -305,7 +307,7 @@ static iree_status_t iree_hal_webgpu_device_create_pipeline_layout(
 
 static iree_status_t iree_hal_webgpu_device_create_semaphore(
     iree_hal_device_t* base_device, uint64_t initial_value,
-    iree_hal_semaphore_t** out_semaphore) {
+    iree_hal_semaphore_flags_t flags, iree_hal_semaphore_t** out_semaphore) {
   iree_hal_webgpu_device_t* device = iree_hal_webgpu_device_cast(base_device);
   return iree_hal_webgpu_nop_semaphore_create(
       initial_value, device->host_allocator, out_semaphore);

integrations/pjrt/src/iree_pjrt/common/api_impl.cc

@@ -528,8 +528,8 @@ iree_status_t BufferInstance::CopyToHost(void* dst, iree_host_size_t dst_size,
   dst_buffer.reset();
 
   iree::vm::ref<iree_hal_semaphore_t> semaphore;
-  IREE_RETURN_IF_ERROR(
-      iree_hal_semaphore_create(device_.device(), 0ull, &semaphore));
+  IREE_RETURN_IF_ERROR(iree_hal_semaphore_create(
+      device_.device(), 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore));
 
   // Signaled when `dst_buffer` is ready to be consumed.
   iree::vm::ref<iree_hal_fence_t> dst_buffer_ready_fence;
@@ -759,10 +759,10 @@ iree_status_t DeviceInstance::OpenDevice() {
       driver_, /*device_id=*/info_.device_id(),
       /*param_count=*/0, /*params=*/nullptr, client_.host_allocator(),
       &device_));
-  IREE_RETURN_IF_ERROR(
-      iree_hal_semaphore_create(device(), 0ull, &main_timeline_));
-  IREE_RETURN_IF_ERROR(
-      iree_hal_semaphore_create(device(), 0ull, &transfer_timeline_));
+  IREE_RETURN_IF_ERROR(iree_hal_semaphore_create(
+      device(), 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &main_timeline_));
+  IREE_RETURN_IF_ERROR(iree_hal_semaphore_create(
+      device(), 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &transfer_timeline_));
 
   return iree_ok_status();
 }

runtime/bindings/python/hal.cc

@@ -371,8 +371,10 @@ void HalDevice::EndProfiling() {
 
 HalSemaphore HalDevice::CreateSemaphore(uint64_t initial_value) {
   iree_hal_semaphore_t* out_sem;
-  CheckApiStatus(iree_hal_semaphore_create(raw_ptr(), initial_value, &out_sem),
-                 "creating semaphore");
+  CheckApiStatus(
+      iree_hal_semaphore_create(raw_ptr(), initial_value,
+                                IREE_HAL_SEMAPHORE_FLAG_NONE, &out_sem),
+      "creating semaphore");
   return HalSemaphore::StealFromRawPtr(out_sem);
 }
 

runtime/bindings/python/loop.cc

@@ -36,7 +36,8 @@ class HalDeviceLoopBridge {
     IREE_PY_TRACEF("new HalDeviceLoopBridge (%p)", this);
     iree_slim_mutex_initialize(&mu_);
     CheckApiStatus(
-        iree_hal_semaphore_create(device_.raw_ptr(), 0, &control_sem_),
+        iree_hal_semaphore_create(device_.raw_ptr(), 0,
+                                  IREE_HAL_SEMAPHORE_FLAG_NONE, &control_sem_),
         "create semaphore");
 
     loop_call_soon_ = loop_.attr("call_soon_threadsafe");

runtime/src/iree/hal/BUILD.bazel

@@ -63,6 +63,7 @@ iree_runtime_cc_library(
         "file.h",
         "pipeline_layout.c",
         "pipeline_layout.h",
+        "queue.h",
         "resource.h",
         "semaphore.c",
         "semaphore.h",

runtime/src/iree/hal/CMakeLists.txt

@@ -56,6 +56,7 @@ iree_cc_library(
     "file.h"
     "pipeline_layout.c"
     "pipeline_layout.h"
+    "queue.h"
     "resource.h"
     "semaphore.c"
     "semaphore.h"

runtime/src/iree/hal/allocator.h

@@ -12,6 +12,7 @@
 
 #include "iree/base/api.h"
 #include "iree/hal/buffer.h"
+#include "iree/hal/queue.h"
 #include "iree/hal/resource.h"
 
 #ifdef __cplusplus
@@ -22,22 +23,6 @@ extern "C" {
 // Types and Enums
 //===----------------------------------------------------------------------===//
 
-// A bitmap indicating logical device queue affinity.
-// Used to direct submissions to specific device queues or locate memory nearby
-// where it will be used. The meaning of the bits in the bitmap is
-// implementation-specific: a bit may represent a logical queue in an underlying
-// API such as a VkQueue or a physical queue such as a discrete virtual device.
-//
-// Bitwise operations can be performed on affinities; for example AND'ing two
-// affinities will produce the intersection and OR'ing will produce the union.
-// This enables just-in-time selection as a command buffer could be made
-// available to some set of queues when recorded and then AND'ed with an actual
-// set of queues to execute on during submission.
-typedef uint64_t iree_hal_queue_affinity_t;
-
-// Specifies that any queue may be selected.
-#define IREE_HAL_QUEUE_AFFINITY_ANY ((iree_hal_queue_affinity_t)(-1))
-
 // TBD: placeholder for reserving unique pools.
 // The intent is that semantically meaningful pools can be defined like
 // "transient" "variable" "constant" "external" (matching what we use in the

runtime/src/iree/hal/api.h

@@ -26,6 +26,7 @@
 #include "iree/hal/fence.h"             // IWYU pragma: export
 #include "iree/hal/file.h"              // IWYU pragma: export
 #include "iree/hal/pipeline_layout.h"   // IWYU pragma: export
+#include "iree/hal/queue.h"             // IWYU pragma: export
 #include "iree/hal/resource.h"          // IWYU pragma: export
 #include "iree/hal/semaphore.h"         // IWYU pragma: export
 #include "iree/hal/string_util.h"       // IWYU pragma: export

runtime/src/iree/hal/buffer_transfer.c

@@ -62,8 +62,8 @@ static iree_status_t iree_hal_device_transfer_and_wait(
   // run out-of-order/overlapped with other work and return earlier than device
   // idle.
   iree_hal_semaphore_t* fence_semaphore = NULL;
-  iree_status_t status =
-      iree_hal_semaphore_create(device, 0ull, &fence_semaphore);
+  iree_status_t status = iree_hal_semaphore_create(
+      device, 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &fence_semaphore);
   uint64_t signal_value = 1ull;
   if (iree_status_is_ok(status)) {
     iree_hal_semaphore_list_t wait_semaphores = {

runtime/src/iree/hal/channel.h

@@ -11,7 +11,7 @@
 #include <stdint.h>
 
 #include "iree/base/api.h"
-#include "iree/hal/allocator.h"
+#include "iree/hal/queue.h"
 #include "iree/hal/resource.h"
 
 #ifdef __cplusplus

runtime/src/iree/hal/command_buffer.h

@@ -17,6 +17,7 @@
 #include "iree/hal/event.h"
 #include "iree/hal/executable.h"
 #include "iree/hal/pipeline_layout.h"
+#include "iree/hal/queue.h"
 #include "iree/hal/resource.h"
 
 #ifdef __cplusplus

runtime/src/iree/hal/cts/cts_test_base.h

@@ -231,8 +231,8 @@ class CTSTestBase : public BaseType, public CTSTestResources {
 
     // One signal semaphore from 0 -> 1.
     iree_hal_semaphore_t* signal_semaphore = NULL;
-    IREE_RETURN_IF_ERROR(
-        iree_hal_semaphore_create(device_, 0ull, &signal_semaphore));
+    IREE_RETURN_IF_ERROR(iree_hal_semaphore_create(
+        device_, 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &signal_semaphore));
     uint64_t target_payload_value = 1ull;
     iree_hal_semaphore_list_t signal_semaphores = {
         /*count=*/1,
@@ -267,7 +267,8 @@ class CTSTestBase : public BaseType, public CTSTestResources {
 
   iree_hal_semaphore_t* CreateSemaphore() {
     iree_hal_semaphore_t* semaphore = NULL;
-    IREE_EXPECT_OK(iree_hal_semaphore_create(device_, 0, &semaphore));
+    IREE_EXPECT_OK(iree_hal_semaphore_create(
+        device_, 0, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore));
     return semaphore;
   }
 

runtime/src/iree/hal/cts/event_test.h

@@ -21,13 +21,15 @@ class EventTest : public CTSTestBase<> {};
 
 TEST_F(EventTest, Create) {
   iree_hal_event_t* event = NULL;
-  IREE_ASSERT_OK(iree_hal_event_create(device_, &event));
+  IREE_ASSERT_OK(iree_hal_event_create(device_, IREE_HAL_QUEUE_AFFINITY_ANY,
+                                       IREE_HAL_EVENT_FLAG_NONE, &event));
   iree_hal_event_release(event);
 }
 
 TEST_F(EventTest, SignalAndReset) {
   iree_hal_event_t* event = NULL;
-  IREE_ASSERT_OK(iree_hal_event_create(device_, &event));
+  IREE_ASSERT_OK(iree_hal_event_create(device_, IREE_HAL_QUEUE_AFFINITY_ANY,
+                                       IREE_HAL_EVENT_FLAG_NONE, &event));
 
   iree_hal_command_buffer_t* command_buffer = NULL;
   IREE_ASSERT_OK(iree_hal_command_buffer_create(
@@ -50,7 +52,8 @@ TEST_F(EventTest, SignalAndReset) {
 
 TEST_F(EventTest, SubmitWithChainedCommandBuffers) {
   iree_hal_event_t* event = NULL;
-  IREE_ASSERT_OK(iree_hal_event_create(device_, &event));
+  IREE_ASSERT_OK(iree_hal_event_create(device_, IREE_HAL_QUEUE_AFFINITY_ANY,
+                                       IREE_HAL_EVENT_FLAG_NONE, &event));
 
   iree_hal_command_buffer_t* command_buffer_1 = NULL;
   iree_hal_command_buffer_t* command_buffer_2 = NULL;

runtime/src/iree/hal/cts/file_test.h

@@ -93,7 +93,8 @@ TEST_F(FileTest, ReadEntireFile) {
   CreatePatternedDeviceBuffer(file_size, 0xCD, &buffer);
 
   iree_hal_semaphore_t* semaphore = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore));
   iree_hal_fence_t* wait_fence = NULL;
   IREE_ASSERT_OK(iree_hal_fence_create_at(
       semaphore, 1ull, iree_allocator_system(), &wait_fence));

runtime/src/iree/hal/cts/semaphore_submission_test.h

@@ -77,7 +77,8 @@ TEST_F(SemaphoreSubmissionTest, SubmitWithWait) {
       wait_payload_values,
   };
   iree_hal_semaphore_t* signal_semaphore = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 100, &signal_semaphore));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 100, IREE_HAL_SEMAPHORE_FLAG_NONE, &signal_semaphore));
   uint64_t signal_payload_values[] = {101};
   iree_hal_semaphore_list_t signal_semaphores = {
       1,

runtime/src/iree/hal/cts/semaphore_test.h

@@ -26,7 +26,8 @@ class SemaphoreTest : public CTSTestBase<> {};
 // Tests that a semaphore that is unused properly cleans itself up.
 TEST_F(SemaphoreTest, NoOp) {
   iree_hal_semaphore_t* semaphore = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 123ull, &semaphore));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 123ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore));
 
   uint64_t value;
   IREE_ASSERT_OK(iree_hal_semaphore_query(semaphore, &value));
@@ -38,7 +39,8 @@ TEST_F(SemaphoreTest, NoOp) {
 // Tests that a semaphore will accept new values as it is signaled.
 TEST_F(SemaphoreTest, NormalSignaling) {
   iree_hal_semaphore_t* semaphore = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 2ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore));
 
   uint64_t value;
   IREE_ASSERT_OK(iree_hal_semaphore_query(semaphore, &value));
@@ -60,7 +62,8 @@ TEST_F(SemaphoreTest, NormalSignaling) {
 // Tests semaphore failure handling.
 TEST_F(SemaphoreTest, Failure) {
   iree_hal_semaphore_t* semaphore = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 2ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore));
 
   IREE_ASSERT_OK(iree_hal_semaphore_signal(semaphore, 3ull));
   uint64_t value;
@@ -98,7 +101,8 @@ TEST_F(SemaphoreTest, EmptyWait) {
 // Tests waiting on a semaphore that has already been signaled.
 TEST_F(SemaphoreTest, WaitAlreadySignaled) {
   iree_hal_semaphore_t* semaphore = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 2ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore));
 
   // Test both previous and current values.
   IREE_ASSERT_OK(iree_hal_semaphore_wait(
@@ -117,7 +121,8 @@ TEST_F(SemaphoreTest, WaitAlreadySignaled) {
 // Tests waiting on a semaphore that has not been signaled.
 TEST_F(SemaphoreTest, WaitUnsignaled) {
   iree_hal_semaphore_t* semaphore = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 2ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore));
 
   // NOTE: we don't actually block here because otherwise we'd lock up.
   // Result status is undefined - some backends may return DeadlineExceededError
@@ -131,7 +136,8 @@ TEST_F(SemaphoreTest, WaitUnsignaled) {
 // Tests waiting on a semaphore that has signals past the desired value.
 TEST_F(SemaphoreTest, WaitLaterSignaledBeyond) {
   iree_hal_semaphore_t* semaphore = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 2ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore));
 
   std::thread thread([&]() {
     // Wait for a short period before signaling.
@@ -154,8 +160,10 @@ TEST_F(SemaphoreTest, WaitLaterSignaledBeyond) {
 TEST_F(SemaphoreTest, WaitAllButNotAllSignaled) {
   iree_hal_semaphore_t* semaphore_a = NULL;
   iree_hal_semaphore_t* semaphore_b = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore_a));
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 1ull, &semaphore_b));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore_a));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 1ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore_b));
 
   iree_hal_semaphore_list_t semaphore_list;
   iree_hal_semaphore_t* semaphore_ptrs[] = {semaphore_a, semaphore_b};
@@ -179,8 +187,10 @@ TEST_F(SemaphoreTest, WaitAllButNotAllSignaled) {
 TEST_F(SemaphoreTest, WaitAllAndAllSignaled) {
   iree_hal_semaphore_t* semaphore_a = NULL;
   iree_hal_semaphore_t* semaphore_b = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 1ull, &semaphore_a));
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 1ull, &semaphore_b));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 1ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore_a));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 1ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore_b));
 
   iree_hal_semaphore_list_t semaphore_list;
   iree_hal_semaphore_t* semaphore_ptrs[] = {semaphore_a, semaphore_b};
@@ -204,8 +214,10 @@ TEST_F(SemaphoreTest, WaitAllAndAllSignaled) {
 TEST_F(SemaphoreTest, WaitAnyAlreadySignaled) {
   iree_hal_semaphore_t* semaphore_a = NULL;
   iree_hal_semaphore_t* semaphore_b = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore_a));
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 1ull, &semaphore_b));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore_a));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 1ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore_b));
 
   iree_hal_semaphore_list_t semaphore_list;
   iree_hal_semaphore_t* semaphore_ptrs[] = {semaphore_a, semaphore_b};
@@ -225,8 +237,10 @@ TEST_F(SemaphoreTest, WaitAnyAlreadySignaled) {
 TEST_F(SemaphoreTest, WaitAnyLaterSignaled) {
   iree_hal_semaphore_t* semaphore_a = NULL;
   iree_hal_semaphore_t* semaphore_b = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore_a));
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore_b));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore_a));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(
+      device_, 0ull, IREE_HAL_SEMAPHORE_FLAG_NONE, &semaphore_b));
 
   iree_hal_semaphore_list_t semaphore_list;
   iree_hal_semaphore_t* semaphore_ptrs[] = {semaphore_a, semaphore_b};
@@ -255,8 +269,10 @@ TEST_F(SemaphoreTest, WaitAnyLaterSignaled) {
 TEST_F(SemaphoreTest, PingPong) {
   iree_hal_semaphore_t* a2b = NULL;
   iree_hal_semaphore_t* b2a = NULL;
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &a2b));
-  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &b2a));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull,
+                                           IREE_HAL_SEMAPHORE_FLAG_NONE, &a2b));
+  IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull,
+                                           IREE_HAL_SEMAPHORE_FLAG_NONE, &b2a));
   std::thread thread([&]() {
     // Should advance right past this because the value is already set.
     IREE_ASSERT_OK(iree_hal_semaphore_wait(