Adding iree_hal_file_from_handle factory for common file impls. (#19513)

This lets us hide the details from HAL implementations. Implementations
are expected to snoop the file handle type to see if they can do better
and otherwise fall back to the common implementations.

experimental/webgpu/BUILD.bazel

@@ -55,7 +55,7 @@ iree_runtime_cc_library(
         "//runtime/src/iree/hal/utils:buffer_transfer",
         "//runtime/src/iree/hal/utils:executable_debug_info",
         "//runtime/src/iree/hal/utils:file_transfer",
-        "//runtime/src/iree/hal/utils:memory_file",
+        "//runtime/src/iree/hal/utils:files",
         "//runtime/src/iree/schemas:executable_debug_info_c_fbs",
         "//runtime/src/iree/schemas:webgpu_executable_def_c_fbs",
         "@webgpu_headers",

experimental/webgpu/CMakeLists.txt

@@ -49,7 +49,7 @@ iree_cc_library(
     iree::experimental::webgpu::platform
     iree::experimental::webgpu::shaders
     iree::hal::utils::file_transfer
-    iree::hal::utils::memory_file
+    iree::hal::utils::files
     iree::schemas::webgpu_executable_def_c_fbs
   PUBLIC
 )

experimental/webgpu/webgpu_device.c

@@ -20,8 +20,8 @@
 #include "experimental/webgpu/simple_allocator.h"
 #include "experimental/webgpu/staging_buffer.h"
 #include "iree/base/internal/arena.h"
+#include "iree/hal/utils/file_registry.h"
 #include "iree/hal/utils/file_transfer.h"
-#include "iree/hal/utils/memory_file.h"
 
 //===----------------------------------------------------------------------===//
 // iree_hal_webgpu_device_t
@@ -283,14 +283,8 @@ static iree_status_t iree_hal_webgpu_device_import_file(
     iree_hal_device_t* base_device, iree_hal_queue_affinity_t queue_affinity,
     iree_hal_memory_access_t access, iree_io_file_handle_t* handle,
     iree_hal_external_file_flags_t flags, iree_hal_file_t** out_file) {
-  if (iree_io_file_handle_type(handle) !=
-      IREE_IO_FILE_HANDLE_TYPE_HOST_ALLOCATION) {
-    return iree_make_status(
-        IREE_STATUS_UNAVAILABLE,
-        "implementation does not support the external file type");
-  }
-  return iree_hal_memory_file_wrap(
-      queue_affinity, access, handle, iree_hal_device_allocator(base_device),
+  return iree_hal_file_from_handle(
+      iree_hal_device_allocator(base_device), queue_affinity, access, handle,
       iree_hal_device_host_allocator(base_device), out_file);
 }
 

runtime/src/iree/hal/drivers/cuda/BUILD.bazel

@@ -63,7 +63,7 @@ iree_runtime_cc_library(
         "//runtime/src/iree/hal/utils:deferred_work_queue",
         "//runtime/src/iree/hal/utils:executable_debug_info",
         "//runtime/src/iree/hal/utils:file_transfer",
-        "//runtime/src/iree/hal/utils:memory_file",
+        "//runtime/src/iree/hal/utils:files",
         "//runtime/src/iree/hal/utils:resource_set",
         "//runtime/src/iree/hal/utils:semaphore_base",
         "//runtime/src/iree/hal/utils:stream_tracing",

runtime/src/iree/hal/drivers/cuda/CMakeLists.txt

@@ -60,7 +60,7 @@ iree_cc_library(
     iree::hal::utils::deferred_work_queue
     iree::hal::utils::executable_debug_info
     iree::hal::utils::file_transfer
-    iree::hal::utils::memory_file
+    iree::hal::utils::files
     iree::hal::utils::resource_set
     iree::hal::utils::semaphore_base
     iree::hal::utils::stream_tracing

runtime/src/iree/hal/drivers/cuda/cuda_device.c

@@ -27,8 +27,8 @@
 #include "iree/hal/drivers/cuda/timepoint_pool.h"
 #include "iree/hal/utils/deferred_command_buffer.h"
 #include "iree/hal/utils/deferred_work_queue.h"
+#include "iree/hal/utils/file_registry.h"
 #include "iree/hal/utils/file_transfer.h"
-#include "iree/hal/utils/memory_file.h"
 #include "iree/hal/utils/stream_tracing.h"
 
 //===----------------------------------------------------------------------===//
@@ -897,14 +897,8 @@ static iree_status_t iree_hal_cuda_device_import_file(
     iree_hal_device_t* base_device, iree_hal_queue_affinity_t queue_affinity,
     iree_hal_memory_access_t access, iree_io_file_handle_t* handle,
     iree_hal_external_file_flags_t flags, iree_hal_file_t** out_file) {
-  if (iree_io_file_handle_type(handle) !=
-      IREE_IO_FILE_HANDLE_TYPE_HOST_ALLOCATION) {
-    return iree_make_status(
-        IREE_STATUS_UNAVAILABLE,
-        "implementation does not support the external file type");
-  }
-  return iree_hal_memory_file_wrap(
-      queue_affinity, access, handle, iree_hal_device_allocator(base_device),
+  return iree_hal_file_from_handle(
+      iree_hal_device_allocator(base_device), queue_affinity, access, handle,
       iree_hal_device_host_allocator(base_device), out_file);
 }
 

runtime/src/iree/hal/drivers/hip/CMakeLists.txt

@@ -70,7 +70,7 @@ iree_cc_library(
     iree::hal::utils::executable_debug_info
     iree::hal::utils::deferred_command_buffer
     iree::hal::utils::file_transfer
-    iree::hal::utils::memory_file
+    iree::hal::utils::files
     iree::hal::utils::resource_set
     iree::hal::utils::semaphore_base
     iree::hal::utils::stream_tracing

runtime/src/iree/hal/drivers/hip/hip_device.c

@@ -31,8 +31,8 @@
 #include "iree/hal/drivers/hip/status_util.h"
 #include "iree/hal/drivers/hip/stream_command_buffer.h"
 #include "iree/hal/utils/deferred_command_buffer.h"
+#include "iree/hal/utils/file_registry.h"
 #include "iree/hal/utils/file_transfer.h"
-#include "iree/hal/utils/memory_file.h"
 #include "iree/hal/utils/stream_tracing.h"
 
 //===----------------------------------------------------------------------===//
@@ -849,15 +849,8 @@ static iree_status_t iree_hal_hip_device_import_file(
     iree_hal_device_t* base_device, iree_hal_queue_affinity_t queue_affinity,
     iree_hal_memory_access_t access, iree_io_file_handle_t* handle,
     iree_hal_external_file_flags_t flags, iree_hal_file_t** out_file) {
-  *out_file = NULL;
-  if (iree_io_file_handle_type(handle) !=
-      IREE_IO_FILE_HANDLE_TYPE_HOST_ALLOCATION) {
-    return iree_make_status(
-        IREE_STATUS_UNAVAILABLE,
-        "implementation does not support the external file type");
-  }
-  return iree_hal_memory_file_wrap(
-      queue_affinity, access, handle, iree_hal_device_allocator(base_device),
+  return iree_hal_file_from_handle(
+      iree_hal_device_allocator(base_device), queue_affinity, access, handle,
       iree_hal_device_host_allocator(base_device), out_file);
 }
 

runtime/src/iree/hal/drivers/local_sync/BUILD.bazel

@@ -37,7 +37,7 @@ iree_runtime_cc_library(
         "//runtime/src/iree/hal/local:executable_environment",
         "//runtime/src/iree/hal/utils:deferred_command_buffer",
         "//runtime/src/iree/hal/utils:file_transfer",
-        "//runtime/src/iree/hal/utils:memory_file",
+        "//runtime/src/iree/hal/utils:files",
         "//runtime/src/iree/hal/utils:semaphore_base",
     ],
 )

runtime/src/iree/hal/drivers/local_sync/CMakeLists.txt

@@ -34,7 +34,7 @@ iree_cc_library(
     iree::hal::local::executable_environment
     iree::hal::utils::deferred_command_buffer
     iree::hal::utils::file_transfer
-    iree::hal::utils::memory_file
+    iree::hal::utils::files
     iree::hal::utils::semaphore_base
   PUBLIC
 )

runtime/src/iree/hal/drivers/local_sync/sync_device.c

@@ -18,8 +18,8 @@
 #include "iree/hal/local/inline_command_buffer.h"
 #include "iree/hal/local/local_executable_cache.h"
 #include "iree/hal/utils/deferred_command_buffer.h"
+#include "iree/hal/utils/file_registry.h"
 #include "iree/hal/utils/file_transfer.h"
-#include "iree/hal/utils/memory_file.h"
 
 typedef struct iree_hal_sync_device_t {
   iree_hal_resource_t resource;
@@ -267,14 +267,8 @@ static iree_status_t iree_hal_sync_device_import_file(
     iree_hal_device_t* base_device, iree_hal_queue_affinity_t queue_affinity,
     iree_hal_memory_access_t access, iree_io_file_handle_t* handle,
     iree_hal_external_file_flags_t flags, iree_hal_file_t** out_file) {
-  if (iree_io_file_handle_type(handle) !=
-      IREE_IO_FILE_HANDLE_TYPE_HOST_ALLOCATION) {
-    return iree_make_status(
-        IREE_STATUS_UNAVAILABLE,
-        "implementation does not support the external file type");
-  }
-  return iree_hal_memory_file_wrap(
-      queue_affinity, access, handle, iree_hal_device_allocator(base_device),
+  return iree_hal_file_from_handle(
+      iree_hal_device_allocator(base_device), queue_affinity, access, handle,
       iree_hal_device_host_allocator(base_device), out_file);
 }
 

runtime/src/iree/hal/drivers/local_task/BUILD.bazel

@@ -49,7 +49,7 @@ iree_runtime_cc_library(
         "//runtime/src/iree/hal/local:executable_library",
         "//runtime/src/iree/hal/utils:deferred_command_buffer",
         "//runtime/src/iree/hal/utils:file_transfer",
-        "//runtime/src/iree/hal/utils:memory_file",
+        "//runtime/src/iree/hal/utils:files",
         "//runtime/src/iree/hal/utils:resource_set",
         "//runtime/src/iree/hal/utils:semaphore_base",
         "//runtime/src/iree/task",

runtime/src/iree/hal/drivers/local_task/CMakeLists.txt

@@ -43,7 +43,7 @@ iree_cc_library(
     iree::hal::local::executable_library
     iree::hal::utils::deferred_command_buffer
     iree::hal::utils::file_transfer
-    iree::hal::utils::memory_file
+    iree::hal::utils::files
     iree::hal::utils::resource_set
     iree::hal::utils::semaphore_base
     iree::task

runtime/src/iree/hal/drivers/local_task/task_device.c

@@ -19,8 +19,8 @@
 #include "iree/hal/local/executable_environment.h"
 #include "iree/hal/local/local_executable_cache.h"
 #include "iree/hal/utils/deferred_command_buffer.h"
+#include "iree/hal/utils/file_registry.h"
 #include "iree/hal/utils/file_transfer.h"
-#include "iree/hal/utils/memory_file.h"
 
 typedef struct iree_hal_task_device_t {
   iree_hal_resource_t resource;
@@ -346,14 +346,8 @@ static iree_status_t iree_hal_task_device_import_file(
     iree_hal_device_t* base_device, iree_hal_queue_affinity_t queue_affinity,
     iree_hal_memory_access_t access, iree_io_file_handle_t* handle,
     iree_hal_external_file_flags_t flags, iree_hal_file_t** out_file) {
-  if (iree_io_file_handle_type(handle) !=
-      IREE_IO_FILE_HANDLE_TYPE_HOST_ALLOCATION) {
-    return iree_make_status(
-        IREE_STATUS_UNAVAILABLE,
-        "implementation does not support the external file type");
-  }
-  return iree_hal_memory_file_wrap(
-      queue_affinity, access, handle, iree_hal_device_allocator(base_device),
+  return iree_hal_file_from_handle(
+      iree_hal_device_allocator(base_device), queue_affinity, access, handle,
       iree_hal_device_host_allocator(base_device), out_file);
 }
 

runtime/src/iree/hal/drivers/metal/CMakeLists.txt

@@ -42,7 +42,7 @@ iree_cc_library(
     iree::hal::utils::deferred_command_buffer
     iree::hal::utils::executable_debug_info
     iree::hal::utils::file_transfer
-    iree::hal::utils::memory_file
+    iree::hal::utils::files
     iree::hal::utils::resource_set
     iree::schemas::executable_debug_info_c_fbs
     iree::schemas::metal_executable_def_c_fbs

runtime/src/iree/hal/drivers/metal/metal_device.m

@@ -17,8 +17,8 @@
 #include "iree/hal/drivers/metal/shared_event.h"
 #include "iree/hal/drivers/metal/staging_buffer.h"
 #include "iree/hal/utils/deferred_command_buffer.h"
+#include "iree/hal/utils/file_registry.h"
 #include "iree/hal/utils/file_transfer.h"
-#include "iree/hal/utils/memory_file.h"
 #include "iree/hal/utils/resource_set.h"
 
 typedef struct iree_hal_metal_device_t {
@@ -288,13 +288,8 @@ static iree_status_t iree_hal_metal_device_import_file(iree_hal_device_t* base_d
                                                        iree_io_file_handle_t* handle,
                                                        iree_hal_external_file_flags_t flags,
                                                        iree_hal_file_t** out_file) {
-  if (iree_io_file_handle_type(handle) != IREE_IO_FILE_HANDLE_TYPE_HOST_ALLOCATION) {
-    return iree_make_status(IREE_STATUS_UNAVAILABLE,
-                            "implementation does not support the external file type");
-  }
-  return iree_hal_memory_file_wrap(queue_affinity, access, handle,
-                                   iree_hal_device_allocator(base_device),
-                                   iree_hal_device_host_allocator(base_device), out_file);
+  return iree_hal_file_from_handle(iree_hal_device_allocator(base_device), queue_affinity, access,
+                                   handle, iree_hal_device_host_allocator(base_device), out_file);
 }
 
 static iree_status_t iree_hal_metal_device_create_semaphore(iree_hal_device_t* base_device,

runtime/src/iree/hal/drivers/null/BUILD.bazel

@@ -44,7 +44,7 @@ iree_runtime_cc_library(
         "//runtime/src/iree/base/internal",
         "//runtime/src/iree/hal",
         "//runtime/src/iree/hal/utils:file_transfer",
-        "//runtime/src/iree/hal/utils:memory_file",
+        "//runtime/src/iree/hal/utils:files",
         "//runtime/src/iree/hal/utils:semaphore_base",
     ],
 )

runtime/src/iree/hal/drivers/null/CMakeLists.txt

@@ -41,7 +41,7 @@ iree_cc_library(
     iree::base::internal
     iree::hal
     iree::hal::utils::file_transfer
-    iree::hal::utils::memory_file
+    iree::hal::utils::files
     iree::hal::utils::semaphore_base
   PUBLIC
 )

runtime/src/iree/hal/drivers/null/device.c

@@ -14,8 +14,8 @@
 #include "iree/hal/drivers/null/executable.h"
 #include "iree/hal/drivers/null/executable_cache.h"
 #include "iree/hal/drivers/null/semaphore.h"
+#include "iree/hal/utils/file_registry.h"
 #include "iree/hal/utils/file_transfer.h"
-#include "iree/hal/utils/memory_file.h"
 
 //===----------------------------------------------------------------------===//
 // iree_hal_null_device_t
@@ -273,14 +273,8 @@ static iree_status_t iree_hal_null_device_import_file(
   // definitely prefer that. The emulated file I/O present here as a default is
   // inefficient. The queue affinity specifies which queues may access the file
   // via read and write queue operations.
-  if (iree_io_file_handle_type(handle) !=
-      IREE_IO_FILE_HANDLE_TYPE_HOST_ALLOCATION) {
-    return iree_make_status(
-        IREE_STATUS_UNAVAILABLE,
-        "implementation does not support the external file type");
-  }
-  return iree_hal_memory_file_wrap(
-      queue_affinity, access, handle, iree_hal_device_allocator(base_device),
+  return iree_hal_file_from_handle(
+      iree_hal_device_allocator(base_device), queue_affinity, access, handle,
       iree_hal_device_host_allocator(base_device), out_file);
 }
 

runtime/src/iree/hal/drivers/vulkan/BUILD.bazel

@@ -81,7 +81,7 @@ iree_runtime_cc_library(
         "//runtime/src/iree/hal/utils:deferred_command_buffer",
         "//runtime/src/iree/hal/utils:executable_debug_info",
         "//runtime/src/iree/hal/utils:file_transfer",
-        "//runtime/src/iree/hal/utils:memory_file",
+        "//runtime/src/iree/hal/utils:files",
         "//runtime/src/iree/hal/utils:resource_set",
         "//runtime/src/iree/hal/utils:semaphore_base",
         "//runtime/src/iree/schemas:executable_debug_info_c_fbs",

runtime/src/iree/hal/drivers/vulkan/CMakeLists.txt

@@ -76,7 +76,7 @@ iree_cc_library(
     iree::hal::utils::deferred_command_buffer
     iree::hal::utils::executable_debug_info
     iree::hal::utils::file_transfer
-    iree::hal::utils::memory_file
+    iree::hal::utils::files
     iree::hal::utils::resource_set
     iree::hal::utils::semaphore_base
     iree::schemas::executable_debug_info_c_fbs

runtime/src/iree/hal/drivers/vulkan/vulkan_device.cc

@@ -31,8 +31,8 @@
 #include "iree/hal/drivers/vulkan/util/arena.h"
 #include "iree/hal/drivers/vulkan/util/ref_ptr.h"
 #include "iree/hal/utils/deferred_command_buffer.h"
+#include "iree/hal/utils/file_registry.h"
 #include "iree/hal/utils/file_transfer.h"
-#include "iree/hal/utils/memory_file.h"
 
 using namespace iree::hal::vulkan;
 
@@ -1599,14 +1599,8 @@ static iree_status_t iree_hal_vulkan_device_import_file(
     iree_hal_device_t* base_device, iree_hal_queue_affinity_t queue_affinity,
     iree_hal_memory_access_t access, iree_io_file_handle_t* handle,
     iree_hal_external_file_flags_t flags, iree_hal_file_t** out_file) {
-  if (iree_io_file_handle_type(handle) !=
-      IREE_IO_FILE_HANDLE_TYPE_HOST_ALLOCATION) {
-    return iree_make_status(
-        IREE_STATUS_UNAVAILABLE,
-        "implementation does not support the external file type");
-  }
-  return iree_hal_memory_file_wrap(
-      queue_affinity, access, handle, iree_hal_device_allocator(base_device),
+  return iree_hal_file_from_handle(
+      iree_hal_device_allocator(base_device), queue_affinity, access, handle,
       iree_hal_device_host_allocator(base_device), out_file);
 }
 

runtime/src/iree/hal/utils/BUILD.bazel

@@ -105,6 +105,23 @@ iree_runtime_cc_library(
     ],
 )
 
+iree_runtime_cc_library(
+    name = "files",
+    srcs = [
+        "file_registry.c",
+        "memory_file.c",
+    ],
+    hdrs = [
+        "file_registry.h",
+        "memory_file.h",
+    ],
+    deps = [
+        "//runtime/src/iree/base",
+        "//runtime/src/iree/hal",
+        "//runtime/src/iree/io:file_handle",
+    ],
+)
+
 iree_runtime_cc_library(
     name = "libmpi",
     srcs = ["libmpi.c"],
@@ -130,17 +147,6 @@ iree_runtime_cc_test(
     ],
 )
 
-iree_runtime_cc_library(
-    name = "memory_file",
-    srcs = ["memory_file.c"],
-    hdrs = ["memory_file.h"],
-    deps = [
-        "//runtime/src/iree/base",
-        "//runtime/src/iree/hal",
-        "//runtime/src/iree/io:file_handle",
-    ],
-)
-
 iree_runtime_cc_library(
     name = "mpi_channel_provider",
     srcs = ["mpi_channel_provider.c"],