feat(EXC-1800): Use Wasmtime deserialize_open_file. (#3412)

EXC-1800

When using the on-disk compilation cache, switch to Wasmtime's new
`deserialize_open_file` API which allows sandboxes to mmap the existing
file so that canister code doesn't take up resident memory in the
sandbox.

rs/embedders/src/wasm_executor.rs

@@ -385,6 +385,29 @@ impl WasmExecutorImpl {
             })
         } else {
             match compilation_cache.get(&wasm_binary.binary) {
+                Some(Ok(StoredCompilation::Disk(on_disk_serialized_module))) => {
+                    // This path is only used when sandboxing is disabled.
+                    // Otherwise the fd is implicitly duplicated when passed to
+                    // the sandbox process over the unix socket.
+                    let instance_pre = self.wasm_embedder.read_file_and_pre_instantiate(
+                        on_disk_serialized_module
+                            .bytes
+                            .try_clone()
+                            .expect("Unable to duplicate serialzed module file descriptor."),
+                    );
+                    let cache = EmbedderCache::new(instance_pre.clone());
+                    *guard = Some(cache.clone());
+                    match instance_pre {
+                        Ok(_) => Ok(CacheLookup {
+                            cache,
+                            serialized_module: Some(StoredCompilation::Disk(
+                                on_disk_serialized_module,
+                            )),
+                            compilation_result: None,
+                        }),
+                        Err(err) => Err(err),
+                    }
+                }
                 Some(Ok(StoredCompilation::Memory(serialized_module))) => {
                     let instance_pre = self
                         .wasm_embedder
@@ -400,10 +423,6 @@ impl WasmExecutorImpl {
                         Err(err) => Err(err),
                     }
                 }
-                Some(Ok(StoredCompilation::Disk(_serialized_module))) => {
-                    // TODO(EXC-1780)
-                    panic!("On disk compilation cache not yet supported");
-                }
                 Some(Err(err)) => {
                     let cache: HypervisorResult<Module> = Err(err.clone());
                     *guard = Some(EmbedderCache::new(cache));

rs/embedders/src/wasmtime_embedder.rs

@@ -11,8 +11,6 @@ use std::{
     convert::TryFrom,
     fs::File,
     mem::size_of,
-    os::fd::{AsRawFd, IntoRawFd},
-    os::unix::fs::MetadataExt,
     sync::{atomic::Ordering, Arc, Mutex},
 };
 
@@ -39,7 +37,6 @@ use ic_types::{
 };
 use ic_wasm_types::{BinaryEncodedWasm, WasmEngineError};
 use memory_tracker::{DirtyPageTracking, PageBitmap, SigsegvMemoryTracker};
-use nix::sys::mman::{mmap, MapFlags, ProtFlags};
 use signal_stack::WasmtimeSignalStack;
 
 use crate::wasm_utils::instrumentation::{
@@ -336,39 +333,26 @@ impl WasmtimeEmbedder {
         self.pre_instantiate(&module)
     }
 
-    /// TODO(EXC-1800): Replace this with `wasmtime::Module::deserialize_open_file`.
-    fn deserialize_from_file(&self, serialized_module: &File) -> HypervisorResult<Module> {
-        let mmap_size = serialized_module.metadata().unwrap().size() as usize;
-        let mmap_ptr = unsafe {
-            mmap(
-                std::ptr::null_mut(),
-                mmap_size,
-                ProtFlags::PROT_READ,
-                MapFlags::MAP_PRIVATE,
-                serialized_module.as_raw_fd(),
-                0,
-            )
-        }
-        .unwrap_or_else(|err| panic!("Module deserialization failed: {:?}", err))
-            as *mut u8;
-        let bytes = unsafe { std::slice::from_raw_parts(mmap_ptr, mmap_size) };
+    fn deserialize_from_file(&self, serialized_module: File) -> HypervisorResult<Module> {
+        // SAFETY: The compilation cache setup guarantees that this file is a
+        // valid serialized module and will not be modified after initial
+        // creation.
         unsafe {
-            Module::deserialize(&self.create_engine()?, bytes).map_err(|err| {
-                HypervisorError::WasmEngineError(WasmEngineError::FailedToDeserializeModule(
-                    format!("{:?}", err),
-                ))
-            })
+            Module::deserialize_open_file(&self.create_engine()?, serialized_module).map_err(
+                |err| {
+                    HypervisorError::WasmEngineError(WasmEngineError::FailedToDeserializeModule(
+                        format!("{:?}", err),
+                    ))
+                },
+            )
         }
     }
 
     pub fn read_file_and_pre_instantiate(
         &self,
         serialized_module: File,
     ) -> HypervisorResult<InstancePre<StoreData>> {
-        // TODO(EXC-1800): Switch to new wasmtime API and remove leaking the
-        // file.
-        let module = self.deserialize_from_file(&serialized_module)?;
-        let _ = serialized_module.into_raw_fd();
+        let module = self.deserialize_from_file(serialized_module)?;
         self.pre_instantiate(&module)
     }