From cf8d8e2c3f64593c630c7b16452bb2d4e01dc303 Mon Sep 17 00:00:00 2001 From: Zhang Tianyang Date: Tue, 9 Jan 2024 14:48:21 +0800 Subject: [PATCH 1/2] vmm: setup hosts, hostname and resolv.conf files If containerd has created these three files, it will append relevant cri mounts to container. If these mounts are defined in Kubernetes Pod Yaml by the end user, they will also exist in container mounts. So anyway, in the above cases, vmm-sandboxer should not append any mounts. If not, vmm-snadboxer should prepare sandbox files and covert them into oci mounts before sandbox starts. The hostname should be set to either pod hostname or host hostname, the hosts should be set as host, and the DNS should be set either from pod spec or host DNS. Moreover, it's better to set guest DNS if we want to mount some nfs sewrver in guest. Signed-off-by: Zhang Tianyang --- vmm/common/src/lib.rs | 8 + vmm/sandbox/Cargo.lock | 32 ++++ vmm/sandbox/Cargo.toml | 6 + vmm/sandbox/src/container/handler/append.rs | 6 +- vmm/sandbox/src/container/handler/spec.rs | 188 ++++++++++++++++++- vmm/sandbox/src/container/handler/storage.rs | 4 +- vmm/sandbox/src/sandbox.rs | 146 +++++++++++++- vmm/sandbox/src/storage/mod.rs | 8 +- vmm/sandbox/src/storage/mount.rs | 3 +- vmm/sandbox/src/utils.rs | 12 +- vmm/task/src/container.rs | 28 ++- vmm/task/src/main.rs | 2 +- vmm/task/src/task.rs | 9 +- 13 files changed, 426 insertions(+), 26 deletions(-) diff --git a/vmm/common/src/lib.rs b/vmm/common/src/lib.rs index 2b5ed4be..c1d9403f 100644 --- a/vmm/common/src/lib.rs +++ b/vmm/common/src/lib.rs @@ -25,3 +25,11 @@ pub const KUASAR_STATE_DIR: &str = "/run/kuasar/state"; pub const IO_FILE_PREFIX: &str = "io"; pub const STORAGE_FILE_PREFIX: &str = "storage"; pub const SHARED_DIR_SUFFIX: &str = "shared"; + +pub const ETC_HOSTS: &str = "/etc/hosts"; +pub const ETC_HOSTNAME: &str = "/etc/hostname"; +pub const ETC_RESOLV: &str = "/etc/resolv.conf"; +pub const DEV_SHM: &str = "/dev/shm"; +pub const HOSTS_FILENAME: &str = "hosts"; +pub const HOSTNAME_FILENAME: &str = "hostname"; +pub const RESOLV_FILENAME: &str = "resolv.conf"; diff --git a/vmm/sandbox/Cargo.lock b/vmm/sandbox/Cargo.lock index 652fb584..edf4b012 100644 --- a/vmm/sandbox/Cargo.lock +++ b/vmm/sandbox/Cargo.lock @@ -753,6 +753,17 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + [[package]] name = "http" version = "0.2.9" @@ -953,6 +964,12 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + [[package]] name = "matchit" version = "0.5.0" @@ -1254,6 +1271,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "path-clean" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" + [[package]] name = "percent-encoding" version = "2.3.0" @@ -1894,6 +1917,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "temp-dir" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af547b166dd1ea4b472165569fc456cfb6818116f854690b0ff205e636523dab" + [[package]] name = "tempfile" version = "3.7.0" @@ -2334,6 +2363,7 @@ dependencies = [ "containerd-shim", "env_logger", "futures-util", + "hostname", "lazy_static", "log", "netlink-packet-core", @@ -2341,6 +2371,7 @@ dependencies = [ "nix 0.26.2", "oci-spec", "os_pipe", + "path-clean", "proc-macro2", "procfs", "prost-types 0.10.1", @@ -2353,6 +2384,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "temp-dir", "time 0.3.25", "tokio", "toml", diff --git a/vmm/sandbox/Cargo.toml b/vmm/sandbox/Cargo.toml index ab0c208b..0dae26c5 100644 --- a/vmm/sandbox/Cargo.toml +++ b/vmm/sandbox/Cargo.toml @@ -43,6 +43,8 @@ ttrpc = { version = "0.7", features = ["async"] } protobuf = "3.2" cgroups-rs = "0.3.2" proc-macro2 = "1.0.66" +hostname = "0.3" +path-clean = "1.0.1" [[bin]] name = "qemu" @@ -55,3 +57,7 @@ path = "src/bin/cloud_hypervisor/main.rs" [[bin]] name = "stratovirt" path = "src/bin/stratovirt/main.rs" + +[dev-dependencies] +temp-dir = "0.1.11" + diff --git a/vmm/sandbox/src/container/handler/append.rs b/vmm/sandbox/src/container/handler/append.rs index 48bf3bd1..2bd3796a 100644 --- a/vmm/sandbox/src/container/handler/append.rs +++ b/vmm/sandbox/src/container/handler/append.rs @@ -17,7 +17,6 @@ limitations under the License. use anyhow::anyhow; use async_trait::async_trait; use containerd_sandbox::{error::Result, ContainerOption}; -use vmm_common::SHARED_DIR_SUFFIX; use crate::{ container::{handler::Handler, KuasarContainer}, @@ -52,8 +51,9 @@ where processes: vec![], }; let bundle = format!( - "{}/{}/{}", - sandbox.base_dir, SHARED_DIR_SUFFIX, self.option.container.id + "{}/{}", + sandbox.get_sandbox_shared_path(), + self.option.container.id ); tokio::fs::create_dir_all(&bundle) .await diff --git a/vmm/sandbox/src/container/handler/spec.rs b/vmm/sandbox/src/container/handler/spec.rs index 02e53355..ade24ddb 100644 --- a/vmm/sandbox/src/container/handler/spec.rs +++ b/vmm/sandbox/src/container/handler/spec.rs @@ -14,10 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ +use std::path::Path; + use anyhow::anyhow; use async_trait::async_trait; -use containerd_sandbox::error::Error; -use vmm_common::SHARED_DIR_SUFFIX; +use containerd_sandbox::{ + error::Error, + spec::{JsonSpec, Mount}, +}; +use path_clean::clean; +use vmm_common::{ + ETC_HOSTNAME, ETC_HOSTS, ETC_RESOLV, HOSTNAME_FILENAME, HOSTS_FILENAME, KUASAR_STATE_DIR, + RESOLV_FILENAME, +}; use crate::{ container::handler::Handler, sandbox::KuasarSandbox, utils::write_file_atomic, vm::VM, @@ -46,6 +55,7 @@ where &self, sandbox: &mut KuasarSandbox, ) -> containerd_sandbox::error::Result<()> { + let shared_path = sandbox.get_sandbox_shared_path(); let container = sandbox.container_mut(&self.container_id)?; let spec = container .data @@ -59,6 +69,8 @@ where if let Some(p) = spec.process.as_mut() { p.apparmor_profile = "".to_string(); } + // Update sandbox files mounts for container + container_mounts(&shared_path, spec); let spec_str = serde_json::to_string(spec) .map_err(|e| anyhow!("failed to parse spec in sandbox, {}", e))?; let config_path = format!("{}/{}", container.data.bundle, CONFIG_FILE_NAME); @@ -71,8 +83,9 @@ where sandbox: &mut KuasarSandbox, ) -> containerd_sandbox::error::Result<()> { let bundle = format!( - "{}/{}/{}", - sandbox.base_dir, SHARED_DIR_SUFFIX, self.container_id + "{}/{}", + sandbox.get_sandbox_shared_path(), + self.container_id ); tokio::fs::remove_dir_all(&*bundle) .await @@ -80,3 +93,170 @@ where Ok(()) } } + +// container_mounts sets up necessary container system file mounts +// including /etc/hostname, /etc/hosts and /etc/resolv.conf. +fn container_mounts(shared_path: &str, spec: &mut JsonSpec) { + let rw_option = if spec.root.as_ref().map(|r| r.readonly).unwrap_or_default() { + "ro" + } else { + "rw" + }; + + let mut extra_mounts: Vec = vec![]; + let cri_mount_handler = |dst, filename, extra_mounts: &mut Vec| { + if !is_in_cri_mounts(dst, &spec.mounts) { + let host_path = format!("{}/{}", shared_path, filename); + // If host path exist, should add it to container mount + if Path::exists(host_path.as_ref()) { + extra_mounts.push(Mount { + destination: dst.to_string(), + r#type: "bind".to_string(), + source: format!("{}/{}", KUASAR_STATE_DIR, filename), + options: vec!["rbind", "rprivate", rw_option] + .into_iter() + .map(String::from) + .collect(), + }); + } + } + }; + + cri_mount_handler(ETC_HOSTNAME, HOSTNAME_FILENAME, &mut extra_mounts); + cri_mount_handler(ETC_HOSTS, HOSTS_FILENAME, &mut extra_mounts); + cri_mount_handler(ETC_RESOLV, RESOLV_FILENAME, &mut extra_mounts); + spec.mounts.append(&mut extra_mounts); +} + +fn is_in_cri_mounts(dst: &str, mounts: &Vec) -> bool { + for mount in mounts { + if clean(&mount.destination) == clean(dst) { + return true; + } + } + false +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use containerd_sandbox::spec::{JsonSpec, Mount, Root}; + use containerd_shim::util::write_str_to_file; + use temp_dir::TempDir; + + use crate::container::handler::spec::{container_mounts, is_in_cri_mounts}; + + fn generate_cri_mounts() -> Vec { + vec![ + Mount { + destination: "/etc/hosts".to_string(), + r#type: "".to_string(), + source: "/run/kuasar-vmm/0d042b22dbaf083f704c5945488c7f63d10a664f743802a7901ac6fae6460d9b/shared/hosts".to_string(), + options: vec![], + }, + Mount { + destination: "/etc/hostname".to_string(), + r#type: "".to_string(), + source: "/run/kuasar-vmm/0d042b22dbaf083f704c5945488c7f63d10a664f743802a7901ac6fae6460d9b/shared/hostname".to_string(), + options: vec![], + }, + Mount { + destination: "/etc/resolv.conf".to_string(), + r#type: "".to_string(), + source: "/run/kuasar-vmm/0d042b22dbaf083f704c5945488c7f63d10a664f743802a7901ac6fae6460d9b/shared/resolv.conf".to_string(), + options: vec![], + }, + Mount { + destination: "/dev/shm".to_string(), + r#type: "".to_string(), + source: "/run/kuasar-vmm/0d042b22dbaf083f704c5945488c7f63d10a664f743802a7901ac6fae6460d9b/shared/shm".to_string(), + options: vec![], + }, + ] + } + + #[test] + fn test_is_in_cri_mounts() { + let cri_mounts = generate_cri_mounts(); + assert!(is_in_cri_mounts("/etc/hostname", &cri_mounts)); + assert!(is_in_cri_mounts("/etc/hosts", &cri_mounts)); + assert!(is_in_cri_mounts("/etc/resolv.conf", &cri_mounts)); + assert!(is_in_cri_mounts("/dev/shm", &cri_mounts)); + assert!(!is_in_cri_mounts("/var/lib/kuasar", &cri_mounts)); + } + + #[tokio::test] + // When no mount defined in spec and kuasar doesn't create these files, expect no mount added. + async fn test_container_mounts_with_target_file_not_exist() { + let mut spec = JsonSpec::default(); + spec.mounts = vec![]; + + let tmp_path = TempDir::new().unwrap(); + let shared_path = tmp_path.path().to_str().unwrap(); + container_mounts(shared_path, &mut spec); + assert_eq!(spec.mounts.len(), 0); + } + + #[tokio::test] + // When no mount defined in spec and kuasar created hostname file, expect hostname mount is added. + async fn test_container_mounts_with_hostname_file_exist() { + let mut spec = JsonSpec::default(); + spec.mounts = vec![]; + spec.root = Some(Root::default()); + assert!(!spec.root.clone().expect("root should not be None").readonly); + + let tmp_path = TempDir::new().unwrap(); + let shared_path = tmp_path.path().to_str().unwrap(); + write_str_to_file(tmp_path.child("hostname"), "kuasar-deno-001") + .await + .unwrap(); + container_mounts(shared_path, &mut spec); + assert_eq!(spec.mounts.len(), 1); + assert!(spec.mounts[0].options.contains(&"rw".to_string())); + let parent_path = Path::new(&spec.mounts[0].source) + .parent() + .unwrap() + .to_str() + .unwrap(); + assert_eq!(parent_path, "/run/kuasar/state"); + } + + #[tokio::test] + // When readonly rootfs defined in spec, expect hostname mount is readonly. + async fn test_container_mounts_with_mounts_and_ro() { + let mut spec = JsonSpec::default(); + spec.mounts = vec![]; + spec.root = Some(Root { + path: "".to_string(), + readonly: true, + }); + assert!(spec.root.clone().expect("root should not be None").readonly); + + let tmp_path = TempDir::new().unwrap(); + let shared_path = tmp_path.path().to_str().unwrap(); + write_str_to_file(tmp_path.child("hostname"), "kuasar-deno-001") + .await + .unwrap(); + container_mounts(shared_path, &mut spec); + assert_eq!(spec.mounts.len(), 1); + assert!(spec.mounts[0].options.contains(&"ro".to_string())); + } + + #[tokio::test] + // When hostname mount already defined, expect hostname mount is no changed. + async fn test_container_mounts_with_mount_predefined() { + let mut spec = JsonSpec::default(); + let cri_mount = generate_cri_mounts(); + spec.mounts = cri_mount.clone(); + + let tmp_path = TempDir::new().unwrap(); + let shared_path = tmp_path.path().to_str().unwrap(); + container_mounts(shared_path, &mut spec); + assert_eq!(spec.mounts.len(), 4); + assert_eq!(cri_mount[0].source, spec.mounts[0].source); + assert_eq!(cri_mount[0].r#type, spec.mounts[0].r#type); + assert_eq!(cri_mount[0].destination, spec.mounts[0].destination); + assert_eq!(cri_mount[0].options, spec.mounts[0].options); + } +} diff --git a/vmm/sandbox/src/container/handler/storage.rs b/vmm/sandbox/src/container/handler/storage.rs index cb9461cf..2a77d982 100644 --- a/vmm/sandbox/src/container/handler/storage.rs +++ b/vmm/sandbox/src/container/handler/storage.rs @@ -21,7 +21,7 @@ use containerd_sandbox::{ Sandbox, }; use log::debug; -use vmm_common::{storage::ANNOTATION_KEY_STORAGE, STORAGE_FILE_PREFIX}; +use vmm_common::{storage::ANNOTATION_KEY_STORAGE, DEV_SHM, STORAGE_FILE_PREFIX}; use crate::{ container::handler::Handler, sandbox::KuasarSandbox, storage::mount::is_bind_shm, @@ -68,7 +68,7 @@ where } // TODO if vmm-task mount shm when startup, then just use the same shm if is_bind_shm(&m) { - m.source = "/dev/shm".to_string(); + m.source = DEV_SHM.to_string(); m.options.push("rbind".to_string()); } handled_mounts.push(m); diff --git a/vmm/sandbox/src/sandbox.rs b/vmm/sandbox/src/sandbox.rs index e9b97f78..7905a248 100644 --- a/vmm/sandbox/src/sandbox.rs +++ b/vmm/sandbox/src/sandbox.rs @@ -25,14 +25,18 @@ use containerd_sandbox::{ utils::cleanup_mounts, ContainerOption, Sandbox, SandboxOption, SandboxStatus, Sandboxer, }; +use containerd_shim::util::write_str_to_file; use log::{debug, error, info, warn}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tokio::{ - fs::{remove_dir_all, OpenOptions}, + fs::{copy, create_dir_all, remove_dir_all, OpenOptions}, io::{AsyncReadExt, AsyncWriteExt}, sync::{Mutex, RwLock}, }; -use vmm_common::{api::sandbox_ttrpc::SandboxServiceClient, storage::Storage, SHARED_DIR_SUFFIX}; +use vmm_common::{ + api::sandbox_ttrpc::SandboxServiceClient, storage::Storage, ETC_HOSTS, ETC_RESOLV, + HOSTNAME_FILENAME, HOSTS_FILENAME, RESOLV_FILENAME, SHARED_DIR_SUFFIX, +}; use crate::{ cgroup::SandboxCgroup, @@ -42,7 +46,7 @@ use crate::{ }, container::KuasarContainer, network::{Network, NetworkConfig}, - utils::{get_resources, get_sandbox_cgroup_parent_path}, + utils::{get_dns_config, get_hostname, get_resources, get_sandbox_cgroup_parent_path}, vm::{Hooks, Recoverable, VMFactory, VM}, }; @@ -236,6 +240,9 @@ where let network = Network::new(network_config).await?; network.attach_to(&mut sandbox).await?; } + + // setup sandbox files: hosts, hostname and resolv.conf for guest + sandbox.setup_sandbox_files().await?; self.hooks.post_create(&mut sandbox).await?; sandbox.dump().await?; self.sandboxes @@ -363,7 +370,7 @@ where async fn remove_container(&mut self, id: &str) -> Result<()> { self.deference_container_storages(id).await?; - let bundle = format!("{}/{}/{}", self.base_dir, SHARED_DIR_SUFFIX, id); + let bundle = format!("{}/{}", self.get_sandbox_shared_path(), &id); if let Err(e) = tokio::fs::remove_dir_all(&*bundle).await { if e.kind() != ErrorKind::NotFound { return Err(anyhow!("failed to remove bundle {}, {}", bundle, e).into()); @@ -540,6 +547,83 @@ where client_sync_clock(client).await; } } + + async fn setup_sandbox_files(&self) -> Result<()> { + let shared_path = self.get_sandbox_shared_path(); + create_dir_all(&shared_path) + .await + .map_err(|e| anyhow!("create host sandbox path({}): {}", shared_path, e))?; + + // Handle hostname + let mut hostname = get_hostname(&self.data); + if hostname.is_empty() { + hostname = hostname::get() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_default(); + } + hostname.push('\n'); + let hostname_path = Path::new(&shared_path).join(HOSTNAME_FILENAME); + write_str_to_file(hostname_path, &hostname) + .await + .map_err(|e| anyhow!("write hostname: {:?}", e))?; + + // handle hosts + let hosts_path = Path::new(&shared_path).join(HOSTS_FILENAME); + copy(ETC_HOSTS, hosts_path) + .await + .map_err(|e| anyhow!("copy hosts: {}", e))?; + + // handle resolv.conf + let resolv_path = Path::new(&shared_path).join(RESOLV_FILENAME); + match get_dns_config(&self.data).map(|dns_config| { + parse_dnsoptions( + &dns_config.servers, + &dns_config.searches, + &dns_config.options, + ) + }) { + Some(resolv_content) if !resolv_content.is_empty() => { + write_str_to_file(resolv_path, &resolv_content) + .await + .map_err(|e| anyhow!("write reslov.conf: {:?}", e))?; + } + _ => { + copy(ETC_RESOLV, resolv_path) + .await + .map_err(|e| anyhow!("copy resolv.conf: {}", e))?; + } + } + + Ok(()) + } + + pub fn get_sandbox_shared_path(&self) -> String { + format!("{}/{}", self.base_dir, SHARED_DIR_SUFFIX) + } +} + +// parse_dnsoptions parse DNS options into resolv.conf format content, +// if none option is specified, will return empty with no error. +fn parse_dnsoptions( + servers: &Vec, + searches: &Vec, + options: &Vec, +) -> String { + let mut resolv_content = String::new(); + + if !searches.is_empty() { + resolv_content.push_str(&format!("search {}\n", searches.join(" "))); + } + + if !servers.is_empty() { + resolv_content.push_str(&format!("nameserver {}\n", servers.join("\nnameserver "))); + } + + if !options.is_empty() { + resolv_content.push_str(&format!("options {}\n", options.join(" "))); + } + + resolv_content } #[derive(Default, Debug, Deserialize)] @@ -595,3 +679,57 @@ fn monitor(sandbox_mutex: Arc>>) { } }); } + +#[cfg(test)] +mod tests { + mod dns { + use crate::sandbox::parse_dnsoptions; + + #[derive(Default)] + struct DnsConfig { + servers: Vec, + searches: Vec, + options: Vec, + } + + #[test] + fn test_parse_empty_dns_option() { + let mut dns_test = DnsConfig::default(); + let resolv_content = + parse_dnsoptions(&dns_test.servers, &dns_test.searches, &dns_test.options); + assert!(resolv_content.is_empty()) + } + + #[test] + fn test_parse_non_empty_dns_option() { + let dns_test = DnsConfig { + servers: vec!["8.8.8.8", "server.google.com"] + .into_iter() + .map(String::from) + .collect(), + searches: vec![ + "server0.google.com", + "server1.google.com", + "server2.google.com", + "server3.google.com", + "server4.google.com", + "server5.google.com", + "server6.google.com", + ] + .into_iter() + .map(String::from) + .collect(), + options: vec!["timeout:1"].into_iter().map(String::from).collect(), + }; + let expected_content = "search server0.google.com server1.google.com server2.google.com server3.google.com server4.google.com server5.google.com server6.google.com +nameserver 8.8.8.8 +nameserver server.google.com +options timeout:1 +".to_string(); + let resolv_content = + parse_dnsoptions(&dns_test.servers, &dns_test.searches, &dns_test.options); + assert!(!resolv_content.is_empty()); + assert_eq!(resolv_content, expected_content) + } + } +} diff --git a/vmm/sandbox/src/storage/mod.rs b/vmm/sandbox/src/storage/mod.rs index 107f64c6..43748681 100644 --- a/vmm/sandbox/src/storage/mod.rs +++ b/vmm/sandbox/src/storage/mod.rs @@ -28,7 +28,7 @@ pub use utils::*; use vmm_common::{ mount::{bind_mount, unmount, MNT_NOFOLLOW}, storage::{Storage, DRIVEREPHEMERALTYPE}, - KUASAR_STATE_DIR, SHARED_DIR_SUFFIX, + KUASAR_STATE_DIR, }; use crate::{ @@ -158,7 +158,7 @@ where } else { m.source.clone() }; - let host_dest = format!("{}/{}/{}", self.base_dir, SHARED_DIR_SUFFIX, &storage_id); + let host_dest = format!("{}/{}", self.get_sandbox_shared_path(), &storage_id); debug!("bind mount storage for mount {:?}, dest: {}", m, &host_dest); let source_path = Path::new(&*source); if source_path.is_dir() { @@ -206,7 +206,7 @@ where m ))); } - let host_dest = format!("{}/{}/{}", self.base_dir, SHARED_DIR_SUFFIX, &storage_id); + let host_dest = format!("{}/{}", self.get_sandbox_shared_path(), &storage_id); debug!("overlay mount storage for {:?}, dest: {}", m, &host_dest); tokio::fs::create_dir_all(&host_dest).await?; mount_rootfs(Some(&m.r#type), Some(&m.source), &m.options, &host_dest) @@ -288,7 +288,7 @@ where if device_id.is_some() { self.vm.hot_detach(&device_id.unwrap()).await?; } else if fs_type == "bind" { - let mount_point = format!("{}/{}/{}", self.base_dir, SHARED_DIR_SUFFIX, id); + let mount_point = format!("{}/{}", self.get_sandbox_shared_path(), &id); unmount(&mount_point, MNT_DETACH | MNT_NOFOLLOW)?; if Path::new(&mount_point).is_dir() { tokio::fs::remove_dir(&mount_point).await.map_err(|e| { diff --git a/vmm/sandbox/src/storage/mount.rs b/vmm/sandbox/src/storage/mount.rs index bc2360d7..ee0cf5fd 100644 --- a/vmm/sandbox/src/storage/mount.rs +++ b/vmm/sandbox/src/storage/mount.rs @@ -16,11 +16,12 @@ limitations under the License. use anyhow::anyhow; use containerd_sandbox::{error::Result, spec::Mount}; +use vmm_common::DEV_SHM; use crate::{storage::MountInfo, utils::read_file}; pub fn is_bind_shm(m: &Mount) -> bool { - is_bind(m) && m.destination == "/dev/shm" + is_bind(m) && m.destination == DEV_SHM } pub fn is_bind(m: &Mount) -> bool { diff --git a/vmm/sandbox/src/utils.rs b/vmm/sandbox/src/utils.rs index 08d561ac..b049e57c 100644 --- a/vmm/sandbox/src/utils.rs +++ b/vmm/sandbox/src/utils.rs @@ -26,7 +26,7 @@ use std::{ use anyhow::anyhow; use containerd_sandbox::{ - cri::api::v1::LinuxContainerResources, + cri::api::v1::{DnsConfig, LinuxContainerResources}, data::SandboxData, error::{Error, Result}, }; @@ -86,6 +86,16 @@ pub fn get_overhead_resources(data: &SandboxData) -> Option<&LinuxContainerResou .and_then(|c| c.linux.as_ref()) .and_then(|l| l.overhead.as_ref()) } +pub fn get_hostname(data: &SandboxData) -> String { + data.config + .as_ref() + .map(|c| c.hostname.clone()) + .unwrap_or_default() +} + +pub fn get_dns_config(data: &SandboxData) -> Option<&DnsConfig> { + data.config.as_ref().and_then(|c| c.dns_config.as_ref()) +} #[allow(dead_code)] pub fn get_total_resources(data: &SandboxData) -> Option { diff --git a/vmm/task/src/container.rs b/vmm/task/src/container.rs index 8ac89465..786b198b 100644 --- a/vmm/task/src/container.rs +++ b/vmm/task/src/container.rs @@ -41,8 +41,8 @@ use containerd_shim::{ util::read_spec, ExitSignal, }; -use log::{debug, error}; -use nix::{sys::signalfd::signal::kill, unistd::Pid}; +use log::{debug, error, warn}; +use nix::{mount, mount::MsFlags, sys::signalfd::signal::kill, unistd::Pid}; use oci_spec::runtime::{LinuxResources, Process, Spec}; use runc::{options::GlobalOpts, Runc, Spawner}; use serde::Deserialize; @@ -52,7 +52,9 @@ use tokio::{ process::Command, sync::Mutex, }; -use vmm_common::{mount::get_mount_type, storage::Storage, KUASAR_STATE_DIR}; +use vmm_common::{ + mount::get_mount_type, storage::Storage, ETC_RESOLV, KUASAR_STATE_DIR, RESOLV_FILENAME, +}; use crate::{ device::rescan_pci_bus, @@ -195,6 +197,26 @@ impl KuasarFactory { Self { sandbox } } + // create_sandbox will do some preparation to provide a livable environment for containers, + // such as adding guest hooks, setting kernel paras, preparing sandbox files and namespaces + pub fn create_sandbox(&self) -> Result<()> { + // Setup DNS, bind mount to /etc/resolv.conf + let dns_file = Path::new(KUASAR_STATE_DIR).join(RESOLV_FILENAME); + if dns_file.exists() { + mount::mount( + Some(&dns_file), + ETC_RESOLV, + Some("bind"), + MsFlags::MS_BIND, + None::<&str>, + )?; + } else { + warn!("unable to find DNS files in kuasar state dir"); + } + + Ok(()) + } + async fn do_create(&self, init: &mut InitProcess) -> Result<()> { let id = init.id.to_string(); let stdio = &init.stdio; diff --git a/vmm/task/src/main.rs b/vmm/task/src/main.rs index 713fe1c0..9600d753 100644 --- a/vmm/task/src/main.rs +++ b/vmm/task/src/main.rs @@ -291,7 +291,7 @@ async fn mount_static_mounts(mounts: Vec) -> containerd_shim::Resul // start_ttrpc_server will create all the ttrpc service and register them to a server that // bind to vsock 1024 port. async fn start_ttrpc_server() -> containerd_shim::Result { - let task = create_task_service().await; + let task = create_task_service().await?; let task_service = create_task(Arc::new(Box::new(task))); let sandbox = SandboxService::new()?; diff --git a/vmm/task/src/task.rs b/vmm/task/src/task.rs index da0c5854..a93eaa59 100644 --- a/vmm/task/src/task.rs +++ b/vmm/task/src/task.rs @@ -35,11 +35,14 @@ use crate::{ sandbox::SandboxResources, }; -pub(crate) async fn create_task_service() -> TaskService { +pub(crate) async fn create_task_service( +) -> containerd_shim::Result> { let (tx, mut rx) = channel(128); let sandbox = Arc::new(Mutex::new(SandboxResources::new().await)); + let factory = KuasarFactory::new(sandbox); + factory.create_sandbox()?; let task = TaskService { - factory: KuasarFactory::new(sandbox), + factory, containers: Arc::new(Default::default()), namespace: "k8s.io".to_string(), exit: Arc::new(Default::default()), @@ -54,7 +57,7 @@ pub(crate) async fn create_task_service() -> TaskService) { From 1df144a431bdbe9dca70d49b096d3f7072edb6be Mon Sep 17 00:00:00 2001 From: Zhang Tianyang Date: Wed, 10 Jan 2024 15:46:22 +0800 Subject: [PATCH 2/2] task: add early and late init calls Add early_init_call() and late_init_call() for VM initiation. Signed-off-by: Zhang Tianyang --- vmm/task/src/container.rs | 28 +++---------------- vmm/task/src/main.rs | 57 ++++++++++++++++++++++++++++++++------- vmm/task/src/task.rs | 9 +++---- 3 files changed, 54 insertions(+), 40 deletions(-) diff --git a/vmm/task/src/container.rs b/vmm/task/src/container.rs index 786b198b..8ac89465 100644 --- a/vmm/task/src/container.rs +++ b/vmm/task/src/container.rs @@ -41,8 +41,8 @@ use containerd_shim::{ util::read_spec, ExitSignal, }; -use log::{debug, error, warn}; -use nix::{mount, mount::MsFlags, sys::signalfd::signal::kill, unistd::Pid}; +use log::{debug, error}; +use nix::{sys::signalfd::signal::kill, unistd::Pid}; use oci_spec::runtime::{LinuxResources, Process, Spec}; use runc::{options::GlobalOpts, Runc, Spawner}; use serde::Deserialize; @@ -52,9 +52,7 @@ use tokio::{ process::Command, sync::Mutex, }; -use vmm_common::{ - mount::get_mount_type, storage::Storage, ETC_RESOLV, KUASAR_STATE_DIR, RESOLV_FILENAME, -}; +use vmm_common::{mount::get_mount_type, storage::Storage, KUASAR_STATE_DIR}; use crate::{ device::rescan_pci_bus, @@ -197,26 +195,6 @@ impl KuasarFactory { Self { sandbox } } - // create_sandbox will do some preparation to provide a livable environment for containers, - // such as adding guest hooks, setting kernel paras, preparing sandbox files and namespaces - pub fn create_sandbox(&self) -> Result<()> { - // Setup DNS, bind mount to /etc/resolv.conf - let dns_file = Path::new(KUASAR_STATE_DIR).join(RESOLV_FILENAME); - if dns_file.exists() { - mount::mount( - Some(&dns_file), - ETC_RESOLV, - Some("bind"), - MsFlags::MS_BIND, - None::<&str>, - )?; - } else { - warn!("unable to find DNS files in kuasar state dir"); - } - - Ok(()) - } - async fn do_create(&self, init: &mut InitProcess) -> Result<()> { let id = init.id.to_string(); let stdio = &init.stdio; diff --git a/vmm/task/src/main.rs b/vmm/task/src/main.rs index 9600d753..2cf25ecc 100644 --- a/vmm/task/src/main.rs +++ b/vmm/task/src/main.rs @@ -22,6 +22,7 @@ use containerd_shim::{ io_error, other, protos::{shim::shim_ttrpc_async::create_task, ttrpc::asynchronous::Server}, util::IntoOption, + Result, }; use futures::StreamExt; use lazy_static::lazy_static; @@ -35,7 +36,10 @@ use nix::{ unistd::Pid, }; use signal_hook_tokio::Signals; -use vmm_common::{api::sandbox_ttrpc::create_sandbox_service, mount::mount, KUASAR_STATE_DIR}; +use vmm_common::{ + api::sandbox_ttrpc::create_sandbox_service, mount::mount, ETC_RESOLV, KUASAR_STATE_DIR, + RESOLV_FILENAME, +}; use crate::{ config::TaskConfig, @@ -67,6 +71,11 @@ pub struct StaticMount { options: Vec<&'static str>, } +const ENVS: [(&str, &str); 2] = [ + ("PATH", "/bin:/sbin/:/usr/bin/:/usr/sbin/"), + ("XDG_RUNTIME_DIR", "/run"), +]; + lazy_static! { pub static ref VM_ROOTFS_MOUNTS: Vec = vec![ StaticMount { @@ -122,9 +131,7 @@ lazy_static! { #[tokio::main] async fn main() { - std::env::set_var("PATH", "/bin:/sbin/:/usr/bin/:/usr/sbin/"); - std::env::set_var("XDG_RUNTIME_DIR", "/run"); - init_vm_rootfs().await.unwrap(); + early_init_call().await.expect("early init call"); let config = TaskConfig::new().await.unwrap(); let log_level = LevelFilter::from_str(&config.log_level).unwrap(); env_logger::Builder::from_default_env() @@ -154,6 +161,8 @@ async fn main() { } } + late_init_call().await.expect("late init call"); + // Start ttrpc server let mut server = start_ttrpc_server() .await @@ -166,6 +175,17 @@ async fn main() { handle_signals(signals).await; } +// Do some initialization before everything starts. +// Such as setting envs, preparing cgroup mounts, setting kernel paras. +async fn early_init_call() -> Result<()> { + // Set environment variables from ENVS vector(ordered). + for (k, v) in ENVS.iter() { + std::env::set_var(k, v); + } + init_vm_rootfs().await?; + Ok(()) +} + async fn handle_signals(signals: Signals) { let mut signals = signals.fuse(); while let Some(sig) = signals.next().await { @@ -248,7 +268,7 @@ async fn handle_signals(signals: Signals) { } } -async fn init_vm_rootfs() -> containerd_shim::Result<()> { +async fn init_vm_rootfs() -> Result<()> { let mounts = VM_ROOTFS_MOUNTS.clone(); mount_static_mounts(mounts).await?; // has to mount /proc before find cgroup mounts @@ -258,11 +278,30 @@ async fn init_vm_rootfs() -> containerd_shim::Result<()> { // For more information see https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt tokio::fs::write("/sys/fs/cgroup/memory/memory.use_hierarchy", "1") .await - .unwrap(); + .map_err(io_error!(e, "failed to set cgroup hierarchy to 1")) +} + +// Continue to do initialization that depend on shared path. +// such as adding guest hook, preparing sandbox files and namespaces. +async fn late_init_call() -> Result<()> { + // Setup DNS, bind mount to /etc/resolv.conf + let dns_file = Path::new(KUASAR_STATE_DIR).join(RESOLV_FILENAME); + if dns_file.exists() { + nix::mount::mount( + Some(&dns_file), + ETC_RESOLV, + Some("bind"), + nix::mount::MsFlags::MS_BIND, + None::<&str>, + )?; + } else { + warn!("unable to find DNS files in kuasar state dir"); + } + Ok(()) } -async fn mount_static_mounts(mounts: Vec) -> containerd_shim::Result<()> { +async fn mount_static_mounts(mounts: Vec) -> Result<()> { for m in mounts { tokio::fs::create_dir_all(Path::new(m.dest)) .await @@ -290,8 +329,8 @@ async fn mount_static_mounts(mounts: Vec) -> containerd_shim::Resul // start_ttrpc_server will create all the ttrpc service and register them to a server that // bind to vsock 1024 port. -async fn start_ttrpc_server() -> containerd_shim::Result { - let task = create_task_service().await?; +async fn start_ttrpc_server() -> Result { + let task = create_task_service().await; let task_service = create_task(Arc::new(Box::new(task))); let sandbox = SandboxService::new()?; diff --git a/vmm/task/src/task.rs b/vmm/task/src/task.rs index a93eaa59..da0c5854 100644 --- a/vmm/task/src/task.rs +++ b/vmm/task/src/task.rs @@ -35,14 +35,11 @@ use crate::{ sandbox::SandboxResources, }; -pub(crate) async fn create_task_service( -) -> containerd_shim::Result> { +pub(crate) async fn create_task_service() -> TaskService { let (tx, mut rx) = channel(128); let sandbox = Arc::new(Mutex::new(SandboxResources::new().await)); - let factory = KuasarFactory::new(sandbox); - factory.create_sandbox()?; let task = TaskService { - factory, + factory: KuasarFactory::new(sandbox), containers: Arc::new(Default::default()), namespace: "k8s.io".to_string(), exit: Arc::new(Default::default()), @@ -57,7 +54,7 @@ pub(crate) async fn create_task_service( debug!("received event {:?}", e); } }); - Ok(task) + task } async fn process_exits(s: Subscription, task: &TaskService) {