support for creating zettacache with multiple disks (openzfs#32)

Add support for creating zettacache with multiple disks.

cmd/zfs_object_agent/server/src/main.rs

@@ -38,12 +38,14 @@ fn main() {
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name("cache-file")
+            Arg::with_name("cache-device")
                 .short("c")
-                .long("cache-file")
-                .value_name("FILE")
+                .long("cache-device")
+                .value_name("PATH")
                 .help("File/device to use for ZettaCache")
-                .takes_value(true),
+                .takes_value(true)
+                .multiple(true)
+                .number_of_values(1),
         )
         .arg(
             Arg::with_name("config-file")
@@ -143,7 +145,9 @@ fn main() {
         }
         _ => {
             let socket_dir = matches.value_of("socket-dir").unwrap();
-            let cache_path = matches.value_of("cache-file");
+            let cache_paths = matches
+                .values_of("cache-device")
+                .map_or(Vec::new(), |values| values.collect());
 
             util::setup_logging(
                 matches.occurrences_of("verbosity"),
@@ -184,7 +188,7 @@ fn main() {
             // trace!() can be used indiscriminately.
             trace!("logging level TRACE enabled");
 
-            zettaobject::init::start(socket_dir, cache_path);
+            zettaobject::init::start(socket_dir, cache_paths);
         }
     }
 }

cmd/zfs_object_agent/util/src/btreemap_ext.rs

@@ -0,0 +1,9 @@
+use std::collections::BTreeMap;
+
+/// Iterate over the values in the BTreeMap, starting with `start`, and looping
+/// around such that all entries will be visited once.
+pub fn iter_wrapping<K: Ord + Copy, V>(map: &BTreeMap<K, V>, start: K) -> impl Iterator<Item = &V> {
+    map.range(start..)
+        .chain(map.iter().take_while(move |(&k, _v)| k != start))
+        .map(|(_k, v)| v)
+}

cmd/zfs_object_agent/util/src/lib.rs

@@ -4,6 +4,7 @@
 #![warn(clippy::cast_sign_loss)]
 
 mod bitmap_range_iterator;
+mod btreemap_ext;
 mod die;
 mod from64;
 mod lock_set;
@@ -15,6 +16,7 @@ mod tunable;
 mod vec_ext;
 
 pub use bitmap_range_iterator::BitmapRangeIterator;
+pub use btreemap_ext::iter_wrapping;
 pub use die::maybe_die_with;
 pub use from64::From64;
 pub use lock_set::LockSet;

cmd/zfs_object_agent/zcdb/src/main.rs

@@ -30,9 +30,14 @@ async fn main() {
         .about("ZFS ZettaCache Debugger")
         .version(GIT_VERSION)
         .arg(
-            Arg::with_name("device")
-                .help("ZettaCache Device")
-                .required(true),
+            Arg::with_name("cache-device")
+                .short("c")
+                .long("cache-device")
+                .value_name("PATH")
+                .help("File/device to use for ZettaCache")
+                .takes_value(true)
+                .multiple(true)
+                .number_of_values(1),
         )
         .subcommand(
             SubCommand::with_name("dump-structures")
@@ -71,7 +76,7 @@ async fn main() {
         .subcommand(SubCommand::with_name("space-usage").about("dump space usage statistics"))
         .get_matches();
 
-    let device = matches.value_of("device").unwrap();
+    let cache_paths = matches.values_of("cache-device").unwrap().collect();
     match matches.subcommand() {
         ("dump-structures", Some(subcommand_matches)) => {
             ZettaCacheDBCommand::issue_command(
@@ -82,7 +87,7 @@ async fn main() {
                         .operation_log_raw(subcommand_matches.is_present("operation-log-raw"))
                         .index_log_raw(subcommand_matches.is_present("index-log-raw")),
                 ),
-                device,
+                cache_paths,
             )
             .await;
         }
@@ -91,12 +96,13 @@ async fn main() {
                 ZettaCacheDBCommand::DumpSlabs(
                     DumpSlabsOptions::default().verbosity(subcommand_matches.occurrences_of("v")),
                 ),
-                device,
+                cache_paths,
             )
             .await;
         }
         ("space-usage", Some(_)) => {
-            ZettaCacheDBCommand::issue_command(ZettaCacheDBCommand::DumpSpaceUsage, device).await;
+            ZettaCacheDBCommand::issue_command(ZettaCacheDBCommand::DumpSpaceUsage, cache_paths)
+                .await;
         }
         _ => {
             matches.usage();

cmd/zfs_object_agent/zettacache/Cargo.toml

@@ -9,6 +9,7 @@ edition = "2021"
 [dependencies]
 anyhow = "1.0"
 async-stream = "0.3.0"
+bimap = "0.6.1"
 bincode = "1.3.2"
 bytes = "1.0"
 chrono = "0.4"
@@ -23,6 +24,7 @@ metered = "0.8"
 more-asserts = "0.2.1"
 nix = "0.23.0"
 num = "0.4.0"
+rand = "0.8.3"
 roaring = "0.7.0"
 seahash = "4.1.0"
 serde = { version = "1.0.125", features = ["derive"] }

cmd/zfs_object_agent/zettacache/src/base_types.rs

@@ -1,6 +1,7 @@
 use more_asserts::*;
 use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
+use std::borrow::Borrow;
 use std::fmt::*;
 use std::ops::Add;
 use std::ops::Sub;
@@ -34,15 +35,19 @@ impl BlockId {
     }
 }
 
+#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd)]
+pub struct DiskId(pub u16);
+
 #[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd)]
 pub struct DiskLocation {
-    // note: will need to add disk ID to support multiple disks
+    pub disk: DiskId,
     pub offset: u64,
 }
 impl Add<u64> for DiskLocation {
     type Output = DiskLocation;
     fn add(self, rhs: u64) -> Self::Output {
         DiskLocation {
+            disk: self.disk,
             offset: self.offset + rhs,
         }
     }
@@ -53,6 +58,14 @@ impl Add<usize> for DiskLocation {
         self + rhs as u64
     }
 }
+impl Sub<DiskLocation> for DiskLocation {
+    type Output = u64;
+
+    fn sub(self, rhs: DiskLocation) -> Self::Output {
+        assert_eq!(self.disk, rhs.disk);
+        self.offset - rhs.offset
+    }
+}
 
 #[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd)]
 pub struct Extent {
@@ -61,13 +74,36 @@ pub struct Extent {
 }
 
 impl Extent {
+    pub fn new(disk: DiskId, offset: u64, size: u64) -> Extent {
+        assert_eq!(offset % 512, 0, "offset {} is not 512-aligned", offset);
+        assert_eq!(size % 512, 0, "size {} is not 512-aligned", size);
+        Extent {
+            location: DiskLocation { disk, offset },
+            size,
+        }
+    }
+
     pub fn range(&self, relative_offset: u64, size: u64) -> Extent {
         assert_ge!(self.size, relative_offset + size);
         Extent {
             location: self.location + relative_offset,
             size,
         }
     }
+
+    /// returns true if `sub` is entirely contained within this extent
+    pub fn contains(&self, sub: &Extent) -> bool {
+        sub.location.disk == self.location.disk
+            && sub.location.offset >= self.location.offset
+            && sub.location.offset + sub.size <= self.location.offset + self.size
+    }
+}
+
+/// This allows Extents to be compared by their `location`s, ignoring the `size`s.
+impl Borrow<DiskLocation> for Extent {
+    fn borrow(&self) -> &DiskLocation {
+        &self.location
+    }
 }
 
 #[derive(Serialize, Deserialize, Default, Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd)]