Merge #35

35: Add Nullable and Validity r=mbrobbel a=mbrobbel

These types are needed to start defining array types.

bors r+

Co-authored-by: Matthijs Brobbel <m1brobbel@gmail.com>

src/bitmap/fmt.rs

@@ -1,3 +1,5 @@
+//! Bitmap formatting.
+
 use std::fmt::{Display, Formatter, Result};
 
 use crate::buffer::Buffer;

src/bitmap/iter.rs

@@ -1,3 +1,5 @@
+//! Bitmap iteration.
+
 use std::{
     borrow::Borrow,
     iter::{Skip, Take},
@@ -10,6 +12,9 @@ use std::{
 /// Bitmap.
 pub type BitmapIter<'a> = Take<Skip<BitUnpacked<slice::Iter<'a, u8>, &'a u8>>>;
 
+/// An iterator over the bits in a Bitmap. Consumes the Bitmap.
+pub type BitmapIntoIter<I> = Take<Skip<BitUnpacked<I, u8>>>;
+
 /// An iterator that packs boolean values as bits in bytes using
 /// least-significant bit (LSB) numbering.
 ///

src/bitmap/mod.rs

@@ -9,10 +9,10 @@ use std::{
 
 use self::{
     fmt::BitsDisplayExt,
-    iter::{BitPackedExt, BitUnpackedExt, BitmapIter},
+    iter::{BitPackedExt, BitUnpackedExt, BitmapIntoIter, BitmapIter},
 };
 use crate::{
-    buffer::{Buffer, BufferAlloc, BufferExtend, BufferMut},
+    buffer::{Buffer, BufferAlloc, BufferExtend, BufferMut, BufferTake},
     DataBuffer, DataBufferMut, Length, Null,
 };
 
@@ -24,6 +24,7 @@ pub mod iter;
 /// The validity bits are stored LSB-first in the bytes of a [Buffer].
 #[derive(Clone, Default, PartialEq, Eq)]
 pub struct Bitmap<T = Vec<u8>>
+// Trait bound added here to avoid confusion about Debug impl.
 where
     T: Buffer<u8>,
 {
@@ -66,6 +67,7 @@ where
     /// Returns the number of trailing padding bits in the last byte of the
     /// buffer that contain no meaningful bits. This bits should be ignored when
     /// inspecting the raw byte buffer.
+    #[inline]
     pub fn trailing_bits(&self) -> usize {
         let trailing_bits = (self.offset + self.bits) % 8;
         if trailing_bits != 0 {
@@ -273,6 +275,22 @@ where
     }
 }
 
+impl<T> IntoIterator for Bitmap<T>
+where
+    T: BufferTake<u8>,
+{
+    type IntoIter = BitmapIntoIter<T::IntoIter>;
+    type Item = bool;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.buffer
+            .into_iter()
+            .bit_unpacked()
+            .skip(self.offset)
+            .take(self.bits)
+    }
+}
+
 impl<'a, T> IntoIterator for &'a Bitmap<T>
 where
     T: Buffer<u8>,
@@ -305,7 +323,7 @@ mod tests {
 
     #[test]
     fn offset_byte_slice() {
-        let mut bitmap: Bitmap = [true; 32].iter().collect();
+        let mut bitmap = [true; 32].iter().collect::<Bitmap>();
         // "unset" first byte
         let slice = bitmap.data_buffer_mut();
         slice[0] = 0;
@@ -330,7 +348,7 @@ mod tests {
 
     #[test]
     fn offset_byte_vec() {
-        let mut bitmap: Bitmap<Vec<u8>> = [true; 32].iter().collect();
+        let mut bitmap = [true; 32].iter().collect::<Bitmap<Vec<u8>>>();
         // "unset" first byte
         let vec: &mut Vec<u8> = bitmap.data_buffer_mut();
         vec[0] = 0;
@@ -352,14 +370,16 @@ mod tests {
 
     #[test]
     fn as_ref() {
-        let bitmap: Bitmap = [false, true, true, false, true].iter().collect();
+        let bitmap = [false, true, true, false, true].iter().collect::<Bitmap>();
         let slice: &[u8] = bitmap.data_buffer().as_ref();
         assert_eq!(&slice[0], &22);
     }
 
     #[test]
     fn as_ref_u8() {
-        let bitmap: Bitmap = [false, true, false, true, false, true].iter().collect();
+        let bitmap = [false, true, false, true, false, true]
+            .iter()
+            .collect::<Bitmap>();
         let bytes = bitmap.data_buffer().as_bytes();
         assert_eq!(bytes.len(), 1);
         assert_eq!(bytes[0], 42);
@@ -368,18 +388,20 @@ mod tests {
     #[test]
     #[should_panic]
     fn as_ref_u8_out_of_bounds() {
-        let bitmap: Bitmap = [false, true, false, true, false, true].iter().collect();
+        let bitmap = [false, true, false, true, false, true]
+            .iter()
+            .collect::<Bitmap>();
         let bits: &[u8] = bitmap.data_buffer().as_ref();
         let _ = bits[std::mem::size_of::<usize>()];
     }
 
     #[test]
     fn as_ref_bitslice() {
-        let bits: Bitmap = [
+        let bits = [
             false, true, false, true, false, true, false, false, false, true,
         ]
         .iter()
-        .collect();
+        .collect::<Bitmap>();
         assert_eq!(bits.len(), 10);
         assert!(!bits[0]);
         assert!(bits[1]);
@@ -396,14 +418,16 @@ mod tests {
     #[test]
     #[should_panic]
     fn as_ref_bitslice_out_of_bounds() {
-        let bitmap: Bitmap = vec![false, true, false, true, false, true].iter().collect();
+        let bitmap = vec![false, true, false, true, false, true]
+            .iter()
+            .collect::<Bitmap>();
         let _ = bitmap[bitmap.bits];
     }
 
     #[test]
     fn count() {
         let vec = vec![false, true, false, true, false, true];
-        let bitmap: Bitmap = vec.iter().collect();
+        let bitmap = vec.iter().collect::<Bitmap>();
         assert_eq!(bitmap.len(), 6);
         assert!(!bitmap.is_empty());
         assert_eq!(bitmap.valid_count(), 3);
@@ -432,7 +456,7 @@ mod tests {
     #[test]
     fn into_iter() {
         let vec = vec![true, false, true, false];
-        let bitmap: Bitmap = vec.iter().collect();
+        let bitmap = vec.iter().collect::<Bitmap>();
         assert_eq!(bitmap.into_iter().collect::<Vec<_>>(), vec);
     }
 }

src/buffer.rs

@@ -88,3 +88,18 @@ where
     U: BufferMut<T> + Extend<T>,
 {
 }
+
+/// A buffer that can be consumed via [IntoIterator].
+pub trait BufferTake<T>
+where
+    T: Primitive,
+    Self: Buffer<T> + IntoIterator<Item = T>,
+{
+}
+
+impl<T, U> BufferTake<T> for U
+where
+    T: Primitive,
+    U: Buffer<T> + IntoIterator<Item = T>,
+{
+}

src/buffers.rs

@@ -1,4 +1,4 @@
-//! Traits for Arrow memory buffers.
+//! Traits for memory buffers.
 
 use crate::{
     bitmap::Bitmap,
@@ -7,13 +7,15 @@ use crate::{
     Primitive,
 };
 
+/// A validity bitmap.
 pub trait ValidityBitmap<T>
 where
     T: Buffer<u8>,
 {
     fn validity_bitmap(&self) -> &Bitmap<T>;
 }
 
+/// A data buffer.
 pub trait DataBuffer<T>
 where
     T: Primitive,
@@ -22,6 +24,7 @@ where
     fn data_buffer(&self) -> &Self::Buffer;
 }
 
+/// A mutable data buffer.
 pub trait DataBufferMut<T>
 where
     T: Primitive,
@@ -30,6 +33,7 @@ where
     fn data_buffer_mut(&mut self) -> &mut Self::Buffer;
 }
 
+/// An offset buffer.
 pub trait OffsetBuffer<T>
 where
     T: OffsetValue,

src/lib.rs

@@ -24,3 +24,4 @@ pub mod bitmap;
 pub mod buffer;
 pub mod nullable;
 pub mod offset;
+pub mod validity;