Add non-nullable to nullable conversion

src/array/boolean.rs

@@ -4,6 +4,7 @@ use super::Array;
 use crate::{
     bitmap::{Bitmap, BitmapRef, BitmapRefMut, ValidityBitmap},
     buffer::{BufferRef, BufferRefMut, BufferType, VecBuffer},
+    nullable::Nullable,
     validity::Validity,
     Length,
 };
@@ -68,6 +69,15 @@ where
     }
 }
 
+impl<Buffer: BufferType> From<BooleanArray<false, Buffer>> for BooleanArray<true, Buffer>
+where
+    Bitmap<Buffer>: FromIterator<bool>,
+{
+    fn from(value: BooleanArray<false, Buffer>) -> Self {
+        Self(Nullable::wrap(value.0))
+    }
+}
+
 impl<const NULLABLE: bool, U, Buffer: BufferType> FromIterator<U> for BooleanArray<NULLABLE, Buffer>
 where
     Bitmap<Buffer>: Validity<NULLABLE>,
@@ -203,6 +213,18 @@ mod tests {
         );
     }
 
+    #[test]
+    fn convert_nullable() {
+        let input = [true, false];
+        let array = input.into_iter().collect::<BooleanArray>();
+        let nullable: BooleanArray<true> = array.into();
+        assert!(nullable.all_valid());
+        assert_eq!(
+            nullable.into_iter().collect::<Vec<_>>(),
+            [Some(true), Some(false)]
+        );
+    }
+
     #[test]
     fn size_of() {
         assert_eq!(mem::size_of::<BooleanArray>(), mem::size_of::<Bitmap>());

src/array/fixed_size_primitive.rs

@@ -2,6 +2,7 @@ use super::Array;
 use crate::{
     bitmap::{Bitmap, BitmapRef, BitmapRefMut, ValidityBitmap},
     buffer::{BufferType, VecBuffer},
+    nullable::Nullable,
     validity::Validity,
     FixedSize, Length,
 };
@@ -69,6 +70,17 @@ where
     }
 }
 
+impl<T: FixedSize, Buffer: BufferType> From<FixedSizePrimitiveArray<T, false, Buffer>>
+    for FixedSizePrimitiveArray<T, true, Buffer>
+where
+    <Buffer as BufferType>::Buffer<T>: Length,
+    Bitmap<Buffer>: FromIterator<bool>,
+{
+    fn from(value: FixedSizePrimitiveArray<T, false, Buffer>) -> Self {
+        Self(Nullable::wrap(value.0))
+    }
+}
+
 impl<T: FixedSize, const NULLABLE: bool, U, Buffer: BufferType> FromIterator<U>
     for FixedSizePrimitiveArray<T, NULLABLE, Buffer>
 where
@@ -141,7 +153,7 @@ mod tests {
     use super::*;
     use crate::{
         bitmap::Bitmap,
-        buffer::{Buffer, BufferRef},
+        buffer::{Buffer, BufferRef, BufferRefMut},
     };
     use std::mem;
 
@@ -197,6 +209,18 @@ mod tests {
         assert_eq!(array.len(), input.len());
     }
 
+    #[test]
+    fn convert_nullable() {
+        let input = [1, 2, 3, 4];
+        let array = input.into_iter().collect::<FixedSizePrimitiveArray<_>>();
+        let mut nullable: FixedSizePrimitiveArray<_, true> = array.into();
+        nullable.bitmap_ref_mut().buffer_ref_mut().as_mut_slice()[0] = 0b00001101;
+        assert_eq!(
+            nullable.into_iter().collect::<Vec<_>>(),
+            [Some(1), None, Some(3), Some(4)]
+        );
+    }
+
     #[test]
     fn size_of() {
         assert_eq!(mem::size_of::<Int8Array>(), mem::size_of::<Vec<i8>>());

src/array/null.rs

@@ -4,6 +4,7 @@ use super::{Array, ArrayType};
 use crate::{
     bitmap::{Bitmap, BitmapRef, BitmapRefMut, ValidityBitmap},
     buffer::{BufferType, VecBuffer},
+    nullable::Nullable,
     validity::Validity,
     Length,
 };
@@ -67,6 +68,15 @@ where
     }
 }
 
+impl<T: Unit, Buffer: BufferType> From<NullArray<T, false, Buffer>> for NullArray<T, true, Buffer>
+where
+    Bitmap<Buffer>: FromIterator<bool>,
+{
+    fn from(value: NullArray<T, false, Buffer>) -> Self {
+        Self(Nullable::wrap(value.0))
+    }
+}
+
 impl<T: Unit, U, const NULLABLE: bool, Buffer: BufferType> FromIterator<U>
     for NullArray<T, NULLABLE, Buffer>
 where

src/array/string.rs

@@ -82,6 +82,17 @@ where
     }
 }
 
+impl<OffsetItem: OffsetElement, Buffer: BufferType> From<StringArray<false, OffsetItem, Buffer>>
+    for StringArray<true, OffsetItem, Buffer>
+where
+    VariableSizeBinaryArray<false, OffsetItem, Buffer>:
+        Into<VariableSizeBinaryArray<true, OffsetItem, Buffer>>,
+{
+    fn from(value: StringArray<false, OffsetItem, Buffer>) -> Self {
+        Self(value.0.into())
+    }
+}
+
 impl<'a, OffsetItem: OffsetElement, Buffer: BufferType> FromIterator<&'a str>
     for StringArray<false, OffsetItem, Buffer>
 where
@@ -163,7 +174,7 @@ impl<OffsetItem: OffsetElement, Buffer: BufferType> ValidityBitmap
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bitmap::BitmapRef;
+    use crate::{bitmap::BitmapRef, buffer::BufferRef};
 
     #[test]
     fn from_iter() {
@@ -203,4 +214,15 @@ mod tests {
             &[true, false, true, true, false]
         );
     }
+
+    #[test]
+    fn convert_nullable() {
+        let input = ["hello", " ", "world"];
+        let array = input
+            .into_iter()
+            .map(ToString::to_string)
+            .collect::<StringArray>();
+        let nullable: StringArray<true> = array.into();
+        assert_eq!(nullable.bitmap_ref().buffer_ref(), &[0b00000111]);
+    }
 }

src/array/struct.rs

@@ -2,6 +2,7 @@ use super::{Array, ArrayType};
 use crate::{
     bitmap::{Bitmap, BitmapRef, BitmapRefMut, ValidityBitmap},
     buffer::{BufferType, VecBuffer},
+    nullable::Nullable,
     validity::Validity,
     Length,
 };
@@ -40,6 +41,17 @@ where
     }
 }
 
+impl<T: StructArrayType, Buffer: BufferType> From<StructArray<T, false, Buffer>>
+    for StructArray<T, true, Buffer>
+where
+    <T as StructArrayType>::Array<Buffer>: Length,
+    Bitmap<Buffer>: FromIterator<bool>,
+{
+    fn from(value: StructArray<T, false, Buffer>) -> Self {
+        Self(Nullable::wrap(value.0))
+    }
+}
+
 impl<T: StructArrayType, U, const NULLABLE: bool, Buffer: BufferType> FromIterator<U>
     for StructArray<T, NULLABLE, Buffer>
 where

src/array/variable_size_binary.rs

@@ -86,6 +86,25 @@ where
     }
 }
 
+impl<OffsetItem: OffsetElement, Buffer: BufferType>
+    From<VariableSizeBinaryArray<false, OffsetItem, Buffer>>
+    for VariableSizeBinaryArray<true, OffsetItem, Buffer>
+where
+    VariableSizeListArray<FixedSizePrimitiveArray<u8, false, Buffer>, false, OffsetItem, Buffer>:
+        Into<
+            VariableSizeListArray<
+                FixedSizePrimitiveArray<u8, false, Buffer>,
+                true,
+                OffsetItem,
+                Buffer,
+            >,
+        >,
+{
+    fn from(value: VariableSizeBinaryArray<false, OffsetItem, Buffer>) -> Self {
+        Self(value.0.into())
+    }
+}
+
 impl<const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType>
     From<StringArray<NULLABLE, OffsetItem, Buffer>>
     for VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>

src/array/variable_size_list.rs

@@ -46,6 +46,17 @@ where
     }
 }
 
+impl<T: Array, OffsetItem: OffsetElement, Buffer: BufferType>
+    From<VariableSizeListArray<T, false, OffsetItem, Buffer>>
+    for VariableSizeListArray<T, true, OffsetItem, Buffer>
+where
+    Offset<T, false, OffsetItem, Buffer>: Into<Offset<T, true, OffsetItem, Buffer>>,
+{
+    fn from(value: VariableSizeListArray<T, false, OffsetItem, Buffer>) -> Self {
+        Self(value.0.into())
+    }
+}
+
 impl<T: Array, U, const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType>
     FromIterator<U> for VariableSizeListArray<T, NULLABLE, OffsetItem, Buffer>
 where

src/bitmap/mod.rs

@@ -136,6 +136,15 @@ impl<Buffer: BufferType> Bitmap<Buffer> {
     pub fn byte_index(&self, index: usize) -> usize {
         (self.offset + index) / 8
     }
+
+    /// Returns a [Bitmap] with `len` bits set.
+    pub fn new_valid(len: usize) -> Self
+    where
+        Self: FromIterator<bool>,
+    {
+        // TODO: optimize
+        Bitmap::from_iter(std::iter::repeat(true).take(len))
+    }
 }
 
 impl<Buffer: BufferType> BufferRef<u8> for Bitmap<Buffer> {

src/nullable.rs

@@ -23,6 +23,18 @@ pub struct Nullable<T, Buffer: BufferType = VecBuffer> {
     pub(crate) validity: Bitmap<Buffer>,
 }
 
+impl<T, Buffer: BufferType> Nullable<T, Buffer> {
+    /// Adds a validity bitmap to `T`, with all bits set.
+    pub(crate) fn wrap(data: T) -> Self
+    where
+        T: Length,
+        Bitmap<Buffer>: FromIterator<bool>,
+    {
+        let validity = Bitmap::new_valid(data.len());
+        Self { data, validity }
+    }
+}
+
 impl<T, Buffer: BufferType> AsRef<T> for Nullable<T, Buffer> {
     fn as_ref(&self) -> &T {
         &self.data

src/offset.rs

@@ -3,10 +3,11 @@
 use crate::{
     bitmap::{Bitmap, BitmapRef, BitmapRefMut, ValidityBitmap},
     buffer::{Buffer, BufferType, VecBuffer},
+    nullable::Nullable,
     validity::Validity,
     FixedSize, Length,
 };
-use std::{num::TryFromIntError, ops::AddAssign};
+use std::{iter, num::TryFromIntError, ops::AddAssign};
 
 /// Types representing offset values.
 ///
@@ -51,7 +52,7 @@ where
 {
     fn default() -> Self {
         let mut offsets = <Buffer as BufferType>::Buffer::<OffsetItem>::default();
-        offsets.extend(std::iter::once(OffsetItem::default()));
+        offsets.extend(iter::once(OffsetItem::default()));
         Self {
             data: Default::default(),
             offsets,
@@ -69,7 +70,7 @@ where
         let mut offsets = <<Buffer as BufferType>::Buffer<OffsetItem> as Validity<true>>::Storage::<
             Buffer,
         >::default();
-        offsets.data.extend(std::iter::once(OffsetItem::default()));
+        offsets.data.extend(iter::once(OffsetItem::default()));
         Self {
             data: Default::default(),
             offsets,
@@ -89,7 +90,7 @@ where
             iter.into_iter()
                 .inspect(|item| {
                     state += OffsetItem::try_from(item.len()).unwrap();
-                    self.offsets.extend(std::iter::once(state));
+                    self.offsets.extend(iter::once(state));
                 })
                 .flatten(),
         );
@@ -109,14 +110,34 @@ where
             iter.into_iter()
                 .inspect(|opt| {
                     state += OffsetItem::try_from(opt.len()).unwrap();
-                    self.offsets.extend(std::iter::once((opt.is_some(), state)));
+                    self.offsets.extend(iter::once((opt.is_some(), state)));
                 })
                 .flatten()
                 .flatten(),
         );
     }
 }
 
+impl<T, OffsetItem: OffsetElement, Buffer: BufferType> From<Offset<T, false, OffsetItem, Buffer>>
+    for Offset<T, true, OffsetItem, Buffer>
+where
+    <Buffer as BufferType>::Buffer<OffsetItem>: Length,
+    Bitmap<Buffer>: FromIterator<bool>,
+{
+    fn from(value: Offset<T, false, OffsetItem, Buffer>) -> Self {
+        // Not using `Nullable::wrap` because the offset buffer has one more
+        // element than the length.
+        let validity = Bitmap::new_valid(value.len());
+        Self {
+            data: value.data,
+            offsets: Nullable {
+                data: value.offsets,
+                validity,
+            },
+        }
+    }
+}
+
 impl<T, U: IntoIterator + Length, OffsetItem: OffsetElement, Buffer: BufferType> FromIterator<U>
     for Offset<T, false, OffsetItem, Buffer>
 where
@@ -131,7 +152,7 @@ where
             .into_iter()
             .inspect(|item| {
                 state += OffsetItem::try_from(item.len()).unwrap();
-                offset.offsets.extend(std::iter::once(state));
+                offset.offsets.extend(iter::once(state));
             })
             .flatten()
             .collect();
@@ -154,9 +175,7 @@ where
             .into_iter()
             .inspect(|opt| {
                 state += OffsetItem::try_from(opt.len()).unwrap();
-                offset
-                    .offsets
-                    .extend(std::iter::once((opt.is_some(), state)));
+                offset.offsets.extend(iter::once((opt.is_some(), state)));
             })
             .flatten()
             .flatten()
@@ -264,4 +283,14 @@ mod tests {
         assert_eq!(offset.offsets.as_ref().as_slice(), &[0, 1, 1, 2, 2]);
         assert_eq!(offset.data, "ab");
     }
+
+    #[test]
+    fn convert_nullable() {
+        let input = vec![vec![1, 2, 3, 4], vec![5, 6], vec![7, 8, 9]];
+        let offset = input.into_iter().collect::<Offset<Vec<u8>>>();
+        assert_eq!(offset.len(), 3);
+        let offset: Offset<Vec<u8>, true> = offset.into();
+        assert_eq!(offset.len(), 3);
+        assert!(offset.all_valid());
+    }
 }