Skip to content

Commit

Permalink
Update arrays with different offset implementation. Data no longer ge…
Browse files Browse the repository at this point in the history 
…ts default values for nulls.
  • Loading branch information
@mbrobbel
mbrobbel committed Jul 25, 2023
1 parent 195c1e9 commit c646637
Show file tree
Hide file tree
Showing 6 changed files with 268 additions and 444 deletions.
20 changes: 4 additions & 16 deletions src/array/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
//! Sequences of values with known length all having the same type.
use std::collections::VecDeque;
// use std::collections::VecDeque;

use crate::{buffer::BufferType, FixedSize};
use std::collections::VecDeque;

mod boolean;
pub use boolean::*;
Expand All @@ -16,8 +17,8 @@ pub use null::*;
mod string;
pub use string::*;

// mod r#struct;
// pub use r#struct::*;
mod r#struct;
pub use r#struct::*;

mod variable_size_binary;
pub use variable_size_binary::*;
Expand Down Expand Up @@ -90,19 +91,6 @@ impl<T: FixedSize, const N: usize> ArrayType for Option<[T; N]> {
type Array<Buffer: BufferType> = FixedSizePrimitiveArray<[T; N], true, Buffer>;
}

// impl<'a> ArrayType for &'a [u8] {
// type Array<Buffer: BufferType> = VariableSizeBinaryArray<false, i32, Buffer>;
// }
// impl<'a> ArrayType for Option<&'a [u8]> {
// type Array<Buffer: BufferType> = VariableSizeBinaryArray<true, i32, Buffer>;
// }
// impl ArrayType for Vec<u8> {
// type Array<Buffer: BufferType> = VariableSizeBinaryArray<false, i32, Buffer>;
// }
// impl ArrayType for Option<Vec<u8>> {
// type Array<Buffer: BufferType> = VariableSizeBinaryArray<true, i32, Buffer>;
// }

impl<'a> ArrayType for &'a str {
type Array<Buffer: BufferType> = StringArray<false, i32, Buffer>;
}
Expand Down
53 changes: 46 additions & 7 deletions src/array/string.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use super::{Array, FixedSizePrimitiveArray, VariableSizeBinaryArray};
use super::{Array, VariableSizeBinaryArray};
use crate::{
buffer::{BufferType, VecBuffer},
offset::OffsetElement,
Expand All @@ -13,19 +13,19 @@ pub struct StringArray<
Buffer: BufferType = VecBuffer,
>(pub VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>)
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>;
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>;

impl<const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType> Array
for StringArray<NULLABLE, OffsetItem, Buffer>
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>,
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
{
}

impl<const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType> Default
for StringArray<NULLABLE, OffsetItem, Buffer>
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>,
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>: Default,
{
fn default() -> Self {
Expand All @@ -36,7 +36,7 @@ where
impl<'a, const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType> Extend<&'a str>
for StringArray<NULLABLE, OffsetItem, Buffer>
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>,
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>: Extend<&'a [u8]>,
{
fn extend<I: IntoIterator<Item = &'a str>>(&mut self, iter: I) {
Expand All @@ -47,7 +47,7 @@ where
impl<const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType> Extend<String>
for StringArray<NULLABLE, OffsetItem, Buffer>
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>,
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>: Extend<Vec<u8>>,
{
fn extend<I: IntoIterator<Item = String>>(&mut self, iter: I) {
Expand Down Expand Up @@ -102,10 +102,49 @@ where
impl<const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType> Length
for StringArray<NULLABLE, OffsetItem, Buffer>
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>,
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>: Length,
{
fn len(&self) -> usize {
self.0.len()
}
}

#[cfg(test)]
mod tests {
use crate::bitmap::BitmapRef;

use super::*;

#[test]
fn from_iter() {
let input = vec!["a".to_string(), "sd".to_string(), "f".to_string()];
let array = input.into_iter().collect::<StringArray>();
assert_eq!(array.len(), 3);
assert_eq!(array.0 .0 .0.data.0, &[97, 115, 100, 102]);
assert_eq!(array.0 .0 .0.offsets, &[0, 1, 3, 4]);

let input = vec![
Some("a".to_string()),
None,
Some("sd".to_string()),
Some("f".to_string()),
None,
];
let array = input.into_iter().collect::<StringArray<true>>();
assert_eq!(array.len(), 5);
assert_eq!(array.0 .0 .0.data.0, &[97, 115, 100, 102]);
assert_eq!(array.0 .0 .0.offsets.as_ref(), &[0, 1, 1, 3, 4, 4]);
assert_eq!(
array
.0
.0
.0
.offsets
.bitmap_ref()
.into_iter()
.collect::<Vec<_>>(),
&[true, false, true, true, false]
);
}
}
77 changes: 64 additions & 13 deletions src/array/variable_size_binary.rs
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
//! Variable-size binary elements.
use super::{Array, FixedSizePrimitiveArray, StringArray, VariableSizeListArray};
use crate::{
buffer::{BufferType, VecBuffer},
offset::OffsetElement,
validity::Validity,
Length,
};

use super::{Array, FixedSizePrimitiveArray, VariableSizeListArray};

/// Variable-size binary elements.
pub struct VariableSizeBinaryArray<
const NULLABLE: bool = false,
Expand All @@ -23,19 +22,19 @@ pub struct VariableSizeBinaryArray<
>,
)
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>;
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>;

impl<const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType> Array
for VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>,
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
{
}

impl<const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType> Default
for VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>,
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
VariableSizeListArray<FixedSizePrimitiveArray<u8, false, Buffer>, NULLABLE, OffsetItem, Buffer>:
Default,
{
Expand All @@ -47,23 +46,56 @@ where
impl<T, const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType> Extend<T>
for VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>,
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
VariableSizeListArray<FixedSizePrimitiveArray<u8, false, Buffer>, NULLABLE, OffsetItem, Buffer>:
Extend<T>,
// Offset<<Buffer as BufferType>::Buffer<u8>, NULLABLE, OffsetItem, Buffer>: Extend<T>,
{
fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
self.0.extend(iter)
}
}

impl<const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType>
From<
VariableSizeListArray<
FixedSizePrimitiveArray<u8, false, Buffer>,
NULLABLE,
OffsetItem,
Buffer,
>,
> for VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>
where
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
{
fn from(
value: VariableSizeListArray<
FixedSizePrimitiveArray<u8, false, Buffer>,
NULLABLE,
OffsetItem,
Buffer,
>,
) -> Self {
Self(value)
}
}

impl<const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType>
From<StringArray<NULLABLE, OffsetItem, Buffer>>
for VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>
where
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
{
fn from(value: StringArray<NULLABLE, OffsetItem, Buffer>) -> Self {
Self(value.0 .0)
}
}

impl<T, const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType> FromIterator<T>
for VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>,
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
VariableSizeListArray<FixedSizePrimitiveArray<u8, false, Buffer>, NULLABLE, OffsetItem, Buffer>:
FromIterator<T>,
// Offset<<Buffer as BufferType>::Buffer<u8>, NULLABLE, OffsetItem, Buffer>: FromIterator<T>,
{
fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
Self(iter.into_iter().collect())
Expand All @@ -73,10 +105,9 @@ where
impl<const NULLABLE: bool, OffsetItem: OffsetElement, Buffer: BufferType> Length
for VariableSizeBinaryArray<NULLABLE, OffsetItem, Buffer>
where
FixedSizePrimitiveArray<u8, false, Buffer>: Validity<NULLABLE>,
<Buffer as BufferType>::Buffer<OffsetItem>: Validity<NULLABLE>,
VariableSizeListArray<FixedSizePrimitiveArray<u8, false, Buffer>, NULLABLE, OffsetItem, Buffer>:
Length,
// Offset<<Buffer as BufferType>::Buffer<u8>, NULLABLE, OffsetItem, Buffer>: Length,
{
fn len(&self) -> usize {
self.0.len()
Expand All @@ -85,6 +116,8 @@ where

#[cfg(test)]
mod tests {
use crate::bitmap::BitmapRef;

use super::*;

#[test]
Expand All @@ -98,7 +131,25 @@ mod tests {
let input = vec![Some(vec![1]), None, Some(vec![2, 3]), Some(vec![4])];
let array = input.into_iter().collect::<VariableSizeBinaryArray<true>>();
assert_eq!(array.len(), 4);
assert_eq!(array.0 .0.data.as_ref().0, &[1, 0, 2, 3, 4]);
assert_eq!(array.0 .0.offsets, &[0, 1, 1, 3, 4]);
assert_eq!(array.0 .0.data.0, &[1, 2, 3, 4]);
assert_eq!(array.0 .0.offsets.as_ref(), &[0, 1, 1, 3, 4]);
assert_eq!(
array
.0
.0
.offsets
.bitmap_ref()
.into_iter()
.collect::<Vec<_>>(),
&[true, false, true, true]
);
}

#[test]
fn convert() {
let input = vec![Some("a".to_string()), None, Some("b".to_string())];
let array = input.into_iter().collect::<StringArray<true>>();
let variable_size_binary: VariableSizeBinaryArray<true> = array.into();
assert_eq!(variable_size_binary.len(), 3);
}
}
Loading

0 comments on commit c646637

Please sign in to comment.