Skip to content

Commit

Permalink
Add encoding/decoding to/from Ion Element (#391)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jpschorr
jpschorr authored Jun 12, 2023
1 parent a2e6bde commit 67777c2
Show file tree
Hide file tree
Showing 4 changed files with 360 additions and 116 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
### Changed
### Added
- Add ability for partiql-extension-ion extension encoding/decoding of `Value` to/from Ion `Element`
### Fixes

## [0.5.0] - 2023-06-06
### Changed
Expand Down
170 changes: 104 additions & 66 deletions extension/partiql-extension-ion/src/decode.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use delegate::delegate;
use ion_rs::{Decimal, Int, IonError, IonReader, IonType, Reader, StreamItem};
use ion_rs::{Decimal, Int, IonError, IonReader, IonType, StreamItem, Symbol};
use once_cell::sync::Lazy;
use partiql_value::{Bag, DateTime, List, Tuple, Value};
use regex::RegexSet;
Expand Down Expand Up @@ -89,7 +89,10 @@ impl IonDecoderBuilder {
}

/// Create a decoder given the previously specified config and an ion [`Reader`].
pub fn build(self, reader: Reader) -> Result<IonValueIter, IonDecodeError> {
pub fn build<'a>(
self,
reader: impl 'a + IonReader<Item = StreamItem, Symbol = Symbol>,
) -> Result<IonValueIter<'a>, IonDecodeError> {
let decoder = SimpleIonValueDecoder {};
let inner: Box<dyn Iterator<Item = IonDecodeResult>> = match self.config.mode {
crate::Encoding::Ion => Box::new(IonValueIterInner { reader, decoder }),
Expand Down Expand Up @@ -123,17 +126,19 @@ impl<'a> Iterator for IonValueIter<'a> {
}
}

struct IonValueIterInner<'a, D>
struct IonValueIterInner<D, R>
where
D: IonValueDecoder,
D: IonValueDecoder<R>,
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
reader: Reader<'a>,
reader: R,
decoder: D,
}

impl<'a, P> Iterator for IonValueIterInner<'a, P>
impl<D, R> Iterator for IonValueIterInner<D, R>
where
P: IonValueDecoder,
D: IonValueDecoder<R>,
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
type Item = IonDecodeResult;

Expand All @@ -147,9 +152,12 @@ where
}
}

trait IonValueDecoder {
trait IonValueDecoder<R>
where
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
#[inline]
fn decode_value(&self, reader: &mut Reader, typ: IonType) -> IonDecodeResult {
fn decode_value(&self, reader: &mut R, typ: IonType) -> IonDecodeResult {
match typ {
IonType::Null => self.decode_null(reader),
IonType::Bool => self.decode_bool(reader),
Expand All @@ -167,19 +175,19 @@ trait IonValueDecoder {
}
}

fn decode_null(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_bool(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_int(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_float(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_decimal(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_timestamp(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_symbol(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_string(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_clob(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_blob(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_list(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_sexp(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_struct(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_null(&self, reader: &mut R) -> IonDecodeResult;
fn decode_bool(&self, reader: &mut R) -> IonDecodeResult;
fn decode_int(&self, reader: &mut R) -> IonDecodeResult;
fn decode_float(&self, reader: &mut R) -> IonDecodeResult;
fn decode_decimal(&self, reader: &mut R) -> IonDecodeResult;
fn decode_timestamp(&self, reader: &mut R) -> IonDecodeResult;
fn decode_symbol(&self, reader: &mut R) -> IonDecodeResult;
fn decode_string(&self, reader: &mut R) -> IonDecodeResult;
fn decode_clob(&self, reader: &mut R) -> IonDecodeResult;
fn decode_blob(&self, reader: &mut R) -> IonDecodeResult;
fn decode_list(&self, reader: &mut R) -> IonDecodeResult;
fn decode_sexp(&self, reader: &mut R) -> IonDecodeResult;
fn decode_struct(&self, reader: &mut R) -> IonDecodeResult;
}

fn ion_decimal_to_decimal(ion_dec: Decimal) -> Result<rust_decimal::Decimal, rust_decimal::Error> {
Expand All @@ -192,38 +200,41 @@ fn ion_decimal_to_decimal(ion_dec: Decimal) -> Result<rust_decimal::Decimal, rus

struct SimpleIonValueDecoder {}

impl IonValueDecoder for SimpleIonValueDecoder {
impl<R> IonValueDecoder<R> for SimpleIonValueDecoder
where
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
#[inline]
fn decode_null(&self, _: &mut Reader) -> IonDecodeResult {
fn decode_null(&self, _: &mut R) -> IonDecodeResult {
Ok(Value::Null)
}

#[inline]
fn decode_bool(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_bool(&self, reader: &mut R) -> IonDecodeResult {
Ok(Value::Boolean(reader.read_bool()?))
}

#[inline]
fn decode_int(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_int(&self, reader: &mut R) -> IonDecodeResult {
match reader.read_int()? {
Int::I64(i) => Ok(Value::Integer(i)),
Int::BigInt(_) => Err(IonDecodeError::UnsupportedType("bigint")),
}
}

#[inline]
fn decode_float(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_float(&self, reader: &mut R) -> IonDecodeResult {
Ok(Value::Real(reader.read_f64()?.into()))
}

#[inline]
fn decode_decimal(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_decimal(&self, reader: &mut R) -> IonDecodeResult {
let dec = ion_decimal_to_decimal(reader.read_decimal()?);
Ok(Value::Decimal(dec?))
}

#[inline]
fn decode_timestamp(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_timestamp(&self, reader: &mut R) -> IonDecodeResult {
let ts = reader.read_timestamp()?;
let offset = ts.offset();
let datetime = DateTime::from_ymdhms_nano_offset_minutes(
Expand All @@ -242,47 +253,50 @@ impl IonValueDecoder for SimpleIonValueDecoder {
}

#[inline]
fn decode_symbol(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_symbol(&self, reader: &mut R) -> IonDecodeResult {
Ok(Value::String(Box::new(
reader.read_symbol()?.text_or_error()?.to_string(),
)))
}

#[inline]
fn decode_string(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_string(&self, reader: &mut R) -> IonDecodeResult {
Ok(Value::String(Box::new(
reader.read_string()?.text().to_string(),
)))
}

#[inline]
fn decode_clob(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_clob(&self, reader: &mut R) -> IonDecodeResult {
Ok(Value::Blob(Box::new(reader.read_clob()?.as_slice().into())))
}

#[inline]
fn decode_blob(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_blob(&self, reader: &mut R) -> IonDecodeResult {
Ok(Value::Blob(Box::new(reader.read_blob()?.as_slice().into())))
}

#[inline]
fn decode_list(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_list(&self, reader: &mut R) -> IonDecodeResult {
decode_list(self, reader)
}

#[inline]
fn decode_sexp(&self, _: &mut Reader) -> IonDecodeResult {
fn decode_sexp(&self, _: &mut R) -> IonDecodeResult {
Err(IonDecodeError::UnsupportedType("sexp"))
}

#[inline]
fn decode_struct(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_struct(&self, reader: &mut R) -> IonDecodeResult {
decode_struct(self, reader)
}
}

#[inline]
fn decode_list(decoder: &impl IonValueDecoder, reader: &mut Reader) -> IonDecodeResult {
fn decode_list<R>(decoder: &impl IonValueDecoder<R>, reader: &mut R) -> IonDecodeResult
where
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
reader.step_in()?;
let mut values = vec![];
'values: loop {
Expand All @@ -299,18 +313,21 @@ fn decode_list(decoder: &impl IonValueDecoder, reader: &mut Reader) -> IonDecode
}

#[inline]
fn decode_struct(decoder: &impl IonValueDecoder, reader: &mut Reader) -> IonDecodeResult {
fn decode_struct<R>(decoder: &impl IonValueDecoder<R>, reader: &mut R) -> IonDecodeResult
where
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
let mut tuple = Tuple::new();
reader.step_in()?;
loop {
'kv: loop {
let item = reader.next()?;
let (key, value) = match item {
StreamItem::Value(typ) => {
let field_name = reader.field_name()?;
(field_name, decoder.decode_value(reader, typ)?)
}
StreamItem::Null(_) => (reader.field_name()?, decoder.decode_null(reader)?),
StreamItem::Nothing => break,
StreamItem::Nothing => break 'kv,
};
tuple.insert(key.text_or_error()?, value);
}
Expand All @@ -323,7 +340,10 @@ struct PartiqlEncodedIonValueDecoder {
}

#[inline]
fn has_annotation(reader: &Reader, annot: &str) -> bool {
fn has_annotation(
reader: &impl IonReader<Item = StreamItem, Symbol = Symbol>,
annot: &str,
) -> bool {
reader
.annotations()
.any(|a| a.map_or(false, |a| a == annot))
Expand All @@ -333,7 +353,10 @@ static TIME_PARTS_PATTERN_SET: Lazy<RegexSet> =
Lazy::new(|| RegexSet::new(RE_SET_TIME_PARTS).unwrap());

impl PartiqlEncodedIonValueDecoder {
fn decode_date(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_date<R>(&self, reader: &mut R) -> IonDecodeResult
where
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
let ts = reader.read_timestamp()?;
let datetime = DateTime::from_ymd(
ts.year(),
Expand All @@ -345,12 +368,18 @@ impl PartiqlEncodedIonValueDecoder {
Ok(datetime.into())
}

fn decode_time(&self, reader: &mut Reader) -> IonDecodeResult {
fn expect_u8(
reader: &mut Reader,
fn decode_time<R>(&self, reader: &mut R) -> IonDecodeResult
where
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
fn expect_u8<R>(
reader: &mut R,
typ: Option<IonType>,
unit: &'static str,
) -> Result<u8, IonDecodeError> {
) -> Result<u8, IonDecodeError>
where
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
match typ {
Some(IonType::Int) => match reader.read_int()? {
Int::I64(i) => Ok(i as u8), // TODO check range
Expand All @@ -363,11 +392,14 @@ impl PartiqlEncodedIonValueDecoder {
))),
}
}
fn maybe_i8(
reader: &mut Reader,
fn maybe_i8<R>(
reader: &mut R,
typ: Option<IonType>,
unit: &'static str,
) -> Result<Option<i8>, IonDecodeError> {
) -> Result<Option<i8>, IonDecodeError>
where
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
match typ {
Some(IonType::Int) => match reader.read_int()? {
Int::I64(i) => Ok(Some(i as i8)), // TODO check range
Expand All @@ -382,11 +414,14 @@ impl PartiqlEncodedIonValueDecoder {
))),
}
}
fn expect_f64(
reader: &mut Reader,
fn expect_f64<R>(
reader: &mut R,
typ: Option<IonType>,
unit: &'static str,
) -> Result<f64, IonDecodeError> {
) -> Result<f64, IonDecodeError>
where
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
match typ {
Some(IonType::Decimal) => {
let dec = ion_decimal_to_decimal(reader.read_decimal()?);
Expand Down Expand Up @@ -457,9 +492,12 @@ impl PartiqlEncodedIonValueDecoder {
}
}

impl IonValueDecoder for PartiqlEncodedIonValueDecoder {
impl<R> IonValueDecoder<R> for PartiqlEncodedIonValueDecoder
where
R: IonReader<Item = StreamItem, Symbol = Symbol>,
{
#[inline]
fn decode_null(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_null(&self, reader: &mut R) -> IonDecodeResult {
if has_annotation(reader, MISSING_ANNOT) {
Ok(Value::Missing)
} else {
Expand All @@ -468,7 +506,7 @@ impl IonValueDecoder for PartiqlEncodedIonValueDecoder {
}

#[inline]
fn decode_timestamp(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_timestamp(&self, reader: &mut R) -> IonDecodeResult {
if has_annotation(reader, DATE_ANNOT) {
self.decode_date(reader)
} else {
Expand All @@ -477,7 +515,7 @@ impl IonValueDecoder for PartiqlEncodedIonValueDecoder {
}

#[inline]
fn decode_list(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_list(&self, reader: &mut R) -> IonDecodeResult {
let is_bag = has_annotation(reader, BAG_ANNOT);
let list = decode_list(self, reader);
if is_bag {
Expand All @@ -488,7 +526,7 @@ impl IonValueDecoder for PartiqlEncodedIonValueDecoder {
}

#[inline]
fn decode_struct(&self, reader: &mut Reader) -> IonDecodeResult {
fn decode_struct(&self, reader: &mut R) -> IonDecodeResult {
if has_annotation(reader, TIME_ANNOT) {
self.decode_time(reader)
} else {
Expand All @@ -498,15 +536,15 @@ impl IonValueDecoder for PartiqlEncodedIonValueDecoder {

delegate! {
to self.inner {
fn decode_bool(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_int(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_float(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_decimal(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_symbol(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_string(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_clob(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_blob(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_sexp(&self, reader: &mut Reader) -> IonDecodeResult;
fn decode_bool(&self, reader: &mut R) -> IonDecodeResult;
fn decode_int(&self, reader: &mut R) -> IonDecodeResult;
fn decode_float(&self, reader: &mut R) -> IonDecodeResult;
fn decode_decimal(&self, reader: &mut R) -> IonDecodeResult;
fn decode_symbol(&self, reader: &mut R) -> IonDecodeResult;
fn decode_string(&self, reader: &mut R) -> IonDecodeResult;
fn decode_clob(&self, reader: &mut R) -> IonDecodeResult;
fn decode_blob(&self, reader: &mut R) -> IonDecodeResult;
fn decode_sexp(&self, reader: &mut R) -> IonDecodeResult;
}
}
}
Loading

1 comment on commit 67777c2

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PartiQL (rust) Benchmark

Benchmark suite Current: 67777c2 Previous: a2e6bde Ratio
parse-1 6140 ns/iter (± 29) 6229 ns/iter (± 389) 0.99
parse-15 57836 ns/iter (± 1200) 58816 ns/iter (± 3210) 0.98
parse-30 115780 ns/iter (± 69) 126349 ns/iter (± 6685) 0.92
compile-1 5132 ns/iter (± 70) 5268 ns/iter (± 326) 0.97
compile-15 36303 ns/iter (± 1136) 41735 ns/iter (± 1991) 0.87
compile-30 73402 ns/iter (± 90) 88240 ns/iter (± 9123) 0.83
plan-1 20075 ns/iter (± 210) 24190 ns/iter (± 1570) 0.83
plan-15 361972 ns/iter (± 2738) 474705 ns/iter (± 28008) 0.76
plan-30 736682 ns/iter (± 1905) 948299 ns/iter (± 40485) 0.78
eval-1 21926548 ns/iter (± 156802) 24985405 ns/iter (± 1548746) 0.88
eval-15 122779721 ns/iter (± 460689) 126878607 ns/iter (± 7686760) 0.97
eval-30 240222097 ns/iter (± 566566) 237218353 ns/iter (± 9472421) 1.01
join 14092 ns/iter (± 36) 14649 ns/iter (± 800) 0.96
simple 7271 ns/iter (± 10) 7046 ns/iter (± 478) 1.03
simple-no 712 ns/iter (± 0) 735 ns/iter (± 53) 0.97
numbers 107 ns/iter (± 0) 145 ns/iter (± 9) 0.74
parse-simple 714 ns/iter (± 6) 768 ns/iter (± 43) 0.93
parse-ion 2681 ns/iter (± 2) 2774 ns/iter (± 183) 0.97
parse-group 8816 ns/iter (± 22) 9263 ns/iter (± 562) 0.95
parse-complex 23207 ns/iter (± 54) 25088 ns/iter (± 1698) 0.93
parse-complex-fexpr 36234 ns/iter (± 89) 42060 ns/iter (± 3227) 0.86

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.