diff --git a/src/serialize/writer/escape.rs b/src/serialize/writer/escape.rs index 7653c7c8..680ac719 100644 --- a/src/serialize/writer/escape.rs +++ b/src/serialize/writer/escape.rs @@ -1,232 +1,293 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) -// This is an adaptation of `src/value/ser.rs` from serde-json. - -use crate::serialize::writer::WriteExt; -use std::io; - -pub enum CharEscape { - /// An escaped quote `"` - Quote, - /// An escaped reverse solidus `\` - ReverseSolidus, - /// An escaped backspace character (usually escaped as `\b`) - Backspace, - /// An escaped form feed character (usually escaped as `\f`) - FormFeed, - /// An escaped line feed character (usually escaped as `\n`) - LineFeed, - /// An escaped carriage return character (usually escaped as `\r`) - CarriageReturn, - /// An escaped tab character (usually escaped as `\t`) - Tab, - /// An escaped ASCII plane control character (usually escaped as - /// `\u00XX` where `XX` are two hex characters) - AsciiControl(u8), -} +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2023-2024 liuq19, ijl +// adapted from sonic-rs' src/util/string.rs -impl CharEscape { - #[inline] - fn from_escape_table(escape: u8, byte: u8) -> CharEscape { - match escape { - self::BB => CharEscape::Backspace, - self::TT => CharEscape::Tab, - self::NN => CharEscape::LineFeed, - self::FF => CharEscape::FormFeed, - self::RR => CharEscape::CarriageReturn, - self::QU => CharEscape::Quote, - self::BS => CharEscape::ReverseSolidus, - self::UU => CharEscape::AsciiControl(byte), - _ => unreachable!(), - } - } -} +#[cfg(feature = "unstable-simd")] +use core::simd::cmp::{SimdPartialEq, SimdPartialOrd}; -#[inline] -fn write_char_escape(writer: &mut W, char_escape: CharEscape) -> io::Result<()> -where - W: ?Sized + io::Write + WriteExt, -{ - use CharEscape::*; - - let s = match char_escape { - Quote => b"\\\"", - ReverseSolidus => b"\\\\", - Backspace => b"\\b", - FormFeed => b"\\f", - LineFeed => b"\\n", - CarriageReturn => b"\\r", - Tab => b"\\t", - AsciiControl(byte) => { - static HEX_DIGITS: [u8; 16] = *b"0123456789abcdef"; - let bytes = &[ - b'\\', - b'u', - b'0', - b'0', - HEX_DIGITS[(byte >> 4) as usize], - HEX_DIGITS[(byte & 0xF) as usize], - ]; - return unsafe { writer.write_reserved_fragment(bytes) }; +#[cfg(feature = "unstable-simd")] +macro_rules! impl_escape_unchecked { + ($src:expr, $dst:expr, $nb:expr, $omask:expr, $cn:expr, $v:expr, $rotate:expr) => { + if $rotate == true { + for _ in 0..$cn { + $v = $v.rotate_elements_left::<1>(); + } + } + $nb -= $cn; + $dst = $dst.add($cn); + $src = $src.add($cn); + $omask >>= $cn; + loop { + if $rotate == true { + $v = $v.rotate_elements_left::<1>(); + } + $nb -= 1; + $omask = $omask >> 1; + let escape = QUOTE_TAB[*($src) as usize]; + core::ptr::copy_nonoverlapping(escape.0.as_ptr(), $dst, 6); + $dst = $dst.add(escape.1 as usize); + $src = $src.add(1); + if likely!($omask & 1 != 1) { + break; + } } }; - - unsafe { writer.write_reserved_fragment(s) } } -#[inline(never)] -pub fn format_escaped_str(writer: &mut W, value: &str) -> io::Result<()> -where - W: ?Sized + io::Write + WriteExt, -{ - let len = value.len(); - - if len == 0 { - reserve_minimum!(writer); - return unsafe { writer.write_reserved_fragment(b"\"\"") }; - } - unsafe { - let mut escapes: u8 = __; - let mut idx = 0; - let as_bytes = value.as_bytes(); - while idx < len.saturating_sub(8) { - escapes |= *ESCAPE.get_unchecked(*as_bytes.get_unchecked(idx) as usize); - escapes |= *ESCAPE.get_unchecked(*as_bytes.get_unchecked(idx + 1) as usize); - escapes |= *ESCAPE.get_unchecked(*as_bytes.get_unchecked(idx + 2) as usize); - escapes |= *ESCAPE.get_unchecked(*as_bytes.get_unchecked(idx + 3) as usize); - escapes |= *ESCAPE.get_unchecked(*as_bytes.get_unchecked(idx + 4) as usize); - escapes |= *ESCAPE.get_unchecked(*as_bytes.get_unchecked(idx + 5) as usize); - escapes |= *ESCAPE.get_unchecked(*as_bytes.get_unchecked(idx + 6) as usize); - escapes |= *ESCAPE.get_unchecked(*as_bytes.get_unchecked(idx + 7) as usize); - if unlikely!(escapes != __) { - return format_escaped_str_with_escapes(writer, as_bytes, idx); +#[cfg(feature = "unstable-simd")] +macro_rules! impl_format_simd { + ($odptr:expr, $value_ptr:expr, $value_len:expr) => { + let mut dptr = $odptr; + let dstart = $odptr; + let mut sptr = $value_ptr; + let mut nb: usize = $value_len; + + const BLASH: StrVector = StrVector::from_array([b'\\'; STRIDE]); + const QUOTE: StrVector = StrVector::from_array([b'"'; STRIDE]); + const X20: StrVector = StrVector::from_array([32; STRIDE]); + + unsafe { + *dptr = b'"'; + dptr = dptr.add(1); + + { + const ROTATE: bool = false; + while nb >= STRIDE { + let mut v = StrVector::from_slice(core::slice::from_raw_parts(sptr, STRIDE)); + let mut mask = + (v.simd_eq(BLASH) | v.simd_eq(QUOTE) | v.simd_lt(X20)).to_bitmask() as u32; + v.copy_to_slice(core::slice::from_raw_parts_mut(dptr, STRIDE)); + + if likely!(mask == 0) { + nb -= STRIDE; + dptr = dptr.add(STRIDE); + sptr = sptr.add(STRIDE); + } else { + let cn = mask.trailing_zeros() as usize; + impl_escape_unchecked!(sptr, dptr, nb, mask, cn, v, ROTATE); + } + } } - idx += 8; - } - while idx < len { - escapes |= *ESCAPE.get_unchecked(*as_bytes.get_unchecked(idx) as usize); - if unlikely!(escapes != __) { - return format_escaped_str_with_escapes(writer, as_bytes, idx); + + { + const ROTATE: bool = true; + let mut v = StrVector::from_slice(core::slice::from_raw_parts( + sptr.add(nb).sub(STRIDE), + STRIDE, + )); + let mut to_skip = STRIDE - nb; + while to_skip >= 4 { + to_skip -= 4; + v = v.rotate_elements_left::<4>(); + } + while to_skip > 0 { + to_skip -= 1; + v = v.rotate_elements_left::<1>(); + } + + let mut mask = (v.simd_eq(BLASH) | v.simd_eq(QUOTE) | v.simd_lt(X20)).to_bitmask() + as u32 + & (STRIDE_SATURATION >> (32 - STRIDE - nb)); + + while nb > 0 { + v.copy_to_slice(core::slice::from_raw_parts_mut(dptr, STRIDE)); + + if likely!(mask == 0) { + dptr = dptr.add(nb); + break; + } else { + let cn = mask.trailing_zeros() as usize; + impl_escape_unchecked!(sptr, dptr, nb, mask, cn, v, ROTATE); + } + } } - idx += 1; + + *dptr = b'"'; + dptr = dptr.add(1); } - } - writer.write_str(value) + return dptr as usize - dstart as usize; + }; } -fn format_escaped_str_with_escapes( - writer: &mut W, - value: &[u8], - initial: usize, -) -> io::Result<()> -where - W: ?Sized + io::Write + WriteExt, -{ - writer.reserve((value.len() * 8) + 2); - unsafe { - writer.write_reserved_punctuation(b'"').unwrap(); - if initial > 0 { - writer - .write_reserved_fragment(value.get_unchecked(0..initial)) - .unwrap(); - } - format_escaped_str_contents(writer, value.get_unchecked(initial..)).unwrap(); - writer.write_reserved_punctuation(b'"').unwrap(); - }; - Ok(()) +#[inline(never)] +#[cfg(feature = "unstable-simd")] +#[cfg_attr(target_arch = "aarch64", target_feature(enable = "neon"))] +#[cfg_attr(target_arch = "x86_64", target_feature(enable = "sse2"))] +pub unsafe fn format_escaped_str_impl_128( + odptr: *mut u8, + value_ptr: *const u8, + value_len: usize, +) -> usize { + const STRIDE: usize = 16; + const STRIDE_SATURATION: u32 = u16::MAX as u32; + type StrVector = core::simd::u8x16; + + impl_format_simd!(odptr, value_ptr, value_len); } -fn format_escaped_str_contents(writer: &mut W, bytes: &[u8]) -> io::Result<()> -where - W: ?Sized + io::Write + WriteExt, -{ - let len = bytes.len(); - let mut start = 0; - let mut idx = 0; - - let mut escape: u8; - loop { - if idx < len.saturating_sub(4) { - escape = 0; - unsafe { - escape |= *ESCAPE.get_unchecked(*bytes.get_unchecked(idx) as usize); - escape |= *ESCAPE.get_unchecked(*bytes.get_unchecked(idx + 1) as usize); - escape |= *ESCAPE.get_unchecked(*bytes.get_unchecked(idx + 2) as usize); - escape |= *ESCAPE.get_unchecked(*bytes.get_unchecked(idx + 3) as usize); - } - if escape == 0 { - idx += 4; - continue; - } - } +#[inline(never)] +#[cfg(not(feature = "unstable-simd"))] +pub unsafe fn format_escaped_str_scalar( + odptr: *mut u8, + value_ptr: *const u8, + value_len: usize, +) -> usize { + unsafe { + let mut dst = odptr; + let mut src = value_ptr; + let mut nb = value_len; + + *dst = b'"'; + dst = dst.add(1); - let byte = unsafe { *bytes.get_unchecked(idx) }; - escape = unsafe { *ESCAPE.get_unchecked(byte as usize) }; - if escape == 0 { - idx += 1; - if idx == len { + let mut clean = 0; + while clean <= value_len.saturating_sub(8) { + let mut escapes = 0; + escapes |= *NEED_ESCAPED.get_unchecked(*(src.add(clean)) as usize); + escapes |= *NEED_ESCAPED.get_unchecked(*(src.add(clean + 1)) as usize); + escapes |= *NEED_ESCAPED.get_unchecked(*(src.add(clean + 2)) as usize); + escapes |= *NEED_ESCAPED.get_unchecked(*(src.add(clean + 3)) as usize); + escapes |= *NEED_ESCAPED.get_unchecked(*(src.add(clean + 4)) as usize); + escapes |= *NEED_ESCAPED.get_unchecked(*(src.add(clean + 5)) as usize); + escapes |= *NEED_ESCAPED.get_unchecked(*(src.add(clean + 6)) as usize); + escapes |= *NEED_ESCAPED.get_unchecked(*(src.add(clean + 7)) as usize); + if unlikely!(escapes != 0) { break; - } else { - continue; } + clean += 8; } - - if start < idx { - unsafe { - writer - .write_reserved_fragment(bytes.get_unchecked(start..idx)) - .unwrap() - }; + if clean > 0 { + core::ptr::copy_nonoverlapping(src, dst, clean); + nb -= clean; + src = src.add(clean); + dst = dst.add(clean); } - - let char_escape = CharEscape::from_escape_table(escape, byte); - write_char_escape(writer, char_escape)?; - - idx += 1; - start = idx; - if idx == len { - break; + for _ in 0..nb { + core::ptr::write(dst, *(src)); + src = src.add(1); + dst = dst.add(1); + if unlikely!(NEED_ESCAPED[*(src.sub(1)) as usize] != 0) { + let escape = QUOTE_TAB[*(src.sub(1)) as usize]; + core::ptr::copy_nonoverlapping(escape.0.as_ptr(), dst.sub(1), 6); + dst = dst.add(escape.1 as usize - 1); + } } - } - if start != len { - unsafe { - writer - .write_reserved_fragment(bytes.get_unchecked(start..len)) - .unwrap() - }; + *dst = b'"'; + dst = dst.add(1); + + dst as usize - odptr as usize } - Ok(()) } -const BB: u8 = b'b'; // \x08 -const TT: u8 = b't'; // \x09 -const NN: u8 = b'n'; // \x0A -const FF: u8 = b'f'; // \x0C -const RR: u8 = b'r'; // \x0D -const QU: u8 = b'"'; // \x22 -const BS: u8 = b'\\'; // \x5C -const UU: u8 = b'u'; // \x00...\x1F except the ones above -const __: u8 = 0; - -// Lookup table of escape sequences. A value of b'x' at index i means that byte -// i is escaped as "\x" in JSON. A value of 0 means that byte i is not escaped. -const ESCAPE: [u8; 256] = [ - // 1 2 3 4 5 6 7 8 9 A B C D E F - UU, UU, UU, UU, UU, UU, UU, UU, BB, TT, NN, UU, FF, RR, UU, UU, // 0 - UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, // 1 - __, __, QU, __, __, __, __, __, __, __, __, __, __, __, __, __, // 2 - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 3 - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 4 - __, __, __, __, __, __, __, __, __, __, __, __, BS, __, __, __, // 5 - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 6 - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 7 - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 8 - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 9 - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // A - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // B - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // C - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // D - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // E - __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // F +#[cfg(not(feature = "unstable-simd"))] +const NEED_ESCAPED: [u8; 256] = [ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +]; + +const QUOTE_TAB: [([u8; 7], u8); 96] = [ + (*b"\\u0000\0", 6), + (*b"\\u0001\0", 6), + (*b"\\u0002\0", 6), + (*b"\\u0003\0", 6), + (*b"\\u0004\0", 6), + (*b"\\u0005\0", 6), + (*b"\\u0006\0", 6), + (*b"\\u0007\0", 6), + (*b"\\b\0\0\0\0\0", 2), + (*b"\\t\0\0\0\0\0", 2), + (*b"\\n\0\0\0\0\0", 2), + (*b"\\u000b\0", 6), + (*b"\\f\0\0\0\0\0", 2), + (*b"\\r\0\0\0\0\0", 2), + (*b"\\u000e\0", 6), + (*b"\\u000f\0", 6), + (*b"\\u0010\0", 6), + (*b"\\u0011\0", 6), + (*b"\\u0012\0", 6), + (*b"\\u0013\0", 6), + (*b"\\u0014\0", 6), + (*b"\\u0015\0", 6), + (*b"\\u0016\0", 6), + (*b"\\u0017\0", 6), + (*b"\\u0018\0", 6), + (*b"\\u0019\0", 6), + (*b"\\u001a\0", 6), + (*b"\\u001b\0", 6), + (*b"\\u001c\0", 6), + (*b"\\u001d\0", 6), + (*b"\\u001e\0", 6), + (*b"\\u001f\0", 6), + ([0; 7], 0), + ([0; 7], 0), + (*b"\\\"\0\0\0\0\0", 2), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), + (*b"\\\\\0\0\0\0\0", 2), + ([0; 7], 0), + ([0; 7], 0), + ([0; 7], 0), ]; diff --git a/src/serialize/writer/json.rs b/src/serialize/writer/json.rs index c603f351..31942b52 100644 --- a/src/serialize/writer/json.rs +++ b/src/serialize/writer/json.rs @@ -575,7 +575,7 @@ where let num_reserved_bytes = value.len() * 8 + 32; writer.reserve(num_reserved_bytes); - let written = crate::serialize::writer::simd::format_escaped_str_impl_128( + let written = crate::serialize::writer::escape::format_escaped_str_impl_128( writer.as_mut_buffer_ptr(), value.as_bytes().as_ptr(), value.len(), @@ -592,7 +592,19 @@ fn format_escaped_str(writer: &mut W, value: &str) -> io::Result<()> where W: ?Sized + io::Write + WriteExt, { - crate::serialize::writer::escape::format_escaped_str(writer, value) + unsafe { + let num_reserved_bytes = value.len() * 8 + 32; + writer.reserve(num_reserved_bytes); + + let written = crate::serialize::writer::escape::format_escaped_str_scalar( + writer.as_mut_buffer_ptr(), + value.as_bytes().as_ptr(), + value.len(), + ); + + writer.set_written(written); + } + Ok(()) } #[inline] diff --git a/src/serialize/writer/mod.rs b/src/serialize/writer/mod.rs index fc489426..47fdf503 100644 --- a/src/serialize/writer/mod.rs +++ b/src/serialize/writer/mod.rs @@ -1,12 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 mod byteswriter; -#[cfg(not(feature = "unstable-simd"))] mod escape; mod formatter; mod json; -#[cfg(feature = "unstable-simd")] -mod simd; pub use byteswriter::{BytesWriter, WriteExt}; pub use json::{to_writer, to_writer_pretty}; diff --git a/src/serialize/writer/simd.rs b/src/serialize/writer/simd.rs deleted file mode 100644 index dc6c500e..00000000 --- a/src/serialize/writer/simd.rs +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright 2023-2024 liuq19, ijl -// adapted from sonic-rs' src/util/string.rs - -use core::simd::cmp::{SimdPartialEq, SimdPartialOrd}; - -macro_rules! impl_escape_unchecked { - ($src:expr, $dst:expr, $nb:expr, $omask:expr, $cn:expr, $v:expr, $rotate:expr) => { - $nb -= $cn; - if $rotate == true { - for _ in 0..$cn { - $v = $v.rotate_elements_left::<1>(); - } - } - $dst = $dst.add($cn); - $src = $src.add($cn); - $omask >>= $cn; - loop { - $nb -= 1; - if $rotate == true { - $v = $v.rotate_elements_left::<1>(); - } - $omask = $omask >> 1; - if *($src) == b'"' { - core::ptr::copy_nonoverlapping(b"\\\"".as_ptr(), $dst, 2); - $dst = $dst.add(2); - } else if *($src) == b'\\' { - core::ptr::copy_nonoverlapping(b"\\\\".as_ptr(), $dst, 2); - $dst = $dst.add(2); - } else { - $dst = write_unusual_escape($src, $dst); - }; - - $src = $src.add(1); - if likely!($omask & 1 != 1) { - break; - } - } - }; -} - -macro_rules! impl_format_simd { - ($odptr:expr, $value_ptr:expr, $value_len:expr) => { - let mut dptr = $odptr; - let dstart = $odptr; - let mut sptr = $value_ptr; - let mut nb: usize = $value_len; - - let blash = StrVector::from_array([b'\\'; STRIDE]); - let quote = StrVector::from_array([b'"'; STRIDE]); - let x20 = StrVector::from_array([32; STRIDE]); - - unsafe { - *dptr = b'"'; - dptr = dptr.add(1); - - { - const ROTATE: bool = false; - while nb >= STRIDE { - let mut v = StrVector::from_slice(core::slice::from_raw_parts(sptr, STRIDE)); - let mut mask = - (v.simd_eq(blash) | v.simd_eq(quote) | v.simd_lt(x20)).to_bitmask() as u32; - v.copy_to_slice(core::slice::from_raw_parts_mut(dptr, STRIDE)); - - if likely!(mask == 0) { - nb -= STRIDE; - dptr = dptr.add(STRIDE); - sptr = sptr.add(STRIDE); - } else { - let cn = mask.trailing_zeros() as usize; - impl_escape_unchecked!(sptr, dptr, nb, mask, cn, v, ROTATE); - } - } - } - - { - const ROTATE: bool = true; - let mut v = StrVector::default(); - { - let vec_ptr = v.as_mut_array().as_mut_ptr(); - for idx in 0..nb { - core::ptr::write(vec_ptr.add(idx), *sptr.add(idx)); - } - } - - let mut mask = (v.simd_eq(blash) | v.simd_eq(quote) | v.simd_lt(x20)).to_bitmask() - as u32 - & (STRIDE_SATURATION >> (32 - STRIDE - nb)); - - while nb > 0 { - v.copy_to_slice(core::slice::from_raw_parts_mut(dptr, STRIDE)); - - if likely!(mask == 0) { - dptr = dptr.add(nb); - break; - } else { - let cn = mask.trailing_zeros() as usize; - impl_escape_unchecked!(sptr, dptr, nb, mask, cn, v, ROTATE); - } - } - } - - *dptr = b'"'; - dptr = dptr.add(1); - } - - return dptr as usize - dstart as usize; - }; -} - -#[cold] -#[inline(never)] -fn write_unusual_escape(sptr: *const u8, dptr: *mut u8) -> *mut u8 { - unsafe { - debug_assert!(*sptr < 32); - let replacement = match *(sptr) { - 0 => (*b"\\u0000\0\0", 6), - 1 => (*b"\\u0001\0\0", 6), - 2 => (*b"\\u0002\0\0", 6), - 3 => (*b"\\u0003\0\0", 6), - 4 => (*b"\\u0004\0\0", 6), - 5 => (*b"\\u0005\0\0", 6), - 6 => (*b"\\u0006\0\0", 6), - 7 => (*b"\\u0007\0\0", 6), - 8 => (*b"\\b\0\0\0\0\0\0", 2), - 9 => (*b"\\t\0\0\0\0\0\0", 2), - 10 => (*b"\\n\0\0\0\0\0\0", 2), - 11 => (*b"\\u000b\0\0", 6), - 12 => (*b"\\f\0\0\0\0\0\0", 2), - 13 => (*b"\\r\0\0\0\0\0\0", 2), - 14 => (*b"\\u000e\0\0", 6), - 15 => (*b"\\u000f\0\0", 6), - 16 => (*b"\\u0010\0\0", 6), - 17 => (*b"\\u0011\0\0", 6), - 18 => (*b"\\u0012\0\0", 6), - 19 => (*b"\\u0013\0\0", 6), - 20 => (*b"\\u0014\0\0", 6), - 21 => (*b"\\u0015\0\0", 6), - 22 => (*b"\\u0016\0\0", 6), - 23 => (*b"\\u0017\0\0", 6), - 24 => (*b"\\u0018\0\0", 6), - 25 => (*b"\\u0019\0\0", 6), - 26 => (*b"\\u001a\0\0", 6), - 27 => (*b"\\u001b\0\0", 6), - 28 => (*b"\\u001c\0\0", 6), - 29 => (*b"\\u001d\0\0", 6), - 30 => (*b"\\u001e\0\0", 6), - 31 => (*b"\\u001f\0\0", 6), - _ => unreachable!(), - }; - core::ptr::copy_nonoverlapping(replacement.0.as_ptr(), dptr, 8); - dptr.add(replacement.1 as usize) - } -} - -#[inline(never)] -pub unsafe fn format_escaped_str_impl_128( - odptr: *mut u8, - value_ptr: *const u8, - value_len: usize, -) -> usize { - const STRIDE: usize = 16; - const STRIDE_SATURATION: u32 = u16::MAX as u32; - type StrVector = core::simd::u8x16; - - impl_format_simd!(odptr, value_ptr, value_len); -}