diff --git a/build.rs b/build.rs index d4cfe0e1c..fb3dc373a 100644 --- a/build.rs +++ b/build.rs @@ -33,6 +33,11 @@ fn main() { println!("cargo:rustc-cfg=feature=\"mem\""); } + // These targets have hardware unaligned access support. + if target.contains("x86_64") || target.contains("i686") || target.contains("aarch64") { + println!("cargo:rustc-cfg=feature=\"mem-unaligned\""); + } + // NOTE we are going to assume that llvm-target, what determines our codegen option, matches the // target triple. This is usually correct for our built-in targets but can break in presence of // custom targets, which can have arbitrary names. diff --git a/src/mem/impls.rs b/src/mem/impls.rs index e60b160fd..9a6d1d04e 100644 --- a/src/mem/impls.rs +++ b/src/mem/impls.rs @@ -16,6 +16,14 @@ const WORD_COPY_THRESHOLD: usize = if 2 * WORD_SIZE > 16 { 16 }; +#[cfg(feature = "mem-unaligned")] +unsafe fn read_usize_unaligned(x: *const usize) -> usize { + // Do not use `core::ptr::read_unaligned` here, since it calls `copy_nonoverlapping` which + // is translated to memcpy in LLVM. + let x_read = (x as *const [u8; core::mem::size_of::()]).read(); + core::mem::transmute(x_read) +} + #[inline(always)] pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) { #[inline(always)] @@ -41,6 +49,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) } } + #[cfg(not(feature = "mem-unaligned"))] #[inline(always)] unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { let mut dest_usize = dest as *mut usize; @@ -69,6 +78,20 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) } } + #[cfg(feature = "mem-unaligned")] + #[inline(always)] + unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + while dest_usize < dest_end { + *dest_usize = read_usize_unaligned(src_usize); + dest_usize = dest_usize.add(1); + src_usize = src_usize.add(1); + } + } + if n >= WORD_COPY_THRESHOLD { // Align dest // Because of n >= 2 * WORD_SIZE, dst_misalignment < n @@ -119,6 +142,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { } } + #[cfg(not(feature = "mem-unaligned"))] #[inline(always)] unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { let mut dest_usize = dest as *mut usize; @@ -147,6 +171,20 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { } } + #[cfg(feature = "mem-unaligned")] + #[inline(always)] + unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + while dest_start < dest_usize { + dest_usize = dest_usize.sub(1); + src_usize = src_usize.sub(1); + *dest_usize = read_usize_unaligned(src_usize as *const u8); + } + } + let mut dest = dest.add(n); let mut src = src.add(n);