From 14cb23e743e02e6923f7e46a14717e9f561f6723 Mon Sep 17 00:00:00 2001 From: Kyrylo Tkachov Date: Tue, 22 Oct 2024 07:52:36 -0700 Subject: [PATCH] aarch64: Emit XAR for vector rotates where possible We can make use of the integrated rotate step of the XAR instruction to implement most vector integer rotates, as long we zero out one of the input registers for it. This allows for a lower-latency sequence than the fallback SHL+USRA, especially when we can hoist the zeroing operation away from loops and hot parts. This should be safe to do for 64-bit vectors as well even though the XAR instructions operate on 128-bit values, as the bottom 64-bit results is later accessed through the right subregs. This strategy is used whenever we have XAR instructions, the logic in aarch64_emit_opt_vec_rotate is adjusted to resort to expand_rotate_as_vec_perm only when it's expected to generate a single REV* instruction or when XAR instructions are not present. With this patch we can gerate for the input: v4si G1 (v4si r) { return (r >> 23) | (r << 9); } v8qi G2 (v8qi r) { return (r << 3) | (r >> 5); } the assembly for +sve2: G1: movi v31.4s, 0 xar z0.s, z0.s, z31.s, #23 ret G2: movi v31.4s, 0 xar z0.b, z0.b, z31.b, #5 ret instead of the current: G1: shl v31.4s, v0.4s, 9 usra v31.4s, v0.4s, 23 mov v0.16b, v31.16b ret G2: shl v31.8b, v0.8b, 3 usra v31.8b, v0.8b, 5 mov v0.8b, v31.8b ret Bootstrapped and tested on aarch64-none-linux-gnu. Signed-off-by: Kyrylo Tkachov gcc/ * config/aarch64/aarch64.cc (aarch64_emit_opt_vec_rotate): Add generation of XAR sequences when possible. gcc/testsuite/ * gcc.target/aarch64/rotate_xar_1.c: New test. --- gcc/config/aarch64/aarch64.cc | 34 +++++-- .../gcc.target/aarch64/rotate_xar_1.c | 93 +++++++++++++++++++ 2 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 gcc/testsuite/gcc.target/aarch64/rotate_xar_1.c diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc index 7388f6b8fdf1a..00f99d5004cac 100644 --- a/gcc/config/aarch64/aarch64.cc +++ b/gcc/config/aarch64/aarch64.cc @@ -16019,17 +16019,39 @@ aarch64_emit_approx_div (rtx quo, rtx num, rtx den) } /* Emit an optimized sequence to perform a vector rotate - of REG by the vector constant amount AMNT and place the result + of REG by the vector constant amount AMNT_VEC and place the result in DST. Return true iff successful. */ bool -aarch64_emit_opt_vec_rotate (rtx dst, rtx reg, rtx amnt) +aarch64_emit_opt_vec_rotate (rtx dst, rtx reg, rtx amnt_vec) { + rtx amnt = unwrap_const_vec_duplicate (amnt_vec); + gcc_assert (CONST_INT_P (amnt)); + HOST_WIDE_INT rotamnt = UINTVAL (amnt); machine_mode mode = GET_MODE (reg); - /* Attempt to expand the rotate as a vector permute. - For some rotate amounts they can be single instructions and - even the general single-vector TBL permute has good throughput. */ - if (expand_rotate_as_vec_perm (mode, dst, reg, amnt)) + /* Rotates by half the element width map down to REV* instructions and should + always be preferred when possible. */ + if (rotamnt == GET_MODE_UNIT_BITSIZE (mode) / 2 + && expand_rotate_as_vec_perm (mode, dst, reg, amnt)) + return true; + /* 64 and 128-bit vector modes can use the XAR instruction + when available. */ + else if (can_create_pseudo_p () + && ((TARGET_SHA3 && mode == V2DImode) + || (TARGET_SVE2 + && (known_eq (GET_MODE_SIZE (mode), 8) + || known_eq (GET_MODE_SIZE (mode), 16))))) + { + rtx zeroes = aarch64_gen_shareable_zero (mode); + rtx xar_op + = gen_rtx_ROTATE (mode, gen_rtx_XOR (mode, reg, zeroes), + amnt_vec); + emit_set_insn (dst, xar_op); + return true; + } + /* If none of the above, try to expand rotates by any byte amount as + permutes. */ + else if (expand_rotate_as_vec_perm (mode, dst, reg, amnt)) return true; return false; } diff --git a/gcc/testsuite/gcc.target/aarch64/rotate_xar_1.c b/gcc/testsuite/gcc.target/aarch64/rotate_xar_1.c new file mode 100644 index 0000000000000..73007701cfb41 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/rotate_xar_1.c @@ -0,0 +1,93 @@ +/* { dg-do compile } */ +/* { dg-options "-O2" } */ +/* { dg-final { check-function-bodies "**" "" } } */ + +typedef char __attribute__ ((vector_size (16))) v16qi; +typedef unsigned short __attribute__ ((vector_size (16))) v8hi; +typedef unsigned int __attribute__ ((vector_size (16))) v4si; +typedef unsigned long long __attribute__ ((vector_size (16))) v2di; +typedef char __attribute__ ((vector_size (8))) v8qi; +typedef unsigned short __attribute__ ((vector_size (8))) v4hi; +typedef unsigned int __attribute__ ((vector_size (8))) v2si; + +#pragma GCC target "+sve2+sha3" + +/* +** G1: +** movi? [vdz][0-9]+\.?(?:[0-9]*[bhsd])?, #?0 +** xar v0\.2d, v[0-9]+\.2d, v[0-9]+\.2d, 39 +** ret +*/ +v2di +G1 (v2di r) { + return (r >> 39) | (r << 25); +} + +/* +** G2: +** movi? [vdz][0-9]+\.?(?:[0-9]*[bhsd])?, #?0 +** xar z0\.s, z[0-9]+\.s, z[0-9]+\.s, #23 +** ret +*/ +v4si +G2 (v4si r) { + return (r >> 23) | (r << 9); +} + +/* +** G3: +** movi? [vdz][0-9]+\.?(?:[0-9]*[bhsd])?, #?0 +** xar z0\.h, z[0-9]+\.h, z[0-9]+\.h, #5 +** ret +*/ +v8hi +G3 (v8hi r) { + return (r >> 5) | (r << 11); +} + +/* +** G4: +** movi? [vdz][0-9]+\.?(?:[0-9]*[bhsd])?, #?0 +** xar z0\.b, z[0-9]+\.b, z[0-9]+\.b, #6 +** ret +*/ +v16qi +G4 (v16qi r) +{ + return (r << 2) | (r >> 6); +} + +/* +** G5: +** movi? [vdz][0-9]+\.?(?:[0-9]*[bhsd])?, #?0 +** xar z0\.s, z[0-9]+\.s, z[0-9]+\.s, #22 +** ret +*/ +v2si +G5 (v2si r) { + return (r >> 22) | (r << 10); +} + +/* +** G6: +** movi? [vdz][0-9]+\.?(?:[0-9]*[bhsd])?, #?0 +** xar z0\.h, z[0-9]+\.h, z[0-9]+\.h, #7 +** ret +*/ +v4hi +G6 (v4hi r) { + return (r >> 7) | (r << 9); +} + +/* +** G7: +** movi? [vdz][0-9]+\.?(?:[0-9]*[bhsd])?, #?0 +** xar z0\.b, z[0-9]+\.b, z[0-9]+\.b, #5 +** ret +*/ +v8qi +G7 (v8qi r) +{ + return (r << 3) | (r >> 5); +} +