Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Micro-optimize fletcher4 calculations #14247

Merged
merged 1 commit into from
Dec 5, 2022
Merged

Micro-optimize fletcher4 calculations #14247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 6 additions & 12 deletions module/zcommon/zfs_fletcher_aarch64_neon.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ ZFS_NO_SANITIZE_UNDEFINED
static void
fletcher_4_aarch64_neon_init(fletcher_4_ctx_t *ctx)
{
kfpu_begin();
memset(ctx->aarch64_neon, 0, 4 * sizeof (zfs_fletcher_aarch64_neon_t));
}

Expand All @@ -69,6 +70,7 @@ fletcher_4_aarch64_neon_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
8 * ctx->aarch64_neon[3].v[1] - 8 * ctx->aarch64_neon[2].v[1] +
ctx->aarch64_neon[1].v[1];
ZIO_SET_CHECKSUM(zcp, A, B, C, D);
kfpu_end();
}

#define NEON_INIT_LOOP() \
Expand Down Expand Up @@ -146,17 +148,13 @@ unsigned char TMP2 __attribute__((vector_size(16)));
unsigned char SRC __attribute__((vector_size(16)));
#endif

kfpu_begin();

NEON_INIT_LOOP();

for (; ip < ipend; ip += 2) {
do {
NEON_MAIN_LOOP(NEON_DONT_REVERSE);
}
} while ((ip += 2) < ipend);

NEON_FINI_LOOP();

kfpu_end();
}

static void
Expand Down Expand Up @@ -185,17 +183,13 @@ unsigned char TMP2 __attribute__((vector_size(16)));
unsigned char SRC __attribute__((vector_size(16)));
#endif

kfpu_begin();

NEON_INIT_LOOP();

for (; ip < ipend; ip += 2) {
do {
NEON_MAIN_LOOP(NEON_DO_REVERSE);
}
} while ((ip += 2) < ipend);

NEON_FINI_LOOP();

kfpu_end();
}

static boolean_t fletcher_4_aarch64_neon_valid(void)
Expand Down
26 changes: 8 additions & 18 deletions module/zcommon/zfs_fletcher_avx512.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ ZFS_NO_SANITIZE_UNDEFINED
static void
fletcher_4_avx512f_init(fletcher_4_ctx_t *ctx)
{
kfpu_begin();
memset(ctx->avx512, 0, 4 * sizeof (zfs_fletcher_avx512_t));
}

Expand Down Expand Up @@ -72,6 +73,7 @@ fletcher_4_avx512f_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
}

ZIO_SET_CHECKSUM(zcp, A, B, C, D);
kfpu_end();
}

#define FLETCHER_4_AVX512_RESTORE_CTX(ctx) \
Expand All @@ -96,21 +98,17 @@ fletcher_4_avx512f_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
const uint32_t *ip = buf;
const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size);

kfpu_begin();

FLETCHER_4_AVX512_RESTORE_CTX(ctx);

for (; ip < ipend; ip += 8) {
do {
__asm("vpmovzxdq %0, %%zmm4"::"m" (*ip));
__asm("vpaddq %zmm4, %zmm0, %zmm0");
__asm("vpaddq %zmm0, %zmm1, %zmm1");
__asm("vpaddq %zmm1, %zmm2, %zmm2");
__asm("vpaddq %zmm2, %zmm3, %zmm3");
}
} while ((ip += 8) < ipend);

FLETCHER_4_AVX512_SAVE_CTX(ctx);

kfpu_end();
}
STACK_FRAME_NON_STANDARD(fletcher_4_avx512f_native);

Expand All @@ -122,16 +120,14 @@ fletcher_4_avx512f_byteswap(fletcher_4_ctx_t *ctx, const void *buf,
const uint32_t *ip = buf;
const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size);

kfpu_begin();

FLETCHER_4_AVX512_RESTORE_CTX(ctx);

__asm("vpbroadcastq %0, %%zmm8" :: "r" (byteswap_mask));
__asm("vpsllq $8, %zmm8, %zmm9");
__asm("vpsllq $16, %zmm8, %zmm10");
__asm("vpsllq $24, %zmm8, %zmm11");

for (; ip < ipend; ip += 8) {
do {
__asm("vpmovzxdq %0, %%zmm5"::"m" (*ip));

__asm("vpsrlq $24, %zmm5, %zmm6");
Expand All @@ -150,11 +146,9 @@ fletcher_4_avx512f_byteswap(fletcher_4_ctx_t *ctx, const void *buf,
__asm("vpaddq %zmm0, %zmm1, %zmm1");
__asm("vpaddq %zmm1, %zmm2, %zmm2");
__asm("vpaddq %zmm2, %zmm3, %zmm3");
}
} while ((ip += 8) < ipend);

FLETCHER_4_AVX512_SAVE_CTX(ctx)

kfpu_end();
}
STACK_FRAME_NON_STANDARD(fletcher_4_avx512f_byteswap);

Expand Down Expand Up @@ -189,13 +183,11 @@ fletcher_4_avx512bw_byteswap(fletcher_4_ctx_t *ctx, const void *buf,
const uint32_t *ip = buf;
const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size);

kfpu_begin();

FLETCHER_4_AVX512_RESTORE_CTX(ctx);

__asm("vmovdqu64 %0, %%zmm5" :: "m" (mask));

for (; ip < ipend; ip += 8) {
do {
__asm("vpmovzxdq %0, %%zmm4"::"m" (*ip));

__asm("vpshufb %zmm5, %zmm4, %zmm4");
Expand All @@ -204,11 +196,9 @@ fletcher_4_avx512bw_byteswap(fletcher_4_ctx_t *ctx, const void *buf,
__asm("vpaddq %zmm0, %zmm1, %zmm1");
__asm("vpaddq %zmm1, %zmm2, %zmm2");
__asm("vpaddq %zmm2, %zmm3, %zmm3");
}
} while ((ip += 8) < ipend);

FLETCHER_4_AVX512_SAVE_CTX(ctx)

kfpu_end();
}
STACK_FRAME_NON_STANDARD(fletcher_4_avx512bw_byteswap);

Expand Down
18 changes: 6 additions & 12 deletions module/zcommon/zfs_fletcher_intel.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ ZFS_NO_SANITIZE_UNDEFINED
static void
fletcher_4_avx2_init(fletcher_4_ctx_t *ctx)
{
kfpu_begin();
memset(ctx->avx, 0, 4 * sizeof (zfs_fletcher_avx_t));
}

Expand Down Expand Up @@ -81,6 +82,7 @@ fletcher_4_avx2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
64 * ctx->avx[3].v[3];

ZIO_SET_CHECKSUM(zcp, A, B, C, D);
kfpu_end();
}

#define FLETCHER_4_AVX2_RESTORE_CTX(ctx) \
Expand All @@ -106,22 +108,18 @@ fletcher_4_avx2_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
const uint64_t *ip = buf;
const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);

kfpu_begin();

FLETCHER_4_AVX2_RESTORE_CTX(ctx);

for (; ip < ipend; ip += 2) {
do {
asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip));
asm volatile("vpaddq %ymm4, %ymm0, %ymm0");
asm volatile("vpaddq %ymm0, %ymm1, %ymm1");
asm volatile("vpaddq %ymm1, %ymm2, %ymm2");
asm volatile("vpaddq %ymm2, %ymm3, %ymm3");
}
} while ((ip += 2) < ipend);

FLETCHER_4_AVX2_SAVE_CTX(ctx);
asm volatile("vzeroupper");

kfpu_end();
}

static void
Expand All @@ -134,26 +132,22 @@ fletcher_4_avx2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
const uint64_t *ip = buf;
const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);

kfpu_begin();

FLETCHER_4_AVX2_RESTORE_CTX(ctx);

asm volatile("vmovdqu %0, %%ymm5" :: "m" (mask));

for (; ip < ipend; ip += 2) {
do {
asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip));
asm volatile("vpshufb %ymm5, %ymm4, %ymm4");

asm volatile("vpaddq %ymm4, %ymm0, %ymm0");
asm volatile("vpaddq %ymm0, %ymm1, %ymm1");
asm volatile("vpaddq %ymm1, %ymm2, %ymm2");
asm volatile("vpaddq %ymm2, %ymm3, %ymm3");
}
} while ((ip += 2) < ipend);

FLETCHER_4_AVX2_SAVE_CTX(ctx);
asm volatile("vzeroupper");

kfpu_end();
}

static boolean_t fletcher_4_avx2_valid(void)
Expand Down
26 changes: 8 additions & 18 deletions module/zcommon/zfs_fletcher_sse.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ ZFS_NO_SANITIZE_UNDEFINED
static void
fletcher_4_sse2_init(fletcher_4_ctx_t *ctx)
{
kfpu_begin();
memset(ctx->sse, 0, 4 * sizeof (zfs_fletcher_sse_t));
}

Expand Down Expand Up @@ -80,6 +81,7 @@ fletcher_4_sse2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
8 * ctx->sse[2].v[1] + ctx->sse[1].v[1];

ZIO_SET_CHECKSUM(zcp, A, B, C, D);
kfpu_end();
}

#define FLETCHER_4_SSE_RESTORE_CTX(ctx) \
Expand All @@ -104,13 +106,11 @@ fletcher_4_sse2_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
const uint64_t *ip = buf;
const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);

kfpu_begin();

FLETCHER_4_SSE_RESTORE_CTX(ctx);

asm volatile("pxor %xmm4, %xmm4");

for (; ip < ipend; ip += 2) {
do {
asm volatile("movdqu %0, %%xmm5" :: "m"(*ip));
asm volatile("movdqa %xmm5, %xmm6");
asm volatile("punpckldq %xmm4, %xmm5");
Expand All @@ -123,11 +123,9 @@ fletcher_4_sse2_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
asm volatile("paddq %xmm0, %xmm1");
asm volatile("paddq %xmm1, %xmm2");
asm volatile("paddq %xmm2, %xmm3");
}
} while ((ip += 2) < ipend);

FLETCHER_4_SSE_SAVE_CTX(ctx);

kfpu_end();
}

static void
Expand All @@ -136,11 +134,9 @@ fletcher_4_sse2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
const uint32_t *ip = buf;
const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size);

kfpu_begin();

FLETCHER_4_SSE_RESTORE_CTX(ctx);

for (; ip < ipend; ip += 2) {
do {
uint32_t scratch1 = BSWAP_32(ip[0]);
uint32_t scratch2 = BSWAP_32(ip[1]);
asm volatile("movd %0, %%xmm5" :: "r"(scratch1));
Expand All @@ -150,11 +146,9 @@ fletcher_4_sse2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
asm volatile("paddq %xmm0, %xmm1");
asm volatile("paddq %xmm1, %xmm2");
asm volatile("paddq %xmm2, %xmm3");
}
} while ((ip += 2) < ipend);

FLETCHER_4_SSE_SAVE_CTX(ctx);

kfpu_end();
}

static boolean_t fletcher_4_sse2_valid(void)
Expand Down Expand Up @@ -186,14 +180,12 @@ fletcher_4_ssse3_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
const uint64_t *ip = buf;
const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);

kfpu_begin();

FLETCHER_4_SSE_RESTORE_CTX(ctx);

asm volatile("movdqu %0, %%xmm7"::"m" (mask));
asm volatile("pxor %xmm4, %xmm4");

for (; ip < ipend; ip += 2) {
do {
asm volatile("movdqu %0, %%xmm5"::"m" (*ip));
asm volatile("pshufb %xmm7, %xmm5");
asm volatile("movdqa %xmm5, %xmm6");
Expand All @@ -207,11 +199,9 @@ fletcher_4_ssse3_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
asm volatile("paddq %xmm0, %xmm1");
asm volatile("paddq %xmm1, %xmm2");
asm volatile("paddq %xmm2, %xmm3");
}
} while ((ip += 2) < ipend);

FLETCHER_4_SSE_SAVE_CTX(ctx);

kfpu_end();
}

static boolean_t fletcher_4_ssse3_valid(void)
Expand Down
8 changes: 4 additions & 4 deletions module/zcommon/zfs_fletcher_superscalar.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ fletcher_4_superscalar_native(fletcher_4_ctx_t *ctx,
c2 = ctx->superscalar[2].v[1];
d2 = ctx->superscalar[3].v[1];

for (; ip < ipend; ip += 2) {
do {
a += ip[0];
a2 += ip[1];
b += a;
Expand All @@ -98,7 +98,7 @@ fletcher_4_superscalar_native(fletcher_4_ctx_t *ctx,
c2 += b2;
d += c;
d2 += c2;
}
} while ((ip += 2) < ipend);

ctx->superscalar[0].v[0] = a;
ctx->superscalar[1].v[0] = b;
Expand Down Expand Up @@ -129,7 +129,7 @@ fletcher_4_superscalar_byteswap(fletcher_4_ctx_t *ctx,
c2 = ctx->superscalar[2].v[1];
d2 = ctx->superscalar[3].v[1];

for (; ip < ipend; ip += 2) {
do {
a += BSWAP_32(ip[0]);
a2 += BSWAP_32(ip[1]);
b += a;
Expand All @@ -138,7 +138,7 @@ fletcher_4_superscalar_byteswap(fletcher_4_ctx_t *ctx,
c2 += b2;
d += c;
d2 += c2;
}
} while ((ip += 2) < ipend);

ctx->superscalar[0].v[0] = a;
ctx->superscalar[1].v[0] = b;
Expand Down
8 changes: 4 additions & 4 deletions module/zcommon/zfs_fletcher_superscalar4.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx,
c4 = ctx->superscalar[2].v[3];
d4 = ctx->superscalar[3].v[3];

for (; ip < ipend; ip += 4) {
do {
a += ip[0];
a2 += ip[1];
a3 += ip[2];
Expand All @@ -130,7 +130,7 @@ fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx,
d2 += c2;
d3 += c3;
d4 += c4;
}
} while ((ip += 4) < ipend);

ctx->superscalar[0].v[0] = a;
ctx->superscalar[1].v[0] = b;
Expand Down Expand Up @@ -179,7 +179,7 @@ fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t *ctx,
c4 = ctx->superscalar[2].v[3];
d4 = ctx->superscalar[3].v[3];

for (; ip < ipend; ip += 4) {
do {
a += BSWAP_32(ip[0]);
a2 += BSWAP_32(ip[1]);
a3 += BSWAP_32(ip[2]);
Expand All @@ -196,7 +196,7 @@ fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t *ctx,
d2 += c2;
d3 += c3;
d4 += c4;
}
} while ((ip += 4) < ipend);

ctx->superscalar[0].v[0] = a;
ctx->superscalar[1].v[0] = b;
Expand Down