diff --git a/module/icp/algs/modes/gcm.c b/module/icp/algs/modes/gcm.c index 3fee18cd7ef0..73286300a32d 100644 --- a/module/icp/algs/modes/gcm.c +++ b/module/icp/algs/modes/gcm.c @@ -1075,8 +1075,13 @@ extern void ASMABI gcm_ghash_vpclmulqdq_avx2(uint64_t ghash[2], static inline void GHASH_AVX(gcm_ctx_t *ctx, const uint8_t *in, size_t len) { if (ctx->gcm_use_avx2) { + uint64_t compare[2]; + memcpy(compare, ctx->gcm_ghash, sizeof(compare)); + gcm_ghash_avx(compare, + (const uint64_t *)ctx->gcm_Htable, in, len); gcm_ghash_vpclmulqdq_avx2(ctx->gcm_ghash, (const uint64_t *)ctx->gcm_Htable, in, len); + ASSERT0F(memcmp(compare, ctx->gcm_ghash, sizeof(compare)), "%s", "ghash mismatch"); } else { gcm_ghash_avx(ctx->gcm_ghash, (const uint64_t *)ctx->gcm_Htable, in, len); @@ -1233,7 +1238,7 @@ gcm_mode_encrypt_contiguous_blocks_avx(gcm_ctx_t *ctx, char *data, size_t done = 0; uint8_t *datap = (uint8_t *)data; size_t chunk_size = (size_t)GCM_CHUNK_SIZE_READ; - aesni_gcm_encrypt_impl *encrypt_blocks = ctx->gcm_use_avx2 ? + aesni_gcm_encrypt_impl *encrypt_blocks = B_FALSE ? aesni_gcm_encrypt_avx2 : aesni_gcm_encrypt_avx; const aes_key_t *key = ((aes_key_t *)ctx->gcm_keysched); uint64_t *ghash = ctx->gcm_ghash; @@ -1473,7 +1478,7 @@ gcm_decrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size) B_FALSE); size_t chunk_size = (size_t)GCM_CHUNK_SIZE_READ; - aesni_gcm_decrypt_impl *decrypt_blocks = ctx->gcm_use_avx2 ? + aesni_gcm_decrypt_impl *decrypt_blocks = B_FALSE ? aesni_gcm_decrypt_avx2 : aesni_gcm_decrypt_avx; size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len; uint8_t *datap = ctx->gcm_pt_buf; @@ -1574,6 +1579,50 @@ gcm_decrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size) return (CRYPTO_SUCCESS); } +static void +dump_bin(char *out, size_t out_len, const void* in, size_t in_len) +{ + const char* in_c = in; + const char *end = in_c + in_len; + for ( ; in_c != end; in_c++) { + int count = snprintf(out, out_len, "%02x ", (unsigned)*in_c); + if (out_len < count) { + break; + } + + out_len -= count; + out += count; + } +} + +static void +assert_buf_eq(const void *l, const void *r, size_t len, const char* msg) +{ + if (!memcmp(l, r, len)) { + return; + } + + size_t compare_str_size = len * 3 + 1; + char *compare_hex = vmem_alloc(compare_str_size, KM_SLEEP); + if (!compare_hex) { + ASSERT0F(memcmp(l, r, len), "%s", msg); + return; + } + char *actual_hex = vmem_alloc(compare_str_size, KM_SLEEP); + if (!actual_hex) { + ASSERT0F(memcmp(l, r, len), "%s", msg); + goto free_compare; + } + + dump_bin(compare_hex, compare_str_size, l, len); + dump_bin(actual_hex, compare_str_size, r, len); + ASSERT0F(memcmp(l, r, len), "%s\nexpected: %s\nactual: %s", msg, compare_hex, actual_hex); + + vmem_free(actual_hex, compare_str_size); +free_compare: + vmem_free(compare_hex, compare_str_size); +} + /* * Initialize the GCM params H, Htabtle and the counter block. Save the * initial counter block. @@ -1602,7 +1651,10 @@ gcm_init_avx(gcm_ctx_t *ctx, const uint8_t *iv, size_t iv_len, (const uint32_t *)H, (uint32_t *)H); if (ctx->gcm_use_avx2) { + uint64_t compare[(2 * 6 * 2)]; gcm_init_vpclmulqdq_avx2((uint128_t *)ctx->gcm_Htable, H); + gcm_init_htab_avx(compare, H); + assert_buf_eq(compare, ctx->gcm_Htable, ctx->gcm_htab_len, "htab differs"); } else { gcm_init_htab_avx(ctx->gcm_Htable, H); }