From c3753f94a90682a59bdd7973a6fee2692ef7e657 Mon Sep 17 00:00:00 2001 From: Didzis Gosko Date: Fri, 12 Apr 2024 14:38:44 +0300 Subject: [PATCH 1/8] make : add AVX512 detection to Makefile and CMakeLists.txt --- CMakeLists.txt | 28 +++++++++++++++++----------- Makefile | 6 ++++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a3c2639939e..8dc4bca65a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,10 +59,11 @@ option(WHISPER_BUILD_EXAMPLES "whisper: build examples" ${WHISPER_STANDA option(WHISPER_SDL2 "whisper: support for libSDL2" OFF) -option(WHISPER_NO_AVX "whisper: disable AVX" OFF) -option(WHISPER_NO_AVX2 "whisper: disable AVX2" OFF) -option(WHISPER_NO_FMA "whisper: disable FMA" OFF) -option(WHISPER_NO_F16C "whisper: disable F16c" OFF) +option(WHISPER_NO_AVX "whisper: disable AVX" OFF) +option(WHISPER_NO_AVX2 "whisper: disable AVX2" OFF) +option(WHISPER_NO_AVX512 "whisper: disable AVX512" OFF) +option(WHISPER_NO_FMA "whisper: disable FMA" OFF) +option(WHISPER_NO_F16C "whisper: disable F16c" OFF) option(WHISPER_OPENVINO "whisper: support for OpenVINO" OFF) @@ -464,16 +465,18 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /utf-8") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /utf-8") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /utf-8") - if(NOT WHISPER_NO_AVX2) + if(NOT WHISPER_NO_AVX512) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX512") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /arch:AVX512") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX512") + elseif(NOT WHISPER_NO_AVX2) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /arch:AVX2") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX2") - else() - if(NOT WHISPER_NO_AVX) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /arch:AVX") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX") - endif() + elseif(NOT WHISPER_NO_AVX) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /arch:AVX") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX") endif() else() if (EMSCRIPTEN) @@ -486,6 +489,9 @@ else() if(NOT WHISPER_NO_AVX2) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") endif() + if(NOT WHISPER_NO_AVX512) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f") + endif() if(NOT WHISPER_NO_FMA) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma") endif() diff --git a/Makefile b/Makefile index e255c52121a..51fe96d3662 100644 --- a/Makefile +++ b/Makefile @@ -144,6 +144,12 @@ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) CXXFLAGS += -mavx2 endif + AVX512F_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512F') + ifneq (,$(AVX512F_M)) + CFLAGS += -mavx512f + CXXFLAGS += -mavx512f + endif + FMA_M := $(shell $(CPUINFO_CMD) | grep -iw 'FMA') ifneq (,$(FMA_M)) CFLAGS += -mfma From e8ee795cddb6b61ddecc875d456ef4595cd948cb Mon Sep 17 00:00:00 2001 From: Didzis Gosko Date: Fri, 12 Apr 2024 18:05:06 +0300 Subject: [PATCH 2/8] make : autodetect more AVX512 instruction subsets --- Makefile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Makefile b/Makefile index 51fe96d3662..f9feb108600 100644 --- a/Makefile +++ b/Makefile @@ -150,6 +150,36 @@ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) CXXFLAGS += -mavx512f endif + AVX512DQ_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512DQ') + ifneq (,$(AVX512DQ_M)) + CFLAGS += -mavx512dq + CXXFLAGS += -mavx512dq + endif + + AVX512CD_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512CD') + ifneq (,$(AVX512CD_M)) + CFLAGS += -mavx512cd + CXXFLAGS += -mavx512cd + endif + + AVX512CD_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512BW') + ifneq (,$(AVX512BW_M)) + CFLAGS += -mavx512bw + CXXFLAGS += -mavx512bw + endif + + AVX512VL_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512VL') + ifneq (,$(AVX512VL_M)) + CFLAGS += -mavx512vl + CXXFLAGS += -mavx512vl + endif + + AVX512VNNI_M := $(shell $(CPUINFO_CMD) | grep -iwE 'AVX512_VNNI|AVX512VNNI') + ifneq (,$(AVX512VNNI_M)) + CFLAGS += -mavx512vnni + CXXFLAGS += -mavx512vnni + endif + FMA_M := $(shell $(CPUINFO_CMD) | grep -iw 'FMA') ifneq (,$(FMA_M)) CFLAGS += -mfma From eb24a776c035f58f6b8052c764d73e10bbe85c66 Mon Sep 17 00:00:00 2001 From: Didzis Gosko Date: Fri, 12 Apr 2024 18:06:03 +0300 Subject: [PATCH 3/8] cmake : do not default to AVX512, must be enabled explicitly --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8dc4bca65a0..296262b6fdc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,7 +61,7 @@ option(WHISPER_SDL2 "whisper: support for libSDL2" OFF) option(WHISPER_NO_AVX "whisper: disable AVX" OFF) option(WHISPER_NO_AVX2 "whisper: disable AVX2" OFF) -option(WHISPER_NO_AVX512 "whisper: disable AVX512" OFF) +option(WHISPER_AVX512 "whisper: enable AVX512" OFF) option(WHISPER_NO_FMA "whisper: disable FMA" OFF) option(WHISPER_NO_F16C "whisper: disable F16c" OFF) @@ -465,7 +465,7 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /utf-8") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /utf-8") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /utf-8") - if(NOT WHISPER_NO_AVX512) + if(WHISPER_AVX512) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX512") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /arch:AVX512") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX512") @@ -489,7 +489,7 @@ else() if(NOT WHISPER_NO_AVX2) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") endif() - if(NOT WHISPER_NO_AVX512) + if(WHISPER_AVX512) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f") endif() if(NOT WHISPER_NO_FMA) From ef25a4a0f8970b0f76b0f5fb67050968b756eb29 Mon Sep 17 00:00:00 2001 From: Didzis Gosko Date: Fri, 12 Apr 2024 18:07:06 +0300 Subject: [PATCH 4/8] cmake : enable a set of AVX512 subsets, when AVX512 is turned on --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 296262b6fdc..ca5b96d9176 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -490,7 +490,7 @@ else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") endif() if(WHISPER_AVX512) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512cd -mavx512vl -mavx512dq -mavx512bw") endif() if(NOT WHISPER_NO_FMA) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma") From dc69460b42fce5e003e367a945b04fddab814c4f Mon Sep 17 00:00:00 2001 From: Didzis Gosko Date: Fri, 12 Apr 2024 20:20:54 +0300 Subject: [PATCH 5/8] make : consolidate AVX512 subsets, add AVX512 VBMI --- Makefile | 36 +++++++++--------------------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index f9feb108600..b481916f346 100644 --- a/Makefile +++ b/Makefile @@ -146,40 +146,22 @@ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) AVX512F_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512F') ifneq (,$(AVX512F_M)) - CFLAGS += -mavx512f - CXXFLAGS += -mavx512f + CFLAGS += -mavx512f -mavx512cd -mavx512vl -mavx512dq -mavx512bw + CXXFLAGS += -mavx512f -mavx512cd -mavx512vl -mavx512dq -mavx512bw endif - AVX512DQ_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512DQ') - ifneq (,$(AVX512DQ_M)) - CFLAGS += -mavx512dq - CXXFLAGS += -mavx512dq - endif - - AVX512CD_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512CD') - ifneq (,$(AVX512CD_M)) - CFLAGS += -mavx512cd - CXXFLAGS += -mavx512cd - endif - - AVX512CD_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512BW') - ifneq (,$(AVX512BW_M)) - CFLAGS += -mavx512bw - CXXFLAGS += -mavx512bw - endif - - AVX512VL_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512VL') - ifneq (,$(AVX512VL_M)) - CFLAGS += -mavx512vl - CXXFLAGS += -mavx512vl - endif - - AVX512VNNI_M := $(shell $(CPUINFO_CMD) | grep -iwE 'AVX512_VNNI|AVX512VNNI') + AVX512VNNI_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512_VNNI') ifneq (,$(AVX512VNNI_M)) CFLAGS += -mavx512vnni CXXFLAGS += -mavx512vnni endif + AVX512VBMI_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512VBMI') + ifneq (,$(AVX512VBMI_M)) + CFLAGS += -mavx512vbmi + CXXFLAGS += -mavx512vbmi + endif + FMA_M := $(shell $(CPUINFO_CMD) | grep -iw 'FMA') ifneq (,$(FMA_M)) CFLAGS += -mfma From f7c4acb0f462c818660f359979c7d1e5e71302d0 Mon Sep 17 00:00:00 2001 From: Didzis Gosko Date: Fri, 12 Apr 2024 20:22:46 +0300 Subject: [PATCH 6/8] cmake : revert to NO AVX512 setting, add settings for AVX512 VNNI and VBMI --- CMakeLists.txt | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ca5b96d9176..841bb623b36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,11 +59,13 @@ option(WHISPER_BUILD_EXAMPLES "whisper: build examples" ${WHISPER_STANDA option(WHISPER_SDL2 "whisper: support for libSDL2" OFF) -option(WHISPER_NO_AVX "whisper: disable AVX" OFF) -option(WHISPER_NO_AVX2 "whisper: disable AVX2" OFF) -option(WHISPER_AVX512 "whisper: enable AVX512" OFF) -option(WHISPER_NO_FMA "whisper: disable FMA" OFF) -option(WHISPER_NO_F16C "whisper: disable F16c" OFF) +option(WHISPER_NO_AVX "whisper: disable AVX" OFF) +option(WHISPER_NO_AVX2 "whisper: disable AVX2" OFF) +option(WHISPER_NO_AVX512 "whisper: disable AVX512" ON) +option(WHISPER_NO_AVX512_VBMI "whisper: disable AVX512-VBMI" ON) +option(WHISPER_NO_AVX512_VNNI "whisper: disable AVX512-VNNI" ON) +option(WHISPER_NO_FMA "whisper: disable FMA" OFF) +option(WHISPER_NO_F16C "whisper: disable F16c" OFF) option(WHISPER_OPENVINO "whisper: support for OpenVINO" OFF) @@ -465,10 +467,23 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /utf-8") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /utf-8") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /utf-8") - if(WHISPER_AVX512) + if(NOT WHISPER_NO_AVX512) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX512") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /arch:AVX512") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX512") + # from llama.cpp: + # MSVC has no compile-time flags enabling specific + # AVX512 extensions, neither it defines the + # macros corresponding to the extensions. + # Do it manually. + if (NOT WHISPER_NO_AVX512_VBMI) + add_compile_definitions($<$:__AVX512VBMI__>) + add_compile_definitions($<$:__AVX512VBMI__>) + endif() + if (NOT WHISPER_NO_AVX512_VNNI) + add_compile_definitions($<$:__AVX512VNNI__>) + add_compile_definitions($<$:__AVX512VNNI__>) + endif() elseif(NOT WHISPER_NO_AVX2) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /arch:AVX2") @@ -489,9 +504,15 @@ else() if(NOT WHISPER_NO_AVX2) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") endif() - if(WHISPER_AVX512) + if(NOT WHISPER_NO_AVX512) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512cd -mavx512vl -mavx512dq -mavx512bw") endif() + if(NOT WHISPER_NO_AVX512_VBMI) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vbmi") + endif() + if(NOT WHISPER_NO_AVX512_VNNI) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vnni") + endif() if(NOT WHISPER_NO_FMA) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma") endif() From 83c902b5217102889d05636a5f9abcc942d2de93 Mon Sep 17 00:00:00 2001 From: Didzis Gosko Date: Fri, 12 Apr 2024 21:45:17 +0300 Subject: [PATCH 7/8] make : re-introduce AVX512VNNI back --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b481916f346..b69628e0374 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) CXXFLAGS += -mavx512f -mavx512cd -mavx512vl -mavx512dq -mavx512bw endif - AVX512VNNI_M := $(shell $(CPUINFO_CMD) | grep -iw 'AVX512_VNNI') + AVX512VNNI_M := $(shell $(CPUINFO_CMD) | grep -iwE 'AVX512_VNNI|AVX512VNNI') ifneq (,$(AVX512VNNI_M)) CFLAGS += -mavx512vnni CXXFLAGS += -mavx512vnni From 9b0b1d184cfcf5c3530f4a61371a02e57dbc30a0 Mon Sep 17 00:00:00 2001 From: Didzis Gosko Date: Fri, 12 Apr 2024 21:46:29 +0300 Subject: [PATCH 8/8] cmake : remove superfluous comment line --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 841bb623b36..2561dc6b65b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -471,7 +471,6 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX512") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /arch:AVX512") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX512") - # from llama.cpp: # MSVC has no compile-time flags enabling specific # AVX512 extensions, neither it defines the # macros corresponding to the extensions.