[CMSIS-NN] Support CMSIS NN from new GitHub location (apache#13656)

CMSIS NN has been moved out of the CMSIS project into a new GitHub project. This pr adds support for CMSIS NN from this new GitHub location. Both CMSIS and CMSIS NN are now downloaded in /opt/arm/ethosu/cmsis I updated the pr with three commits: - First, I supported CMSIS NN from the new GitHub location, as previously done. - Second, I prevented the definition of the `cmsis_path` project API option from an environment variable. Before, thanks to the environment variable, the `cmsis_path` option was always enabled. That was not a problem, but now CMSIS NN uses a new header (`arm_acle.h`) which is not always present, so we need to explicitly enable CMSIS when we need it. - In the end, I re-added support for the old location of CMSIS NN because the docker image is not yet updated, and we need the tests to pass to accept this pr. In this way, tvm will use the new CMSIS NN project when we will update the docker image, but for now, it uses the old one. I'll create a pr that reverts this last commit when the docker image is updated.
fzi-peccia · Mar 27, 2023 · b53706e · b53706e
1 parent dcf05ef
commit b53706e
Show file tree

Hide file tree

Showing 12 changed files with 82 additions and 34 deletions.
diff --git a/apps/microtvm/cmsisnn/Makefile b/apps/microtvm/cmsisnn/Makefile
@@ -31,14 +31,19 @@ CMAKE ?= cmake
 CC = arm-none-eabi-gcc
 AR = arm-none-eabi-ar
 RANLIB = arm-none-eabi-ranlib
+ifeq ($(shell [ -d ${CMSIS_PATH}/CMSIS-NN ]; echo $$?), 0)
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS-NN
+else
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS/NN
+endif
 PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	-I${STANDALONE_CRT_PATH}/include \
 	-I${STANDALONE_CRT_PATH}/src/runtime/crt/include \
 	-I${PWD}/include \
 	-I${CORSTONE_300_PATH} \
 	-I${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Include/ \
 	-I${CMSIS_PATH}/CMSIS/Core/Include \
-	-I${CMSIS_PATH}/CMSIS/NN/Include \
+	-I${CMSIS_NN_PATH}/Include \
 	-I${CMSIS_PATH}/CMSIS/DSP/Include \
 	-I$(abspath $(BUILD_DIR))/codegen/host/include
 CMSIS_NN_CMAKE_FLAGS = -DCMAKE_TOOLCHAIN_FILE=$(abspath $(BUILD_DIR))/../arm-none-eabi-gcc.cmake \
@@ -56,7 +61,7 @@ DEMO_MAIN = src/demo_bare_metal.c
 CODEGEN_SRCS = $(wildcard $(abspath $(BUILD_DIR))/codegen/host/src/*.c)
 CODEGEN_OBJS = $(subst .c,.o,$(CODEGEN_SRCS))
 CMSIS_STARTUP_SRCS = $(wildcard ${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Source/*.c)
-CMSIS_NN_SRCS = $(shell find ${CMSIS_PATH}/CMSIS/NN/Source/*/*.c)
+CMSIS_NN_SRCS = $(shell find ${CMSIS_NN_PATH}/Source/*/*.c)
 UART_SRCS = $(wildcard ${CORSTONE_300_PATH}/*.c)
 
 demo: $(BUILD_DIR)/demo

diff --git a/apps/microtvm/cmsisnn/README.md b/apps/microtvm/cmsisnn/README.md
@@ -35,6 +35,7 @@ If the demo is not run in the ci_cpu Docker container, then you will need the fo
   - [GCC toolchain from Arm(R)](https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2)
   - [Arm(R) Ethos(TM)-U NPU driver stack](https://review.mlplatform.org)
   - [CMSIS](https://github.com/ARM-software/CMSIS_5)
+  - [CMSIS NN](https://github.com/ARM-software/CMSIS-NN)
 - The python libraries listed in the requirements.txt of this directory
   - These can be installed by running the following from the current directory:
     ```bash

diff --git a/apps/microtvm/ethosu/Makefile b/apps/microtvm/ethosu/Makefile
@@ -32,6 +32,11 @@ CMAKE ?= cmake
 CC = arm-none-eabi-gcc
 AR = arm-none-eabi-ar
 RANLIB = arm-none-eabi-ranlib
+ifeq ($(shell [ -d ${CMSIS_PATH}/CMSIS-NN ]; echo $$?), 0)
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS-NN
+else
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS/NN
+endif
 PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	-I${STANDALONE_CRT_PATH}/include \
 	-I${STANDALONE_CRT_PATH}/src/runtime/crt/include \
@@ -40,7 +45,7 @@ PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	-I${ETHOSU_PATH}/core_driver/include \
 	-I${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Include/ \
 	-I${CMSIS_PATH}/CMSIS/Core/Include \
-	-I${CMSIS_PATH}/CMSIS/NN/Include \
+	-I${CMSIS_NN_PATH}/Include \
 	-I${CMSIS_PATH}/CMSIS/DSP/Include \
 	-I$(abspath $(BUILD_DIR))/codegen/host/include \
 	-DETHOSU_TEST_RUNNER_TOL=${ETHOSU_TEST_RUNNER_TOL}
@@ -78,7 +83,7 @@ endif
 CODEGEN_SRCS = $(wildcard $(abspath $(BUILD_DIR))/codegen/host/src/*.c)
 CODEGEN_OBJS = $(subst .c,.o,$(CODEGEN_SRCS))
 CMSIS_STARTUP_SRCS = $(wildcard ${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Source/*.c)
-CMSIS_NN_SOFTMAX_SRCS = $(shell find ${CMSIS_PATH}/CMSIS/NN/Source/SoftmaxFunctions/*.c)
+CMSIS_NN_SOFTMAX_SRCS = $(shell find ${CMSIS_NN_PATH}/Source/SoftmaxFunctions/*.c)
 UART_SRCS = $(wildcard ${CORSTONE_300_PATH}/*.c)
 
 demo: $(BUILD_DIR)/demo

diff --git a/apps/microtvm/ethosu/README.md b/apps/microtvm/ethosu/README.md
@@ -35,6 +35,7 @@ If the demo is not run in the ci_cpu Docker container, then you will need the fo
   - [GCC toolchain from Arm(R)](https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2)
   - [Arm(R) Ethos(TM)-U NPU driver stack](https://review.mlplatform.org)
   - [CMSIS](https://github.com/ARM-software/CMSIS_5)
+  - [CMSIS NN](https://github.com/ARM-software/CMSIS-NN)
 - The python libraries listed in the requirements.txt of this directory
   - These can be installed by running the following from the current directory:
     ```bash

diff --git a/apps/microtvm/zephyr/template_project/CMakeLists.txt.template b/apps/microtvm/zephyr/template_project/CMakeLists.txt.template
@@ -31,21 +31,27 @@ find_package(Zephyr HINTS $ENV{ZEPHYR_BASE})
 project(microtvm_autogenerated_project)
 
 if(DEFINED CMSIS_PATH)
+  if (EXISTS ${CMSIS_PATH}/CMSIS-NN)
+      set(CMSIS_NN_PATH ${CMSIS_PATH}/CMSIS-NN)
+  else()
+      set(CMSIS_NN_PATH ${CMSIS_PATH}/CMSIS/NN)
+  endif()
+
   file(GLOB_RECURSE cmsis_lib_srcs
-    ${CMSIS_PATH}/CMSIS/NN/Source/ActivationFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/BasicMathFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConcatenationFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/FullyConnectedFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/PoolingFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ReshapeFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/SoftmaxFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/ActivationFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/BasicMathFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/ConcatenationFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/FullyConnectedFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/PoolingFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/ReshapeFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/SoftmaxFunctions/*.c
   )
 
   set(cmsis_includes
     ${CMSIS_PATH}/CMSIS/Core/Include
-    ${CMSIS_PATH}/CMSIS/NN/Include
+    ${CMSIS_NN_PATH}/Include
     ${CMSIS_PATH}/CMSIS/DSP/Include
     ${CMSIS_PATH}/CMSIS/DSP/Include/dsp
   )

diff --git a/apps/microtvm/zephyr_cmsisnn/CMakeLists.txt b/apps/microtvm/zephyr_cmsisnn/CMakeLists.txt
@@ -51,22 +51,34 @@ set(DATA_FILES
     ${CMAKE_CURRENT_BINARY_DIR}/outputs.c
     ${CMAKE_CURRENT_BINARY_DIR}/labels.c
 )
+
+if (EXISTS ${CMSIS_PATH}/CMSIS-NN)
+    set(CMSIS_NN_PATH ${CMSIS_PATH}/CMSIS-NN)
+    set(CMSIS_NN_ADDITIONAL_SOURCES
+        ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_convolve_1x1_s8.c
+        ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_mat_mult_nt_t_s8.c
+    )
+else()
+    set(CMSIS_NN_PATH ${CMSIS_PATH}/CMSIS/NN)
+endif()
+
 set(CMSIS_SOURCES
-    ${CMSIS_PATH}/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_wrapper_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_s8_opt.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_depthwise_conv_nt_t_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_depthwise_conv_nt_t_padded_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_wrapper_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1_x_n_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1x1_s8_fast.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mat_mul_core_4x_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_vec_mat_mult_t_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mat_mul_core_1x_s8.c
+    ${CMSIS_NN_PATH}/Source/SoftmaxFunctions/arm_softmax_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_depthwise_conv_wrapper_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_depthwise_conv_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_depthwise_conv_s8_opt.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_depthwise_conv_nt_t_s8.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_depthwise_conv_nt_t_padded_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_convolve_wrapper_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_convolve_1_x_n_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_convolve_1x1_s8_fast.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_convolve_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_nn_mat_mult_s8.c
+    ${CMSIS_NN_PATH}/Source/FullyConnectedFunctions/arm_fully_connected_s8.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_mat_mul_core_4x_s8.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_vec_mat_mult_t_s8.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_mat_mul_core_1x_s8.c
+    ${CMSIS_NN_ADDITIONAL_SOURCES}
 )
 
 add_custom_command(
@@ -93,5 +105,5 @@ target_sources(app PRIVATE
 target_include_directories(app
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/runtime/include ${CMAKE_CURRENT_BINARY_DIR}/codegen/host/include
-    PUBLIC ${CMSIS_PATH}/CMSIS/NN/Include/ ${CMSIS_PATH}/CMSIS/DSP/Include
+    PUBLIC ${CMSIS_NN_PATH}/Include/ ${CMSIS_PATH}/CMSIS/DSP/Include
 )
diff --git a/apps/microtvm/zephyr_cmsisnn/README.md b/apps/microtvm/zephyr_cmsisnn/README.md
@@ -37,6 +37,11 @@ Checkout [CMSIS_5](https://github.com/ARM-software/CMSIS_5.git) (default is `/op
 git clone "https://github.com/ARM-software/CMSIS_5.git" cmsis
 ```
 
+Checkout [CMSIS NN](https://github.com/ARM-software/CMSIS-NN.git) (default is `/opt/arm/ethosu/cmsis/CMSIS-NN` to reflect `tlcpack/ci_cortexm`):
+```
+git clone "https://github.com/ARM-software/CMSIS-NN.git" cmsis/CMSIS-NN
+```
+
 And run the demo using `west`, with the path to CMSIS:
 ```
 west build -t run -- -DCMSIS_PATH=/opt/arm/ethosu/cmsis

diff --git a/docker/install/ubuntu_install_cmsis.sh b/docker/install/ubuntu_install_cmsis.sh
@@ -48,4 +48,8 @@ wget ${CMSIS_URL} -O "${DOWNLOAD_PATH}"
 echo "$CMSIS_SHASUM" ${DOWNLOAD_PATH} | sha512sum -c
 tar -xf "${DOWNLOAD_PATH}" -C "${INSTALLATION_PATH}" --strip-components=1
 touch "${INSTALLATION_PATH}"/"${CMSIS_SHA}".sha
+
+CMSIS_NN_TAG="v4.0.0"
+CMSIS_NN_URL="https://github.com/ARM-software/CMSIS-NN.git"
+git clone ${CMSIS_NN_URL} --branch ${CMSIS_NN_TAG} --single-branch ${INSTALLATION_PATH}/CMSIS-NN
 echo "SUCCESS"
diff --git a/docker/install/ubuntu_install_ethosu_driver_stack.sh b/docker/install/ubuntu_install_ethosu_driver_stack.sh
@@ -85,6 +85,9 @@ cmake -DCMAKE_TOOLCHAIN_FILE=${ethosu_dir}/core_platform/cmake/toolchain/arm-non
 make
 
 # Build NN Library
+mkdir ${CMSIS_PATH}/CMSIS-NN/build/ && cd ${CMSIS_PATH}/CMSIS-NN/build/
+cmake .. -DCMAKE_TOOLCHAIN_FILE=${ethosu_dir}/core_platform/cmake/toolchain/arm-none-eabi-gcc.cmake -DTARGET_CPU=cortex-m55 -DBUILD_CMSIS_NN_FUNCTIONS=YES -DCMSIS_PATH=${CMSIS_PATH}
+
 mkdir ${CMSIS_PATH}/CMSIS/NN/build/ && cd ${CMSIS_PATH}/CMSIS/NN/build/
 cmake .. -DCMAKE_TOOLCHAIN_FILE=${ethosu_dir}/core_platform/cmake/toolchain/arm-none-eabi-gcc.cmake -DTARGET_CPU=cortex-m55 -DBUILD_CMSIS_NN_FUNCTIONS=YES
 make
diff --git a/python/tvm/micro/project_api/server.py b/python/tvm/micro/project_api/server.py
@@ -804,7 +804,7 @@ def default_project_options(**kw) -> typing.List[ProjectOption]:
             "cmsis_path",
             optional=["generate_project"],
             type="str",
-            default=os.environ.get("CMSIS_PATH", None),
+            default=None,
             help="Path to the CMSIS directory.",
         ),
         ProjectOption(

diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py
@@ -605,6 +605,7 @@ def test_schedule_build_with_cmsis_dependency(workspace_dir, board, microtvm_deb
         "project_type": "host_driven",
         "verbose": bool(build_config.get("debug")),
         "board": board,
+        "cmsis_path": os.getenv("CMSIS_PATH"),
         "use_fvp": bool(use_fvp),
     }
 
@@ -623,7 +624,7 @@ def test_schedule_build_with_cmsis_dependency(workspace_dir, board, microtvm_deb
     assert "CMSIS/DSP/Include" in cmake_content
     assert "CMSIS/DSP/Include/dsp" in cmake_content
     assert "CMSIS/DSP/Include" in cmake_content
-    assert "CMSIS/NN/Include" in cmake_content
+    # assert "CMSIS-NN/Include" in cmake_content
 
 
 if __name__ == "__main__":

diff --git a/tests/python/relay/aot/corstone300.mk b/tests/python/relay/aot/corstone300.mk
@@ -48,6 +48,11 @@ CC = arm-none-eabi-gcc
 AR = arm-none-eabi-ar
 RANLIB = arm-none-eabi-ranlib
 CC_OPTS = CC=$(CC) AR=$(AR) RANLIB=$(RANLIB)
+ifeq ($(shell [ -d ${CMSIS_PATH}/CMSIS-NN ]; echo $$?), 0)
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS-NN
+else
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS/NN
+endif
 PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	${CFLAGS} \
 	-I$(build_dir)/../include \
@@ -57,7 +62,7 @@ PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	-I${DRIVER_PATH}/include \
 	-I${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Include/ \
 	-I${CMSIS_PATH}/CMSIS/Core/Include \
-	-I${CMSIS_PATH}/CMSIS/NN/Include \
+	-I${CMSIS_NN_PATH}/Include \
 	-I${CMSIS_PATH}/CMSIS/DSP/Include \
 	-isystem$(STANDALONE_CRT_DIR)/include
 DRIVER_CMAKE_FLAGS = -DCMAKE_TOOLCHAIN_FILE=$(ETHOSU_TEST_ROOT)/arm-none-eabi-gcc.cmake \
@@ -78,7 +83,7 @@ CC_CODEGEN_SRCS = $(shell find $(abspath $(CODEGEN_ROOT)/host/src/*.cc))
 C_CODEGEN_OBJS = $(subst .c,.o,$(C_CODEGEN_SRCS))
 CC_CODEGEN_OBJS = $(subst .cc,.o,$(CC_CODEGEN_SRCS))
 CMSIS_STARTUP_SRCS = $(shell find ${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Source/*.c)
-CMSIS_NN_SRCS = $(shell find ${CMSIS_PATH}/CMSIS/NN/Source/*/*.c)
+CMSIS_NN_SRCS = $(shell find ${CMSIS_NN_PATH}/Source/*/*.c)
 UART_SRCS = $(shell find ${PLATFORM_PATH}/*.c)
 
 ifdef ETHOSU_TEST_ROOT