Ascend Add docker build support

Dockerfile

@@ -4,6 +4,7 @@ ARG CUDA_VERSION_12=12.4.0
 ARG ROCM_VERSION=6.1.2
 ARG JETPACK_6=r36.2.0
 ARG JETPACK_5=r35.4.1
+ARG ASCEND_VERSION=8.0.rc2.beta1-910b-openeuler22.03-py3.10
 
 ### To create a local image for building linux binaries on mac or windows with efficient incremental builds
 #
@@ -119,6 +120,48 @@ RUN --mount=type=cache,target=/root/.ccache \
         DIST_LIB_DIR=/go/src/github.com/ollama/ollama/dist/linux-arm64-jetpack6/lib/ollama \
         DIST_GPU_RUNNER_DEPS_DIR=/go/src/github.com/ollama/ollama/dist/linux-arm64-jetpack6/lib/ollama/cuda_jetpack6
 
+FROM quay.io/ascend/cann:8.0.rc2.beta1-910b-openeuler22.03-py3.10 AS cann-builder-arm64
+ARG GOLANG_VERSION
+COPY ./scripts/rh_linux_deps.sh /
+RUN GOLANG_VERSION=${GOLANG_VERSION} sh -x /rh_linux_deps.sh
+RUN dnf clean all && \
+    dnf install -y \
+    zsh
+ARG CANN_INSTALL_DIR
+ENV GOARCH arm64
+ENV CGO_ENABLED 1
+ENTRYPOINT [ "zsh" ]
+
+#ENV LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/8.0.RC2/aarch64-linux/devlib/linux/aarch64:${LD_LIBRARY_PATH}
+FROM --platform=linux/arm64 cann-builder-arm64 AS cann_build-arm64
+ARG OLLAMA_SKIP_CUDA_GENERATE=1
+ARG OLLAMA_SKIP_ROCM_GENERATE=1
+ARG OLLAMA_FAST_BUILD=1
+ARG VERSION
+WORKDIR /go/src/github.com/ollama/ollama/
+COPY . .
+ENV CUSTOM_CPU_FLAGS=cann
+RUN --mount=type=cache,target=/root/.ccache \
+    cann_in_sys_path=/usr/local/Ascend/ascend-toolkit; \
+    cann_in_user_path=$HOME/Ascend/ascend-toolkit; \
+    if [ -f "${cann_in_sys_path}/set_env.sh" ]; then \
+        source ${cann_in_sys_path}/set_env.sh; \
+        export LD_LIBRARY_PATH=${cann_in_sys_path}/latest/lib64:${cann_in_sys_path}/latest/aarch64-linux/devlib:${LD_LIBRARY_PATH} ; \
+    elif [ -f "${cann_in_user_path}/set_env.sh" ]; then \
+        source "$HOME/Ascend/ascend-toolkit/set_env.sh"; \
+        export LD_LIBRARY_PATH=${cann_in_user_path}/latest/lib64:${cann_in_user_path}/latest/aarch64-linux/devlib:${LD_LIBRARY_PATH}; \ 
+    else \
+        echo "No Ascend Toolkit found"; \
+        exit 1; \
+    fi && \
+    make --trace dist
+RUN cd dist/linux-$GOARCH && \
+    tar -cf - . | pigz --best > ../ollama-linux-$GOARCH-cann.tgz
+
+FROM --platform=linux/amd64 cann_build-arm64 AS dist-arm64-cann
+COPY --from=cann_build-arm64 /go/src/github.com/ollama/ollama/dist/linux-arm64/bin/ /bin/
+COPY --from=cann_build-arm64 /go/src/github.com/ollama/ollama/dist/linux-arm64/lib/ /lib/
+
 FROM --platform=linux/arm64 unified-builder-arm64 AS build-arm64
 COPY . .
 ARG OLLAMA_SKIP_CUDA_GENERATE
@@ -188,6 +231,16 @@ ENV OLLAMA_HOST 0.0.0.0
 ENTRYPOINT ["/bin/ollama"]
 CMD ["serve"]
 
+FROM quay.io/ascend/cann:8.0.rc2.beta1-910b-openeuler22.03-py3.10 AS runtime-cann
+EXPOSE 11434
+ENV OLLAMA_HOST 0.0.0.0
+COPY --from=cann_build-arm64 /go/src/github.com/ollama/ollama/dist/linux-arm64/bin/ /bin/
+COPY --from=cann_build-arm64 /go/src/github.com/ollama/ollama/dist/linux-arm64/lib/ /lib/
+ENV LD_LIBRARY_PATH=/lib/ollama:/lib/ollama/runners/cann:${LD_LIBRARY_PATH}
+RUN echo "/bin/ollama serve" >> /usr/local/Ascend/ascend-toolkit/set_env.sh
+ENTRYPOINT ["/bin/bash", "-c", "source /usr/local/Ascend/ascend-toolkit/set_env.sh && exec \"$@\"", "--"]
+CMD ["bash"]
+
 FROM runtime-$TARGETARCH
 EXPOSE 11434
 ENV OLLAMA_HOST 0.0.0.0

llama/ggml-cann/kernels/CMakeLists.txt

@@ -1,8 +1,7 @@
 cmake_minimum_required(VERSION 3.14) # for add_link_options and implicit target directories.
-# 设置政策 CMP0057
+# let cmake could find C++ compiler successfully 
 cmake_policy(SET CMP0057 NEW)
 
-# 其他 CMake 配置
 project("llama.cann.kernel")
 
 file(GLOB SRC_FILES

make/Makefile.cann

@@ -10,6 +10,7 @@ ifeq ($(OS),windows)
 endif
 
 # Check if CANN_INSTALL_DIR is unset or empty, and ASCEND_TOOLKIT_HOME is defined
+$(info CANN: CANN_INSTALL_DIR=$(CANN_INSTALL_DIR) ASCEND_TOOLKIT_HOME=$(ASCEND_TOOLKIT_HOME) PATH=$(PATH))
 ifeq ($(strip $(CANN_INSTALL_DIR)),)
     ifneq ($(strip $(ASCEND_TOOLKIT_HOME)),)
         CANN_INSTALL_DIR := $(ASCEND_TOOLKIT_HOME)
@@ -81,10 +82,10 @@ include make/gpu.make
 # $(GPU_RUNNER_NAME): $(CANN_BUILD_DIR)/$(SHARED_PREFIX)ascendc_kernels.$(SHARED_EXT) WAIT
 # First detect Ascend SOC type
 SOC_VERSION :=
-SOC_TYPE :=
-detect_ascend_soc_type = $(shell npu-smi info | awk -F' ' 'NF > 0 && NR==7 {print $$3}')
-$(info CANN detect_ascend_soc_type auto-detected is $(detect_ascend_soc_type))
+SOC_TYPE := Ascend910B3
 ifeq ($(SOC_TYPE),)
+	detect_ascend_soc_type = $(shell npu-smi info | awk -F' ' 'NF > 0 && NR==7 {print $$3}')
+	$(info CANN detect_ascend_soc_type auto-detected is $(detect_ascend_soc_type))
     SOC_VERSION := $(call detect_ascend_soc_type)
     ifeq ($(SOC_VERSION),)
         $(error Auto-detect ascend soc type failed, please specify manually or check ascend device working normally.)
@@ -100,7 +101,7 @@ SOC_VERSION := $(shell echo $(SOC_TYPE) | tr '[:upper:]' '[:lower:]')
 SOC_TYPE_MAJOR_SN := $(shell echo $(SOC_VERSION) | grep -o [0-9][0-9][0-9][a-zA-Z]*)
 SOC_TYPE_COMPILE_OPTION := ASCEND_$(SOC_TYPE_MAJOR_SN)
 SOC_TYPE_COMPILE_OPTION := $(shell echo $(SOC_TYPE_COMPILE_OPTION) | tr '[:lower:]' '[:upper:]')
-CANN_KERNEL_CMAKE_FLAGS +=  -DSOC_TYPE=$(SOC_TYPE) -DSOC_VERSION=$(SOC_VERSION) -DSOC_TYPE_COMPILE_OPTION=$(SOC_TYPE_COMPILE_OPTION)
+CANN_KERNEL_CMAKE_FLAGS += -DSOC_TYPE=$(SOC_TYPE) -DSOC_VERSION=$(SOC_VERSION) -DSOC_TYPE_COMPILE_OPTION=$(SOC_TYPE_COMPILE_OPTION)
 # Second build cann llama.cpp custom kernel
 $(CANN_BUILD_DIR)/$(SHARED_PREFIX)ascendc_kernels.$(SHARED_EXT):
 	@rm -rf $(CANN_KERNEL_BUILD_DIR)

scripts/build_linux.sh

@@ -7,23 +7,23 @@
 # docker buildx create --name mybuilder --append desktop-linux --platform linux/arm64
 # docker buildx use mybuilder
 
-
+#        --output type=local,dest=./dist/ \
 set -eu
 
 . $(dirname $0)/env.sh
 
 mkdir -p dist
 
-docker buildx build \
-        --output type=local,dest=./dist/ \
-        --platform=${PLATFORM} \
+HTTP_PROXY=172.17.0.1:7890 HTTPS_PROXY=172.17.0.1:7890 docker --debug buildx build --load --progress=plain \
+        --platform=linux/arm64 \
         ${OLLAMA_COMMON_BUILD_ARGS} \
-        --target dist \
+        --target runtime-cann \
         -f Dockerfile \
+        -t ollama_ascend_cann:test_v0 \
         .
 
 # buildx behavior changes for single vs. multiplatform
 if echo $PLATFORM | grep "," > /dev/null ; then 
         mv -f ./dist/linux_*64/ollama* ./dist/
-        rmdir ./dist/linux_*64
+        # rmdir ./dist/linux_*64
 fi

scripts/rh_linux_deps.sh

@@ -5,7 +5,8 @@
 set -ex
 set -o pipefail
 MACHINE=$(uname -m)
-
+export HTTP_PROXY=172.17.0.1:7890
+export HTTPS_PROXY=172.17.0.1:7890
 if grep -i "centos" /etc/system-release >/dev/null; then
     # As of 7/1/2024 mirrorlist.centos.org has been taken offline, so adjust accordingly
     sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
@@ -31,6 +32,16 @@ if grep -i "centos" /etc/system-release >/dev/null; then
         ln -s /opt/rh/rh-git227/root/usr/bin/git /usr/local/bin/git
     fi
     dnf install -y devtoolset-10-gcc devtoolset-10-gcc-c++ pigz findutils
+elif grep -i "openEuler" /etc/system-release >/dev/null; then
+    # just for openeuler build envrionment, does not need to push to ollama github
+    dnf install -y git \
+        gcc \
+        gcc-c++ \
+        make \
+        cmake \
+        findutils \
+        yum \
+        pigz
 elif grep -i "rocky" /etc/system-release >/dev/null; then
     # Temporary workaround until rocky 8 AppStream ships GCC 10.4 (10.3 is incompatible with NVCC)
     cat << EOF > /etc/yum.repos.d/Rocky-Vault.repo
@@ -57,7 +68,7 @@ if [ "${MACHINE}" = "x86_64" ] ; then
     curl -s -L https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz | tar -Jx -C /tmp --strip-components 1 && \
     mv /tmp/ccache /usr/local/bin/
 else
-    yum -y install epel-release
+    # yum -y install epel-release
     yum install -y ccache
 fi