Add binary build support

zhongTao99 · Jan 26, 2025 · f792c4c · f792c4c
1 parent e88c7f7
commit f792c4c
Show file tree

Hide file tree

Showing 8 changed files with 159 additions and 74 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -84,7 +84,7 @@ RUN if [ -z ${OLLAMA_SKIP_ROCM_GENERATE} ] ; then \
 # Jetsons need to be built in discrete stages
 FROM --platform=linux/arm64 nvcr.io/nvidia/l4t-jetpack:${JETPACK_5} AS runners-jetpack5-arm64
 ARG GOLANG_VERSION
-RUN apt-get update && apt-get install -y git curl ccache && \
+RUN export http_proxy=http://172.17.0.1:7890 && https_proxy=http://172.17.0.1:7890 && apt-get update && apt-get install -y git curl ccache && \
     curl -s -L https://dl.google.com/go/go${GOLANG_VERSION}.linux-arm64.tar.gz | tar xz -C /usr/local && \
     ln -s /usr/local/go/bin/go /usr/local/bin/go && \
     ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt && \
@@ -103,7 +103,7 @@ RUN --mount=type=cache,target=/root/.ccache \
 
 FROM --platform=linux/arm64 nvcr.io/nvidia/l4t-jetpack:${JETPACK_6} AS runners-jetpack6-arm64
 ARG GOLANG_VERSION
-RUN apt-get update && apt-get install -y git curl ccache && \
+RUN export http_proxy=http://172.17.0.1:7890 && https_proxy=http://172.17.0.1:7890 && apt-get update && apt-get install -y git curl ccache && \
     curl -s -L https://dl.google.com/go/go${GOLANG_VERSION}.linux-arm64.tar.gz | tar xz -C /usr/local && \
     ln -s /usr/local/go/bin/go /usr/local/bin/go && \
     ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt && \
@@ -154,7 +154,7 @@ RUN --mount=type=cache,target=/root/.ccache \
         echo "No Ascend Toolkit found"; \
         exit 1; \
     fi && \
-    make --trace dist
+    make dist
 RUN cd dist/linux-$GOARCH && \
     tar -cf - . | pigz --best > ../ollama-linux-$GOARCH-cann.tgz
 
@@ -182,6 +182,7 @@ FROM --platform=linux/amd64 scratch AS dist-amd64
 COPY --from=build-amd64 /go/src/github.com/ollama/ollama/dist/ollama-linux-*.tgz /
 FROM --platform=linux/arm64 scratch AS dist-arm64
 COPY --from=build-arm64 /go/src/github.com/ollama/ollama/dist/ollama-linux-*.tgz /
+COPY --from=cann_build-arm64 /go/src/github.com/ollama/ollama/dist/ollama-linux-*.tgz /
 FROM dist-$TARGETARCH AS dist
 
 

diff --git a/Makefile b/Makefile
@@ -6,6 +6,7 @@ include make/common-defs.make
 include make/cuda-v11-defs.make
 include make/cuda-v12-defs.make
 include make/rocm-defs.make
+include make/cann-defs.make
 
 ifeq ($(CUSTOM_CPU_FLAGS),)
 ifeq ($(ARCH),amd64)
@@ -35,8 +36,10 @@ endif
 endif
 
 ifeq ($(OLLAMA_SKIP_CANN_GENERATE),)
+ifneq ($(CANN_INSTALL_DIR),)
 	RUNNER_TARGETS += cann
 endif
+endif
 
 all: runners exe
 

diff --git a/llama/Makefile.detect b/llama/Makefile.detect
diff --git a/make/Makefile.cann b/make/Makefile.cann
@@ -3,6 +3,7 @@
 # Note: at present we only support a single CANN version
 
 include make/common-defs.make
+include make/cann-defs.make
 
 ifeq ($(OS),windows)
 	@echo "CANN does not currently support windows."
@@ -38,7 +39,8 @@ CANN_KERNEL_CMAKE_FLAGS = -DCANN_INSTALL_DIR=$(CANN_INSTALL_DIR)
 GGML_SHARED_BUILD_PRE_JOB := $(CANN_BUILD_DIR)/$(SHARED_PREFIX)ascendc_kernels.$(SHARED_EXT)
 
 GPU_COMPILER_LINUX := $(shell X=$$(which g++ 2>/dev/null) && echo $$X)
-GPU_COMPILER:=$(GPU_COMPILER_LINUX)
+GPU_COMPILER:=$(shell X=$$(which g++ 2>/dev/null) && echo $$X)
+GPU_C_COMPILER:=$(shell X=$$(which gcc 2>/dev/null) && echo $$X)
 GPU_RUNNER_LIBS_SHORT := ascendcl nnopbase opapi acl_op_compiler ascendc_kernels
 GPU_LIBS = $(wildcard $(addsuffix .$(SHARED_EXT),$(addprefix $(GPU_LIB_DIR)/$(SHARED_PREFIX),$(GPU_RUNNER_LIBS_SHORT))))
 CANN_DIST_DEPS_DIR = ./dist/$(OS)-$(ARCH)-cann/lib/ollama
@@ -107,8 +109,8 @@ $(CANN_BUILD_DIR)/$(SHARED_PREFIX)ascendc_kernels.$(SHARED_EXT):
 	@rm -rf $(CANN_KERNEL_BUILD_DIR)
 	@-mkdir -p $(CANN_KERNEL_BUILD_DIR)
 	$(info CANN_INSTALL_DIR: $(CANN_INSTALL_DIR))
-	cmake $(CANN_KERNEL_SRC_DIR) -G "Unix Makefiles" $(CANN_KERNEL_CMAKE_FLAGS) -B $(CANN_KERNEL_BUILD_DIR) -C $(CANN_KERNEL_SRC_DIR)/CMakeLists.txt -DCMAKE_BUILD_TYPE=debug
-	cmake --build $(CANN_KERNEL_BUILD_DIR) --config debug
+	@cmake $(CANN_KERNEL_SRC_DIR) -G "Unix Makefiles" $(CANN_KERNEL_CMAKE_FLAGS) -B $(CANN_KERNEL_BUILD_DIR) -C $(CANN_KERNEL_SRC_DIR)/CMakeLists.txt -DCMAKE_BUILD_TYPE=debug
+	@cmake --build $(CANN_KERNEL_BUILD_DIR) --config debug
 	@-mkdir -p $(DIST_LIB_DIR)
 	$(CP) $(CANN_KERNEL_BUILD_DIR)/lib/$(SHARED_PREFIX)ascendc_kernels.$(SHARED_EXT) $(DIST_LIB_DIR)
 	$(CP) $(CANN_KERNEL_BUILD_DIR)/lib/$(SHARED_PREFIX)ascendc_kernels.$(SHARED_EXT) $(RUNNERS_BUILD_DIR)/$(GPU_RUNNER_NAME)/

diff --git a/make/cann-defs.make b/make/cann-defs.make
@@ -0,0 +1,9 @@
+# Common definitions for the various Makefiles which set cann settings
+# No rules are defined here so this is safe to include at the beginning of other makefiles
+
+ifeq ($(OS),linux)
+	CANN_INSTALL_DIR?=$(shell ls -d /usr/local/Ascend/ascend-toolkit/latest 2>/dev/null)
+ifeq ($(strip $(CANN_INSTALL_DIR)),)
+	CANN_INSTALL_DIR=$(shell ls -d "$(HOME)/Ascend/ascend-toolkit/latest" 2>/dev/null)
+endif
+endif
diff --git a/make/gpu.make b/make/gpu.make
@@ -5,6 +5,11 @@ dummy:
 	$(error This makefile is not meant to build directly, but instead included in other Makefiles that set required variables)
 endif
 
+# Allow specify compiler for c source 
+ifndef GPU_C_COMPILER
+	GPU_C_COMPILER := $(GPU_COMPILER)
+endif
+
 GPU_GOFLAGS="-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$(VERSION)\" $(EXTRA_GOLDFLAGS) $(TARGET_LDFLAGS)"
 
 # TODO Unify how we handle dependencies in the dist/packaging and install flow
@@ -57,7 +62,7 @@ $(BUILD_DIR)/%.$(GPU_RUNNER_NAME).$(OBJ_EXT): %.cu
 	$(CCACHE) $(GPU_COMPILER) -c $(GPU_COMPILER_CFLAGS) $(GPU_COMPILER_CUFLAGS) $(GPU_RUNNER_ARCH_FLAGS) -o $@ $<
 $(BUILD_DIR)/%.$(GPU_RUNNER_NAME).$(OBJ_EXT): %.c
 	@-mkdir -p $(dir $@)
-	$(CCACHE) gcc -c $(GPU_COMPILER_CFLAGS) -o $@ $<
+	$(CCACHE) $(GPU_C_COMPILER) -c $(GPU_COMPILER_CFLAGS) -o $@ $<
 $(BUILD_DIR)/%.$(GPU_RUNNER_NAME).$(OBJ_EXT): %.cpp
 	@-mkdir -p $(dir $@)
 	$(CCACHE) $(GPU_COMPILER) -c $(GPU_COMPILER_CXXFLAGS) -o $@ $<

diff --git a/scripts/install.sh b/scripts/install.sh
@@ -199,22 +199,31 @@ check_gpu() {
             case $2 in
                 nvidia) available lspci && lspci -d '10de:' | grep -q 'NVIDIA' || return 1 ;;
                 amdgpu) available lspci && lspci -d '1002:' | grep -q 'AMD' || return 1 ;;
+                ascend) available lspci && lspci -d '19e5:' | grep -q 'Processing accelerators: Huawei' || return 1 ;;
             esac ;;
         lshw)
             case $2 in
                 nvidia) available lshw && $SUDO lshw -c display -numeric -disable network | grep -q 'vendor: .* \[10DE\]' || return 1 ;;
                 amdgpu) available lshw && $SUDO lshw -c display -numeric -disable network | grep -q 'vendor: .* \[1002\]' || return 1 ;;
+                ascend) available lshw && $SUDO lshw -c display -numeric -disable network | grep -q 'vendor: .* \[19E5\]' || return 1 ;;
             esac ;;
         nvidia-smi) available nvidia-smi || return 1 ;;
+        npu-smi) available npu-smi || return 1 ;;
     esac
 }
 
 if check_gpu nvidia-smi; then
     status "NVIDIA GPU installed."
     exit 0
 fi
+: << 'COMMENT'
+if check_gpu npu-smi; then
+    status "ASCEND GPU installed."
+    exit 0
+fi
+COMMENT
 
-if ! check_gpu lspci nvidia && ! check_gpu lshw nvidia && ! check_gpu lspci amdgpu && ! check_gpu lshw amdgpu; then
+if ! check_gpu lspci nvidia && ! check_gpu lshw nvidia && ! check_gpu lspci amdgpu && ! check_gpu lshw amdgpu && ! check_gpu lspci ascend && ! check_gpu lshw ascend; then
     install_success
     warning "No NVIDIA/AMD GPU detected. Ollama will run in CPU-only mode."
     exit 0
@@ -322,51 +331,131 @@ if [ -z "$PACKAGE_MANAGER" ]; then
     error "Unknown package manager. Skipping CUDA installation."
 fi
 
-if ! check_gpu nvidia-smi || [ -z "$(nvidia-smi | grep -o "CUDA Version: [0-9]*\.[0-9]*")" ]; then
-    case $OS_NAME in
-        centos|rhel) install_cuda_driver_yum 'rhel' $(echo $OS_VERSION | cut -d '.' -f 1) ;;
-        rocky) install_cuda_driver_yum 'rhel' $(echo $OS_VERSION | cut -c1) ;;
-        fedora) [ $OS_VERSION -lt '39' ] && install_cuda_driver_yum $OS_NAME $OS_VERSION || install_cuda_driver_yum $OS_NAME '39';;
-        amzn) install_cuda_driver_yum 'fedora' '37' ;;
-        debian) install_cuda_driver_apt $OS_NAME $OS_VERSION ;;
-        ubuntu) install_cuda_driver_apt $OS_NAME $(echo $OS_VERSION | sed 's/\.//') ;;
-        *) exit ;;
-    esac
-fi
+if check_gpu lspci nvidia || check_gpu lshw nvidia; then
+    if ! check_gpu nvidia-smi || [ -z "$(nvidia-smi | grep -o "CUDA Version: [0-9]*\.[0-9]*")" ]; then
+        case $OS_NAME in
+            centos|rhel) install_cuda_driver_yum 'rhel' $(echo $OS_VERSION | cut -d '.' -f 1) ;;
+            rocky) install_cuda_driver_yum 'rhel' $(echo $OS_VERSION | cut -c1) ;;
+            fedora) [ $OS_VERSION -lt '39' ] && install_cuda_driver_yum $OS_NAME $OS_VERSION || install_cuda_driver_yum $OS_NAME '39';;
+            amzn) install_cuda_driver_yum 'fedora' '37' ;;
+            debian) install_cuda_driver_apt $OS_NAME $OS_VERSION ;;
+            ubuntu) install_cuda_driver_apt $OS_NAME $(echo $OS_VERSION | sed 's/\.//') ;;
+            *) exit ;;
+        esac
+    fi
 
-if ! lsmod | grep -q nvidia || ! lsmod | grep -q nvidia_uvm; then
-    KERNEL_RELEASE="$(uname -r)"
-    case $OS_NAME in
-        rocky) $SUDO $PACKAGE_MANAGER -y install kernel-devel kernel-headers ;;
-        centos|rhel|amzn) $SUDO $PACKAGE_MANAGER -y install kernel-devel-$KERNEL_RELEASE kernel-headers-$KERNEL_RELEASE ;;
-        fedora) $SUDO $PACKAGE_MANAGER -y install kernel-devel-$KERNEL_RELEASE ;;
-        debian|ubuntu) $SUDO apt-get -y install linux-headers-$KERNEL_RELEASE ;;
-        *) exit ;;
-    esac
+    if ! lsmod | grep -q nvidia || ! lsmod | grep -q nvidia_uvm; then
+        KERNEL_RELEASE="$(uname -r)"
+        case $OS_NAME in
+            rocky) $SUDO $PACKAGE_MANAGER -y install kernel-devel kernel-headers ;;
+            centos|rhel|amzn) $SUDO $PACKAGE_MANAGER -y install kernel-devel-$KERNEL_RELEASE kernel-headers-$KERNEL_RELEASE ;;
+            fedora) $SUDO $PACKAGE_MANAGER -y install kernel-devel-$KERNEL_RELEASE ;;
+            debian|ubuntu) $SUDO apt-get -y install linux-headers-$KERNEL_RELEASE ;;
+            *) exit ;;
+        esac
+
+        NVIDIA_CUDA_VERSION=$($SUDO dkms status | awk -F: '/added/ { print $1 }')
+        if [ -n "$NVIDIA_CUDA_VERSION" ]; then
+            $SUDO dkms install $NVIDIA_CUDA_VERSION
+        fi
 
-    NVIDIA_CUDA_VERSION=$($SUDO dkms status | awk -F: '/added/ { print $1 }')
-    if [ -n "$NVIDIA_CUDA_VERSION" ]; then
-        $SUDO dkms install $NVIDIA_CUDA_VERSION
+        if lsmod | grep -q nouveau; then
+            status 'Reboot to complete NVIDIA CUDA driver install.'
+            exit 0
+        fi
+
+        $SUDO modprobe nvidia
+        $SUDO modprobe nvidia_uvm
+    fi
+    # make sure the NVIDIA modules are loaded on boot with nvidia-persistenced
+    if available nvidia-persistenced; then
+        $SUDO touch /etc/modules-load.d/nvidia.conf
+        MODULES="nvidia nvidia-uvm"
+        for MODULE in $MODULES; do
+            if ! grep -qxF "$MODULE" /etc/modules-load.d/nvidia.conf; then
+                echo "$MODULE" | $SUDO tee -a /etc/modules-load.d/nvidia.conf > /dev/null
+            fi
+        done
     fi
+fi
+
+install_ascend_driver_yum() {
+    status 'Installing ASCNED driver version: $ASCEND_DRIVER_VERSION ,firmware version: $ASCEND_FIRMWARE_VERSION...'
+    $SUDO $PACKAGE_MANAGER -y install gcc gcc-c++ make cmake unzip zlib-devel libffi-devel openssl-devel pciutils net-tools sqlite-devel lapack-devel gcc-gfortran python3-devel
+    $SUDO groupadd -g HwHiAiUser
+    $SUDO useradd -g HwHiAiUser -d /home/HwHiAiUser -m HwHiAiUser -s /bin/bash
+    $SUDO usermod -aG HwHiAiUser $USER
+
+    # driver version, mabey get from it
+    # npu-smi info
+    # +------------------------------------------------------------------------------------------------+
+    # | npu-smi 24.1.rc1.b060            Version: 24.1.rc1.b060                                        |
+    wget "https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Ascend HDK/Ascend HDK $ASCEND_DRIVER_VERSION/Ascend-hdk-$1-npu-driver_$(echo "$ASCEND_DRIVER_VERSION" | tr '[:upper:]' '[:lower:]')_linux-$(uname -m).run"
+    $SUDO sh Ascend-hdk-$1-npu-driver_$(echo "$ASCEND_DRIVER_VERSION" | tr '[:upper:]' '[:lower:]')_linux-$(uname -m).run --full --install-for-all
+    rm -rf ./Ascend-hdk-$1-npu-driver_$(echo "$ASCEND_DRIVER_VERSION" | tr '[:upper:]' '[:lower:]')_linux-$(uname -m).run
+
+    wget "https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Ascend HDK/Ascend HDK $ASCEND_DRIVER_VERSION/Ascend-hdk-$1-npu-firmware_$ASCEND_FIRMWARE_VERSION.231.run"
+    sudo sh Ascend-hdk-$1-npu-firmware_$ASCEND_FIRMWARE_VERSION.231.run --full
+    rm -rf ./Ascend-hdk-$1-npu-firmware_$ASCEND_FIRMWARE_VERSION.231.run
+}
+
+install_ascend_driver_apt() {
+    status 'Installing ASCNED driver version: $ASCEND_DRIVER_VERSION ,firmware version: $ASCEND_FIRMWARE_VERSION...'
+    apt-get -y install gcc g++ make cmake zlib1g zlib1g-dev openssl libsqlite3-dev libssl-dev libffi-dev unzip pciutils net-tools libblas-dev gfortran libblas3 python3-dev
+    groupadd -g HwHiAiUser
+    useradd -g HwHiAiUser -d /home/HwHiAiUser -m HwHiAiUser -s /bin/bash
+    usermod -aG HwHiAiUser $USER
+
+    # driver version,mabey get from it
+    # npu-smi info
+    # +------------------------------------------------------------------------------------------------+
+    # | npu-smi 24.1.rc1.b060            Version: 24.1.rc1.b060                                        |
+    wget "https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Ascend HDK/Ascend HDK $ASCEND_DRIVER_VERSION/Ascend-hdk-$1-npu-driver_$(echo "$ASCEND_DRIVER_VERSION" | tr '[:upper:]' '[:lower:]')_linux-$(uname -m).run"
+    sh Ascend-hdk-$1-npu-driver_$(echo "$ASCEND_DRIVER_VERSION" | tr '[:upper:]' '[:lower:]')_linux-$(uname -m).run --full --install-for-all
+    rm -rf ./Ascend-hdk-$1-npu-driver_$(echo "$ASCEND_DRIVER_VERSION" | tr '[:upper:]' '[:lower:]')_linux-$(uname -m).run
+
+    wget "https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Ascend HDK/Ascend HDK $ASCEND_DRIVER_VERSION/Ascend-hdk-$1-npu-firmware_$ASCEND_FIRMWARE_VERSION.231.run"
+    sudo sh Ascend-hdk-$1-npu-firmware_$ASCEND_FIRMWARE_VERSION.231.run --full
+    rm -rf ./Ascend-hdk-$1-npu-firmware_$ASCEND_FIRMWARE_VERSION.231.run
+}
+
+install_ascend_cann() {
+    status 'Installing ASCNED CANN version: $ASCEND_CANN_VERSION...'
+    echo "ASCEND_CANN_VERSION=$ASCEND_CANN_VERSION, 1st paramenter=$1"
+    pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple attrs numpy==1.24.0 decorator sympy cffi pyyaml pathlib2 psutil protobuf scipy requests absl-py wheel typing_extensions
+    wget "https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/CANN/CANN $ASCEND_CANN_VERSION/Ascend-cann-toolkit_${ASCEND_CANN_VERSION}_linux-$(uname -m).run"
+    bash Ascend-cann-toolkit_${ASCEND_CANN_VERSION}_linux-$(uname -m).run --install
+    rm -rf ./Ascend-cann-toolkit_${ASCEND_CANN_VERSION}_linux-$(uname -m).run
+
+    wget "https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/CANN/CANN $ASCEND_CANN_VERSION/Ascend-cann-kernels-$1_${ASCEND_CANN_VERSION}_linux-$(uname -m).run"
+    bash Ascend-cann-kernels-$1_${ASCEND_CANN_VERSION}_linux-$(uname -m).run --install
+    rm -rf ./Ascend-cann-kernels-$1_${ASCEND_CANN_VERSION}_linux.run
+}
 
-    if lsmod | grep -q nouveau; then
-        status 'Reboot to complete NVIDIA CUDA driver install.'
-        exit 0
+# use env val: ASCEND_DRIVER_VERSION ASCEND_FIRMWARE_VERSIO and ASCEND_CANN_VERSION to get version 
+# ref:https://ascend.github.io/docs/sources/ascend/quick_install.html
+if check_gpu lspci ascend || check_gpu lshw ascend; then
+    if [ -z "$ASCEND_DRIVER_VERSION" ]; then
+        ASCEND_DRIVER_VERSION="24.1.RC2"
+    fi
+    if [ -z "$ASCEND_FIRMWARE_VERSION" ]; then
+        ASCEND_FIRMWARE_VERSION="7.3.0.1"
     fi
+    if [ -z "$ASCEND_CANN_VERSION" ]; then
+        ASCEND_CANN_VERSION="8.0.RC2"
+    fi
+    echo "after set ASCEND_DRIVER_VERSION=${ASCEND_DRIVER_VERSION}";
+    type=$(npu-smi info -m | grep 'Ascend' | awk '{print $5}' | head -n 1 | tr '[:upper:]' '[:lower:]'|sed 's/[0-9]$//')
+    case $OS_NAME in
+        openeuler) install_ascend_driver_yum $type ;;
+        ubuntu) install_ascend_driver_apt $type ;;
+        *) exit ;;
+    esac
 
-    $SUDO modprobe nvidia
-    $SUDO modprobe nvidia_uvm
-fi
+    install_ascend_cann $type
 
-# make sure the NVIDIA modules are loaded on boot with nvidia-persistenced
-if available nvidia-persistenced; then
-    $SUDO touch /etc/modules-load.d/nvidia.conf
-    MODULES="nvidia nvidia-uvm"
-    for MODULE in $MODULES; do
-        if ! grep -qxF "$MODULE" /etc/modules-load.d/nvidia.conf; then
-            echo "$MODULE" | $SUDO tee -a /etc/modules-load.d/nvidia.conf > /dev/null
-        fi
-    done
+    echo "source ~/Ascend/ascend-toolkit/set_env.sh" >> ~/.bashrc
+    source ~/.bashrc
 fi
 
 status "NVIDIA GPU ready."

diff --git a/scripts/rh_linux_deps.sh b/scripts/rh_linux_deps.sh
@@ -68,7 +68,10 @@ if [ "${MACHINE}" = "x86_64" ] ; then
     curl -s -L https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz | tar -Jx -C /tmp --strip-components 1 && \
     mv /tmp/ccache /usr/local/bin/
 else
-    # yum -y install epel-release
+    # openEuler does not have epel-release
+    if ! grep -i "openEuler" /etc/system-release >/dev/null; then
+        yum -y install epel-release
+    fi
     yum install -y ccache
 fi