ppc64le support with CI integration

.github/workflows/ppc64le.yaml

@@ -0,0 +1,19 @@
+on: [pull_request]
+
+jobs:
+  ppc64le-build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger a Buildkite Build
+        uses: "buildkite/trigger-pipeline-action@v2.3.0"
+        with:
+          buildkite_api_access_token: "bkua_702df6e3ef4a32a8dff371e75e8db64d3286fad0"
+          pipeline: "ibm-14/ppc64le-build"
+          message:  ":github: Triggered from a GitHub Action"
+          build_env_vars: '{"TRIGGERED_FROM_GHA": "true"}'
+          build_meta_data: '{"FOO": "bar"}'
+          ignore_pipeline_branch_filter: 'true'
+          send_pull_request: 'true'
+          wait: 'true'
+          wait_interval: '60'
+          wait_timeout: '7200'

Makefile

@@ -81,9 +81,7 @@ ifeq (${ENABLE_AZURE}, false)
 	AZURE_OPTION := -Z
 endif
 
-milvus: build-cpp print-build-info build-go
-
-build-go:
+milvus: build-cpp print-build-info
 	@echo "Building Milvus ..."
 	@source $(PWD)/scripts/setenv.sh && \
 		mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \

internal/core/conanfile.py

@@ -14,11 +14,11 @@ class MilvusConan(ConanFile):
         "snappy/1.1.9#0519333fef284acd04806243de7d3070",
         "lzo/2.10#9517fc1bcc4d4cc229a79806003a1baa",
         "arrow/15.0.0#0456d916ff25d509e0724c5b219b4c45",
-        "openssl/3.1.2#02594c4c0a6e2b4feb3cd15119993597",
+        "openssl/3.1.6#977e6192420f30b53dd1fe3862835827",
         "aws-sdk-cpp/1.9.234#28d6d2c175975900ce292bafe8022c88",
         "googleapis/cci.20221108#65604e1b3b9a6b363044da625b201a2a",
         "benchmark/1.7.0#459f3bb1a64400a886ba43047576df3c",
-        "gtest/1.13.0#f9548be18a41ccc6367efcb8146e92be",
+        "gtest/1.14.0#25e2a474b4d1aecf5ff4f0555dcdf72c",
         "protobuf/3.21.4#fd372371d994b8585742ca42c12337f9",
         "rapidxml/1.13#10c11a4bfe073e131ed399d5c4f2e075",
         "yaml-cpp/0.7.0#9c87b3998de893cf2e5a08ad09a7a6e0",
@@ -74,6 +74,7 @@ class MilvusConan(ConanFile):
         "boost:without_locale": False,
         "glog:with_gflags": True,
         "glog:shared": True,
+        "libunwind:shared": True,
         "prometheus-cpp:with_pull": False,
         "fmt:header_only": True,
         "onetbb:tbbmalloc": False,
@@ -90,7 +91,7 @@ def configure(self):
 
     def requirements(self):
         if self.settings.os != "Macos":
-            self.requires("libunwind/1.7.2")
+            self.requires("libunwind/1.8.1")
 
     def imports(self):
         self.copy("*.dylib", "../lib", "lib")

internal/core/thirdparty/versions.txt

@@ -4,4 +4,4 @@ OPENTRACING_VERSION=v1.5.1
 PROTOBUF_VERSION=3.9.0
 LIBUNWIND_VERSION=1.6.2
 GPERFTOOLS_VERSION=2.9.1
-MILVUS_JEMALLOC_BUILD_VERSION=5.2.1
+MILVUS_JEMALLOC_BUILD_VERSION=5.3.0

internal/json/sonic.go

@@ -13,6 +13,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+// +build !ppc64le
 
 package json
 

internal/json/sonic_ppc64le.go

@@ -0,0 +1,26 @@
+package json
+
+import (
+       "encoding/json"
+)
+
+var (
+       // Marshal is exported by gin/json package.
+       Marshal = json.Marshal
+       // Unmarshal is exported by gin/json package.
+       Unmarshal = json.Unmarshal
+       // MarshalIndent is exported by gin/json package.
+       MarshalIndent = json.MarshalIndent
+       // NewDecoder is exported by gin/json package.
+       NewDecoder = json.NewDecoder
+       // NewEncoder is exported by gin/json package.
+       NewEncoder = json.NewEncoder
+)
+
+type (
+        Delim      = json.Delim
+        Decoder    = json.Decoder
+        Number     = json.Number
+        RawMessage = json.RawMessage
+)
+

scripts/3rdparty_build.sh

@@ -72,7 +72,7 @@ case "${unameOut}" in
     if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
       conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -r default-conan-local -u || { echo 'conan install failed'; exit 1; }
     else
-      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 -r default-conan-local -u || { echo 'conan install failed'; exit 1; }
+      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -c tools.build:jobs=$(nproc) -s compiler.libcxx=libstdc++11 -u || { echo 'conan install failed'; exit 1; }
     fi
     ;;
   *)

scripts/ppc64le-build-ubi9.3.sh

@@ -0,0 +1,123 @@
+#!/bin/bash -ex
+
+CMAKE_VERSION=3.30.5
+PYTHON_VERSION=3.10.2
+GO_VERSION=1.22.6
+SCRIPT_PATH=$(dirname $(realpath $0))
+wdir=`pwd`
+
+create_cmake_conanfile()
+{
+touch /usr/local/cmake/conanfile.py
+cat <<EOT >> /usr/local/cmake/conanfile.py
+from conans import ConanFile, tools
+class CmakeConan(ConanFile):
+  name = "cmake"
+  package_type = "application"
+  version = "${CMAKE_VERSION}"
+  description = "CMake, the cross-platform, open-source build system."
+  homepage = "https://github.com/Kitware/CMake"
+  license = "BSD-3-Clause"
+  topics = ("build", "installer")
+  settings = "os", "arch"
+  def package(self):
+    self.copy("*")
+  def package_info(self):
+    self.cpp_info.libs = tools.collect_libs(self)
+EOT
+}
+
+#Install centos and epel repos
+yum config-manager --add-repo https://mirror.stream.centos.org/9-stream/CRB/ppc64le/os
+yum config-manager --add-repo https://mirror.stream.centos.org/9-stream/AppStream//ppc64le/os
+yum config-manager --add-repo https://mirror.stream.centos.org/9-stream/BaseOS/ppc64le/os
+#rpm --import http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-Official
+rpm --import https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official
+dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+
+#Install and setup deps
+yum install -y --allowerasing make wget git sudo curl zip unzip tar pkg-config perl-IPC-Cmd perl-Digest-SHA perl-FindBin perl-File-Compare openssl-devel scl-utils openblas-devel rust cargo gcc gcc-c++ libstdc++-static which libaio libuuid-devel ncurses-devel ccache libtool m4 autoconf automake ninja-build zlib-devel libffi-devel java-11-openjdk-devel gfortran yum-utils patchelf  hdf5-devel sqlite-devel bzip2-devel xz-devel perl-open.noarch diffutils texinfo
+export JAVA_HOME=$(compgen -G '/usr/lib/jvm/java-11-openjdk-*')
+export JRE_HOME=${JAVA_HOME}/jre
+export PATH=${JAVA_HOME}/bin:$PATH
+
+#Install cmake
+cd $wdir
+if [ -z "$(ls -A $wdir/cmake-${CMAKE_VERSION})" ]; then
+        wget -c https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz
+        tar -zxvf cmake-${CMAKE_VERSION}.tar.gz
+        rm -rf cmake-${CMAKE_VERSION}.tar.gz
+        cd cmake-${CMAKE_VERSION}
+        ./bootstrap --prefix=/usr/local/cmake --parallel=2 -- -DBUILD_TESTING:BOOL=OFF -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_USE_OPENSSL:BOOL=ON
+else
+        cd cmake-${CMAKE_VERSION}
+fi
+make install -j$(nproc)
+export PATH=/usr/local/cmake/bin:$PATH
+cmake --version
+
+#Install Python from source
+cd $wdir
+if [ -z "$(ls -A $wdir/Python-${PYTHON_VERSION})" ]; then
+    wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
+    tar xzf Python-${PYTHON_VERSION}.tgz
+    rm -rf Python-${PYTHON_VERSION}.tgz
+    cd Python-${PYTHON_VERSION}
+    ./configure --enable-loadable-sqlite-extensions
+    make -j ${nproc}
+else
+    cd Python-${PYTHON_VERSION}
+fi
+make altinstall
+ln -sf $(which python3.10) /usr/bin/python3
+ln -sf $(which python3.10) /usr/bin/python
+ln -sf $(which pip3.10) /usr/bin/pip3
+python3 -V && pip3 -V
+
+#Install Python dependencies
+pip3 install conan==1.64.1 setuptools==59.5.0
+
+#Install Golang
+cd $wdir
+wget https://go.dev/dl/go${GO_VERSION}.linux-ppc64le.tar.gz
+rm -rf /usr/local/go && tar -C /usr/local -xzf go${GO_VERSION}.linux-ppc64le.tar.gz
+rm -rf go${GO_VERSION}.linux-ppc64le.tar.gz
+export PATH=/usr/local/go/bin:$PATH
+go version
+export PATH=$PATH:$HOME/go/bin
+
+#Build
+cd $wdir
+pushd /usr/local/cmake
+create_cmake_conanfile
+conan export-pkg . cmake/${CMAKE_REQUIRED_VERSION}@ -s os="Linux" -s arch="ppc64le"
+conan profile update settings.compiler.libcxx=libstdc++11 default
+popd
+export VCPKG_FORCE_SYSTEM_BINARIES=1
+export ENABLE_AZURE=false
+mkdir -p $HOME/.cargo/bin/
+ret=0
+make -j$(nproc) || ret=$?
+if [ "$ret" -ne 0 ]
+then
+        echo "FAIL: Build failed."
+        exit 1
+fi
+export MILVUS_BIN=$wdir/${PACKAGE_NAME}/bin/milvus
+
+#Disable Azure tests
+sed -i "s#test_azure_chunk_manager.cpp##g" ./internal/core/unittest/CMakeLists.txt
+
+#Cpp unit tests
+cd $wdir/${PACKAGE_NAME}/
+make test-cpp -j$(nproc) || ret=$?
+if [ "$ret" -ne 0 ]
+then
+        echo "Cpp Tests fail."
+        exit 2
+fi
+
+# Conclude
+set +ex
+echo "Complete: Build and Test successful! Milvus binary available at [$MILVUS_BIN]"
+echo "10 Azure related tests were disabled."

scripts/ppc64le-docker-build-test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -ex
+
+# Setup cleanup
+remove_docker_container() { docker rm -f milvus-build || true; docker system prune -f --all; }
+trap remove_docker_container EXIT
+remove_docker_container
+
+docker run --network host --privileged -t -d --name milvus-build registry.access.redhat.com/ubi9/ubi:9.3
+docker cp ./. milvus-build:/
+docker exec -it milvus-build bash -c "chmod 777 scripts/ppc64le-build-ubi9.3.sh && ./scripts/ppc64le-build-ubi9.3.sh"