diff --git a/presto-native-execution/Makefile b/presto-native-execution/Makefile index d8ef40a7433ff..f77cff05fc1a1 100644 --- a/presto-native-execution/Makefile +++ b/presto-native-execution/Makefile @@ -9,7 +9,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: all cmake build clean debug release unit submodules velox-submodule +.PHONY: all cmake build clean debug release unittest submodules velox-submodule BUILD_BASE_DIR=_build BUILD_DIR=release @@ -19,15 +19,19 @@ ENABLE_WALL ?= 1 NUM_THREADS ?= $(shell getconf _NPROCESSORS_CONF 2>/dev/null || echo 1) CPU_TARGET ?= "avx" CMAKE_PREFIX_PATH ?= "/usr/local" +PRESTOCPP_ROOT_DIR="$(shell pwd)" PRESTO_ENABLE_PARQUET ?= "OFF" +PRESTO_ENABLE_S3 ?= "OFF" PRESTO_ENABLE_HDFS ?= "OFF" +EXTRA_CMAKE_FLAGS ?= "" CMAKE_FLAGS := -DTREAT_WARNINGS_AS_ERRORS=${TREAT_WARNINGS_AS_ERRORS} CMAKE_FLAGS += -DENABLE_ALL_WARNINGS=${ENABLE_WALL} CMAKE_FLAGS += -DCMAKE_PREFIX_PATH=$(CMAKE_PREFIX_PATH) CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) CMAKE_FLAGS += -DPRESTO_ENABLE_PARQUET=$(PRESTO_ENABLE_PARQUET) +CMAKE_FLAGS += -DPRESTO_ENABLE_S3=$(PRESTO_ENABLE_S3) CMAKE_FLAGS += -DPRESTO_ENABLE_HDFS=$(PRESTO_ENABLE_HDFS) SHELL := /bin/bash @@ -58,7 +62,7 @@ velox-submodule: #: Check out code for velox submodule submodules: velox-submodule cmake: submodules #: Use CMake to create a Makefile build system - cmake -B "$(BUILD_BASE_DIR)/$(BUILD_DIR)" $(FORCE_COLOR) $(CMAKE_FLAGS) + cmake -B "$(BUILD_BASE_DIR)/$(BUILD_DIR)" $(FORCE_COLOR) $(CMAKE_FLAGS) $(EXTRA_CMAKE_FLAGS) build: #: Build the software based in BUILD_DIR and BUILD_TYPE variables cmake --build $(BUILD_BASE_DIR)/$(BUILD_DIR) -j $(NUM_THREADS) @@ -109,6 +113,13 @@ linux-container: cd /tmp/docker && \ docker build --build-arg cpu_target=$(CPU_TARGET) --tag "prestocpp/prestocpp-$(CPU_TARGET)-$(CONTAINER_NAME):$(USER)-$(shell date +%Y%m%d)" -f $(CONTAINER_NAME)-container.dockfile . +runtime-container: + rm -rf /tmp/release-centos-dockerfile && \ + mkdir -p /tmp/release-centos-dockerfile && \ + cp -r scripts/build-centos.sh scripts/release-centos-dockerfile /tmp/release-centos-dockerfile && \ + cd /tmp/release-centos-dockerfile && \ + PRESTOCPP_ROOT_DIR=$(PRESTOCPP_ROOT_DIR) ./build-centos.sh + help: #: Show the help messages @cat $(firstword $(MAKEFILE_LIST)) | \ awk '/^[-a-z]+:/' | \ diff --git a/presto-native-execution/README.md b/presto-native-execution/README.md index cb486b2745a79..21fc1d0b9c73d 100644 --- a/presto-native-execution/README.md +++ b/presto-native-execution/README.md @@ -21,6 +21,11 @@ Run `make` in the root directory to compile the sources. For development, use `make debug` to build a non-optimized debug version. Use `make unittest` to build and run tests. +## Building Dockerfile from source +Run `make runtime-container` in the presto-native-execution root directory +to build run-ready containerized version of PrestoCpp. Information on available +configuration options can be found in [scripts/release-centos-dockerfile/README.md](scripts/release-centos-dockerfile/README.md) + ### Makefile targets A reminder of the available Makefile targets can be obtained using `make help` ``` @@ -39,6 +44,7 @@ A reminder of the available Makefile targets can be obtained using `make help` tidy-fix Fix clang-tidy issues in the current branch tidy-check Check clang-tidy issues in the current branch linux-container Build the CircleCi linux container from scratch + runtime-container Build the software in container using current git commit help Show the help messages ``` @@ -48,12 +54,12 @@ A reminder of the available Makefile targets can be obtained using `make help` Makefile targets exist for showing, fixing and checking formatting, license headers and clang-tidy warnings. These targets are shortcuts for calling -`./scripts/check.py`. +`./scripts/check.py`. GitHub Actions run `make format-check`, `make header-check` and `make tidy-check` as part of our continuous integration. Pull requests should pass linux-build, format-check, header-check and other jobs without errors -before being accepted. +before being accepted. Formatting issues found on the changed lines in the current commit can be displayed using `make format-show`. These issues can be fixed by using `make @@ -68,13 +74,13 @@ commit. Similar commands `make tidy-show`, `make-tidy-fix`, `make tidy-check` exist for running clang-tidy, but these checks are currently advisory only. -An entire directory tree of files can be formatted and have license headers added +An entire directory tree of files can be formatted and have license headers added using the `tree` variant of the format.sh commands: ``` - ./scripts/check.py format tree + ./scripts/check.py format tree ./scripts/check.py format tree --fix - ./scripts/check.py header tree + ./scripts/check.py header tree ./scripts/check.py header tree --fix ``` @@ -103,7 +109,7 @@ export PATH=$HOME/bin:$HOME/Library/Python/3.7/bin:$PATH * Clone the whole Presto repository. * Run `presto/presto-native-execution/scripts/setup-centos.sh` -### Setting up local Presto environment on MacOS (with optional IntelliJ and CLion) +### Setting up local Presto environment on MacOS (with optional IntelliJ and CLion) Clone the whole Presto repository. Close IntelliJ and CLion @@ -133,7 +139,7 @@ Run IntelliJ In "presto/presto-native-execution" run: * `make -C presto-native-execution submodules` * `git/presto/presto-native-execution % sudo chown -R $(whoami) /usr/local/bin /usr/local/lib /usr/local/sbin; sudo chown -R $(whoami) /usr/local/bin /usr/local/lib /usr/local/bin; sudo chown -R $(whoami) /usr/local/bin /usr/local/lib /usr/local/lib; chmod u+w /usr/local/bin /usr/local/lib /usr/local/sbin; chmod u+w /usr/local/bin /usr/local/lib /usr/local/bin; chmod u+w /usr/local/bin /usr/local/lib /usr/local/lib; ./scripts/setup-macos.sh` - + Run CLion: * File->Close Project if any is open. * Open `presto/presto-native-execution` folder as CMake project and wait till CLion loads/generates cmake files, symbols, etc. @@ -166,7 +172,7 @@ Running Presto Coordinator + Worker on MacOS ### Running Integration (End to End or E2E) Tests on MacOS (with optional IntelliJ and CLion) * Note that everything below can be done w/o using IDEs by running command line commands (not in this readme). * Open a test file which has the test(s) you want to run in IntelliJ from `presto/presto-native-execution/src/test/java/com/facebook/presto/nativeworker` path. -* Click the green arrow to the left of the test class line of code and chose if you want to Run or Debug. This will run all tests in this class. +* Click the green arrow to the left of the test class line of code and chose if you want to Run or Debug. This will run all tests in this class. * Alternatively click the green arrow to the left of the test class' test method line of code and chose if you want tor Run or Debug. This will run all tests only in this class's member. * The framework will launch single Coordinator and four native workers to test-run the queries. * Similarly, the unit tests of Velox and presto_cpp can be run from CLion. @@ -181,5 +187,5 @@ Running Presto Coordinator + Worker on MacOS ### Creating PRs for presto/presto-native-execution/ * Submit PRs as usual following [Presto repository guidelines](https://github.com/prestodb/presto/wiki/Review-and-Commit-guidelines). -* On top of it please add `[native]` prefix in the `title` as well as to the `commit message` for PRs modifying anything in `presto-native-execution`. -* PRs that only change files in `presto-native-execution` should be approved by a Code Owner ([team-velox](https://github.com/orgs/prestodb/teams/team-velox)) to have merging enabled. +* On top of it please add `[native]` prefix in the `title` as well as to the `commit message` for PRs modifying anything in `presto-native-execution`. +* PRs that only change files in `presto-native-execution` should be approved by a Code Owner ([team-velox](https://github.com/orgs/prestodb/teams/team-velox)) to have merging enabled. diff --git a/presto-native-execution/scripts/build-centos.sh b/presto-native-execution/scripts/build-centos.sh new file mode 100755 index 0000000000000..8b81f98c291f6 --- /dev/null +++ b/presto-native-execution/scripts/build-centos.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") +source $SCRIPT_DIR/release-centos-dockerfile/opt/common.sh + +set -eE -o pipefail +trap 'error "Stage failed, exiting"; exit 5' SIGSTOP SIGINT SIGTERM SIGQUIT ERR + +print_logo + +export CPU_TARGET=${CPU_TARGET:-'avx'} +export BASE_IMAGE=${BASE_IMAGE:-'quay.io/centos/centos:stream8'} +export IMAGE_NAME=${IMAGE_NAME:-"presto/prestissimo-${CPU_TARGET}-centos"} +export IMAGE_TAG=${IMAGE_TAG:-"latest"} +export IMAGE_REGISTRY=${IMAGE_REGISTRY:-''} +export IMAGE_PUSH=${IMAGE_PUSH:-'0'} + +export USER_FLAGS=${USER_FLAGS:-''} +export PRESTOCPP_ROOT_DIR=${PRESTOCPP_ROOT_DIR:-"$(readlink -f "$SCRIPT_DIR/..")"} +export PRESTODB_REPOSITORY=${PRESTODB_REPOSITORY:-"$(cd "${PRESTOCPP_ROOT_DIR}/.." && git config --get remote.origin.url)"} +export PRESTODB_CHECKOUT=${PRESTODB_CHECKOUT:-"$(cd "${PRESTOCPP_ROOT_DIR}/.." && git show -s --format="%H" HEAD)"} +BUILD_LOGS_FILEPATH="${SCRIPT_DIR}/$(date +%Y%m%d%H%M%S)-${USER}-${CPU_TARGET}-build-centos.log" +( + prompt "Using build time variables:" + prompt "------------" + prompt "\tIMAGE_NAME=${IMAGE_NAME}" + prompt "\tIMAGE_TAG=${IMAGE_TAG}" + prompt "\tIMAGE_REGISTRY=${IMAGE_REGISTRY}" + prompt "\tBASE_IMAGE=${BASE_IMAGE}" + prompt "\tCPU_TARGET=${CPU_TARGET}" + prompt "\tUSER_FLAGS=${USER_FLAGS}" + prompt "------------" + prompt "\tPRESTODB_REPOSITORY=${PRESTODB_REPOSITORY}" + prompt "\tPRESTODB_CHECKOUT=${PRESTODB_CHECKOUT}" + prompt "------------" + prompt "Using build time computed variables:" + prompt "\t[1/2] Base build image: ${BASE_IMAGE}" + prompt "\t[2/2] Release image tag: ${IMAGE_REGISTRY}${IMAGE_NAME}:${IMAGE_TAG}" + prompt "------------" + prompt "Build logs: ${BUILD_LOGS_FILEPATH}" + prompt "------------" +) 2>&1 | +tee "${BUILD_LOGS_FILEPATH}" +( + prompt "[1/2] Preflight Git checks stage starting $(txt_yellow remote commit exisitance)" && + prompt "[1/2] Fetching remote repository" && + cd "${PRESTOCPP_ROOT_DIR}/.." > /dev/null && + git fetch --all > /dev/null && + prompt "[1/2] Checking if local hash is available on remote repository" && + git branch -r --contains $PRESTODB_CHECKOUT > /dev/null || + ( error '[1/2] Preflight stage failed, commit not found. Exiting.' && exit 1 ) +) 2>&1 | +tee -a "${BUILD_LOGS_FILEPATH}" +( + prompt "[2/2] Preflight CPU checks stage starting $(txt_yellow processor instructions)" + error=0 + check=$(txt_green success) + prompt "Velox build requires bellow CPU instructions to be available:" + for flag in 'bmi|bmi1' 'bmi2' 'f16c'; + do + echo $(cat /proc/cpuinfo) | grep -E -q " $flag " && check=$(txt_green success) || check=$(txt_red failed) error=1 + prompt "Testing (${flag}): \t$check" + done + prompt "Velox build suggest bellow CPU instructions to be available:" + for flag in avx avx2 sse; + do + echo $(cat /proc/cpuinfo) | grep -q " $flag " && check=$(txt_green success) || check=$(txt_yellow failed) + prompt "Testing (${flag}): \t$check" + done + [ $error -eq 0 ] || ( error 'Preflight checks failed, lack of CPU functionality. Exiting.' && exit 1 ) + prompt "[2/2] Preflight CPU checks $(txt_green success)" +) 2>&1 | +tee -a "${BUILD_LOGS_FILEPATH}" +( + prompt "[1/1] Build stage starting $(txt_yellow ${IMAGE_REGISTRY}${IMAGE_NAME}:${IMAGE_TAG})" && + cd "${SCRIPT_DIR}" && + docker build $USER_FLAGS \ + --network=host \ + --build-arg http_proxy \ + --build-arg https_proxy \ + --build-arg no_proxy \ + --build-arg CPU_TARGET \ + --build-arg BASE_IMAGE \ + --build-arg PRESTODB_REPOSITORY \ + --build-arg PRESTODB_CHECKOUT \ + --tag "${IMAGE_REGISTRY}${IMAGE_NAME}:${IMAGE_TAG}" \ + ./release-centos-dockerfile && + prompt "[1/1] Build finished" && + ( + [ "$IMAGE_PUSH" == "1" ] && + prompt "[1/1] Pushing image $(txt_yellow ${IMAGE_REGISTRY}${IMAGE_NAME}:${IMAGE_TAG})" && + docker push "${IMAGE_REGISTRY}${IMAGE_NAME}:${IMAGE_TAG}" || true + ) && + prompt "[1/1] Build finished $(txt_green ${IMAGE_REGISTRY}${IMAGE_NAME}:${IMAGE_TAG})" || + ( error '[1/1] Build failed. Exiting' && exit 2 ) +) 2>&1 | +tee -a "${BUILD_LOGS_FILEPATH}" + +prompt "Prestissimo is ready for deployment" +prompt "Image tagged as: ${IMAGE_REGISTRY}${IMAGE_NAME}:${IMAGE_TAG}" diff --git a/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile b/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile new file mode 100644 index 0000000000000..ca3ab17416626 --- /dev/null +++ b/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile @@ -0,0 +1,162 @@ +# prestissimo + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#============================================================ +# ///////////////////////////////// +# BUILD TASK DEFINITION +# ///////////////////////////////// +#============================================================ +ARG BASE_IMAGE=quay.io/centos/centos:stream8 +FROM ${BASE_IMAGE} AS build-stage + +ARG CPU_TARGET=avx +ARG PRESTODB_REPOSITORY=https://github.com/prestodb/presto +ARG PRESTODB_CHECKOUT=origin/master + +USER root:root + +ENV CPU_TARGET=${CPU_TARGET} +ENV TZ=Europe/Warsaw + +ENV CC=/opt/rh/gcc-toolset-9/root/bin/gcc +ENV CXX=/opt/rh/gcc-toolset-9/root/bin/g++ + +ENV PRESTODB_HOME=/opt/presto +ENV DEPENDENCY_DIR=/opt/dependency +ENV INSTALL_PREFIX=/usr/local +ENV CMAKE_PREFIX_PATH=/usr/local +ENV PROMPT_ALWAYS_RESPOND=n + +ENV PATH=${DEPENDENCY_DIR}/install/bin:${DEPENDENCY_DIR}/install/bin/hadoop-2.10.1/bin:$PATH + +ENV PRESTO_ENABLE_PARQUET=ON +ENV PRESTO_ENABLE_S3=ON +ENV PRESTO_ENABLE_HDFS=ON +ENV PRESTO_ENABLE_TESTING=ON +ENV CLEANUP_BUILD_ARTIFACTS=1 + +SHELL ["/bin/bash", "-c"] +#----------------------------------------------------------- +#---( START of PrestoDB and Velox dependency tree build scripts +#----------------------------------------------------------- +WORKDIR ${PRESTODB_HOME} +RUN set -exu && \ + dnf -y install git wget unzip pkg-config python3-devel java clang-tools-extra bzip2 bzip2-libs && \ + git clone --progress ${PRESTODB_REPOSITORY} "${PRESTODB_HOME}/_repo" && \ + git -C "${PRESTODB_HOME}/_repo" checkout "${PRESTODB_CHECKOUT}" && \ + make --directory="${PRESTODB_HOME}/_repo/presto-native-execution" submodules && \ + mkdir -p "${DEPENDENCY_DIR}/install/bin" \ + "${PRESTODB_HOME}/_repo/presto-native-execution/_build/release" \ + "${PRESTODB_HOME}/catalog" \ + "${PRESTODB_HOME}/tests" && \ + dnf clean all + +WORKDIR ${DEPENDENCY_DIR} +RUN set -exu && \ + bash "${PRESTODB_HOME}/_repo/presto-native-execution/velox/scripts/setup-circleci.sh" && \ + bash "${PRESTODB_HOME}/_repo/presto-native-execution/scripts/setup-centos.sh" && \ + python3 -m pip install six && \ + set +exu && \ + source /opt/rh/gcc-toolset-9/enable && \ + set -exu && \ +#----------------------------------------------------------- +#---( START wget valid antlr java runtime +#----------------------------------------------------------- + wget -q 'https://www.antlr.org/download/antlr-4.9.3-complete.jar' -O '/usr/local/lib/antlr-4.9.3-complete.jar' && \ + printf '#!/usr/bin/bash\n\njava -jar /usr/local/lib/antlr-4.9.3-complete.jar "$@"' > '/usr/local/bin/antlr' && \ + chmod +x "/usr/local/bin/antlr" && \ + bash "${PRESTODB_HOME}/_repo/presto-native-execution/velox/scripts/setup-adapters.sh" && \ + dnf clean all + +#----------------------------------------------------------- +#---( START of PrestoCpp and Velox main build process +#----------------------------------------------------------- +WORKDIR ${PRESTODB_HOME} +RUN source /opt/rh/gcc-toolset-9/enable && \ + set -exu && \ + make --directory="${PRESTODB_HOME}/_repo/presto-native-execution" release \ + PRESTO_ENABLE_PARQUET=ON \ + PRESTO_ENABLE_S3=ON \ + PRESTO_ENABLE_HDFS=ON \ + EXTRA_CMAKE_FLAGS=" -DFBTHRIFT_INCLUDE_DIR=${DEPENDENCY_DIR}/fbthrift/thrift/lib/cpp2 " +#----------------------------------------------------------- +#---( START of clean-up artifacts task +#----------------------------------------------------------- +RUN set -exu && \ + [ ${CLEANUP_BUILD_ARTIFACTS} -eq 0 ] || \ + ( \ + rm -rf $(find "${PRESTODB_HOME}/_repo/presto-native-execution/_build/release" | grep -E '\.o$|\.cpp$|\.tcc$|\.h$|CMakeFiles|\.cmake$|\.cs$|\.rb$|\.cc$|\.R$|\.c$|\.hpp$|\.rb$') && \ + rm -rf $(find "${PRESTODB_HOME}/_repo/presto-native-execution/_build/release" | grep -E '\.ninja_deps|\.ninja_log|\.ninja|CMakeCache|\.tcl') \ + ) && \ + ( \ + mv ${PRESTODB_HOME}/_repo/presto-native-execution/_build/release/presto_cpp/main/* ${PRESTODB_HOME}/ && \ + mv ${PRESTODB_HOME}/_repo/presto-native-execution/_build/release/presto_cpp/presto_protocol/tests/presto_protocol_test $${PRESTODB_HOME}/ || true \ + ) && \ + [ ${CLEANUP_BUILD_ARTIFACTS} -eq 0 ] || \ + rm -rf ${PRESTODB_HOME}/_repo/presto-native-execution/_repo /opt/dependency/aws-sdk-cpp /tmp/* && \ + dnf clean all + +#============================================================ +#---) END of PrestoCpp and Velox main build process +#============================================================ +# ///////////////////////////////// +# RUNTIME TASK DEFINITION +# ///////////////////////////////// +#============================================================ +ARG BASE_IMAGE=quay.io/centos/centos:stream8 +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="Prestissimo" \ + org.opencontainers.image.description="Prestissimo Runtime Docker image" \ + org.opencontainers.image.version="1.0.0" + +ARG PRESTO_UID=186 + +SHELL ["/bin/bash", "-c"] + +ENV PRESTO_HOME="/opt/presto/" +ENV CPU_TARGET=${CPU_TARGET} +ENV TZ=Europe/Warsaw + +WORKDIR /opt/presto/ + +COPY --chown=${PRESTO_UID} opt/ /opt/ +COPY --chown=${PRESTO_UID} --from=build-stage /opt/presto/ /opt/presto/ + +RUN set -exu && \ + dnf -y install uuid hostname vim git && \ + #---( INFO create a system account (-r) and same name group (-U) with UID and home path at /opt (-d) named presto + useradd -r -U --uid=${PRESTO_UID} -d /opt presto && \ + chgrp ${PRESTO_UID} /etc/passwd && \ + chown -R ${PRESTO_UID} /opt && \ + chmod -R ug+rw /etc/passwd /opt && \ + chmod ug+x /opt/*.sh /opt/presto/presto_server && \ + rm -rf /root/.cache /tmp && \ + dnf clean all + +USER ${PRESTO_UID} + +COPY --chown=${PRESTO_UID} etc/ /opt/presto/etc/ +COPY --chown=${PRESTO_UID} --from=build-stage /usr/local/lib64/lib* /usr/lib64/lib* /usr/local/lib64/ +COPY --chown=${PRESTO_UID} --from=build-stage /usr/local/lib/lib* /usr/local/lib/ + +ENTRYPOINT [ "/opt/entrypoint.sh" ] + +EXPOSE 22 8080 + +#============================================================ +# ///////////////////////////////// +# END OF RUNTIME TASK DEFINITION +# ///////////////////////////////// +#============================================================ diff --git a/presto-native-execution/scripts/release-centos-dockerfile/README.md b/presto-native-execution/scripts/release-centos-dockerfile/README.md new file mode 100644 index 0000000000000..95577b8f6c95d --- /dev/null +++ b/presto-native-execution/scripts/release-centos-dockerfile/README.md @@ -0,0 +1,492 @@ +# Prestissimo - Dockerfile build + +> ๐Ÿ’ก _**PrestoDB** repository: [Presto - https://github.com/prestodb/presto](https://github.com/prestodb/presto)_ + +> ๐Ÿ’ก _**Velox** repository: [Velox - https://github.com/facebookincubator/velox](https://github.com/facebookincubator/velox)_ + +## Practical Velox implementation using PrestoCpp + +> ๐Ÿ“ _**Note:** This readme and the build process was adapted from internal pipeline. You can e-mail the author if you've got questions [milosz.linkiewicz@intel.com](mailto:milosz.linkiewicz@intel.com)_ + +Prestissimo, marked in PrestoDB GitHub repository as 'presto-native-execution', is effort of making PrestoDB even better using Velox library as a starting point. Both of mentioned - PrestoCpp and Velox - are mainly written using low level `C` and `C++ 17` languages, which makes the build-from-scratch process humongously complicated. To make this process simple, Intel Cloud Native Data Services Team is introducing 3-stage, fully automated Docker build process based on unmodified project GitHub repository. + +## Quick Start + +### 1. Clone this repository + +```bash +git clone https://github.com/prestodb/presto prestodb +``` + +### 2. (Optional) Define and export Docker registry, image name and image tag variables + +> ๐Ÿ“ _**Note:** Remember to end your `IMAGE_REGISTRY` with `/` as this is required for full tag generation._ + +> ๐Ÿ’ก _**Tip:** Depending on your configuration you may need to run all bellow commands as root user, to switch type as your first command `sudo su`_ + +> ๐Ÿ’ก _**Tip:** If `IMAGE_REGISTRY` is not specified `IMAGE_PUSH` should be set `'0'` or docker image push stage will fail._ + +Type in you console, changing variables values to meet your needs: + +```bash +# defaults to 'avx', more info on Velox GitHub +export CPU_TARGET="avx" +# defaults to 'presto/prestissimo-${CPU_TARGET}-centos' +export IMAGE_NAME='presto/prestissimo-${CPU_TARGET}-centos' +# defaults to 'latest' +export IMAGE_TAG='latest' +# defaults to '' +export IMAGE_REGISTRY='https://my_docker_registry.com/' +# defaults to '0' +export IMAGE_PUSH='0' +``` + +### 3. Make sure Docker daemon is running + +(Ubuntu users) Type in your console: + +```bash +systemctl status docker +``` + +### 4. Build Dockerfile repo + +Type in your console: + +```bash +cd prestodb/presto-native-execution +make runtime-container +``` + +The process is fully automated and require no interaction for user. The process of building images for the first time can take up to couple of hours (~1-2h using 10 processor cores). + +### 5. Run container + +> ๐Ÿ“ _**Note:** Remember that you should start Presto JAVA server first_ + +Depending on values you have set the container tag is defined as + +`PRESTO_CPP_TAG="${IMAGE_REGISTRY}${IMAGE_NAME}:${IMAGE_TAG}"` + +for default values this will be just: + +`PRESTO_CPP_TAG=presto/prestissimo-avx-centos:latest` + +to run container build with default tag execute: + +```bash +docker run "presto/prestissimo-avx-centos:latest" \ + --use-env-params \ + --discovery-uri=http://localhost:8080 \ + --http-server-port=8080" +``` + +to run container interactively, not executing entrypoint file: + +```bash +docker run -it --entrypoint=/bin/bash "presto/prestissimo-avx-centos:latest" +``` + +## Container manual build + +For manual build outside Intel network or without access to Cloud Native Data Services Poland Docker registry follow the steps bellow. +In you terminal - with the same session that you want to build the images - define and export environment variables: + +```bash +export CPU_TARGET="avx" +export IMAGE_NAME='presto/prestissimo-${CPU_TARGET}-centos' +export IMAGE_TAG='latest' +export IMAGE_REGISTRY='some-registry.my-domain.com/' +export IMAGE_PUSH='0' +export PRESTODB_REPOSITORY=$(git config --get remote.origin.url) +export PRESTODB_CHECKOUT=$(git show -s --format="%H" HEAD) +``` + +Where `IMAGE_NAME` and `IMAGE_TAG` will be the prestissimo release image name and tag, `IMAGE_REGISTRY` will be the registry that the image will be tagged with and witch will be used to download the images from previous stages in case there are no cached images locally. The `CPU_TARGET` will be unchanged for most of the cases, for more info read the Velox documentation. The `PRESTODB_REPOSITORY` and `PRESTODB_CHECKOUT` will be used as a build repository and branch inside the container. You can set them manually or as provided using git commands. + +Then for example to build containers when being behind a proxy server, change dir to and type: + +```bash +cd presto-native-execution/scripts/release-centos-dockerfile +docker build \ + --network=host \ + --build-arg http_proxy \ + --build-arg https_proxy \ + --build-arg no_proxy \ + --build-arg CPU_TARGET \ + --build-arg PRESTODB_REPOSITORY \ + --build-arg PRESTODB_CHECKOUT \ + --tag "${IMAGE_REGISTRY}${IMAGE_NAME}:${IMAGE_TAG}" . +``` + + +## Build process - more info - `prestissimo` (with artifacts ~35 GB, without ~10 GB) + +Most of runtime and build time dependencies are downloaded, configured and installed in this step. The result from this step is a starting point for both second and third stage. This container will be build 'once per breaking change' in any of repositories. It can be used as starting point for Ci/Cd integrated systems. +This step install Maven, Java 8, Python3-Dev, libboost-dev and lots of other massive frameworks, libraries and applications and ensures that all of steps from 2 stage will run with no errors. + +On-top of container from step 1 repository is initialized, Velox and submodules are updated, adapters, connectors and side-dependencies are build and configured. PrestoDB native, full repository build, using Meta wrapper mvnw for Maven is being done. After all of those partial steps, make and build are being run for PrestoCpp and Velox with Parquet, ORC, Hive connector with Thrift with S3-EMRFS filesystem implementation (schema `s3://`) and Hadoop filesystem implementation. + +```bash +### DIRECTORY AND MAIN BUILD ARTIFACTS +## Native Presto JAVA build artifacts: +/root/.m2/ + +## Build, third party dependencies, mostly for adapters +/opt/dependency/ +/opt/dependency/aws-sdk-cpp +/opt/dependency/install/ +/opt/dependency/install/run/ +/opt/dependency/install/bin/ +/opt/dependency/install/lib64/ + +## Root PrestoDB application directory +/opt/presto/ + +## Root GitHub clone of PrestoDB repository +/opt/presto/_repo/ + +## Root PrestoCpp subdirectory +/opt/presto/_repo/presto-native-execution/ + +## Root Velox GitHub repository directory, as PrestoDB submodule +/opt/presto/_repo/presto-native-execution/Velox + +## Root build results directory for PrestoCpp with Velox +/opt/presto/_repo/presto-native-execution/_build/release/ +/opt/presto/_repo/presto-native-execution/_build/release/velox/ +/opt/presto/_repo/presto-native-execution/_build/release/presto_cpp/ +``` + +Release container build - mostly with only the must-have runtime files, including presto_server build presto executable and some libraries. What will be used in the final released container depends on user needs and can be adjusted. + +# Prestissimo - runtime configuration and settings + +Presto server with all dependencies can be found inside `/opt/presto/`, runtime name is `presto_server`. There are 2 ways of starting PrestoCpp using provided entry point `/opt/entrypoint.sh`. + +## 1) Quick start - pass parameters to entrypoint + +This is valid when running using docker and using kubernetes. It is not advised to use this method. User should prefer mounting configuration files using Kubernetes. + +``` +"/opt/entrypoint.sh --use-env-params --discovery-uri=http://presto-coordinaator.default.svc.cluster.local:8080 --http-server-port=8080" +``` + + +## 2) Using in Kubernetes environment: + +Mount config file inside a container as `/opt/presto/node.properties.template`. Replace each variable with you configuration values or leave it as is: + +> _**Notice:** set up same values for JAVA coordinator as for prestoCpp - version, location and environment should be the same or you will get connection errors._ + +```ini +presto.version=0.273.3 +node.location=datacenter-warsaw +node.environment=test-environment +node.data-dir=/var/presto/data +catalog.config-dir=/opt/presto/catalog +plugin.dir=/opt/presto/plugin +# node.id is generated and filled during machine startup if not specified +``` + +Mount config file inside a container as `/opt/presto/config.properties.template`. Replace each variable with you configuration values: + +```ini +coordinator=false +http-server.http.port=8080 +discovery.uri=http://presto-coordinaator.default.svc.cluster.local:8080 +``` + +## 3) Hive-Metastore connector and S3 configuration: + +For minimum required configuration just mount file `/opt/presto/catalog/hive.properties` inside container at give path (fill `hive.metastore.uri` with you metastore endpoint address): + +```ini +connector.name=hive-hadoop2 +hive.metastore.uri=thrift://hive-metastore-service.default.svc:9098 +hive.pushdown-filter-enabled=true +cache.enabled=true +``` + +Setting required by S3 connector and Velox query engine, replace with your values, reefer to presto hive connector settings help: +```ini +hive.s3.path-style-access={{ isPathstyle }} +hive.s3.endpoint={{ scheme }}://{{ serviceFqdnTpl . }}:{{ portRest }} +hive.s3.aws-access-key={{ accessKey }} +hive.s3.aws-secret-key={{ secretKey }} +hive.s3.ssl.enabled={{ sslEnabled }} +hive.s3select-pushdown.enabled={{ s3selectPushdownFilterEnabled }} +hive.parquet.pushdown-filter-enabled={{ parquetPushdownFilterEnabled }} +``` + +# Appendix A - Possible flags that can be passed to runtime server +## Flags available for `/opt/presto/presto_server`: +### Flags from ./src/logging.cc: + + - alsologtoemail (log messages go to these email addresses in addition to logfiles) + - type: string + - default: "" + - alsologtostderr (log messages go to stderr in addition to logfiles) + - type: bool + - default: false + - colorlogtostderr (color messages logged to stderr (if supported by terminal)) + - type: bool + - default: false + - drop_log_memory (Drop in-memory buffers of log contents. Logs can grow very quickly and they are rarely read before they need to be evicted from memory. Instead, drop them from memory as soon as they are flushed to disk.) + - type: bool + - default: true + - log_backtrace_at (Emit a backtrace when logging at file:linenum.) + - type: string + - default: "" + - log_dir (If specified, logfiles are written into this directory instead of the default logging directory.) + - type: string + - default: "" + - log_link (Put additional links to the log files in this directory) + - type: string + - default: "" + - log_prefix (Prepend the log prefix to the start of each log line) + - type: bool + - default: true + - logbuflevel (Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; ...)) + - type: int32 + - default: 0 + - logbufsecs (Buffer log messages for at most this many seconds) + - type: int32 + - default: 30 + - logemaillevel (Email log messages logged at this level or higher (0 means email all; 3 means email FATAL only; ...)) + - type: int32 + - default: 999 + - logfile_mode (Log file mode/permissions.) + - type: int32 + - default: 436 + - logmailer (Mailer used to send logging email) + - type: string + - default: "/bin/mail" + - logtostderr (log messages go to stderr instead of logfiles) + - type: bool + - default: false + - max_log_size (approx. maximum log file size (in MB). A value of 0 will be silently overridden to 1.) + - type: int32 + - default: 1800 + - minloglevel (Messages logged at a lower level than this don't actually get logged anywhere) + - type: int32 + - default: 0 + - stderrthreshold (log messages at or above this level are copied to stderr in addition to logfiles. This flag obsoletes --alsologtostderr.) + - type: int32 + - default: 2 + - stop_logging_if_full_disk (Stop attempting to log to disk if the disk is full.) + - type: bool + - default: false + +### Flags from ./src/utilities.cc: + + - symbolize_stacktrace (Symbolize the stack trace in the tombstone) + - type: bool + - default: true + +### Flags from ./src/vlog_is_on.cc: + + - v (Show all VLOG(m) messages for m <= this. Overridable by --vmodule.) + - type: int32 + - default: 0 + - vmodule (per-module verbose level. Argument is a comma-separated list of =. is a glob pattern, matched against the filename base (that is, name ignoring .cc/.h./-inl.h). overrides any value given by --v.) + - type: string + - default: "" + +### Flags from /build/gflags-WDCpEz/gflags-2.2.2/src/gflags.cc: + + - flagfile (load flags from file) + - type: string + - default: "" + - fromenv (set flags from the environment [use 'export FLAGS_flag1=value']) + - type: string + - default: "" + - tryfromenv (set flags from the environment if present) + - type: string + - default: "" + - undefok (comma-separated list of flag names that it is okay to specify on the command line even if the program does not define a flag with that name. IMPORTANT: flags in this list that have arguments MUST use the flag=value format) + - type: string + - default: "" + +### Flags from /build/gflags-WDCpEz/gflags-2.2.2/src/gflags_completions.cc: + + - tab_completion_columns (Number of columns to use in output for tab completion) + - type: int32 + - default: 80 + - tab_completion_word (If non-empty, HandleCommandLineCompletions() will hijack the process and attempt to do bash-style command line flag completion on this value.) + - type: string + - default: "" + +### Flags from /build/gflags-WDCpEz/gflags-2.2.2/src/gflags_reporting.cc: + + - help (show help on all flags [tip: all flags can have two dashes]) + - type: bool + - default: false currently: true + - helpfull (show help on all flags -- same as -help) + - type: bool + - default: false + - helpmatch (show help on modules whose name contains the specified substr) + - type: string + - default: "" + - helpon (show help on the modules named by this flag value) + - type: string + - default: "" + - helppackage (show help on all modules in the main package) + - type: bool + - default: false + - helpshort (show help on only the main module for this program) + - type: bool + - default: false + - helpxml (produce an xml version of help) + - type: bool + - default: false + - version (show version and build info and exit) + - type: bool + - default: false + +### Flags from /opt/prestodb/presto-native-execution/fbthrift/thrift/lib/cpp2/protocol/Protocol.cpp: + + - thrift_protocol_max_depth (How many nested struct/list/set/map are allowed) + - type: int32 + - default: 15000 + +### Flags from /opt/prestodb/presto-native-execution/folly/folly/detail/MemoryIdler.cpp: + + - folly_memory_idler_purge_arenas (if enabled, folly memory-idler purges jemalloc arenas on thread idle) + - type: bool + - default: true + +### Flags from /opt/prestodb/presto-native-execution/folly/folly/executors/CPUThreadPoolExecutor.cpp: + + - dynamic_cputhreadpoolexecutor (CPUThreadPoolExecutor will dynamically create and destroy threads) + - type: bool + - default: true + +### Flags from /opt/prestodb/presto-native-execution/folly/folly/executors/IOThreadPoolExecutor.cpp: + + - dynamic_iothreadpoolexecutor (IOThreadPoolExecutor will dynamically create threads) + - type: bool + - default: true + +### Flags from /opt/prestodb/presto-native-execution/folly/folly/executors/ThreadPoolExecutor.cpp: + + - threadtimeout_ms (Idle time before ThreadPoolExecutor threads are joined) + - type: int64 + - default: 60000 + +### Flags from /opt/prestodb/presto-native-execution/folly/folly/experimental/observer/detail/ObserverManager.cpp: + + - observer_manager_pool_size (How many internal threads ObserverManager should use) + - type: int32 + - default: 4 + +### Flags from /opt/prestodb/presto-native-execution/folly/folly/init/Init.cpp: + + - logging (Logging configuration) + - type: string + - default: "" + +### Flags from /opt/prestodb/presto-native-execution/folly/folly/synchronization/Hazptr.cpp: + + - folly_hazptr_use_executor (Use an executor for hazptr asynchronous reclamation) + - type: bool + - default: true + +### Flags from /opt/prestodb/presto-native-execution/presto_cpp/main/PrestoMain.cpp: + + - etc_dir (etc directory for presto configuration) + - type: string + - default: "." + +### Flags from /opt/prestodb/presto-native-execution/presto_cpp/main/QueryContextManager.cpp: + + - num_query_threads (Process-wide number of query execution threads) + - type: int32 + - default: 80 + +### Flags from /opt/prestodb/presto-native-execution/presto_cpp/main/TaskManager.cpp: + + - old_task_ms (Time (ms) since the task execution ended, when task is considered old.) + - type: int32 + - default: 60000 + +### Flags from /opt/prestodb/presto-native-execution/proxygen/proxygen/lib/utils/ZlibStreamCompressor.cpp: + + - zlib_compressor_buffer_growth (The buffer growth size to use during IOBuf zlib deflation) + - type: int64 + - default: 2024 + +### Flags from /opt/prestodb/presto-native-execution/velox/velox/common/caching/SsdFile.cpp: + + - ssd_odirect (Use O_DIRECT for SSD cache IO) + - type: bool + - default: true + - ssd_verify_write (Read back data after writing to SSD) + - type: bool + - default: false + +### Flags from /opt/prestodb/presto-native-execution/velox/velox/connectors/hive/HiveConnector.cpp: + + - file_handle_cache_mb (Amount of space for the file handle cache in mb.) + - type: int32 + - default: 1024 + +### Flags from /opt/prestodb/presto-native-execution/velox/velox/dwio/common/BufferedInput.cpp: + + - wsVRLoad (Use WS VRead API to load) + - type: bool + - default: false + +### Flags from /opt/prestodb/presto-native-execution/velox/velox/dwio/common/CachedBufferedInput.cpp: + + - cache_prefetch_min_pct (Minimum percentage of actual uses over references to a column for prefetching. No prefetch if > 100) + - type: int32 + - default: 80 + +### Flags from /opt/prestodb/presto-native-execution/velox/velox/expression/Expr.cpp: + + - force_eval_simplified (Whether to overwrite queryCtx and force the use of simplified expression evaluation path.) + - type: bool + - default: false + +### Flags from /opt/prestodb/presto-native-execution/velox/velox/flag_definitions/flags.cpp: + + - avx2 (Enables use of AVX2 when available) + - type: bool + - default: true + - bmi2 (Enables use of BMI2 when available) + - type: bool + - default: true + - max_block_value_set_length (Max entries per column that the block meta-record stores for pre-flight filtering optimization) + - type: int32 + - default: 5 + - memory_usage_aggregation_interval_millis (Interval to compute aggregate memory usage for all nodes) + - type: int32 + - default: 2 + - velox_exception_system_stacktrace_enabled (Enable the stacktrace for system type of VeloxException) + - type: bool + - default: true + - velox_exception_system_stacktrace_rate_limit_ms (Min time interval in milliseconds between stack traces captured in system type of VeloxException; off when set to 0 (the default)) + - type: int32 + - default: 0 + - velox_exception_user_stacktrace_enabled (Enable the stacktrace for user type of VeloxException) + - type: bool + - default: false + - velox_exception_user_stacktrace_rate_limit_ms (Min time interval in milliseconds between stack traces captured in user type of VeloxException; off when set to 0 (the default)) + - type: int32 + - default: 0 + - velox_memory_pool_mb (Size of file cache/operator working memory in MB) + - type: int32 + - default: 4096 + - velox_time_allocations (Record time and volume for large allocation/free) + - type: bool + - default: true + - velox_use_malloc (Use malloc for file cache and large operator allocations) + - type: bool + - default: true + +### Flags from /opt/prestodb/presto-native-execution/wangle/wangle/ssl/SSLSessionCacheManager.cpp: + + - dcache_unit_test (All VIPs share one session cache) + - type: bool + - default: false diff --git a/presto-native-execution/scripts/release-centos-dockerfile/etc/config.properties.template b/presto-native-execution/scripts/release-centos-dockerfile/etc/config.properties.template new file mode 100644 index 0000000000000..9731149720d7a --- /dev/null +++ b/presto-native-execution/scripts/release-centos-dockerfile/etc/config.properties.template @@ -0,0 +1 @@ +# presto.version=0.273.3 diff --git a/presto-native-execution/scripts/release-centos-dockerfile/etc/node.properties.template b/presto-native-execution/scripts/release-centos-dockerfile/etc/node.properties.template new file mode 100644 index 0000000000000..001da00a5987d --- /dev/null +++ b/presto-native-execution/scripts/release-centos-dockerfile/etc/node.properties.template @@ -0,0 +1,3 @@ +# node.environment=cnds-intel +# node.location=cnds-cluster +# node.id diff --git a/presto-native-execution/scripts/release-centos-dockerfile/opt/common.sh b/presto-native-execution/scripts/release-centos-dockerfile/opt/common.sh new file mode 100644 index 0000000000000..59fcb6824747c --- /dev/null +++ b/presto-native-execution/scripts/release-centos-dockerfile/opt/common.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash + +function message() { + local type=$1 + shift + echo -e "$type: $@" >&2 +} + +function prompt() { + local PomptHBlue='\e[38;05;33m'; + local PomptTBlue='\e[38;05;45m'; + message "${PomptHBlue}INFO${PomptTBlue}" "$@\e[m" +} + +function error() { + local ErrorHRed='\e[38;05;1m'; + local PomptTBlue='\e[38;05;45m'; + message "${ErrorHRed}ERROR${PomptTBlue}" "$@\e[m" +} + +function warning() { + local WarningHPurple='\e[38;05;61m'; + local PomptTBlue='\e[38;05;45m'; + message "${WarningHPurple}WARN${PomptTBlue}" "$@\e[m" +} + +function txt_green() { echo "\e[38;05;2m$@\e[m"; } +function txt_yellow() { echo "\e[38;05;11m$@\e[m"; } +function txt_red() { echo "\e[38;05;9m$@\e[m"; } + +function get_filename() { + local path=$1 + echo ${path##*/} +} + +function get_dirname() { + local path=$1 + echo "${path%/*}/" +} + +function get_extension() { + local filename=$(get_filename $1) + echo "${filename#*.}" +} + +function get_basename() { + local filename=$(get_filename $1) + echo "${filename%%.*}" +} + +# Takes ID as first param and full path to output file as second for creating benchmark specific output file +# input: [path]/[file_base].[extension] +# output: [path]/[file_base][id].[extension] +function get_path_with_id() { + local id="${1}" + local path="${2}" + local job_dir=$(get_dirname "${path}") + local job_base=$(get_basename "${path}") + local job_ext=$(get_extension "${path}") + echo "${job_dir}${job_base}${id}.${job_ext}" +} + +# For using file customization user can use date formatting workflow +# File name is evaluated once at the begigning of sequence run +# %Y year, %m month, %d day, %H hour, %M minute, %S second, example: +# - given path "/home/ubuntu/.hidden/plik_%Y%m%d_%H-%M-%S.scv" +# will give "/home/ubuntu/.hidden/plik_20220415_20-48-33.scv" +function eval_date_path() { + local path=${1} + echo $(date +"${path}") +} + +# Execute any number of commands for output dump to file passed as first param +function get_debug_dump_info() { + local out_path="${1}/debug.log" + shift + local cmds=(${@:-"env uname lspci"}) + + for cmd in ${cmds[@]}; do + echo "-----start---${cmd}-----" >> "${out_path}" + $cmd >> "${out_path}" + echo "------end----${cmd}-----" >> "${out_path}" + done +} + +function failure() { + local -n _lineno="${1:-LINENO}" + local -n _bash_lineno="${2:-BASH_LINENO}" + local _last_command="${3:-${BASH_COMMAND}}" + local _code="${4:-0}" + local _last_command_height="$(wc -l <<<"${_last_command}")" + local -a _output_array=() + + _output_array+=( + '---' + "lines_history: [${_lineno} ${_bash_lineno[*]}]" + "function_trace: [${FUNCNAME[*]}]" + "exit_code: ${_code}" + ) + + if [[ "${#BASH_SOURCE[@]}" -gt '1' ]]; then + _output_array+=('source_trace:') + for _item in "${BASH_SOURCE[@]}"; do + _output_array+=(" - ${_item}") + done + else + _output_array+=("source_trace: [${BASH_SOURCE[*]}]") + fi + + if [[ "${_last_command_height}" -gt '1' ]]; then + _output_array+=( + 'last_command: ->' + "${_last_command}" + ) + else + _output_array+=("last_command: ${_last_command}") + fi + + _output_array+=('---') + _output_array+=('--- CRITICAL ERROR! ---') + _output_array+=('---') + printf '%s\n' "${_output_array[@]}" >&2 +} + +function print_logo() { + local blue_code=(17 17 18 19 20 20 21 04 27 26 32 12 33 06 39 38 44 45) + local IFS='\n\t' + local logo_string="$(cat <<- EOF + .-----------------------------------------------------------. + | * . .. . * | + | . . . : . . . | + | . . . . . . | + | . . *:. . . | + | . . . .. . . | + | . . . . ... . . | + | . . . . . . . . . | + | . . . ... .. . . | + | . . . *. . . | + | :. . . | + | . . . . | + | . . . . ^ | + | . .. :. . | . . | + |. ... . | | + | :. . . *. | . . | + | *. We are here. | + | . . *. | + .---------------------------------------ascii-author-unknown. + = = + = 88 88 = + = "" ,d 88 = + = 88 88 88 = + = 88 8b,dPPYba, MM88MMM ,adPPYba, 88 = + = 88 88P' '"8a 88 a8P_____88 88 = + = 88 88 88 88 8PP""""""" 88 = + = 88 88 88 88, "8b, ,aa 88 = + = 88 88 88 "Y888 '"Ybbd8"' 88 = + = = + ============================================================= + = Intelยฎ Technology Poland 2022 = + = by Cloud Native Data Services Team = + =-----------------------------------------------------------= + = PROJECT: Cloud Native Data Services = + =-----------------------------------------------------------= + ============================================================= + EOF + )" + + local colorized_logo_string=() + for (( i=0; i<${#logo_string}; i++ )) + do + colorized_logo_string+="\e[38;05;${blue_code[$(( (i-(i/64)*64)/4 ))]}m"; + colorized_logo_string+="${logo_string:$i:1}"; + done; + colorized_logo_string+='\e[m\n' + + echo -e "$colorized_logo_string" >&2 +} diff --git a/presto-native-execution/scripts/release-centos-dockerfile/opt/entrypoint.sh b/presto-native-execution/scripts/release-centos-dockerfile/opt/entrypoint.sh new file mode 100755 index 0000000000000..b4ccba3e65b3f --- /dev/null +++ b/presto-native-execution/scripts/release-centos-dockerfile/opt/entrypoint.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -eExv -o functrace + +SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") +PRESTO_HOME="${PRESTO_HOME:-"/opt/presto/"}" +USE_ENV_PARAMS=${USE_ENV_PARAMS:-0} + +source "${SCRIPT_DIR}/common.sh" + +trap 'exit 2' SIGSTOP SIGINT SIGTERM SIGQUIT +trap 'failure "LINENO" "BASH_LINENO" "${BASH_COMMAND}" "${?}"; [ -z "${DEBUG}" ] && exit 1 || sleep 3600' ERR + +if [[ "${DEBUG}" == "0" || "${DEBUG}" == "false" || "${DEBUG}" == "False" ]]; then DEBUG=""; fi + +while getopts ':-:' optchar; do + case "$optchar" in + -) + case "$OPTARG" in + discovery-uri=*) discovery_uri="${OPTARG#*=}" ;; + http-server-port=*) http_server_port="${OPTARG#*=}" ;; + use-env-params) USE_ENV_PARAMS=1 ;; + *) + presto_args+=($optchar) + ;; + esac + ;; + *) + presto_args+=($optchar) + ;; + esac +done + +function node_command_line_config() +{ + printf "presto.version=0.273.3\n" > "${PRESTO_HOME}/config.properties" + printf "discovery.uri=${discovery_uri}\n" >> "${PRESTO_HOME}/config.properties" + printf "http-server.http.port=${http_server_port}\n" >> "${PRESTO_HOME}/config.properties" + + printf "node.environment=intel-poland\n" > "${PRESTO_HOME}/node.properties" + printf "node.location=torun-cluster\n" >> "${PRESTO_HOME}/node.properties" + printf "node.id=${NODE_UUID}\n" >> "${PRESTO_HOME}/node.properties" + printf "node.ip=$(hostname -I)\n" >> "${PRESTO_HOME}/node.properties" +} + +function node_configuration() +{ + cat "${PRESTO_HOME}/config.properties.template" > "${PRESTO_HOME}/config.properties" + cat "${PRESTO_HOME}/node.properties.template" > "${PRESTO_HOME}/node.properties" + + [ -z "$NODE_UUID" ] && NODE_UUID=$(uuid) || return -2 + + if [[ -z "$(grep -E '^ *node\.id=' "${PRESTO_HOME}/node.properties" | cut -d'=' -f2)" ]] + then + printf "node.id=${NODE_UUID}\n" >> "${PRESTO_HOME}/node.properties" + fi + printf "node.ip=$(hostname -I)\n" >> "${PRESTO_HOME}/node.properties" +} + +[ $USE_ENV_PARAMS == "1" ] && node_command_line_config || node_configuration + +cd "${PRESTO_HOME}" +"${PRESTO_HOME}/presto_server" --logtostderr=1 --v=1 "${presto_args[@]}"