1.15: CI fixes backport (#13697)

Backport following commits to 1.15:
748b2ab (mac ci: try ignoring update failure (#13658), 2020-10-20)
f95f539 (ci: various improvements (#13660), 2020-10-20)
73d78f8 (ci: use multiple stage (#13557), 2020-10-15)
b7a4756 (ci: use azp for api and go-control-plane sync (#13550), 2020-10-14)
876a6bb (ci use azp to sync filter example (#13501), 2020-10-12)
a0f31ee (ci: use azp to generate docs (#13481), 2020-10-12)

Signed-off-by: Lizan Zhou <lizan@tetrate.io>
Co-authored-by: asraa <asraa@google.com>

ci/api_mirror.sh

@@ -3,16 +3,15 @@
 set -e
 
 CHECKOUT_DIR=../data-plane-api
+MAIN_BRANCH="refs/heads/master"
+API_MAIN_BRANCH="master"
 
-if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
+if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then
   echo "Cloning..."
-  git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR"
+  git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR" -b "${API_MAIN_BRANCH}"
 
-  git -C "$CHECKOUT_DIR" config user.name "data-plane-api(CircleCI)"
+  git -C "$CHECKOUT_DIR" config user.name "data-plane-api(Azure Pipelines)"
   git -C "$CHECKOUT_DIR" config user.email data-plane-api@users.noreply.github.com
-  git -C "$CHECKOUT_DIR" fetch
-  git -C "$CHECKOUT_DIR" checkout -B master origin/master
 
   # Determine last envoyproxy/envoy SHA in envoyproxy/data-plane-api
   MIRROR_MSG="Mirrored from https://github.com/envoyproxy/envoy"
@@ -40,6 +39,6 @@ then
   done
 
   echo "Pushing..."
-  git -C "$CHECKOUT_DIR" push origin master
+  git -C "$CHECKOUT_DIR" push origin "${API_MAIN_BRANCH}"
   echo "Done"
 fi

ci/do_ci.sh

@@ -246,9 +246,16 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then
 
   exit 0
 elif [[ "$CI_TARGET" == "bazel.api" ]]; then
+  # Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib,
+  # which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild.
+  ENVOY_STDLIB="libstdc++"
   setup_clang_toolchain
+  export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config
   echo "Validating API structure..."
   ./tools/api/validate_structure.py
+  echo "Testing API and API Boosting..."
+  bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \
+    @envoy_api_canonical//tools:tap2pcap_test @envoy_dev//clang_tools/api_booster/...
   echo "Building API..."
   bazel build ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api_canonical//envoy/...
   echo "Testing API..."
@@ -258,7 +265,7 @@ elif [[ "$CI_TARGET" == "bazel.api" ]]; then
   bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_dev//clang_tools/api_booster/...
   echo "Testing API boosting (golden C++ tests)..."
   # We use custom BAZEL_BUILD_OPTIONS here; the API booster isn't capable of working with libc++ yet.
-  LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config BAZEL_BUILD_OPTIONS="--config=clang" python3.8 ./tools/api_boost/api_boost_test.py
+  BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" python3.8 ./tools/api_boost/api_boost_test.py
   exit 0
 elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then
   setup_clang_toolchain

ci/filter_example_mirror.sh

@@ -4,16 +4,15 @@ set -e
 
 ENVOY_SRCDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd)
 CHECKOUT_DIR=../envoy-filter-example
+MAIN_BRANCH="refs/heads/master"
+FILTER_EXAMPLE_MAIN_BRANCH="master"
 
-if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
+if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then
   echo "Cloning..."
-  git clone git@github.com:envoyproxy/envoy-filter-example "$CHECKOUT_DIR"
+  git clone git@github.com:envoyproxy/envoy-filter-example "$CHECKOUT_DIR" -b "${FILTER_EXAMPLE_MAIN_BRANCH}"
 
-  git -C "$CHECKOUT_DIR" config user.name "envoy-filter-example(CircleCI)"
+  git -C "$CHECKOUT_DIR" config user.name "envoy-filter-example(Azure Pipelines)"
   git -C "$CHECKOUT_DIR" config user.email envoy-filter-example@users.noreply.github.com
-  git -C "$CHECKOUT_DIR" fetch
-  git -C "$CHECKOUT_DIR" checkout -B master origin/master
 
   echo "Updating Submodule..."
   # Update submodule to latest Envoy SHA
@@ -26,6 +25,6 @@ then
 
   echo "Committing, and Pushing..."
   git -C "$CHECKOUT_DIR" commit -a -m "Update Envoy submodule to $ENVOY_SHA"
-  git -C "$CHECKOUT_DIR" push origin master
+  git -C "$CHECKOUT_DIR" push origin "${FILTER_EXAMPLE_MAIN_BRANCH}"
   echo "Done"
 fi

ci/go_mirror.sh

@@ -2,7 +2,11 @@
 
 set -e
 
-if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
-  tools/api/generate_go_protobuf.py
+MAIN_BRANCH="refs/heads/master"
+
+# shellcheck source=ci/setup_cache.sh
+. "$(dirname "$0")"/setup_cache.sh
+
+if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then
+  BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS}" tools/api/generate_go_protobuf.py
 fi

ci/mac_ci_setup.sh

@@ -20,9 +20,23 @@ function install {
     fi
 }
 
-if ! brew update; then
-    echo "Failed to update homebrew"
-    exit 1
+function retry () {
+    local returns=1 i=1
+    while ((i<=HOMEBREW_RETRY_ATTEMPTS)); do
+	if "$@"; then
+	    returns=0
+	    break
+	else
+	    sleep "$HOMEBREW_RETRY_INTERVAL";
+	    ((i++))
+	fi
+    done
+    return "$returns"
+}
+
+if ! retry brew update; then
+  # Do not exit early if update fails.
+  echo "Failed to update homebrew"
 fi
 
 DEPS="automake cmake coreutils go libtool wget ninja"

ci/run_envoy_docker.sh

@@ -18,13 +18,15 @@ USER_GROUP=root
 
 [[ -t 1 ]] && ENVOY_DOCKER_OPTIONS+=" -it"
 [[ -f .git ]] && [[ ! -d .git ]] && ENVOY_DOCKER_OPTIONS+=" -v $(git rev-parse --git-common-dir):$(git rev-parse --git-common-dir)"
+[[ -n "${SSH_AUTH_SOCK}" ]] && ENVOY_DOCKER_OPTIONS+="-v ${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK} -e SSH_AUTH_SOCK"
 
 export ENVOY_BUILD_IMAGE="${IMAGE_NAME}:${IMAGE_ID}"
 
 mkdir -p "${ENVOY_DOCKER_BUILD_DIR}"
 # Since we specify an explicit hash, docker-run will pull from the remote repo if missing.
 docker run --rm ${ENVOY_DOCKER_OPTIONS} -e HTTP_PROXY=${http_proxy} -e HTTPS_PROXY=${https_proxy} -e NO_PROXY=${no_proxy} \
   -u "${USER}":"${USER_GROUP}" -v "${ENVOY_DOCKER_BUILD_DIR}":/build -v /var/run/docker.sock:/var/run/docker.sock  \
+  -e AZP_BRANCH \
   -e BAZEL_BUILD_EXTRA_OPTIONS -e BAZEL_EXTRA_TEST_OPTIONS -e BAZEL_REMOTE_CACHE -e ENVOY_STDLIB -e BUILD_REASON \
   -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS -e ENVOY_RBE -e FUZZIT_API_KEY -e ENVOY_BUILD_IMAGE \
   -e ENVOY_SRCDIR -e ENVOY_BUILD_TARGET -e SYSTEM_PULLREQUEST_TARGETBRANCH -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \

docs/build.sh

@@ -4,25 +4,30 @@
 
 set -e
 
+RELEASE_TAG_REGEX="^refs/tags/v.*"
+
+if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
+  DOCS_TAG="${AZP_BRANCH/refs\/tags\//}"
+fi
+
 # We need to set ENVOY_DOCS_VERSION_STRING and ENVOY_DOCS_RELEASE_LEVEL for Sphinx.
 # We also validate that the tag and version match at this point if needed.
-if [ -n "$CIRCLE_TAG" ]
-then
+if [[ -n "${DOCS_TAG}" ]]; then
   # Check the git tag matches the version number in the VERSION file.
   VERSION_NUMBER=$(cat VERSION)
-  if [ "v${VERSION_NUMBER}" != "${CIRCLE_TAG}" ]; then
+  if [[ "v${VERSION_NUMBER}" != "${DOCS_TAG}" ]]; then
     echo "Given git tag does not match the VERSION file content:"
-    echo "${CIRCLE_TAG} vs $(cat VERSION)"
+    echo "${DOCS_TAG} vs $(cat VERSION)"
     exit 1
   fi
   # Check the version_history.rst contains current release version.
   grep --fixed-strings "$VERSION_NUMBER" docs/root/version_history/current.rst \
     || (echo "Git tag not found in version_history/current.rst" && exit 1)
 
   # Now that we know there is a match, we can use the tag.
-  export ENVOY_DOCS_VERSION_STRING="tag-$CIRCLE_TAG"
+  export ENVOY_DOCS_VERSION_STRING="tag-${DOCS_TAG}"
   export ENVOY_DOCS_RELEASE_LEVEL=tagged
-  export ENVOY_BLOB_SHA="$CIRCLE_TAG"
+  export ENVOY_BLOB_SHA="${DOCS_TAG}"
 else
   BUILD_SHA=$(git rev-parse HEAD)
   VERSION_NUM=$(cat VERSION)

docs/publish.sh

@@ -10,35 +10,36 @@
 set -e
 
 DOCS_DIR=generated/docs
-CHECKOUT_DIR=../envoy-docs
-BUILD_SHA=`git rev-parse HEAD`
-
-if [ -n "$CIRCLE_TAG" ]
-then
-  PUBLISH_DIR="$CHECKOUT_DIR"/docs/envoy/"$CIRCLE_TAG"
-elif [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
-  PUBLISH_DIR="$CHECKOUT_DIR"/docs/envoy/latest
+CHECKOUT_DIR=envoy-docs
+BUILD_SHA=$(git rev-parse HEAD)
+
+MAIN_BRANCH="refs/heads/master"
+RELEASE_TAG_REGEX="^refs/tags/v.*"
+
+if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
+  PUBLISH_DIR="${CHECKOUT_DIR}"/docs/envoy/"${AZP_BRANCH/refs\/tags\//}"
+elif [[ "$AZP_BRANCH" == "${MAIN_BRANCH}" ]]; then
+  PUBLISH_DIR="${CHECKOUT_DIR}"/docs/envoy/latest
 else
   echo "Ignoring docs push"
   exit 0
 fi
 
+DOCS_MAIN_BRANCH="master"
+
 echo 'cloning'
-git clone git@github.com:envoyproxy/envoyproxy.github.io "$CHECKOUT_DIR"
+git clone git@github.com:envoyproxy/envoyproxy.github.io "${CHECKOUT_DIR}" -b "${DOCS_MAIN_BRANCH}" --depth 1
 
-git -C "$CHECKOUT_DIR" fetch
-git -C "$CHECKOUT_DIR" checkout -B master origin/master
 rm -fr "$PUBLISH_DIR"
 mkdir -p "$PUBLISH_DIR"
 cp -r "$DOCS_DIR"/* "$PUBLISH_DIR"
-cd "$CHECKOUT_DIR"
+cd "${CHECKOUT_DIR}"
 
-git config user.name "envoy-docs(travis)"
+git config user.name "envoy-docs(Azure Pipelines)"
 git config user.email envoy-docs@users.noreply.github.com
-echo 'add'
+
+set -x
+
 git add .
-echo 'commit'
 git commit -m "docs envoy@$BUILD_SHA"
-echo 'push'
-git push origin master
+git push origin "${DOCS_MAIN_BRANCH}"

tools/api/generate_go_protobuf.py

@@ -4,17 +4,21 @@
 from subprocess import check_call
 import glob
 import os
+import shlex
 import shutil
 import sys
 import re
 
+# Needed for CI to pass down bazel options.
+BAZEL_BUILD_OPTIONS = shlex.split(os.environ.get('BAZEL_BUILD_OPTIONS', ''))
+
 TARGETS = '@envoy_api//...'
 IMPORT_BASE = 'github.com/envoyproxy/go-control-plane'
 OUTPUT_BASE = 'build_go'
 REPO_BASE = 'go-control-plane'
 BRANCH = 'master'
 MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ '
-USER_NAME = 'go-control-plane(CircleCI)'
+USER_NAME = 'go-control-plane(Azure Pipelines)'
 USER_EMAIL = 'go-control-plane@users.noreply.github.com'
 
 
@@ -28,7 +32,7 @@ def generateProtobufs(output):
 
   # Each rule has the form @envoy_api//foo/bar:baz_go_proto.
   # First build all the rules to ensure we have the output files.
-  check_call(['bazel', 'build', '-c', 'fastbuild'] + go_protos)
+  check_call(['bazel', 'build', '-c', 'fastbuild'] + BAZEL_BUILD_OPTIONS + go_protos)
 
   for rule in go_protos:
     # Example rule:
@@ -63,9 +67,7 @@ def git(repo, *args):
 
 def cloneGoProtobufs(repo):
   # Create a local clone of go-control-plane
-  git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo)
-  git(repo, 'fetch')
-  git(repo, 'checkout', '-B', BRANCH, 'origin/master')
+  git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo, '-b', BRANCH)
 
 
 def findLastSyncSHA(repo):

tools/api_boost/api_boost.py

@@ -133,7 +133,6 @@ def ApiBoostTree(target_paths,
     sp.run([
         'bazel',
         'build',
-        '--config=libc++',
         '--strip=always',
     ] + BAZEL_BUILD_OPTIONS + dep_lib_build_targets,
            check=True)