ci: disable cache in AWS pool (envoyproxy#11633)

Bottleneck of coverage build was network instead of CPU, while network transfer is expensive. The coverage build artifacts is huge, saving cache storage + network by CPU. At least until we have cache inside AWS.

Along with some CI optimization:
- remove workarounds for self run agents
- tar coverage report for artifact to reduce upload time
- append arm64 for arm64 artifacts

Signed-off-by: Lizan Zhou <lizan@tetrate.io>

.azure-pipelines/bazel.yml

@@ -3,25 +3,22 @@ parameters:
     displayName: "CI target"
     type: string
     default: bazel.release
+  - name: artifactSuffix
+    displayName: "Suffix of artifact"
+    type: string
+    default: ""
   - name: rbe
     displayName: "Enable RBE"
-    type: string
-    default: "true"
-  - name: bazelBuildExtraOptions
-    type: string
-    # Use https://docs.bazel.build/versions/master/command-line-reference.html#flag--experimental_repository_cache_hardlinks
-    # to save disk space.
-    default: "--config=remote-ci --jobs=$(RbeJobs) --curses=no --experimental_repository_cache_hardlinks"
+    type: boolean
+    default: true
   - name: managedAgent
     type: boolean
     default: true
+  - name: bazelBuildExtraOptions
+    type: string
+    default: ""
 
 steps:
-  - bash: |
-      /usr/local/bin/set-instance-protection.sh on
-    displayName: "Set Instance Protection on Agent to prevent scale in"
-    condition:  eq(false, ${{ parameters.managedAgent }})
-
   - task: Cache@2
     inputs:
       key: '"${{ parameters.ciTarget }}" | ./WORKSPACE | **/*.bzl'
@@ -51,11 +48,17 @@ steps:
     workingDirectory: $(Build.SourcesDirectory)
     env:
       ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-      ENVOY_RBE: "${{ parameters.rbe }}"
-      BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelBuildExtraOptions }}"
-      BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
-      BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
-      GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+      ${{ if parameters.rbe }}:
+        ENVOY_RBE: "1"
+        # Use https://docs.bazel.build/versions/master/command-line-reference.html#flag--experimental_repository_cache_hardlinks
+        # to save disk space.
+        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs) --curses=no --experimental_repository_cache_hardlinks ${{ parameters.bazelBuildExtraOptions }}"
+        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+      ${{ if eq(parameters.rbe, false) }}:
+        BAZEL_BUILD_EXTRA_OPTIONS: "--curses=no --experimental_repository_cache_hardlinks ${{ parameters.bazelBuildExtraOptions }}"
+
     displayName: "Run CI script"
 
   - bash: |
@@ -74,17 +77,5 @@ steps:
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: "$(Build.StagingDirectory)/envoy"
-      artifactName: ${{ parameters.ciTarget }}
+      artifactName: ${{ parameters.ciTarget }}${{ parameters.artifactSuffix }}
     condition: always()
-
-  # TODO(lizan): This is a workaround for self hosted azure agent can't clean up bazel local cache due to
-  # permission. Remove this once it is resolved.
-  - bash: |
-      chmod -R u+w $(Build.StagingDirectory)
-    displayName: "Self hosted agent clean up"
-    condition: eq(false, ${{ parameters.managedAgent }})
-
-  - bash: |
-      /usr/local/bin/set-instance-protection.sh off
-    displayName: "Set Instance Protection on Agent to prevent scale in"
-    condition:  eq(false, ${{ parameters.managedAgent }})

.azure-pipelines/pipelines.yml

@@ -60,8 +60,8 @@ jobs:
         parameters:
           managedAgent: false
           ciTarget: bazel.release.server_only
-          rbe: ""
-          bazelBuildExtraOptions: "--curses=no"
+          rbe: false
+          artifactSuffix: ".arm64"
 
   - job: bazel
     displayName: "Linux-x64"
@@ -99,10 +99,11 @@ jobs:
         parameters:
           managedAgent: false
           ciTarget: bazel.coverage
-          rbe: ""
-          bazelBuildExtraOptions: "--test_env=ENVOY_IP_TEST_VERSIONS=v4only --curses=no"
+          rbe: false
+          # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces
+          bazelBuildExtraOptions: "--test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base"
 
-      - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /build/envoy/generated/coverage coverage'
+      - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/coverage coverage'
         displayName: "Upload Report to GCS"
         env:
           ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)

.bazelrc

@@ -115,8 +115,8 @@ coverage --config=coverage
 build:coverage --action_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1
 build:coverage --action_env=GCOV=llvm-profdata
 build:coverage --copt=-DNDEBUG
-# Doubling timeout in all categories
-build:coverage --test_timeout=120,600,1800,7200
+# 1.5x original timeout + 300s for trace merger in all categories
+build:coverage --test_timeout=390,750,1500,5700
 build:coverage --define=ENVOY_CONFIG_COVERAGE=1
 build:coverage --cxxopt="-DENVOY_CONFIG_COVERAGE=1"
 build:coverage --coverage_support=@envoy//bazel/coverage:coverage_support

ci/build_setup.sh

@@ -103,8 +103,7 @@ export ENVOY_DELIVERY_DIR="${ENVOY_BUILD_DIR}"/source/exe
 mkdir -p "${ENVOY_DELIVERY_DIR}"
 
 # This is where we copy the coverage report to.
-export ENVOY_COVERAGE_DIR="${ENVOY_BUILD_DIR}"/generated/coverage
-mkdir -p "${ENVOY_COVERAGE_DIR}"
+export ENVOY_COVERAGE_ARTIFACT="${ENVOY_BUILD_DIR}"/generated/coverage.tar.gz
 
 # This is where we dump failed test logs for CI collection.
 export ENVOY_FAILED_TEST_LOGS="${ENVOY_BUILD_DIR}"/generated/failed-testlogs

test/run_envoy_bazel_coverage.sh

@@ -30,7 +30,11 @@ else
   BAZEL_BUILD_OPTIONS+=" --config=test-coverage --test_tag_filters=-nocoverage,-fuzz_target"
 fi
 
-bazel coverage ${BAZEL_BUILD_OPTIONS} --test_output=all ${COVERAGE_TARGETS}
+bazel coverage ${BAZEL_BUILD_OPTIONS} ${COVERAGE_TARGETS}
+
+# Collecting profile and testlogs
+[[ -z "${ENVOY_BUILD_PROFILE}" ]] || cp -f "$(bazel info output_base)/command.profile.gz" "${ENVOY_BUILD_PROFILE}/coverage.profile.gz" || true
+[[ -z "${ENVOY_BUILD_DIR}" ]] || find bazel-testlogs/ -name test.log | tar zcf "${ENVOY_BUILD_DIR}/testlogs.tar.gz" -T -
 
 COVERAGE_DIR="${SRCDIR}"/generated/coverage
 
@@ -43,7 +47,7 @@ cp bazel-out/_coverage/_coverage_report.dat "${COVERAGE_DATA}"
 COVERAGE_VALUE=$(genhtml --prefix ${PWD} --output "${COVERAGE_DIR}" "${COVERAGE_DATA}" | tee /dev/stderr | grep lines... | cut -d ' ' -f 4)
 COVERAGE_VALUE=${COVERAGE_VALUE%?}
 
-[[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}"
+[[ -z "${ENVOY_COVERAGE_ARTIFACT}" ]] || tar zcf "${ENVOY_COVERAGE_ARTIFACT}" -C ${COVERAGE_DIR} --transform 's/^\./coverage/' .
 
 if [[ "$VALIDATE_COVERAGE" == "true" ]]; then
   if [[ "${FUZZ_COVERAGE}" == "true" ]]; then