Merge remote-tracking branch 'upstream/main' into fuzz_bug_67870

Signed-off-by: Adi Suissa-Peleg <adip@google.com>

.dockerignore

@@ -6,8 +6,10 @@
 !/configs/*yaml
 !/linux/amd64/release.tar.zst
 !/linux/amd64/schema_validator_tool
+!/linux/amd64/router_check_tool
 !/linux/arm64/release.tar.zst
 !/linux/arm64/schema_validator_tool
+!/linux/arm64/router_check_tool
 !/local
 !/test/config/integration/certs
 !/windows

.github/workflows/mobile-ios_build.yml

@@ -42,7 +42,7 @@ jobs:
       command: ./bazelw
       container-command:
       request: ${{ needs.load.outputs.request }}
-      runs-on: macos-12
+      runs-on: macos-12-large
       source: |
         source ./ci/mac_ci_setup.sh
         ./bazelw shutdown
@@ -81,7 +81,7 @@ jobs:
       command: ./bazelw
       container-command:
       request: ${{ needs.load.outputs.request }}
-      runs-on: macos-12
+      runs-on: macos-12-large
       source: |
         source ./ci/mac_ci_setup.sh
         ./bazelw shutdown
@@ -126,7 +126,7 @@ jobs:
       command: ./bazelw
       container-command:
       request: ${{ needs.load.outputs.request }}
-      runs-on: macos-12
+      runs-on: macos-12-large
       source: |
         source ./ci/mac_ci_setup.sh
       steps-post: |
@@ -137,6 +137,7 @@ jobs:
             expected: >-
               ${{ matrix.expected
                   || format('received headers with status {0}', matrix.expected-status) }}
+            timeout: ${{ matrix.timeout || '5m' }}
           env:
             ANDROID_NDK_HOME:
             ANDROID_HOME:
@@ -163,6 +164,7 @@ jobs:
           expected: >-
             \[2\] Uploaded 7 MB of data
           target: swift-async-await
+          timeout: 10m
         - name: Build objc hello world
           app: //examples/objective-c/hello_world:app
           expected-status: 301

.github/workflows/mobile-perf.yml

@@ -61,17 +61,9 @@ jobs:
             --config=mobile-remote-release-clang
             //test/performance:test_binary_size
           # Ensure files don't leak back into the main binary
-          source: >-
-            rm
-            source/common/http/route_config_update_requster.h
-            source/common/listener_manager/listener_manager_impl.h
-            source/server/overload_manager_impl.cc
-            source/common/network/listen_socket_impl.h
-            source/common/network/tcp_listener_impl.h
-            source/server/guarddog_impl.h
-            source/server/watchdog_impl.h
-            source/server/options_impl.cc
-            source/extensions/access_loggers/common/file_access_log_impl.h
+          source: |-
+            TOREMOVE=$(cat mobile/test/performance/files_em_does_not_use | grep -vE '^#')
+            rm $TOREMOVE
           target: size-current
         - name: Main size
           args: >-

api/envoy/extensions/filters/http/composite/v3/composite.proto

@@ -53,9 +53,11 @@ message ExecuteFilterAction {
   config.core.v3.TypedExtensionConfig typed_config = 1
       [(udpa.annotations.field_migrate).oneof_promotion = "config_type"];
 
-  // Dynamic configuration of filter obtained via extension configuration discovery
-  // service.
+  // Dynamic configuration of filter obtained via extension configuration discovery service.
   // Only one of ``typed_config`` or ``dynamic_config`` can be set.
+  // When composite filter is in upstream filter chain, the ``dynamic_config``
+  // can not be set. TBD: Refactor cluster manager init sequence to Support this.
+  // Please check https://github.com/envoyproxy/envoy/issues/33218 for details.
   DynamicConfig dynamic_config = 2
       [(udpa.annotations.field_migrate).oneof_promotion = "config_type"];
 }

bazel/coverage/fuzz_coverage_wrapper.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+
+set -e
 
 TEST_BINARY=$1
 shift

bazel/external/boringssl_fips.genrule_cmd

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

bazel/gen_sh_test_runner.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Used in a genrule to wrap sh_test script for execution in
 # //test/coverage:coverage_tests single binary.

bazel/get_workspace_status

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This file was imported from https://github.com/bazelbuild/bazel at d6fec93.
 

bazel/setup_clang.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+
+set -e
 
 BAZELRC_FILE="${BAZELRC_FILE:-./clang.bazelrc}"
 

bazel/setup_local_tsan.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 BAZELRC_FILE="${BAZELRC_FILE:-$(bazel info workspace)/local_tsan.bazelrc}"
 

bazel/sh_test_wrapper.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Dummy shell implementation for nooping tests.
 # TODO(lizan): remove when we have a solution for

bazel/test/verify_tap_test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -ex
 

bazel/test_for_benchmark_wrapper.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Set the benchmark time to 0 to just verify that the benchmark runs to
 # completion.  We're interacting with two different flag parsers, so the order

bazel/version_update_post.sh

@@ -1,5 +1,6 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
 
+set -e
 set -o pipefail
 
 

bazel/volatile_env.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+
+set -e
 
 if [[ ! -f bazel-out/volatile-status.txt ]]; then
     # shellcheck disable=SC2016

changelogs/current.yaml

@@ -408,6 +408,12 @@ new_features:
 - area: http
   change: |
     Added DownstreamRemoteReset to CoreResponseFlag, and it is set when stream is remote reset.
+- area: match_delegate
+  change: |
+    Convert match_delegate fiter into dual filter so it can be installed in the upstream filter chain.
+- area: composite
+  change: |
+    Convert composite fiter into dual filter so it can be installed in the upstream filter chain.
 - area: tracing
   change: |
     Added support for variant span attribute type for the OpenTelemetry tracer.

ci/Dockerfile-envoy

@@ -54,7 +54,7 @@ FROM envoy AS envoy-tools
 ARG TARGETPLATFORM
 ENV TARGETPLATFORM="${TARGETPLATFORM:-linux/amd64}"
 COPY --chown=0:0 --chmod=755 \
-    "${TARGETPLATFORM}/schema_validator_tool" /usr/local/bin/schema_validator_tool
+    "${TARGETPLATFORM}/schema_validator_tool" "${TARGETPLATFORM}/router_check_tool" /usr/local/bin/
 
 
 # STAGE: envoy-distroless

ci/build_setup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Configure environment variables for Bazel build and test.
 

ci/check_and_fix_format.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

ci/do_ci.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Run a CI build/test target, e.g. docs, asan.
 
@@ -134,6 +134,8 @@ function cp_binary_for_image_build() {
   # Tools for the tools image. Strip to save size.
   strip bazel-bin/test/tools/schema_validator/schema_validator_tool \
     -o "${BASE_TARGET_DIR}"/"${TARGET_DIR}"/schema_validator_tool
+  strip bazel-bin/test/tools/router_check/router_check_tool \
+    -o "${BASE_TARGET_DIR}"/"${TARGET_DIR}"/router_check_tool
 
   # Copy the su-exec utility binary into the image
   cp -f bazel-bin/external/com_github_ncopa_suexec/su-exec "${BASE_TARGET_DIR}"/"${TARGET_DIR}"
@@ -199,6 +201,8 @@ function bazel_binary_build() {
   # Validation tools for the tools image.
   bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" \
     //test/tools/schema_validator:schema_validator_tool "${CONFIG_ARGS[@]}"
+  bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" \
+    //test/tools/router_check:router_check_tool "${CONFIG_ARGS[@]}"
 
   # Build su-exec utility
   bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" external:su-exec
@@ -853,6 +857,12 @@ case $CI_TARGET in
         cp -a \
            bazel-bin/test/tools/schema_validator/schema_validator_tool.stripped \
            "${ENVOY_BINARY_DIR}/schema_validator_tool"
+        bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" \
+              --remote_download_toplevel \
+              //test/tools/router_check:router_check_tool.stripped
+        cp -a \
+           bazel-bin/test/tools/router_check/router_check_tool.stripped \
+           "${ENVOY_BINARY_DIR}/router_check_tool"
         echo "Release files created in ${ENVOY_BINARY_DIR}"
         ;;
 

ci/docker_ci.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Do not ever set -x here, it is a security hazard as it will place the credentials below in the
 # CI logs.

ci/docker_rebuild_google-vrp.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Script to rebuild Dockerfile-envoy-google-vrp locally (i.e. not in CI) for development purposes.
 # This makes use of the latest envoy:dev base image on Docker Hub as the base and takes an

ci/envoy_build_sha.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 
 ENVOY_BUILD_CONTAINER="$(grep envoyproxy/envoy-build-ubuntu "$(dirname "$0")"/../.bazelrc | sed -e 's#.*envoyproxy/envoy-build-ubuntu:\(.*\)#\1#' | uniq)"

ci/filter_example_setup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Configure environment for Envoy Filter Example build and test.
 

ci/format_pre.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -E
+#!/usr/bin/env bash
+
+set -E
 
 # Pre-checks for validation and linting
 #

ci/mac_ci_setup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Installs the dependencies required for a macOS build via homebrew.
 # Tools are not upgraded to new versions.

ci/mac_ci_steps.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

ci/reopen_branch.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+
+set -e
 
 ENVOY_GIT_USERNAME="${ENVOY_GIT_USERNAME:-envoy-bot}"
 ENVOY_GIT_EMAIL="${ENVOY_GIT_EMAIL:-envoy-bot@users.noreply.github.com}"

ci/run_clang_tidy.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -eo pipefail
 

ci/run_envoy_docker.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

ci/test_docker_ci.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Run this with `./ci/test_docker_ci.sh`.
 #

ci/upload_gcs_artifact.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e -o pipefail
 

ci/verify_examples.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -E
+#!/usr/bin/env bash
+
+set -E
 
 TESTFILTER="${1:-*}"
 TESTEXCLUDES="${2}"

configs/configgen.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

configs/google-vrp/launch_envoy.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 cd /etc/envoy || exit
 envoy "$@"

contrib/cryptomb/private_key_providers/test/test_data/generate-keys.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

contrib/qat/private_key_providers/test/test_data/generate-keys.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

distribution/distrotest.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -E
+#!/usr/bin/env bash
+
+set -E
 
 FAILED=()
 TESTNAME=

docs/root/configuration/operations/overload_manager/overload_manager.rst

@@ -382,4 +382,5 @@ with the following statistics:
   :widths: 1, 1, 2
 
   scale_percent, Gauge, "Scaled value of the action as a percent (0-99=scaling, 100=saturated)"
+  shed_load_count, Counter, "Total count the load is sheded"
 

docs/root/operations/tools/route_table_check_tool.rst

@@ -72,6 +72,8 @@ Output
   If the ``--only-show-failures`` flag is provided, only the failed test cases are written to a file.
 
 Building
+  The tool is included in the :ref:`tools image <install_tools>`.
+
   The tool can be built locally using Bazel. ::
 
     bazel build //test/tools/router_check:router_check_tool

envoy/upstream/cluster_manager.h

@@ -214,6 +214,21 @@ class ClusterManager {
 
   virtual ~ClusterManager() = default;
 
+  // API to initialize the ClusterManagerImpl instance based on the given Bootstrap config.
+  //
+  // This method *must* be called prior to invoking any other methods on the class and *must* only
+  // be called once. This method should be called immediately after ClusterManagerImpl construction
+  // and from the same thread in which the ClusterManagerImpl was constructed.
+  //
+  // The initialization is separated from the constructor because lots of work, including ADS
+  // initialization, is done in this method. If the contents of this method are invoked during
+  // construction, a derived class cannot override any of the virtual methods and have them invoked
+  // instead, since the base class's methods are used when in a base class constructor.
+  virtual absl::Status initialize(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) PURE;
+
+  // API to return whether the ClusterManagerImpl instance is initialized.
+  virtual bool initialized() PURE;
+
   /**
    * Add or update a cluster via API. The semantics of this API are:
    * 1) The hash of the config is used to determine if an already existing cluster has changed.
@@ -491,6 +506,8 @@ class ClusterManagerFactory {
 
   /**
    * Allocate a cluster manager from configuration proto.
+   * The cluster manager init() method needs to be called right after this method.
+   * Please check https://github.com/envoyproxy/envoy/issues/33218 for details.
    */
   virtual ClusterManagerPtr
   clusterManagerFromProto(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) PURE;