Merge envoy/release/v1.16 into release-1.8

VERSION

@@ -1 +1 @@
-1.16.1
+1.16.3-dev

ci/Dockerfile-envoy-google-vrp

@@ -16,6 +16,9 @@ ADD configs/google-vrp/supervisor.conf /etc/supervisor.conf
 ADD test/config/integration/certs/serverkey.pem /etc/envoy/certs/serverkey.pem
 ADD test/config/integration/certs/servercert.pem /etc/envoy/certs/servercert.pem
 # ADD %local envoy bin% /usr/local/bin/envoy
+RUN chmod 777 /var/log/supervisor
+RUN chmod a+r /etc/supervisor.conf /etc/envoy/* /etc/envoy/certs/*
+RUN chmod a+rx /usr/local/bin/launch_envoy.sh
 
 EXPOSE 10000
 EXPOSE 10001

configs/google-vrp/supervisor.conf

@@ -1,5 +1,6 @@
 [supervisord]
 nodaemon=true
+logfile=/var/log/supervisor/supervisord.log
 
 [program:envoy-edge]
 command=launch_envoy.sh -c /etc/envoy/envoy-edge.yaml %(ENV_ENVOY_EDGE_EXTRA_ARGS)s

docs/root/version_history/current.rst

@@ -1,5 +1,5 @@
-1.16.1 (November 20, 2020)
-==========================
+1.16.3 (Pending)
+================
 
 Incompatible Behavior Changes
 -----------------------------
@@ -74,12 +74,10 @@ Bug Fixes
 Bug Fixes
 ---------
 *Changes expected to improve the state of the world and are unlikely to have negative effects*
-* examples: examples use v3 configs.
-* listener: fix crash when disabling or re-enabling listeners due to overload while processing LDS updates.
-* proxy_proto: fixed a bug where the wrong downstream address got sent to upstream connections.
-* proxy_proto: fixed a bug where network filters would not have the correct downstreamRemoteAddress() when accessed from the StreamInfo. This could result in incorrect enforcement of RBAC rules in the RBAC network filter (but not in the RBAC HTTP filter), or incorrect access log addresses from tcp_proxy.
-* tls: fix read resumption after triggering buffer high-watermark and all remaining request/response bytes are stored in the SSL connection's internal buffers.
-* udp: fixed issue in which receiving truncated UDP datagrams would cause Envoy to crash.
+
+* aggregate cluster: fixed a crash due to a TLS initialization issue.
+* lua: fixed crash when Lua script contains streamInfo():downstreamSslConnection().
+* tls: fix detection of the upstream connection close event.
 
 Removed Config or Runtime
 -------------------------

docs/root/version_history/v1.16.1.rst

@@ -0,0 +1,31 @@
+1.16.1 (November 20, 2020)
+==========================
+
+Incompatible Behavior Changes
+-----------------------------
+*Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required*
+
+Minor Behavior Changes
+----------------------
+*Changes that may cause incompatibilities for some users, but should not for most*
+
+Bug Fixes
+---------
+*Changes expected to improve the state of the world and are unlikely to have negative effects*
+
+* examples: examples use v3 configs.
+* listener: fix crash when disabling or re-enabling listeners due to overload while processing LDS updates.
+* proxy_proto: fixed a bug where the wrong downstream address got sent to upstream connections.
+* proxy_proto: fixed a bug where network filters would not have the correct downstreamRemoteAddress() when accessed from the StreamInfo. This could result in incorrect enforcement of RBAC rules in the RBAC network filter (but not in the RBAC HTTP filter), or incorrect access log addresses from tcp_proxy.
+* tls: fix read resumption after triggering buffer high-watermark and all remaining request/response bytes are stored in the SSL connection's internal buffers.
+* udp: fixed issue in which receiving truncated UDP datagrams would cause Envoy to crash.
+
+Removed Config or Runtime
+-------------------------
+*Normally occurs at the end of the* :ref:`deprecation period <deprecated>`
+
+New Features
+------------
+
+Deprecated
+----------

docs/root/version_history/v1.16.2.rst

@@ -0,0 +1,27 @@
+1.16.2 (December 7, 2020)
+=========================
+
+Incompatible Behavior Changes
+-----------------------------
+*Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required*
+
+Minor Behavior Changes
+----------------------
+*Changes that may cause incompatibilities for some users, but should not for most*
+
+Bug Fixes
+---------
+*Changes expected to improve the state of the world and are unlikely to have negative effects*
+
+* http: fixed URL parsing for HTTP/1.1 fully qualified URLs and connect requests containing IPv6 addresses.
+* vrp: allow supervisord to open its log file.
+
+Removed Config or Runtime
+-------------------------
+*Normally occurs at the end of the* :ref:`deprecation period <deprecated>`
+
+New Features
+------------
+
+Deprecated
+----------

docs/root/version_history/version_history.rst

@@ -7,6 +7,8 @@ Version history
   :titlesonly:
 
   current
+  v1.16.2
+  v1.16.1
   v1.16.0
   v1.15.2
   v1.15.1

include/envoy/registry/registry.h

@@ -346,6 +346,7 @@ template <class Base> class FactoryRegistry : public Logger::Loggable<Logger::Id
     auto mapping = std::make_unique<absl::flat_hash_map<std::string, Base*>>();
 
     for (const auto& [factory_name, factory] : factories()) {
+      UNREFERENCED_PARAMETER(factory_name);
       if (factory == nullptr) {
         continue;
       }

source/common/config/new_grpc_mux_impl.cc

@@ -90,6 +90,7 @@ void NewGrpcMuxImpl::onDiscoveryResponse(
 
 void NewGrpcMuxImpl::onStreamEstablished() {
   for (auto& [type_url, subscription] : subscriptions_) {
+    UNREFERENCED_PARAMETER(type_url);
     subscription->sub_state_.markStreamFresh();
   }
   trySendDiscoveryRequests();

source/common/http/utility.cc

@@ -228,6 +228,30 @@ namespace Http {
 
 static const char kDefaultPath[] = "/";
 
+// If http_parser encounters an IP address [address] as the host it will set the offset and
+// length to point to 'address' rather than '[address]'. Fix this by adjusting the offset
+// and length to include the brackets.
+// @param absolute_url the absolute URL. This is usually of the form // http://host/path
+//        but may be host:port for CONNECT requests
+// @param offset the offset for the first character of the host. For IPv6 hosts
+//        this will point to the first character inside the brackets and will be
+//        adjusted to point at the brackets
+// @param len the length of the host-and-port field. For IPv6 hosts this will
+//        not include the brackets and will be adjusted to do so.
+bool maybeAdjustForIpv6(absl::string_view absolute_url, uint64_t& offset, uint64_t& len) {
+  // According to https://tools.ietf.org/html/rfc3986#section-3.2.2 the only way a hostname
+  // may begin with '[' is if it's an ipv6 address.
+  if (offset == 0 || *(absolute_url.data() + offset - 1) != '[') {
+    return false;
+  }
+  // Start one character sooner and end one character later.
+  offset--;
+  len += 2;
+  // HTTP parser ensures that any [ has a closing ]
+  ASSERT(absolute_url.length() >= offset + len);
+  return true;
+}
+
 bool Utility::Url::initialize(absl::string_view absolute_url, bool is_connect) {
   struct http_parser_url u;
   http_parser_url_init(&u);
@@ -244,20 +268,27 @@ bool Utility::Url::initialize(absl::string_view absolute_url, bool is_connect) {
   scheme_ = absl::string_view(absolute_url.data() + u.field_data[UF_SCHEMA].off,
                               u.field_data[UF_SCHEMA].len);
 
-  uint16_t authority_len = u.field_data[UF_HOST].len;
+  uint64_t authority_len = u.field_data[UF_HOST].len;
   if ((u.field_set & (1 << UF_PORT)) == (1 << UF_PORT)) {
     authority_len = authority_len + u.field_data[UF_PORT].len + 1;
   }
-  host_and_port_ =
-      absl::string_view(absolute_url.data() + u.field_data[UF_HOST].off, authority_len);
+
+  uint64_t authority_beginning = u.field_data[UF_HOST].off;
+  const bool is_ipv6 = maybeAdjustForIpv6(absolute_url, authority_beginning, authority_len);
+  host_and_port_ = absl::string_view(absolute_url.data() + authority_beginning, authority_len);
+  if (is_ipv6 && !parseAuthority(host_and_port_).is_ip_address_) {
+    return false;
+  }
 
   // RFC allows the absolute-uri to not end in /, but the absolute path form
-  // must start with
-  uint64_t path_len = absolute_url.length() - (u.field_data[UF_HOST].off + hostAndPort().length());
-  if (path_len > 0) {
-    uint64_t path_beginning = u.field_data[UF_HOST].off + hostAndPort().length();
-    path_and_query_params_ = absl::string_view(absolute_url.data() + path_beginning, path_len);
+  // must start with. Determine if there's a non-zero path, and if so determine
+  // the length of the path, query params etc.
+  uint64_t path_etc_len = absolute_url.length() - (authority_beginning + hostAndPort().length());
+  if (path_etc_len > 0) {
+    uint64_t path_beginning = authority_beginning + hostAndPort().length();
+    path_and_query_params_ = absl::string_view(absolute_url.data() + path_beginning, path_etc_len);
   } else if (!is_connect) {
+    ASSERT((u.field_set & (1 << UF_PATH)) == 0);
     path_and_query_params_ = absl::string_view(kDefaultPath, 1);
   }
   return true;

source/common/init/manager_impl.cc

@@ -71,6 +71,7 @@ void ManagerImpl::dumpUnreadyTargets(envoy::admin::v3::UnreadyTargetsDumps& unre
   auto& message = *unready_targets_dumps.mutable_unready_targets_dumps()->Add();
   message.set_name(name_);
   for (const auto& [target_name, count] : target_names_count_) {
+    UNREFERENCED_PARAMETER(count);
     message.add_target_names(target_name);
   }
 }

source/common/router/scoped_rds.cc

@@ -427,6 +427,7 @@ ScopedRdsConfigSubscription::detectUpdateConflictAndCleanupRemoved(
   absl::flat_hash_map<uint64_t, std::string> scope_name_by_hash = scope_name_by_hash_;
   absl::erase_if(scope_name_by_hash, [&updated_or_removed_scopes](const auto& key_name) {
     auto const& [key, name] = key_name;
+    UNREFERENCED_PARAMETER(key);
     return updated_or_removed_scopes.contains(name);
   });
   absl::flat_hash_map<std::string, envoy::config::route::v3::ScopedRouteConfiguration>

source/extensions/clusters/aggregate/cluster.cc

@@ -20,7 +20,9 @@ Cluster::Cluster(const envoy::config::cluster::v3::Cluster& cluster,
     : Upstream::ClusterImplBase(cluster, runtime, factory_context, std::move(stats_scope),
                                 added_via_api),
       cluster_manager_(cluster_manager), runtime_(runtime), random_(random),
-      tls_(tls.allocateSlot()), clusters_(config.clusters().begin(), config.clusters().end()) {}
+      tls_(tls.allocateSlot()), clusters_(config.clusters().begin(), config.clusters().end()) {
+  tls_->set([](Event::Dispatcher&) { return nullptr; });
+}
 
 PriorityContextPtr
 Cluster::linearizePrioritySet(const std::function<bool(const std::string&)>& skip_predicate) {

source/extensions/filters/http/lua/wrappers.h

@@ -207,7 +207,10 @@ class StreamInfoWrapper : public Filters::Common::Lua::BaseLuaObject<StreamInfoW
   DECLARE_LUA_FUNCTION(StreamInfoWrapper, luaDownstreamSslConnection);
 
   // Envoy::Lua::BaseLuaObject
-  void onMarkDead() override { dynamic_metadata_wrapper_.reset(); }
+  void onMarkDead() override {
+    dynamic_metadata_wrapper_.reset();
+    downstream_ssl_connection_.reset();
+  }
 
   StreamInfo::StreamInfo& stream_info_;
   Filters::Common::Lua::LuaDeathRef<DynamicMetadataMapWrapper> dynamic_metadata_wrapper_;

source/server/admin/config_dump_handler.cc

@@ -155,6 +155,7 @@ ConfigDumpHandler::addResourceToDump(envoy::admin::v3::ConfigDump& dump,
   }
 
   for (const auto& [name, callback] : callbacks_map) {
+    UNREFERENCED_PARAMETER(name);
     ProtobufTypes::MessagePtr message = callback();
     ASSERT(message);
 
@@ -200,6 +201,7 @@ void ConfigDumpHandler::addAllConfigToDump(envoy::admin::v3::ConfigDump& dump,
   }
 
   for (const auto& [name, callback] : callbacks_map) {
+    UNREFERENCED_PARAMETER(name);
     ProtobufTypes::MessagePtr message = callback();
     ASSERT(message);
 
@@ -220,6 +222,7 @@ ProtobufTypes::MessagePtr ConfigDumpHandler::dumpEndpointConfigs() const {
   auto endpoint_config_dump = std::make_unique<envoy::admin::v3::EndpointsConfigDump>();
 
   for (const auto& [name, cluster_ref] : server_.clusterManager().clusters()) {
+    UNREFERENCED_PARAMETER(name);
     const Upstream::Cluster& cluster = cluster_ref.get();
     Upstream::ClusterInfoConstSharedPtr cluster_info = cluster.info();
     envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment;

source/server/filter_chain_manager_impl.cc

@@ -600,6 +600,7 @@ const Network::FilterChain* FilterChainManagerImpl::findFilterChainForSourceIpAn
 
 void FilterChainManagerImpl::convertIPsToTries() {
   for (auto& [destination_port, destination_ips_pair] : destination_ports_map_) {
+    UNREFERENCED_PARAMETER(destination_port);
     // These variables are used as we build up the destination CIDRs used for the trie.
     auto& [destination_ips_map, destination_ips_trie] = destination_ips_pair;
     std::vector<std::pair<ServerNamesMapSharedPtr, std::vector<Network::Address::CidrRange>>>
@@ -613,8 +614,11 @@ void FilterChainManagerImpl::convertIPsToTries() {
       // We need to get access to all of the source IP strings so that we can convert them into
       // a trie like we did for the destination IPs above.
       for (auto& [server_name, transport_protocols_map] : *server_names_map_ptr) {
+        UNREFERENCED_PARAMETER(server_name);
         for (auto& [transport_protocol, application_protocols_map] : transport_protocols_map) {
+          UNREFERENCED_PARAMETER(transport_protocol);
           for (auto& [application_protocol, source_arrays] : application_protocols_map) {
+            UNREFERENCED_PARAMETER(application_protocol);
             for (auto& [source_ips_map, source_ips_trie] : source_arrays) {
               std::vector<
                   std::pair<SourcePortsMapSharedPtr, std::vector<Network::Address::CidrRange>>>

test/common/http/utility_test.cc

@@ -1251,6 +1251,9 @@ TEST(Url, ParsingFails) {
   EXPECT_FALSE(url.initialize("random_scheme://host.com/path", false));
   EXPECT_FALSE(url.initialize("http://www.foo.com", true));
   EXPECT_FALSE(url.initialize("foo.com", true));
+  EXPECT_FALSE(url.initialize("http://[notaddress]:80/?query=param", false));
+  EXPECT_FALSE(url.initialize("http://[1::z::2]:80/?query=param", false));
+  EXPECT_FALSE(url.initialize("http://1.2.3.4:65536/?query=param", false));
 }
 
 void validateUrl(absl::string_view raw_url, absl::string_view expected_scheme,
@@ -1262,12 +1265,17 @@ void validateUrl(absl::string_view raw_url, absl::string_view expected_scheme,
   EXPECT_EQ(url.pathAndQueryParams(), expected_path);
 }
 
-void validateConnectUrl(absl::string_view raw_url, absl::string_view expected_host_port) {
+void validateConnectUrl(absl::string_view raw_url) {
   Utility::Url url;
   ASSERT_TRUE(url.initialize(raw_url, true)) << "Failed to initialize " << raw_url;
   EXPECT_TRUE(url.scheme().empty());
   EXPECT_TRUE(url.pathAndQueryParams().empty());
-  EXPECT_EQ(url.hostAndPort(), expected_host_port);
+  EXPECT_EQ(url.hostAndPort(), raw_url);
+}
+
+void invalidConnectUrl(absl::string_view raw_url) {
+  Utility::Url url;
+  ASSERT_FALSE(url.initialize(raw_url, true)) << "Unexpectedly initialized " << raw_url;
 }
 
 TEST(Url, ParsingTest) {
@@ -1302,6 +1310,14 @@ TEST(Url, ParsingTest) {
   validateUrl("http://www.host.com:80/?query=param", "http", "www.host.com:80", "/?query=param");
   validateUrl("http://www.host.com/?query=param", "http", "www.host.com", "/?query=param");
 
+  // Test with an ipv4 host address.
+  validateUrl("http://1.2.3.4/?query=param", "http", "1.2.3.4", "/?query=param");
+  validateUrl("http://1.2.3.4:80/?query=param", "http", "1.2.3.4:80", "/?query=param");
+
+  // Test with an ipv6 address
+  validateUrl("http://[1::2:3]/?query=param", "http", "[1::2:3]", "/?query=param");
+  validateUrl("http://[1::2:3]:80/?query=param", "http", "[1::2:3]:80", "/?query=param");
+
   // Test url with query parameter but without slash
   validateUrl("http://www.host.com:80?query=param", "http", "www.host.com:80", "?query=param");
   validateUrl("http://www.host.com?query=param", "http", "www.host.com", "?query=param");
@@ -1324,8 +1340,16 @@ TEST(Url, ParsingTest) {
 }
 
 TEST(Url, ParsingForConnectTest) {
-  validateConnectUrl("host.com:443", "host.com:443");
-  validateConnectUrl("host.com:80", "host.com:80");
+  validateConnectUrl("host.com:443");
+  validateConnectUrl("host.com:80");
+  validateConnectUrl("1.2.3.4:80");
+  validateConnectUrl("[1:2::3:4]:80");
+
+  invalidConnectUrl("[::12345678]:80");
+  invalidConnectUrl("[1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1]:80");
+  invalidConnectUrl("[1:1]:80");
+  invalidConnectUrl("[:::]:80");
+  invalidConnectUrl("[::1::]:80");
 }
 
 void validatePercentEncodingEncodeDecode(absl::string_view source,

test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc

@@ -287,8 +287,7 @@ TEST_P(ProxyFilterIntegrationTest, UpstreamTlsWithIpHost) {
       {":method", "POST"},
       {":path", "/test/long/url"},
       {":scheme", "http"},
-      {":authority", fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(GetParam()),
-                                 fake_upstreams_[0]->localAddress()->ip()->port())}};
+      {":authority", fake_upstreams_[0]->localAddress()->asString()}};
 
   auto response = codec_client_->makeHeaderOnlyRequest(request_headers);
   waitForNextUpstreamRequest();

test/extensions/filters/http/lua/lua_filter_test.cc

@@ -1881,7 +1881,7 @@ TEST_F(LuaHttpFilterTest, InspectStreamInfoDowstreamSslConnection) {
 
   Http::TestRequestHeaderMapImpl request_headers{{":path", "/"}};
 
-  auto connection_info = std::make_shared<Ssl::MockConnectionInfo>();
+  const auto connection_info = std::make_shared<Ssl::MockConnectionInfo>();
   EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(stream_info_));
   EXPECT_CALL(stream_info_, downstreamSslConnection()).WillRepeatedly(Return(connection_info));
 
@@ -1989,6 +1989,35 @@ TEST_F(LuaHttpFilterTest, InspectStreamInfoDowstreamSslConnectionOnPlainConnecti
   EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true));
 }
 
+// Should survive from multiple streamInfo():downstreamSslConnection() calls.
+// This is a regression test for #14091.
+TEST_F(LuaHttpFilterTest, SurviveMultipleDownstreamSslConnectionCalls) {
+  const std::string SCRIPT{R"EOF(
+    function envoy_on_request(request_handle)
+      if request_handle:streamInfo():downstreamSslConnection() ~= nil then
+         request_handle:logTrace("downstreamSslConnection is present")
+      end
+    end
+  )EOF"};
+
+  setup(SCRIPT);
+
+  const auto connection_info = std::make_shared<Ssl::MockConnectionInfo>();
+  EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(stream_info_));
+  EXPECT_CALL(stream_info_, downstreamSslConnection()).WillRepeatedly(Return(connection_info));
+
+  for (uint64_t i = 0; i < 200; i++) {
+    EXPECT_CALL(*filter_,
+                scriptLog(spdlog::level::trace, StrEq("downstreamSslConnection is present")));
+
+    Http::TestRequestHeaderMapImpl request_headers{{":path", "/"}};
+    EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true));
+
+    filter_->onDestroy();
+    setupFilter();
+  }
+}
+
 TEST_F(LuaHttpFilterTest, ImportPublicKey) {
   const std::string SCRIPT{R"EOF(
     function string.fromhex(str)