set_metadata: allow setting typed metadata (envoyproxy#31244)

Signed-off-by: Jacob Bohanon <jacob.bohanon@solo.io>

api/envoy/extensions/filters/http/set_metadata/v3/BUILD

@@ -5,5 +5,8 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
 licenses(["notice"])  # Apache 2
 
 api_proto_package(
-    deps = ["@com_github_cncf_xds//udpa/annotations:pkg"],
+    deps = [
+        "//envoy/annotations:pkg",
+        "@com_github_cncf_xds//udpa/annotations:pkg",
+    ],
 )

api/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto

@@ -2,8 +2,10 @@ syntax = "proto3";
 
 package envoy.extensions.filters.http.set_metadata.v3;
 
+import "google/protobuf/any.proto";
 import "google/protobuf/struct.proto";
 
+import "envoy/annotations/deprecation.proto";
 import "udpa/annotations/status.proto";
 import "validate/validate.proto";
 
@@ -19,13 +21,41 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 //
 // [#extension: envoy.filters.http.set_metadata]
 
-message Config {
+message Metadata {
   // The metadata namespace.
   string metadata_namespace = 1 [(validate.rules).string = {min_len: 1}];
 
-  // The value to update the namespace with. See
+  // Allow the filter to overwrite or merge with an existing value in the namespace.
+  bool allow_overwrite = 2;
+
+  // The value to place at the namespace. If ``allow_overwrite``, this will
+  // overwrite or merge with any existing values in that namespace. See
+  // :ref:`the filter documentation <config_http_filters_set_metadata>` for
+  // more information on how this value is merged with potentially existing
+  // ones if ``allow_overwrite`` is configured. Only one of ``value`` and
+  // ``typed_value`` may be set.
+  google.protobuf.Struct value = 3;
+
+  // The value to place at the namespace. If ``allow_overwrite``, this will
+  // overwrite any existing values in that namespace. Only one of ``value`` and
+  // ``typed_value`` may be set.
+  google.protobuf.Any typed_value = 4;
+}
+
+message Config {
+  // The metadata namespace.
+  // This field is deprecated; please use ``metadata`` as replacement.
+  string metadata_namespace = 1
+      [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"];
+
+  // The untyped value to update the dynamic metadata namespace with. See
   // :ref:`the filter documentation <config_http_filters_set_metadata>` for
   // more information on how this value is merged with potentially existing
   // ones.
-  google.protobuf.Struct value = 2;
+  // This field is deprecated; please use ``metadata`` as replacement.
+  google.protobuf.Struct value = 2
+      [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"];
+
+  // Defines changes to be made to dynamic metadata.
+  repeated Metadata metadata = 3;
 }

changelogs/current.yaml

@@ -311,8 +311,21 @@ new_features:
 - area: attributes
   change: |
     Added support for node data in ``%CEL%`` formatter.
+- area: set_metadata
+  change: |
+    Added support for injecting typed and untyped dynamic metadata with this filter, also adds the ability
+    to add multiple namespaces with one filter and config to overwrite existing metadata is opt-in.
+    :ref:`untyped_metadata <envoy_v3_api_field_extensions.filters.http.set_metadata.v3.Config.metadata>`
+    may now be used to configure the ``set_metadata`` filter.
 
 deprecated:
 - area: wasm
   change: |
     Wasm-specific configuration attributes are deprecated in favor of ``xds`` attributes.
+- area: set_metadata
+  change: |
+    :ref:`metadata_namespace <envoy_v3_api_field_extensions.filters.http.set_metadata.v3.Config.metadata_namespace>`
+    and :ref:`value <envoy_v3_api_field_extensions.filters.http.set_metadata.v3.Config.value>`
+    are deprecated. Please use the new field
+    :ref:`untyped_metadata <envoy_v3_api_field_extensions.filters.http.set_metadata.v3.Config.metadata>`
+    to configure static metadata to inject.

docs/root/configuration/http/http_filters/set_metadata_filter.rst

@@ -8,13 +8,14 @@ Set Metadata
 
 This filters adds or updates dynamic metadata with static data.
 
-Dynamic metadata values are updated with the following scheme. If a key
-does not exists, it's just copied into the current metadata. If the key exists
-but has a different type, it is replaced by the new value. Otherwise:
+Dynamic metadata values are updated with the following rules. If a key does not exist, it is copied into the current metadata. If the key exists, then following rules will be used:
 
- * for scalar values (null, string, number, boolean) are replaced with the new value
- * for lists: new values are added to the current list
- * for structures: recursively apply this scheme
+* if :ref:`typed metadata value <envoy_v3_api_field_extensions.filters.http.set_metadata.v3.Metadata.typed_value>` is used, it will overwrite existing values iff :ref:`allow_overwrite <envoy_v3_api_field_extensions.filters.http.set_metadata.v3.Metadata.allow_overwrite>` is set to true, otherwise nothing is done.
+* if :ref:`untyped metadata value <envoy_v3_api_field_extensions.filters.http.set_metadata.v3.Metadata.value>` is used and ``allow_overwrite`` is set to true, or if deprecated :ref:`value <envoy_v3_api_field_extensions.filters.http.set_metadata.v3.Config.value>` field is used, the values are updated with the following scheme:
+  - existing value with different type: the existing value is replaced.
+  - scalar values (null, string, number, boolean): the existing value is replaced.
+  - lists: new values are appended to the current list.
+  - structures: recursively apply this scheme.
 
 For instance, if the namespace already contains this structure:
 
@@ -50,4 +51,12 @@ After applying this filter, the namespace will contain:
 Statistics
 ----------
 
-Currently, this filter generates no statistics.
+The ``set_metadata`` filter outputs statistics in the ``http.<stat_prefix>.set_metadata.`` namespace. The :ref:`stat prefix
+<envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.stat_prefix>` comes from the
+owning HTTP connection manager.
+
+.. csv-table::
+  :header: Name, Type, Description
+  :widths: 1, 1, 2
+
+  overwrite_denied, Counter, Total number of denied attempts to overwrite an existing metadata value

source/extensions/filters/http/set_metadata/config.cc

@@ -16,8 +16,9 @@ namespace SetMetadataFilter {
 
 Http::FilterFactoryCb SetMetadataConfig::createFilterFactoryFromProtoTyped(
     const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config,
-    const std::string&, Server::Configuration::FactoryContext&) {
-  ConfigSharedPtr filter_config(std::make_shared<Config>(proto_config));
+    const std::string& stats_prefix, Server::Configuration::FactoryContext& context) {
+  ConfigSharedPtr filter_config(
+      std::make_shared<Config>(proto_config, context.scope(), stats_prefix));
 
   return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void {
     callbacks.addStreamDecoderFilter(
@@ -27,8 +28,9 @@ Http::FilterFactoryCb SetMetadataConfig::createFilterFactoryFromProtoTyped(
 
 Http::FilterFactoryCb SetMetadataConfig::createFilterFactoryFromProtoWithServerContextTyped(
     const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config,
-    const std::string&, Server::Configuration::ServerFactoryContext&) {
-  ConfigSharedPtr filter_config(std::make_shared<Config>(proto_config));
+    const std::string& stats_prefix, Server::Configuration::ServerFactoryContext& server_context) {
+  ConfigSharedPtr filter_config(
+      std::make_shared<Config>(proto_config, server_context.scope(), stats_prefix));
 
   return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void {
     callbacks.addStreamDecoderFilter(

source/extensions/filters/http/set_metadata/set_metadata_filter.cc

@@ -13,23 +13,83 @@ namespace Extensions {
 namespace HttpFilters {
 namespace SetMetadataFilter {
 
-Config::Config(const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config) {
-  namespace_ = proto_config.metadata_namespace();
-  value_ = proto_config.value();
+Config::Config(const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config,
+               Stats::Scope& scope, const std::string& stats_prefix)
+    : stats_(generateStats(stats_prefix, scope)) {
+  if (proto_config.has_value() && !proto_config.metadata_namespace().empty()) {
+    UntypedMetadataEntry deprecated_api_val{true, proto_config.metadata_namespace(),
+                                            proto_config.value()};
+    untyped_.emplace_back(deprecated_api_val);
+  }
+
+  for (const auto& metadata : proto_config.metadata()) {
+    if (metadata.has_value()) {
+      UntypedMetadataEntry untyped_entry{metadata.allow_overwrite(), metadata.metadata_namespace(),
+                                         metadata.value()};
+      untyped_.emplace_back(untyped_entry);
+    } else if (metadata.has_typed_value()) {
+      TypedMetadataEntry typed_entry{metadata.allow_overwrite(), metadata.metadata_namespace(),
+                                     metadata.typed_value()};
+      typed_.emplace_back(typed_entry);
+    } else {
+      ENVOY_LOG(warn, "set_metadata filter configuration contains metadata entries without value "
+                      "or typed_value");
+    }
+  }
+}
+
+FilterStats Config::generateStats(const std::string& prefix, Stats::Scope& scope) {
+  std::string final_prefix = prefix + "set_metadata.";
+  return {ALL_SET_METADATA_FILTER_STATS(POOL_COUNTER_PREFIX(scope, final_prefix))};
 }
 
 SetMetadataFilter::SetMetadataFilter(const ConfigSharedPtr config) : config_(config) {}
 
 SetMetadataFilter::~SetMetadataFilter() = default;
 
 Http::FilterHeadersStatus SetMetadataFilter::decodeHeaders(Http::RequestHeaderMap&, bool) {
-  const absl::string_view metadata_namespace = config_->metadataNamespace();
-  auto& metadata = *decoder_callbacks_->streamInfo().dynamicMetadata().mutable_filter_metadata();
-  ProtobufWkt::Struct& org_fields =
-      metadata[toStdStringView(metadata_namespace)]; // NOLINT(std::string_view)
-  const ProtobufWkt::Struct& to_merge = config_->value();
 
-  StructUtil::update(org_fields, to_merge);
+  // Add configured untyped metadata.
+  if (!config_->untyped().empty()) {
+    auto& mut_untyped_metadata =
+        *decoder_callbacks_->streamInfo().dynamicMetadata().mutable_filter_metadata();
+
+    for (const auto& entry : config_->untyped()) {
+      if (!mut_untyped_metadata.contains(entry.metadata_namespace)) {
+        // Insert the new entry.
+        mut_untyped_metadata[entry.metadata_namespace] = entry.value;
+      } else if (entry.allow_overwrite) {
+        // Get the existing metadata at this key for merging.
+        ProtobufWkt::Struct& orig_fields = mut_untyped_metadata[entry.metadata_namespace];
+        const auto& to_merge = entry.value;
+
+        // Merge the new metadata into the existing metadata.
+        StructUtil::update(orig_fields, to_merge);
+      } else {
+        // The entry exists, and we are not allowed to overwrite -- emit a stat.
+        config_->stats().overwrite_denied_.inc();
+      }
+    }
+  }
+
+  // Add configured typed metadata.
+  if (!config_->typed().empty()) {
+    auto& mut_typed_metadata =
+        *decoder_callbacks_->streamInfo().dynamicMetadata().mutable_typed_filter_metadata();
+
+    for (const auto& entry : config_->typed()) {
+      if (!mut_typed_metadata.contains(entry.metadata_namespace)) {
+        // Insert the new entry.
+        mut_typed_metadata[entry.metadata_namespace] = entry.value;
+      } else if (entry.allow_overwrite) {
+        // Overwrite the existing typed metadata at this key.
+        mut_typed_metadata[entry.metadata_namespace] = entry.value;
+      } else {
+        // The entry exists, and we are not allowed to overwrite -- emit a stat.
+        config_->stats().overwrite_denied_.inc();
+      }
+    }
+  }
 
   return Http::FilterHeadersStatus::Continue;
 }

source/extensions/filters/http/set_metadata/set_metadata_filter.h

@@ -5,28 +5,51 @@
 #include <vector>
 
 #include "envoy/extensions/filters/http/set_metadata/v3/set_metadata.pb.h"
+#include "envoy/stats/stats_macros.h"
 
 #include "source/common/common/logger.h"
 #include "source/extensions/filters/http/common/pass_through_filter.h"
 
 #include "absl/strings/string_view.h"
+#include "absl/types/variant.h"
 
 namespace Envoy {
 namespace Extensions {
 namespace HttpFilters {
 namespace SetMetadataFilter {
 
+#define ALL_SET_METADATA_FILTER_STATS(COUNTER) COUNTER(overwrite_denied)
+
+struct FilterStats {
+  ALL_SET_METADATA_FILTER_STATS(GENERATE_COUNTER_STRUCT)
+};
+
+struct UntypedMetadataEntry {
+  bool allow_overwrite{};
+  std::string metadata_namespace;
+  ProtobufWkt::Struct value;
+};
+struct TypedMetadataEntry {
+  bool allow_overwrite{};
+  std::string metadata_namespace;
+  ProtobufWkt::Any value;
+};
 class Config : public ::Envoy::Router::RouteSpecificFilterConfig,
                public Logger::Loggable<Logger::Id::config> {
 public:
-  Config(const envoy::extensions::filters::http::set_metadata::v3::Config& config);
+  Config(const envoy::extensions::filters::http::set_metadata::v3::Config& config,
+         Stats::Scope& scope, const std::string& stats_prefix);
 
-  absl::string_view metadataNamespace() const { return namespace_; }
-  const ProtobufWkt::Struct& value() { return value_; }
+  const std::vector<UntypedMetadataEntry>& untyped() { return untyped_; }
+  const std::vector<TypedMetadataEntry>& typed() { return typed_; }
+  const FilterStats& stats() const { return stats_; }
 
 private:
-  std::string namespace_;
-  ProtobufWkt::Struct value_;
+  static FilterStats generateStats(const std::string& prefix, Stats::Scope& scope);
+
+  std::vector<UntypedMetadataEntry> untyped_;
+  std::vector<TypedMetadataEntry> typed_;
+  FilterStats stats_;
 };
 
 using ConfigSharedPtr = std::shared_ptr<Config>;

test/extensions/filters/http/set_metadata/config_test.cc

@@ -22,12 +22,21 @@ using SetMetadataProtoConfig = envoy::extensions::filters::http::set_metadata::v
 
 TEST(SetMetadataFilterConfigTest, SimpleConfig) {
   const std::string yaml = R"EOF(
-metadata_namespace: thenamespace
-value:
-  mynumber: 20
-  mylist: ["b"]
-  tags:
-    mytag1: 1
+metadata:
+- metadata_namespace: thenamespace
+  value:
+    mynumber: 20
+    mylist: ["b"]
+    tags:
+      mytag1: 1
+  allow_overwrite: true
+- metadata_namespace: thenamespace
+  typed_value:
+    '@type': type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config
+    metadata_namespace: foo_namespace
+    value:
+      foo: bar
+  allow_overwrite: true
   )EOF";
 
   SetMetadataProtoConfig proto_config;
@@ -45,12 +54,21 @@ metadata_namespace: thenamespace
 
 TEST(SetMetadataFilterConfigTest, SimpleConfigServerContext) {
   const std::string yaml = R"EOF(
-metadata_namespace: thenamespace
-value:
-  mynumber: 20
-  mylist: ["b"]
-  tags:
-    mytag1: 1
+metadata:
+- metadata_namespace: thenamespace
+  value:
+    mynumber: 20
+    mylist: ["b"]
+    tags:
+      mytag1: 1
+  allow_overwrite: true
+- metadata_namespace: thenamespace
+  typed_value:
+    '@type': type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config
+    metadata_namespace: foo_namespace
+    value:
+      foo: bar
+  allow_overwrite: true
   )EOF";
 
   SetMetadataProtoConfig proto_config;