Skip to content

Commit d999605

Browse files
venilnoronhafredlas
venilnoronha
authored and
fredlas
committed
mongo_proxy: add dynamic metadata to stream info (envoyproxy#5027)
This adds dynamic metadata to the stream info while processing data in the mongo_proxy filter. Signed-off-by: Venil Noronha <veniln@vmware.com> Signed-off-by: Fred Douglas <fredlas@google.com>
1 parent 2b3a9bf commit d999605

File tree

22 files changed

+212
-28
lines changed

22 files changed

+212
-28
lines changed

‎api/envoy/config/filter/network/mongo_proxy/v2/mongo_proxy.proto

+4
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,8 @@ message MongoProxy {
2525
// and KillCursors. Once an active delay is in progress, all incoming
2626
// data up until the timer event fires will be a part of the delay.
2727
envoy.config.filter.fault.v2.FaultDelay delay = 3;
28+
29+
// Flag to specify whether :ref:`dynamic metadata
30+
// <config_network_filters_mongo_proxy_dynamic_metadata>` should be emitted. Defaults to false.
31+
bool emit_dynamic_metadata = 4;
2832
}

‎docs/root/configuration/network_filters/mongo_proxy_filter.rst

+16
Original file line numberDiff line numberDiff line change
@@ -173,3 +173,19 @@ message
173173
upstream_host
174174
The upstream host that the connection is proxying to, if available. This is populated if the
175175
filter is used along with the :ref:`TCP proxy filter <config_network_filters_tcp_proxy>`.
176+
177+
.. _config_network_filters_mongo_proxy_dynamic_metadata:
178+
179+
Dynamic Metadata
180+
----------------
181+
182+
The Mongo filter emits the following dynamic metadata when enabled via the
183+
:ref:`configuration <envoy_api_field_config.filter.network.mongo_proxy.v2.MongoProxy.emit_dynamic_metadata>`.
184+
This dynamic metadata is available as one struct per message under a list named *messages*.
185+
186+
.. csv-table::
187+
:header: Name, Type, Description
188+
:widths: 1, 1, 2
189+
190+
operation, string, The operation to be executed.
191+
resource, string, The resource name in *db.collection* format on which the operation is to be executed.

‎docs/root/configuration/well_known_dynamic_metadata.rst

+1
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@ by looking at the operational metadata emitted by the MongoDB filter.
1313

1414
The following Envoy filters emit dynamic metadata that other filters can leverage.
1515

16+
* :ref:`Mongo Proxy Filter <config_network_filters_mongo_proxy_dynamic_metadata>`
1617
* :ref:`Role Based Access Control (RBAC) Filter <config_http_filters_rbac_dynamic_metadata>`
1718
* :ref:`Role Based Access Control (RBAC) Network Filter <config_network_filters_rbac_dynamic_metadata>`

‎docs/root/intro/version_history.rst

+2-1
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ Version history
2828
* http: no longer close the TCP connection when a HTTP/1 request is retried due
2929
to a response with empty body.
3030
* load balancer: added a `configuration <envoy_api_msg_Cluster.LeastRequestLbConfig>` option to specify the number of choices made in P2C.
31-
* network: removed the reference to `FilterState` in `Connection` in favor of `StreamInfo`.
3231
* logging: added missing [ in log prefix.
32+
* mongo_proxy: added :ref:`dynamic metadata <config_network_filters_mongo_proxy_dynamic_metadata>`.
33+
* network: removed the reference to `FilterState` in `Connection` in favor of `StreamInfo`.
3334
* rate-limit: added :ref:`configuration <envoy_api_field_config.filter.http.rate_limit.v2.RateLimit.rate_limited_as_resource_exhausted>`
3435
to specify whether the `GrpcStatus` status returned should be `RESOURCE_EXHAUSTED` or
3536
`UNAVAILABLE` when a gRPC call is rate limited.

‎include/envoy/stream_info/stream_info.h

+1
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,7 @@ class StreamInfo {
296296
/**
297297
* @return const envoy::api::v2::core::Metadata& the dynamic metadata associated with this request
298298
*/
299+
virtual envoy::api::v2::core::Metadata& dynamicMetadata() PURE;
299300
virtual const envoy::api::v2::core::Metadata& dynamicMetadata() const PURE;
300301

301302
/**

‎source/common/stream_info/stream_info_impl.h

+1
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ struct StreamInfoImpl : public StreamInfo {
176176

177177
const Router::RouteEntry* routeEntry() const override { return route_entry_; }
178178

179+
envoy::api::v2::core::Metadata& dynamicMetadata() override { return metadata_; };
179180
const envoy::api::v2::core::Metadata& dynamicMetadata() const override { return metadata_; };
180181

181182
void setDynamicMetadata(const std::string& name, const ProtobufWkt::Struct& value) override {

‎source/extensions/filters/network/mongo_proxy/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ envoy_cc_library(
7575
"//source/common/network:filter_lib",
7676
"//source/common/protobuf:utility_lib",
7777
"//source/common/singleton:const_singleton",
78+
"//source/extensions/filters/network:well_known_names",
7879
"@envoy_api//envoy/config/filter/network/mongo_proxy/v2:mongo_proxy_cc",
7980
],
8081
)

‎source/extensions/filters/network/mongo_proxy/config.cc

+5-3
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,13 @@ Network::FilterFactoryCb MongoProxyFilterConfigFactory::createFilterFactoryFromP
3434
fault_config = std::make_shared<FaultConfig>(proto_config.delay());
3535
}
3636

37-
return [stat_prefix, &context, access_log,
38-
fault_config](Network::FilterManager& filter_manager) -> void {
37+
const bool emit_dynamic_metadata = proto_config.emit_dynamic_metadata();
38+
return [stat_prefix, &context, access_log, fault_config,
39+
emit_dynamic_metadata](Network::FilterManager& filter_manager) -> void {
3940
filter_manager.addFilter(std::make_shared<ProdProxyFilter>(
4041
stat_prefix, context.scope(), context.runtime(), access_log, fault_config,
41-
context.drainDecision(), context.random(), context.dispatcher().timeSystem()));
42+
context.drainDecision(), context.random(), context.dispatcher().timeSystem(),
43+
emit_dynamic_metadata));
4244
};
4345
}
4446

‎source/extensions/filters/network/mongo_proxy/proxy.cc

+54-2
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,26 @@
1515
#include "common/common/utility.h"
1616

1717
#include "extensions/filters/network/mongo_proxy/codec_impl.h"
18+
#include "extensions/filters/network/well_known_names.h"
19+
20+
#include "absl/strings/str_split.h"
1821

1922
namespace Envoy {
2023
namespace Extensions {
2124
namespace NetworkFilters {
2225
namespace MongoProxy {
2326

27+
class DynamicMetadataKeys {
28+
public:
29+
const std::string MessagesField{"messages"};
30+
const std::string OperationField{"operation"};
31+
const std::string OperationInsert{"OP_INSERT"};
32+
const std::string OperationQuery{"OP_QUERY"};
33+
const std::string ResourceField{"resource"};
34+
};
35+
36+
typedef ConstSingleton<DynamicMetadataKeys> DynamicMetadataKeysSingleton;
37+
2438
AccessLog::AccessLog(const std::string& file_name, Envoy::AccessLog::AccessLogManager& log_manager,
2539
TimeSource& time_source)
2640
: time_source_(time_source) {
@@ -44,10 +58,12 @@ ProxyFilter::ProxyFilter(const std::string& stat_prefix, Stats::Scope& scope,
4458
Runtime::Loader& runtime, AccessLogSharedPtr access_log,
4559
const FaultConfigSharedPtr& fault_config,
4660
const Network::DrainDecision& drain_decision,
47-
Runtime::RandomGenerator& generator, Event::TimeSystem& time_system)
61+
Runtime::RandomGenerator& generator, Event::TimeSystem& time_system,
62+
bool emit_dynamic_metadata)
4863
: stat_prefix_(stat_prefix), scope_(scope), stats_(generateStats(stat_prefix, scope)),
4964
runtime_(runtime), drain_decision_(drain_decision), generator_(generator),
50-
access_log_(access_log), fault_config_(fault_config), time_system_(time_system) {
65+
access_log_(access_log), fault_config_(fault_config), time_system_(time_system),
66+
emit_dynamic_metadata_(emit_dynamic_metadata) {
5167
if (!runtime_.snapshot().featureEnabled(MongoRuntimeConfig::get().ConnectionLoggingEnabled,
5268
100)) {
5369
// If we are not logging at the connection level, just release the shared pointer so that we
@@ -58,6 +74,23 @@ ProxyFilter::ProxyFilter(const std::string& stat_prefix, Stats::Scope& scope,
5874

5975
ProxyFilter::~ProxyFilter() { ASSERT(!delay_timer_); }
6076

77+
void ProxyFilter::setDynamicMetadata(std::string operation, std::string resource) {
78+
ProtobufWkt::Struct metadata(
79+
(*read_callbacks_->connection()
80+
.streamInfo()
81+
.dynamicMetadata()
82+
.mutable_filter_metadata())[NetworkFilterNames::get().MongoProxy]);
83+
auto& fields = *metadata.mutable_fields();
84+
auto& list = *fields[DynamicMetadataKeysSingleton::get().MessagesField].mutable_list_value();
85+
auto& message = *list.add_values()->mutable_struct_value()->mutable_fields();
86+
87+
message[DynamicMetadataKeysSingleton::get().OperationField].set_string_value(operation);
88+
message[DynamicMetadataKeysSingleton::get().ResourceField].set_string_value(resource);
89+
90+
read_callbacks_->connection().streamInfo().setDynamicMetadata(
91+
NetworkFilterNames::get().MongoProxy, metadata);
92+
}
93+
6194
void ProxyFilter::decodeGetMore(GetMoreMessagePtr&& message) {
6295
tryInjectDelay();
6396

@@ -69,6 +102,11 @@ void ProxyFilter::decodeGetMore(GetMoreMessagePtr&& message) {
69102
void ProxyFilter::decodeInsert(InsertMessagePtr&& message) {
70103
tryInjectDelay();
71104

105+
if (emit_dynamic_metadata_) {
106+
setDynamicMetadata(DynamicMetadataKeysSingleton::get().OperationInsert,
107+
message->fullCollectionName());
108+
}
109+
72110
stats_.op_insert_.inc();
73111
logMessage(*message, true);
74112
ENVOY_LOG(debug, "decoded INSERT: {}", message->toString(true));
@@ -85,6 +123,11 @@ void ProxyFilter::decodeKillCursors(KillCursorsMessagePtr&& message) {
85123
void ProxyFilter::decodeQuery(QueryMessagePtr&& message) {
86124
tryInjectDelay();
87125

126+
if (emit_dynamic_metadata_) {
127+
setDynamicMetadata(DynamicMetadataKeysSingleton::get().OperationQuery,
128+
message->fullCollectionName());
129+
}
130+
88131
stats_.op_query_.inc();
89132
logMessage(*message, true);
90133
ENVOY_LOG(debug, "decoded QUERY: {}", message->toString(true));
@@ -255,6 +298,15 @@ void ProxyFilter::doDecode(Buffer::Instance& buffer) {
255298
return;
256299
}
257300

301+
// Clear dynamic metadata
302+
if (emit_dynamic_metadata_) {
303+
auto& metadata = (*read_callbacks_->connection()
304+
.streamInfo()
305+
.dynamicMetadata()
306+
.mutable_filter_metadata())[NetworkFilterNames::get().MongoProxy];
307+
metadata.mutable_fields()->clear();
308+
}
309+
258310
if (!decoder_) {
259311
decoder_ = createDecoder(*this);
260312
}

‎source/extensions/filters/network/mongo_proxy/proxy.h

+4-1
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ class ProxyFilter : public Network::Filter,
128128
ProxyFilter(const std::string& stat_prefix, Stats::Scope& scope, Runtime::Loader& runtime,
129129
AccessLogSharedPtr access_log, const FaultConfigSharedPtr& fault_config,
130130
const Network::DrainDecision& drain_decision, Runtime::RandomGenerator& generator,
131-
Event::TimeSystem& time_system);
131+
Event::TimeSystem& time_system, bool emit_dynamic_metadata);
132132
~ProxyFilter();
133133

134134
virtual DecoderPtr createDecoder(DecoderCallbacks& callbacks) PURE;
@@ -158,6 +158,8 @@ class ProxyFilter : public Network::Filter,
158158
void onAboveWriteBufferHighWatermark() override {}
159159
void onBelowWriteBufferLowWatermark() override {}
160160

161+
void setDynamicMetadata(std::string operation, std::string resource);
162+
161163
private:
162164
struct ActiveQuery {
163165
ActiveQuery(ProxyFilter& parent, const QueryMessage& query)
@@ -207,6 +209,7 @@ class ProxyFilter : public Network::Filter,
207209
Event::TimerPtr delay_timer_;
208210
Event::TimerPtr drain_close_timer_;
209211
Event::TimeSystem& time_system_;
212+
const bool emit_dynamic_metadata_;
210213
};
211214

212215
class ProdProxyFilter : public ProxyFilter {

‎test/common/access_log/BUILD

+2-13
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,6 @@ load(
1212

1313
envoy_package()
1414

15-
envoy_cc_test_library(
16-
name = "test_util",
17-
hdrs = ["test_util.h"],
18-
deps = [
19-
"//include/envoy/stream_info:stream_info_interface",
20-
"//source/common/common:assert_lib",
21-
"//source/common/stream_info:filter_state_lib",
22-
"//test/test_common:test_time_lib",
23-
],
24-
)
25-
2615
envoy_proto_library(
2716
name = "access_log_formatter_fuzz_proto",
2817
srcs = ["access_log_formatter_fuzz.proto"],
@@ -58,10 +47,10 @@ envoy_cc_test(
5847
name = "access_log_impl_test",
5948
srcs = ["access_log_impl_test.cc"],
6049
deps = [
61-
":test_util",
6250
"//source/common/access_log:access_log_lib",
6351
"//source/common/config:filter_json_lib",
6452
"//source/extensions/access_loggers/file:config",
53+
"//test/common/stream_info:test_util",
6554
"//test/common/upstream:utility_lib",
6655
"//test/mocks/access_log:access_log_mocks",
6756
"//test/mocks/event:event_mocks",
@@ -95,10 +84,10 @@ envoy_cc_binary(
9584
"benchmark",
9685
],
9786
deps = [
98-
":test_util",
9987
"//source/common/access_log:access_log_formatter_lib",
10088
"//source/common/http:header_map_lib",
10189
"//source/common/network:address_lib",
90+
"//test/common/stream_info:test_util",
10291
"//test/mocks/http:http_mocks",
10392
"//test/mocks/stream_info:stream_info_mocks",
10493
"//test/test_common:printers_lib",

‎test/common/access_log/access_log_formatter_speed_test.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#include "common/access_log/access_log_formatter.h"
22
#include "common/network/address_impl.h"
33

4-
#include "test/common/access_log/test_util.h"
4+
#include "test/common/stream_info/test_util.h"
55
#include "test/mocks/http/mocks.h"
66

77
#include "testing/base/public/benchmark.h"

‎test/common/access_log/access_log_formatter_test.cc

+3
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include "gtest/gtest.h"
1818

1919
using testing::_;
20+
using testing::Const;
2021
using testing::NiceMock;
2122
using testing::Return;
2223
using testing::ReturnRef;
@@ -504,6 +505,7 @@ TEST(AccessLogFormatterTest, JsonFormatterDynamicMetadataTest) {
504505
envoy::api::v2::core::Metadata metadata;
505506
populateMetadataTestData(metadata);
506507
EXPECT_CALL(stream_info, dynamicMetadata()).WillRepeatedly(ReturnRef(metadata));
508+
EXPECT_CALL(Const(stream_info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata));
507509

508510
std::unordered_map<std::string, std::string> expected_json_map = {
509511
{"test_key", "\"test_value\""},
@@ -622,6 +624,7 @@ TEST(AccessLogFormatterTest, CompositeFormatterSuccess) {
622624
envoy::api::v2::core::Metadata metadata;
623625
populateMetadataTestData(metadata);
624626
EXPECT_CALL(stream_info, dynamicMetadata()).WillRepeatedly(ReturnRef(metadata));
627+
EXPECT_CALL(Const(stream_info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata));
625628
const std::string format = "%DYNAMIC_METADATA(com.test:test_key)%|%DYNAMIC_METADATA(com.test:"
626629
"test_obj)%|%DYNAMIC_METADATA(com.test:test_obj:inner_key)%";
627630
FormatterImpl formatter(format);

‎test/common/access_log/access_log_impl_test.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
#include "common/runtime/runtime_impl.h"
1212
#include "common/runtime/uuid_util.h"
1313

14-
#include "test/common/access_log/test_util.h"
14+
#include "test/common/stream_info/test_util.h"
1515
#include "test/common/upstream/utility.h"
1616
#include "test/mocks/access_log/mocks.h"
1717
#include "test/mocks/event/mocks.h"

‎test/common/stream_info/BUILD

+11
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,17 @@ envoy_cc_test_library(
3939
],
4040
)
4141

42+
envoy_cc_test_library(
43+
name = "test_util",
44+
hdrs = ["test_util.h"],
45+
deps = [
46+
"//include/envoy/stream_info:stream_info_interface",
47+
"//source/common/common:assert_lib",
48+
"//source/common/stream_info:filter_state_lib",
49+
"//test/test_common:test_time_lib",
50+
],
51+
)
52+
4253
envoy_cc_test(
4354
name = "utility_test",
4455
srcs = ["utility_test.cc"],

‎test/common/access_log/test_util.h ‎test/common/stream_info/test_util.h

+1
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ class TestStreamInfo : public StreamInfo::StreamInfo {
153153
return duration(end_time_);
154154
}
155155

156+
envoy::api::v2::core::Metadata& dynamicMetadata() override { return metadata_; };
156157
const envoy::api::v2::core::Metadata& dynamicMetadata() const override { return metadata_; };
157158

158159
void setDynamicMetadata(const std::string& name, const ProtobufWkt::Struct& value) override {

‎test/extensions/filters/network/mongo_proxy/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ envoy_extension_cc_test(
4343
"//source/extensions/filters/network/mongo_proxy:bson_lib",
4444
"//source/extensions/filters/network/mongo_proxy:codec_lib",
4545
"//source/extensions/filters/network/mongo_proxy:proxy_lib",
46+
"//test/common/stream_info:test_util",
4647
"//test/mocks/access_log:access_log_mocks",
4748
"//test/mocks/event:event_mocks",
4849
"//test/mocks/filesystem:filesystem_mocks",

0 commit comments

Comments
 (0)