From b6a28df5c46810728c1eb17ca797391695628e64 Mon Sep 17 00:00:00 2001 From: Lalit Kumar Bhasin Date: Fri, 4 Feb 2022 18:46:06 -0800 Subject: [PATCH] Metrics SDK: Filtering metrics attributes (#1191) --- .../sdk/common/attribute_utils.h | 51 +++++++++++++++ .../sdk/common/attributemap_hash.h | 62 +++++++++++++++++++ .../sdk/metrics/view/attributes_processor.h | 47 +++++++++++++- sdk/test/common/BUILD | 23 +++++++ sdk/test/common/CMakeLists.txt | 5 ++ sdk/test/common/attribute_utils_test.cc | 32 ++++++++-- .../common/attributemap_hash_benchmark.cc | 22 +++++++ sdk/test/common/attributemap_hash_test.cc | 32 ++++++++++ sdk/test/metrics/BUILD | 35 +++++++++++ sdk/test/metrics/CMakeLists.txt | 7 ++- .../metrics/attributes_processor_benchmark.cc | 27 ++++++++ sdk/test/metrics/attributes_processor_test.cc | 49 +++++++++++++++ 12 files changed, 386 insertions(+), 6 deletions(-) create mode 100644 sdk/include/opentelemetry/sdk/common/attributemap_hash.h create mode 100644 sdk/test/common/attributemap_hash_benchmark.cc create mode 100644 sdk/test/common/attributemap_hash_test.cc create mode 100644 sdk/test/metrics/attributes_processor_benchmark.cc create mode 100644 sdk/test/metrics/attributes_processor_test.cc diff --git a/sdk/include/opentelemetry/sdk/common/attribute_utils.h b/sdk/include/opentelemetry/sdk/common/attribute_utils.h index e8935b8f62..68b09f044c 100644 --- a/sdk/include/opentelemetry/sdk/common/attribute_utils.h +++ b/sdk/include/opentelemetry/sdk/common/attribute_utils.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -143,6 +144,56 @@ class AttributeMap : public std::unordered_map private: AttributeConverter converter_; }; + +/** + * Class for storing attributes. + */ +class OrderedAttributeMap : public std::map +{ +public: + // Contruct empty attribute map + OrderedAttributeMap() : std::map(){}; + + // Contruct attribute map and populate with attributes + OrderedAttributeMap(const opentelemetry::common::KeyValueIterable &attributes) + : OrderedAttributeMap() + { + attributes.ForEachKeyValue( + [&](nostd::string_view key, opentelemetry::common::AttributeValue value) noexcept { + SetAttribute(key, value); + return true; + }); + } + + // Construct map from initializer list by applying `SetAttribute` transform for every attribute + OrderedAttributeMap( + std::initializer_list> + attributes) + : OrderedAttributeMap() + { + for (auto &kv : attributes) + { + SetAttribute(kv.first, kv.second); + } + } + + // Returns a reference to this map + const std::map &GetAttributes() const noexcept + { + return (*this); + } + + // Convert non-owning key-value to owning std::string(key) and OwnedAttributeValue(value) + void SetAttribute(nostd::string_view key, + const opentelemetry::common::AttributeValue &value) noexcept + { + (*this)[std::string(key)] = nostd::visit(converter_, value); + } + +private: + AttributeConverter converter_; +}; + } // namespace common } // namespace sdk OPENTELEMETRY_END_NAMESPACE diff --git a/sdk/include/opentelemetry/sdk/common/attributemap_hash.h b/sdk/include/opentelemetry/sdk/common/attributemap_hash.h new file mode 100644 index 0000000000..573f57eb1b --- /dev/null +++ b/sdk/include/opentelemetry/sdk/common/attributemap_hash.h @@ -0,0 +1,62 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include "opentelemetry/sdk/common/attribute_utils.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace common +{ + +template +inline void GetHashForAttributeValue(size_t &seed, const T arg) +{ + std::hash hasher; + // reference - + // https://www.boost.org/doc/libs/1_37_0/doc/html/hash/reference.html#boost.hash_combine + seed ^= hasher(arg) + 0x9e3779b9 + (seed << 6) + (seed >> 2); +} + +template +inline void GetHashForAttributeValue(size_t &seed, const std::vector &arg) +{ + for (auto v : arg) + { + GetHashForAttributeValue(seed, v); + } +} + +struct GetHashForAttributeValueVisitor +{ + GetHashForAttributeValueVisitor(size_t &seed) : seed_(seed) {} + template + void operator()(T &v) + { + GetHashForAttributeValue(seed_, v); + } + size_t &seed_; +}; + +// Calculate hash of keys and values of attribute map +inline size_t GetHashForAttributeMap(const OrderedAttributeMap &attribute_map) +{ + size_t seed = 0UL; + for (auto &kv : attribute_map) + { + std::hash hasher; + // reference - + // https://www.boost.org/doc/libs/1_37_0/doc/html/hash/reference.html#boost.hash_combine + seed ^= hasher(kv.first) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + nostd::visit(GetHashForAttributeValueVisitor(seed), kv.second); + } + return seed; +} + +} // namespace common +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE \ No newline at end of file diff --git a/sdk/include/opentelemetry/sdk/metrics/view/attributes_processor.h b/sdk/include/opentelemetry/sdk/metrics/view/attributes_processor.h index 1ec28b3752..6611abe97f 100644 --- a/sdk/include/opentelemetry/sdk/metrics/view/attributes_processor.h +++ b/sdk/include/opentelemetry/sdk/metrics/view/attributes_processor.h @@ -9,15 +9,27 @@ namespace sdk { namespace metrics { -using MetricAttributes = opentelemetry::sdk::common::AttributeMap; +using MetricAttributes = opentelemetry::sdk::common::OrderedAttributeMap; + +/** + * The AttributesProcessor is responsible for customizing which + * attribute(s) are to be reported as metrics dimension(s). + */ class AttributesProcessor { public: + // Process the metric instrument attributes. + // @returns The processed attributes virtual MetricAttributes process( const opentelemetry::common::KeyValueIterable &attributes) noexcept = 0; }; +/** + * DefaultAttributesProcessor returns copy of input instrument attributes without + * any modification. + */ + class DefaultAttributesProcessor : public AttributesProcessor { MetricAttributes process( @@ -28,6 +40,39 @@ class DefaultAttributesProcessor : public AttributesProcessor } }; +/** + * FilteringAttributesProcessor filters by allowed attribute names and drops any names + * that are not in the allow list. + */ + +class FilteringAttributesProcessor : public AttributesProcessor +{ +public: + FilteringAttributesProcessor( + const std::unordered_map allowed_attribute_keys = {}) + : allowed_attribute_keys_(std::move(allowed_attribute_keys)) + {} + + MetricAttributes process( + const opentelemetry::common::KeyValueIterable &attributes) noexcept override + { + MetricAttributes result; + attributes.ForEachKeyValue( + [&](nostd::string_view key, opentelemetry::common::AttributeValue value) noexcept { + if (allowed_attribute_keys_.find(key.data()) != allowed_attribute_keys_.end()) + { + result.SetAttribute(key, value); + return true; + } + return true; + }); + return result; + } + +private: + std::unordered_map allowed_attribute_keys_; +}; + } // namespace metrics } // namespace sdk OPENTELEMETRY_END_NAMESPACE diff --git a/sdk/test/common/BUILD b/sdk/test/common/BUILD index 57a84339fd..8a98e5617d 100644 --- a/sdk/test/common/BUILD +++ b/sdk/test/common/BUILD @@ -127,3 +127,26 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +cc_test( + name = "attributemap_hash_test", + srcs = [ + "attributemap_hash_test.cc", + ], + tags = ["test"], + deps = [ + "//api", + "//sdk:headers", + "@com_google_googletest//:gtest_main", + ], +) + +otel_cc_benchmark( + name = "attributemap_hash_benchmark", + srcs = ["attributemap_hash_benchmark.cc"], + tags = ["test"], + deps = [ + "//api", + "//sdk:headers", + ], +) diff --git a/sdk/test/common/CMakeLists.txt b/sdk/test/common/CMakeLists.txt index 40a195804e..0fefc86b10 100644 --- a/sdk/test/common/CMakeLists.txt +++ b/sdk/test/common/CMakeLists.txt @@ -6,6 +6,7 @@ foreach( circular_buffer_range_test circular_buffer_test attribute_utils_test + attributemap_hash_test global_log_handle_test) add_executable(${testname} "${testname}.cc") @@ -30,3 +31,7 @@ target_link_libraries(random_benchmark benchmark::benchmark add_executable(circular_buffer_benchmark circular_buffer_benchmark.cc) target_link_libraries(circular_buffer_benchmark benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT} opentelemetry_api) + +add_executable(attributemap_hash_benchmark attributemap_hash_benchmark.cc) +target_link_libraries(attributemap_hash_benchmark benchmark::benchmark + ${CMAKE_THREAD_LIBS_INIT} opentelemetry_common) diff --git a/sdk/test/common/attribute_utils_test.cc b/sdk/test/common/attribute_utils_test.cc index 1499a03e78..b7ef17244f 100644 --- a/sdk/test/common/attribute_utils_test.cc +++ b/sdk/test/common/attribute_utils_test.cc @@ -7,8 +7,15 @@ TEST(AttributeMapTest, DefaultConstruction) { - opentelemetry::sdk::common::AttributeMap map; - EXPECT_EQ(map.GetAttributes().size(), 0); + + opentelemetry::sdk::common::AttributeMap attribute_map; + EXPECT_EQ(attribute_map.GetAttributes().size(), 0); +} + +TEST(OrderedAttributeMapTest, DefaultConstruction) +{ + opentelemetry::sdk::common::OrderedAttributeMap attribute_map; + EXPECT_EQ(attribute_map.GetAttributes().size(), 0); } TEST(AttributeMapTest, AttributesConstruction) @@ -20,10 +27,27 @@ TEST(AttributeMapTest, AttributesConstruction) {keys[0], values[0]}, {keys[1], values[1]}, {keys[2], values[2]}}; opentelemetry::common::KeyValueIterableView> iterable(attributes); - opentelemetry::sdk::common::AttributeMap map(iterable); + opentelemetry::sdk::common::AttributeMap attribute_map(iterable); + + for (int i = 0; i < kNumAttributes; i++) + { + EXPECT_EQ(opentelemetry::nostd::get(attribute_map.GetAttributes().at(keys[i])), values[i]); + } +} + +TEST(OrderedAttributeMapTest, AttributesConstruction) +{ + const int kNumAttributes = 3; + std::string keys[kNumAttributes] = {"attr1", "attr2", "attr3"}; + int values[kNumAttributes] = {15, 24, 37}; + std::map attributes = { + {keys[0], values[0]}, {keys[1], values[1]}, {keys[2], values[2]}}; + + opentelemetry::common::KeyValueIterableView> iterable(attributes); + opentelemetry::sdk::common::OrderedAttributeMap attribute_map(iterable); for (int i = 0; i < kNumAttributes; i++) { - EXPECT_EQ(opentelemetry::nostd::get(map.GetAttributes().at(keys[i])), values[i]); + EXPECT_EQ(opentelemetry::nostd::get(attribute_map.GetAttributes().at(keys[i])), values[i]); } } diff --git a/sdk/test/common/attributemap_hash_benchmark.cc b/sdk/test/common/attributemap_hash_benchmark.cc new file mode 100644 index 0000000000..811ecb23dd --- /dev/null +++ b/sdk/test/common/attributemap_hash_benchmark.cc @@ -0,0 +1,22 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include +#include "opentelemetry/sdk/common/attributemap_hash.h" + +using namespace opentelemetry::sdk::common; +namespace +{ +void BM_AttributeMapHash(benchmark::State &state) +{ + OrderedAttributeMap map1 = {{"k1", "v1"}, {"k2", "v2"}, {"k3", "v3"}, {"k4", "v4"}, + {"k5", true}, {"k6", 12}, {"k7", 12.209}}; + while (state.KeepRunning()) + { + benchmark::DoNotOptimize(GetHashForAttributeMap(map1)); + } +} +BENCHMARK(BM_AttributeMapHash); + +} // namespace +BENCHMARK_MAIN(); diff --git a/sdk/test/common/attributemap_hash_test.cc b/sdk/test/common/attributemap_hash_test.cc new file mode 100644 index 0000000000..7d2748670d --- /dev/null +++ b/sdk/test/common/attributemap_hash_test.cc @@ -0,0 +1,32 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/sdk/common/attributemap_hash.h" +#include + +using namespace opentelemetry::sdk::common; +TEST(AttributeMapHashTest, BasicTests) +{ + { + OrderedAttributeMap map1 = {{"k1", "v1"}, {"k2", "v2"}, {"k3", "v3"}}; + OrderedAttributeMap map2 = {{"k1", "v1"}, {"k2", "v2"}, {"k3", "v3"}, {"k4", "v4"}}; + OrderedAttributeMap map3 = {{"k3", "v3"}, {"k1", "v1"}, {"k2", "v2"}}; + + EXPECT_TRUE(GetHashForAttributeMap(map1) != 0); + EXPECT_TRUE(GetHashForAttributeMap(map1) == GetHashForAttributeMap(map1)); + EXPECT_TRUE(GetHashForAttributeMap(map1) != GetHashForAttributeMap(map2)); + EXPECT_TRUE(GetHashForAttributeMap(map1) == GetHashForAttributeMap(map3)); + } + + { + OrderedAttributeMap map1 = {{"k1", 10}, {"k2", true}, {"k3", 12.22}}; + OrderedAttributeMap map2 = {{"k3", 12.22}, {"k1", 10}, {"k2", true}}; + EXPECT_TRUE(GetHashForAttributeMap(map1) == GetHashForAttributeMap(map2)); + EXPECT_TRUE(GetHashForAttributeMap(map1) != 0); + } + + { + OrderedAttributeMap map1 = {}; + EXPECT_TRUE(GetHashForAttributeMap(map1) == 0); + } +} \ No newline at end of file diff --git a/sdk/test/metrics/BUILD b/sdk/test/metrics/BUILD index 6ba99b4f7a..1e4df5a481 100644 --- a/sdk/test/metrics/BUILD +++ b/sdk/test/metrics/BUILD @@ -1,11 +1,46 @@ +load("//bazel:otel_cc_benchmark.bzl", "otel_cc_benchmark") + cc_test( name = "meter_provider_sdk_test", srcs = [ "meter_provider_sdk_test.cc", ], + tags = [ + "metrics", + "test", + ], deps = [ "//sdk/src/metrics", "//sdk/src/resource", "@com_google_googletest//:gtest_main", ], ) + +cc_test( + name = "attributes_processor_test", + srcs = [ + "attributes_processor_test.cc", + ], + tags = [ + "metrics", + "test", + ], + deps = [ + "//sdk/src/metrics", + "@com_google_googletest//:gtest_main", + ], +) + +otel_cc_benchmark( + name = "attributes_processor_benchmark", + srcs = [ + "attributes_processor_benchmark.cc", + ], + tags = [ + "metrics", + "test", + ], + deps = [ + "//sdk/src/metrics", + ], +) diff --git a/sdk/test/metrics/CMakeLists.txt b/sdk/test/metrics/CMakeLists.txt index f65f3b8bc8..36deef5f05 100644 --- a/sdk/test/metrics/CMakeLists.txt +++ b/sdk/test/metrics/CMakeLists.txt @@ -1,4 +1,5 @@ -foreach(testname meter_provider_sdk_test view_registry_test) +foreach(testname meter_provider_sdk_test view_registry_test + attributes_processor_test) add_executable(${testname} "${testname}.cc") target_link_libraries( ${testname} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} @@ -8,3 +9,7 @@ foreach(testname meter_provider_sdk_test view_registry_test) TEST_PREFIX metrics. TEST_LIST ${testname}) endforeach() + +add_executable(attributes_processor_benchmark attributes_processor_benchmark.cc) +target_link_libraries(attributes_processor_benchmark benchmark::benchmark + ${CMAKE_THREAD_LIBS_INIT} opentelemetry_common) diff --git a/sdk/test/metrics/attributes_processor_benchmark.cc b/sdk/test/metrics/attributes_processor_benchmark.cc new file mode 100644 index 0000000000..d558a668f7 --- /dev/null +++ b/sdk/test/metrics/attributes_processor_benchmark.cc @@ -0,0 +1,27 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include +#ifndef ENABLE_METRICS_PREVIEW +# include +# include "opentelemetry/sdk/metrics/view/attributes_processor.h" +using namespace opentelemetry::sdk::metrics; +namespace +{ +void BM_AttributseProcessorFilter(benchmark::State &state) +{ + std::map attributes = { + {"att1", 10}, {"attr1", 20}, {"attr3", 30}, {"attr4", 40}}; + FilteringAttributesProcessor attributes_processor( + {{"attr2", true}, {"attr4", true}, {"attr6", true}}); + opentelemetry::common::KeyValueIterableView> iterable(attributes); + while (state.KeepRunning()) + { + auto filtered_attributes = attributes_processor.process(iterable); + } +} + +BENCHMARK(BM_AttributseProcessorFilter); +} // namespace +#endif +BENCHMARK_MAIN(); diff --git a/sdk/test/metrics/attributes_processor_test.cc b/sdk/test/metrics/attributes_processor_test.cc new file mode 100644 index 0000000000..d496cc7b09 --- /dev/null +++ b/sdk/test/metrics/attributes_processor_test.cc @@ -0,0 +1,49 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#ifndef ENABLE_METRICS_PREVIEW +# include "opentelemetry/sdk/metrics/view/attributes_processor.h" +# include + +using namespace opentelemetry::sdk::metrics; +using namespace opentelemetry::common; +using namespace opentelemetry::sdk::common; + +TEST(AttributesProcessor, FilteringAttributesProcessor) +{ + const int kNumFilterAttributes = 3; + std::unordered_map filter = { + {"attr2", true}, {"attr4", true}, {"attr6", true}}; + const int kNumAttributes = 6; + std::string keys[kNumAttributes] = {"attr1", "attr2", "attr3", "attr4", "attr5", "attr6"}; + int values[kNumAttributes] = {10, 20, 30, 40, 50, 60}; + std::map attributes = {{keys[0], values[0]}, {keys[1], values[1]}, + {keys[2], values[2]}, {keys[3], values[3]}, + {keys[4], values[4]}, {keys[5], values[5]}}; + FilteringAttributesProcessor attributes_processor(filter); + opentelemetry::common::KeyValueIterableView> iterable(attributes); + auto filtered_attributes = attributes_processor.process(iterable); + for (auto &e : filtered_attributes) + { + EXPECT_FALSE(filter.find(e.first) == filter.end()); + } + EXPECT_EQ(filter.size(), kNumFilterAttributes); +} + +TEST(AttributesProcessor, FilteringAllAttributesProcessor) +{ + const int kNumFilterAttributes = 0; + std::unordered_map filter = {}; + const int kNumAttributes = 6; + std::string keys[kNumAttributes] = {"attr1", "attr2", "attr3", "attr4", "attr5", "attr6"}; + int values[kNumAttributes] = {10, 20, 30, 40, 50, 60}; + std::map attributes = {{keys[0], values[0]}, {keys[1], values[1]}, + {keys[2], values[2]}, {keys[3], values[3]}, + {keys[4], values[4]}, {keys[5], values[5]}}; + FilteringAttributesProcessor attributes_processor(filter); + opentelemetry::common::KeyValueIterableView> iterable(attributes); + auto filtered_attributes = attributes_processor.process(iterable); + EXPECT_EQ(filter.size(), kNumFilterAttributes); +} + +#endif