From 9a3e1de14fedc0ccd39eb37955b780c767cc6537 Mon Sep 17 00:00:00 2001 From: Christian Mergenthaler <377953+cmergenthaler@users.noreply.github.com> Date: Thu, 11 Jul 2024 16:47:13 +0200 Subject: [PATCH] [receiver/azuremonitor] Add parameter to overwrite azure's max record size (#32380) **Description:** When having lots of different records of one dimension in the metric, azure by default only returns 10 of them. This setting adds the possibility to overwrite the default and specify a custom number in the config of the receiver. Fixes #32165 **Testing:** Tested fetching metrics with different configs. Do we need a unit test for this? **Documentation:** Added parameter to README --- .chloggen/azuremonitorreceiver-top.yaml | 27 ++++++++++++++++ receiver/azuremonitorreceiver/README.md | 1 + receiver/azuremonitorreceiver/config.go | 31 ++++++++++--------- receiver/azuremonitorreceiver/factory.go | 17 +++++----- receiver/azuremonitorreceiver/factory_test.go | 15 ++++----- receiver/azuremonitorreceiver/scraper.go | 3 ++ 6 files changed, 64 insertions(+), 30 deletions(-) create mode 100644 .chloggen/azuremonitorreceiver-top.yaml diff --git a/.chloggen/azuremonitorreceiver-top.yaml b/.chloggen/azuremonitorreceiver-top.yaml new file mode 100644 index 000000000000..202c341ba419 --- /dev/null +++ b/.chloggen/azuremonitorreceiver-top.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: azuremonitorreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add `maximum_number_of_records_per_resource` config parameter in order to overwrite default + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [32165] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/receiver/azuremonitorreceiver/README.md b/receiver/azuremonitorreceiver/README.md index 8c01eabd1b3f..c973cc1f1d23 100644 --- a/receiver/azuremonitorreceiver/README.md +++ b/receiver/azuremonitorreceiver/README.md @@ -28,6 +28,7 @@ The following settings are optional: - `cache_resources` (default = 86400): List of resources will be cached for the provided amount of time in seconds. - `cache_resources_definitions` (default = 86400): List of metrics definitions will be cached for the provided amount of time in seconds. - `maximum_number_of_metrics_in_a_call` (default = 20): Maximum number of metrics to fetch in per API call, current limit in Azure is 20 (as of 03/27/2023). +- `maximum_number_of_records_per_resource` (default = 10): Maximum number of records to fetch per resource. - `initial_delay` (default = `1s`): defines how long this receiver waits before starting. - `cloud` (default = `AzureCloud`): defines which Azure cloud to use. Either `AzureCloud` or `AzureUSGovernment` diff --git a/receiver/azuremonitorreceiver/config.go b/receiver/azuremonitorreceiver/config.go index 7078813512a8..90f7d7ef6292 100644 --- a/receiver/azuremonitorreceiver/config.go +++ b/receiver/azuremonitorreceiver/config.go @@ -229,21 +229,22 @@ var ( // Config defines the configuration for the various elements of the receiver agent. type Config struct { - scraperhelper.ControllerConfig `mapstructure:",squash"` - MetricsBuilderConfig metadata.MetricsBuilderConfig `mapstructure:",squash"` - Cloud string `mapstructure:"cloud"` - SubscriptionID string `mapstructure:"subscription_id"` - Authentication string `mapstructure:"auth"` - TenantID string `mapstructure:"tenant_id"` - ClientID string `mapstructure:"client_id"` - ClientSecret string `mapstructure:"client_secret"` - FederatedTokenFile string `mapstructure:"federated_token_file"` - ResourceGroups []string `mapstructure:"resource_groups"` - Services []string `mapstructure:"services"` - CacheResources float64 `mapstructure:"cache_resources"` - CacheResourcesDefinitions float64 `mapstructure:"cache_resources_definitions"` - MaximumNumberOfMetricsInACall int `mapstructure:"maximum_number_of_metrics_in_a_call"` - AppendTagsAsAttributes bool `mapstructure:"append_tags_as_attributes"` + scraperhelper.ControllerConfig `mapstructure:",squash"` + MetricsBuilderConfig metadata.MetricsBuilderConfig `mapstructure:",squash"` + Cloud string `mapstructure:"cloud"` + SubscriptionID string `mapstructure:"subscription_id"` + Authentication string `mapstructure:"auth"` + TenantID string `mapstructure:"tenant_id"` + ClientID string `mapstructure:"client_id"` + ClientSecret string `mapstructure:"client_secret"` + FederatedTokenFile string `mapstructure:"federated_token_file"` + ResourceGroups []string `mapstructure:"resource_groups"` + Services []string `mapstructure:"services"` + CacheResources float64 `mapstructure:"cache_resources"` + CacheResourcesDefinitions float64 `mapstructure:"cache_resources_definitions"` + MaximumNumberOfMetricsInACall int `mapstructure:"maximum_number_of_metrics_in_a_call"` + MaximumNumberOfRecordsPerResource int32 `mapstructure:"maximum_number_of_records_per_resource"` + AppendTagsAsAttributes bool `mapstructure:"append_tags_as_attributes"` } const ( diff --git a/receiver/azuremonitorreceiver/factory.go b/receiver/azuremonitorreceiver/factory.go index bb024957b966..2da418bf3c00 100644 --- a/receiver/azuremonitorreceiver/factory.go +++ b/receiver/azuremonitorreceiver/factory.go @@ -36,14 +36,15 @@ func createDefaultConfig() component.Config { cfg.CollectionInterval = defaultCollectionInterval return &Config{ - ControllerConfig: cfg, - MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig(), - CacheResources: 24 * 60 * 60, - CacheResourcesDefinitions: 24 * 60 * 60, - MaximumNumberOfMetricsInACall: 20, - Services: monitorServices, - Authentication: servicePrincipal, - Cloud: defaultCloud, + ControllerConfig: cfg, + MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig(), + CacheResources: 24 * 60 * 60, + CacheResourcesDefinitions: 24 * 60 * 60, + MaximumNumberOfMetricsInACall: 20, + MaximumNumberOfRecordsPerResource: 10, + Services: monitorServices, + Authentication: servicePrincipal, + Cloud: defaultCloud, } } diff --git a/receiver/azuremonitorreceiver/factory_test.go b/receiver/azuremonitorreceiver/factory_test.go index e2a07e56f7d0..8366c3324198 100644 --- a/receiver/azuremonitorreceiver/factory_test.go +++ b/receiver/azuremonitorreceiver/factory_test.go @@ -39,13 +39,14 @@ func TestNewFactory(t *testing.T) { CollectionInterval: 10 * time.Second, InitialDelay: time.Second, }, - MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig(), - Services: monitorServices, - CacheResources: 24 * 60 * 60, - CacheResourcesDefinitions: 24 * 60 * 60, - MaximumNumberOfMetricsInACall: 20, - Authentication: servicePrincipal, - Cloud: defaultCloud, + MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig(), + Services: monitorServices, + CacheResources: 24 * 60 * 60, + CacheResourcesDefinitions: 24 * 60 * 60, + MaximumNumberOfMetricsInACall: 20, + MaximumNumberOfRecordsPerResource: 10, + Authentication: servicePrincipal, + Cloud: defaultCloud, } require.Equal(t, expectedCfg, factory.CreateDefaultConfig()) diff --git a/receiver/azuremonitorreceiver/scraper.go b/receiver/azuremonitorreceiver/scraper.go index c20ea33afaa4..8e5440dae0cd 100644 --- a/receiver/azuremonitorreceiver/scraper.go +++ b/receiver/azuremonitorreceiver/scraper.go @@ -393,6 +393,7 @@ func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID compositeKey.timeGrain, start, end, + s.cfg.MaximumNumberOfRecordsPerResource, ) start = end @@ -441,6 +442,7 @@ func getResourceMetricsValuesRequestOptions( timeGrain string, start int, end int, + top int32, ) armmonitor.MetricsClientListOptions { resType := strings.Join(metrics[start:end], ",") filter := armmonitor.MetricsClientListOptions{ @@ -448,6 +450,7 @@ func getResourceMetricsValuesRequestOptions( Interval: to.Ptr(timeGrain), Timespan: to.Ptr(timeGrain), Aggregation: to.Ptr(strings.Join(aggregations, ",")), + Top: to.Ptr(top), } if len(dimensionsStr) > 0 {