From dec042cca15f2fa885f02b262deb7d74e923de6c Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Mon, 10 Aug 2020 19:20:53 +0200 Subject: [PATCH] Update vm_compute metricset with more info on guest metrics (#20448) * mofidy doc * update doc * changelog --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/azure.asciidoc | 1 - .../module/azure/_meta/docs.asciidoc | 1 - .../module/azure/compute_vm/_meta/data.json | 632 +++--------------- .../azure/compute_vm/_meta/docs.asciidoc | 4 + 5 files changed, 102 insertions(+), 537 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 565da0496622..883783b9c715 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -310,6 +310,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Modify doc for app_insights metricset to contain example of config. {pull}20185[20185] - Add required option for `metrics` in app_insights. {pull}20406[20406] - Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403] +- Updates vm_compute metricset with more info on guest metrics. {pull}20448[20448] *Packetbeat* diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index e4cd48354737..4db38120041e 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -16,7 +16,6 @@ The azure monitor metrics are numerical values that describe some aspect of a sy The azure module will periodically retrieve the azure monitor metrics using the Azure REST APIs as MetricList. Additional azure API calls will be executed in order to retrieve information regarding the resources targeted by the user. -The azure module mericsets are `monitor`, `compute_vm` and `compute_vm_scaleset` IMPORTANT: Extra Azure charges on metric queries may be generated by this module. Please see <> for more details. diff --git a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc index e828c89e41a4..b0f76ecb6230 100644 --- a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc @@ -8,7 +8,6 @@ The azure monitor metrics are numerical values that describe some aspect of a sy The azure module will periodically retrieve the azure monitor metrics using the Azure REST APIs as MetricList. Additional azure API calls will be executed in order to retrieve information regarding the resources targeted by the user. -The azure module mericsets are `monitor`, `compute_vm` and `compute_vm_scaleset` IMPORTANT: Extra Azure charges on metric queries may be generated by this module. Please see <> for more details. diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index 5858cf0407e0..960eb88cf7e4 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -1,569 +1,131 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "azure": { - "compute_vm": { - "asp_net_application_restarts": { - "avg": 0 - }, - "asp_net_applications_anonymous_requests": { - "avg": 0 - }, - "asp_net_applications_anonymous_requests_per_sec": { - "avg": 0 - }, - "asp_net_applications_cache_api_entries": { - "avg": 0 - }, - "asp_net_applications_cache_api_hit_ratio": { - "avg": 0 - }, - "asp_net_applications_cache_api_hits": { - "avg": 0 - }, - "asp_net_applications_cache_api_misses": { - "avg": 0 - }, - "asp_net_applications_cache_api_turnover_rate": { - "avg": 0 - }, - "asp_net_applications_cache_total_entries": { - "avg": 0 - }, - "asp_net_applications_cache_total_hit_ratio": { - "avg": 0 - }, - "asp_net_applications_cache_total_hits": { - "avg": 0 - }, - "asp_net_applications_cache_total_misses": { - "avg": 0 - }, - "asp_net_applications_cache_total_turnover_rate": { - "avg": 0 - }, - "asp_net_applications_compilations_total": { - "avg": 0 - }, - "asp_net_applications_debugging_requests": { - "avg": 0 - }, - "asp_net_applications_errors_during_compilation": { - "avg": 0 - }, - "asp_net_applications_errors_during_execution": { - "avg": 0 - }, - "asp_net_applications_errors_during_preprocessing": { - "avg": 0 - }, - "asp_net_applications_errors_total": { - "avg": 0 - }, - "asp_net_applications_errors_total_per_sec": { - "avg": 0 - }, - "asp_net_applications_errors_unhandled_during_execution": { - "avg": 0 - }, - "asp_net_applications_errors_unhandled_during_execution_per_sec": { - "avg": 0 - }, - "asp_net_applications_output_cache_entries": { - "avg": 0 - }, - "asp_net_applications_output_cache_hit_ratio": { - "avg": 0 - }, - "asp_net_applications_output_cache_hits": { - "avg": 0 - }, - "asp_net_applications_output_cache_misses": { - "avg": 0 - }, - "asp_net_applications_output_cache_turnover_rate": { - "avg": 0 - }, - "asp_net_applications_pipeline_instance_count": { - "avg": 0 - }, - "asp_net_applications_request_bytes_in_total": { - "avg": 0 - }, - "asp_net_applications_request_bytes_out_total": { - "avg": 0 - }, - "asp_net_applications_requests_executing": { - "avg": 0 - }, - "asp_net_applications_requests_failed": { - "avg": 0 - }, - "asp_net_applications_requests_in_application_queue": { - "avg": 0 - }, - "asp_net_applications_requests_not_authorized": { - "avg": 0 - }, - "asp_net_applications_requests_not_found": { - "avg": 0 - }, - "asp_net_applications_requests_per_sec": { - "avg": 0 - }, - "asp_net_applications_requests_succeeded": { - "avg": 0 - }, - "asp_net_applications_requests_timed_out": { - "avg": 0 - }, - "asp_net_applications_requests_total": { - "avg": 0 - }, - "asp_net_applications_running": { - "avg": 0 - }, - "asp_net_applications_sessions_abandoned": { - "avg": 0 - }, - "asp_net_applications_sessions_active": { - "avg": 0 - }, - "asp_net_applications_sessions_timed_out": { - "avg": 0 - }, - "asp_net_applications_sessions_total": { - "avg": 0 - }, - "asp_net_applications_transactions_aborted": { - "avg": 0 - }, - "asp_net_applications_transactions_committed": { - "avg": 0 - }, - "asp_net_applications_transactions_pending": { - "avg": 0 - }, - "asp_net_applications_transactions_per_sec": { - "avg": 0 - }, - "asp_net_applications_transactions_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_anonymous_requests": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_anonymous_requests_per_sec": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_api_entries": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_api_hit_ratio": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_api_hits": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_api_misses": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_api_turnover_rate": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_total_entries": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_total_hit_ratio": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_total_hits": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_total_misses": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_total_turnover_rate": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_compilations_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_debugging_requests": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_during_compilation": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_during_execution": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_during_preprocessing": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_total_per_sec": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_unhandled_during_execution": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_unhandled_during_execution_per_sec": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_output_cache_entries": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_output_cache_hit_ratio": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_output_cache_hits": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_output_cache_misses": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_output_cache_turnover_rate": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_pipeline_instance_count": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_request_bytes_in_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_request_bytes_out_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_executing": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_failed": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_in_application_queue": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_not_authorized": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_not_found": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_per_sec": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_succeeded": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_timed_out": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_sessions_abandoned": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_sessions_active": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_sessions_timed_out": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_sessions_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_transactions_aborted": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_transactions_committed": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_transactions_pending": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_transactions_per_sec": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_transactions_total": { - "avg": 0 - }, - "asp_net_request_execution_time": { - "avg": 0 - }, - "asp_net_request_wait_time": { - "avg": 0 - }, - "asp_net_requests_current": { - "avg": 0 - }, - "asp_net_requests_disconnected": { - "avg": 0 - }, - "asp_net_requests_queued": { - "avg": 0 - }, - "asp_net_requests_rejected": { - "avg": 0 - }, - "asp_net_v4_0_30319_application_restarts": { - "avg": 0 - }, - "asp_net_v4_0_30319_applications_running": { - "avg": 0 - }, - "asp_net_v4_0_30319_request_execution_time": { - "avg": 0 - }, - "asp_net_v4_0_30319_request_wait_time": { - "avg": 0 - }, - "asp_net_v4_0_30319_requests_current": { - "avg": 0 - }, - "asp_net_v4_0_30319_requests_disconnected": { - "avg": 0 - }, - "asp_net_v4_0_30319_requests_queued": { - "avg": 0 - }, - "asp_net_v4_0_30319_requests_rejected": { - "avg": 0 - }, - "asp_net_v4_0_30319_worker_process_restarts": { - "avg": 0 - }, - "asp_net_v4_0_30319_worker_processes_running": { - "avg": 0 - }, - "asp_net_worker_process_restarts": { - "avg": 0 - }, - "asp_net_worker_processes_running": { - "avg": 0 - }, - "logical_disk_avg_disk_queue_length": { - "avg": 0 - }, - "logical_disk_avg_disk_read_queue_length": { - "avg": 0 - }, - "logical_disk_avg_disk_sec_per_read": { - "avg": 0 - }, - "logical_disk_avg_disk_sec_per_transfer": { - "avg": 0 - }, - "logical_disk_avg_disk_sec_per_write": { - "avg": 0 - }, - "logical_disk_avg_disk_write_queue_length": { - "avg": 0 - }, - "logical_disk_disk_bytes_per_sec": { - "avg": 449380 - }, - "logical_disk_disk_read_bytes_per_sec": { - "avg": 4783 - }, - "logical_disk_disk_reads_per_sec": { - "avg": 0 - }, - "logical_disk_disk_transfers_per_sec": { - "avg": 6.6 - }, - "logical_disk_disk_write_bytes_per_sec": { - "avg": 444596.8 - }, - "logical_disk_disk_writes_per_sec": { - "avg": 6.4 - }, - "logical_disk_free_megabytes": { - "avg": 10433 - }, - "logical_disk_pct_disk_read_time": { - "avg": 0 - }, - "logical_disk_pct_disk_time": { - "avg": 1.6 - }, - "logical_disk_pct_disk_write_time": { - "avg": 1.6 - }, - "logical_disk_pct_free_space": { - "avg": 22 - }, - "logical_disk_pct_idle_time": { - "avg": 98.6 - }, - "memory_available_bytes": { - "avg": 2951829094.4 - }, - "memory_cache_bytes": { - "avg": 60126003.2 - }, - "memory_committed_bytes": { - "avg": 8922399539.2 - }, - "memory_page_faults_per_sec": { - "avg": 1484.2 - }, - "memory_pages_per_sec": { - "avg": 6 - }, - "memory_pct_committed_bytes_in_use": { - "avg": 82.4 - }, - "memory_pool_nonpaged_bytes": { - "avg": 204028313.6 - }, - "memory_pool_paged_bytes": { - "avg": 359613235.2 - }, - "network_interface_bytes_received_per_sec": { - "avg": 7922.333333333333 - }, - "network_interface_bytes_sent_per_sec": { - "avg": 60510.666666666664 - }, - "network_interface_bytes_total_per_sec": { - "avg": 68433.26666666666 - }, - "network_interface_packets_outbound_errors": { - "avg": 0 - }, - "network_interface_packets_per_sec": { - "avg": 43.4 - }, - "network_interface_packets_received_errors": { - "avg": 0 - }, - "network_interface_packets_received_per_sec": { - "avg": 27.533333333333335 + "@timestamp":"2020-08-05T10:36:00.000Z", + "cloud":{ + "provider":"azure", + "region":"westeurope", + "instance":{ + "id":"/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak", + "name":"obstestmemleak" + }, + "machine":{ + "type":"Standard_B2ms" + } + }, + "event":{ + "dataset":"azure.compute_vm", + "module":"azure", + "duration":5364973100 + }, + "metricset":{ + "name":"compute_vm", + "period":300000 + }, + "azure":{ + "timegrain":"PT5M", + "resource":{ + "type":"Microsoft.Compute/virtualMachines", + "group":"obs-infrastructure", + "tags":{ + "vmtest":"value1, value 2", + "vmtest1":"value3" + } + }, + "subscription_id":"fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "namespace":"Microsoft.Compute/virtualMachines", + "compute_vm":{ + "outbound_flows":{ + "avg":291.2 }, - "network_interface_packets_sent_per_sec": { - "avg": 15.733333333333333 + "outbound_flows_maximum_creation_rate":{ + "avg":6 }, - "process_handle_count": { - "avg": 119553 + "inbound_flows_maximum_creation_rate":{ + "avg":6 }, - "process_thread_count": { - "avg": 1705.8 + "os_per_disk_read_bytes_per_sec":{ + "avg":341.35875 }, - "process_working_set": { - "avg": 5566961254.4 + "per_disk_read_bytes_per_sec":{ + "avg":0 }, - "process_working_set_private": { - "avg": 4168472985.6 + "os_disk_queue_depth":{ + "avg":0 }, - "processor_information_pct_privileged_time": { - "avg": 2.6 + "os_per_disk_qd":{ + "avg":0 }, - "processor_information_pct_processor_time": { - "avg": 6.4 + "os_disk_read_bytes_per_sec":{ + "avg":341.35875 }, - "processor_information_pct_user_time": { - "avg": 3.4 + "network_in":{ + "total":3933481.0 }, - "processor_information_processor_frequency": { - "avg": 2295 + "network_out":{ + "total":7169752.0 }, - "sqlserver_buffer_manager_buffer_cache_hit_ratio": { - "avg": 100 + "os_disk_write_operations_per_sec":{ + "avg":4.994285714285715 }, - "sqlserver_buffer_manager_checkpoint_pages_per_sec": { - "avg": 0 + "percentage_cpu":{ + "avg":8.780625 }, - "sqlserver_buffer_manager_database_pages": { - "avg": 1483 + "cpu_credits_remaining":{ + "avg":864 }, - "sqlserver_buffer_manager_lazy_writes_per_sec": { - "avg": 0 + "network_out_total":{ + "total":9249694.0 }, - "sqlserver_buffer_manager_page_reads_per_sec": { - "avg": 0 + "disk_read_operations_per_sec":{ + "avg":0.022857142857142857 }, - "sqlserver_buffer_manager_page_writes_per_sec": { - "avg": 0 + "disk_write_bytes":{ + "total":1.759878456E7 }, - "sqlserver_general_statistics_user_connections": { - "avg": 1 + "os_disk_write_bytes_per_sec":{ + "avg":73050.70375 }, - "sqlserver_memory_manager_memory_grants_pending": { - "avg": 0 + "cpu_credits_consumed":{ + "avg":0.175 }, - "sqlserver_memory_manager_total_server_memory": { - "avg": 317976 + "per_disk_qd":{ + "avg":0 }, - "sqlserver_sql_statistics_batch_requests_per_sec": { - "avg": 0 + "disk_write_operations_per_sec":{ + "avg":2.5221428571428572 }, - "sqlserver_sql_statistics_sql_compilations_per_sec": { - "avg": 0 + "os_per_disk_write_operations_per_sec":{ + "avg":4.994285714285715 }, - "sqlserver_sql_statistics_sql_re-compilations_per_sec": { - "avg": 0 + "per_disk_write_operations_per_sec":{ + "avg":0.049999999999999996 }, - "system_context_switches_per_sec": { - "avg": 7782.6 + "inbound_flows":{ + "avg":291.2 }, - "system_processes": { - "avg": 114 + "os_disk_read_operations_per_sec":{ + "avg":0.045714285714285714 }, - "system_processor_queue_length": { - "avg": 0.2 + "disk_read_bytes":{ + "total":81926.52 }, - "system_system_up_time": { - "avg": 12530764 + "network_in_total":{ + "total":4920267.0 }, - "web_service_bytes_total_per_sec": { - "avg": 0 + "per_disk_read_operations_per_sec":{ + "avg":0 }, - "web_service_current_connections": { - "avg": 0 + "per_disk_write_bytes_per_sec":{ + "avg":277.56 }, - "web_service_isapi_extension_requests_per_sec": { - "avg": 0 + "os_per_disk_write_bytes_per_sec":{ + "avg":73050.70375 }, - "web_service_total_method_requests_per_sec": { - "avg": 0 + "os_per_disk_read_operations_per_sec":{ + "avg":0.045714285714285714 } - }, - "namespace": "Azure.VM.Windows.GuestMetrics", - "resource": { - "group": "obs-infrastructure", - "tags": { - "vmtest": "value1, value 2", - "vmtest1": "value3" - }, - "type": "Microsoft.Compute/virtualMachines" - }, - "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", - "timegrain": "PT5M" - }, - "cloud": { - "instance": { - "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak", - "name": "obstestmemleak" - }, - "machine": { - "type": "Standard_B2ms" - }, - "provider": "azure", - "region": "westeurope" - }, - "event": { - "dataset": "azure.compute_vm", - "duration": 115000, - "module": "azure" - }, - "metricset": { - "name": "compute_vm", - "period": 10000 + } }, - "service": { - "type": "azure" + "service":{ + "type":"azure" } } diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc index e704e9ec6643..f30db5e06d85 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc @@ -16,4 +16,8 @@ If none of the options are entered then all virtual machine inside the subscript For each metric the primary aggregation assigned will be retrieved. A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. +To populate the VM Guest Metrics Overview dashboard, users will have to enable the Virtual Machine Guest namespace for the monitored resources. +More on sending guest OS metrics to Azure Monitor here https://docs.microsoft.com/en-us/azure/azure-monitor/platform/collect-custom-metrics-guestos-resource-manager-vm. + +image::./images/metricbeat-azure-vm-guestmetrics-overview.png[]