From 2a662639f8960e6afe2a31fbe82228415722c686 Mon Sep 17 00:00:00 2001 From: EamonnTP Date: Tue, 30 Jun 2020 19:14:38 +0100 Subject: [PATCH] Split fields and metrics content (#1210) --- docs/en/metrics/aws-ec2-metricset.asciidoc | 12 +- docs/en/metrics/aws-rds-metricset.asciidoc | 11 +- docs/en/metrics/aws-s3-metricset.asciidoc | 11 +- docs/en/metrics/aws-sqs-metricset.asciidoc | 9 +- docs/en/metrics/docker-metricset.asciidoc | 12 +- docs/en/metrics/host-metricset.asciidoc | 11 +- docs/en/metrics/index.asciidoc | 7 +- .../metrics/infrastructure-metrics.asciidoc | 24 +- docs/en/metrics/kubernetes-metricset.asciidoc | 12 +- .../metrics/metrics-fields-reference.asciidoc | 403 ++++++++++++++++++ docs/en/metrics/metrics-installation.asciidoc | 21 +- 11 files changed, 454 insertions(+), 79 deletions(-) create mode 100644 docs/en/metrics/metrics-fields-reference.asciidoc diff --git a/docs/en/metrics/aws-ec2-metricset.asciidoc b/docs/en/metrics/aws-ec2-metricset.asciidoc index 9f21c54d4..cb242ac70 100644 --- a/docs/en/metrics/aws-ec2-metricset.asciidoc +++ b/docs/en/metrics/aws-ec2-metricset.asciidoc @@ -1,14 +1,7 @@ [[aws-ec2-metricset]] [role="xpack"] -=== AWS EC2 Instance Fields - -*ID*:: `cloud.instance.id` -*Name*:: `cloud.instance.name` -*IP Address*:: `aws.ec2.instance.public.ip` - -[float] -=== AWS ECS Instance Metrics +== AWS ECS Instance metrics *CPU Usage*:: Average of `aws.ec2.cpu.total.pct` @@ -20,3 +13,6 @@ *Disk Writes (Bytes)*:: Average of `aws.ec2.diskio.write.bytes_per_sec` + +For information about which required fields the {metrics-app} uses to display EC2 instance metrics, see the <>. + diff --git a/docs/en/metrics/aws-rds-metricset.asciidoc b/docs/en/metrics/aws-rds-metricset.asciidoc index 343232525..cd794a997 100644 --- a/docs/en/metrics/aws-rds-metricset.asciidoc +++ b/docs/en/metrics/aws-rds-metricset.asciidoc @@ -1,13 +1,7 @@ [[aws-rds-metricset]] [role="xpack"] -=== AWS RDS Database Fields - -*ID*:: `aws.rds.db_instance.arn` -*Name*:: `aws.rds.db_instance.identifier` - -[float] -=== AWS RDS Database Metrics +== AWS RDS database metrics *CPU Usage*:: Average of `aws.rds.cpu.total.pct` @@ -19,3 +13,6 @@ *Latency*:: Average of `aws.rds.latency.dml` + +For information about which required fields the {metrics-app} uses to display RDS database metrics, see the <>. + diff --git a/docs/en/metrics/aws-s3-metricset.asciidoc b/docs/en/metrics/aws-s3-metricset.asciidoc index 7ffeb9ebe..6f3a2d914 100644 --- a/docs/en/metrics/aws-s3-metricset.asciidoc +++ b/docs/en/metrics/aws-s3-metricset.asciidoc @@ -1,13 +1,7 @@ [[aws-s3-metricset]] [role="xpack"] -=== AWS S3 Bucket Fields - -*ID*:: `aws.s3.bucket.name` -*Name*:: `aws.s3.bucket.name` - -[float] -=== AWS S3 Bucket Metrics +== AWS S3 bucket metrics *Bucket Size*:: Average of `aws.s3_daily_storage.bucket.size.bytes` @@ -19,3 +13,6 @@ *Uploads (Bytes)*:: Average of `aws.s3_request.uploaded.bytes` + +For information about which required fields the {metrics-app} uses to display S3 bucket metrics, see the <>. + diff --git a/docs/en/metrics/aws-sqs-metricset.asciidoc b/docs/en/metrics/aws-sqs-metricset.asciidoc index 0478cb968..eda4f6a1e 100644 --- a/docs/en/metrics/aws-sqs-metricset.asciidoc +++ b/docs/en/metrics/aws-sqs-metricset.asciidoc @@ -1,13 +1,7 @@ [[aws-sqs-metricset]] [role="xpack"] -=== AWS SQS Queue Fields - -*ID*:: `aws.sqs.queue.name` -*Name*:: `aws.sqs.queue.name` - -[float] -=== AWS SQS Queue Metrics +== AWS SQS queue metrics *Messages Available*:: Max of `aws.sqs.messages.visible` @@ -20,3 +14,4 @@ *Oldest Message*:: Max of `aws.sqs.oldest_message_age.sec` +For information about which required fields the {metrics-app} uses to display SQS queue metrics, see the <>. \ No newline at end of file diff --git a/docs/en/metrics/docker-metricset.asciidoc b/docs/en/metrics/docker-metricset.asciidoc index 8c9330c8a..4f6ca6912 100644 --- a/docs/en/metrics/docker-metricset.asciidoc +++ b/docs/en/metrics/docker-metricset.asciidoc @@ -1,14 +1,7 @@ [[docker-metricset]] [role="xpack"] -=== Docker Container Fields - -*ID*:: `container.id` -*Name*:: `container.name` -*IP Address*:: `container.ip_address` - -[float] -=== Docker Container Metrics +== Docker container metrics *CPU Usage*:: Average of `docker.cpu.total.pct` @@ -17,3 +10,6 @@ *Inbound Traffic*:: Derivative of the maximum of `docker.network.in.bytes` scaled to a 1 second rate *Outbound Traffic*:: Derivative of the maximum of `docker.network.out.bytes` scaled to a 1 second rate + + +For information about which required fields the {metrics-app} uses to display Docker metrics, see the <>. \ No newline at end of file diff --git a/docs/en/metrics/host-metricset.asciidoc b/docs/en/metrics/host-metricset.asciidoc index 1315d3afc..791e0119f 100644 --- a/docs/en/metrics/host-metricset.asciidoc +++ b/docs/en/metrics/host-metricset.asciidoc @@ -1,14 +1,7 @@ [[host-metricset]] [role="xpack"] -=== Hosts Fields - -*ID*:: `host.name` -*Name*:: `host.name` -*IP Address*:: `host.ip` - -[float] -=== Host Metrics +== Host metrics *CPU Usage*:: Average of `system.cpu.user.pct` added to the average of `system.cpu.system.pct` divided by `system.cpu.cores` @@ -22,3 +15,5 @@ *Log Rate*:: Derivative of the cumulative sum of the document count scaled to a 1 second rate. This metric relies on the same indices as the logs. + +For information about which required fields the {metrics-app} uses to display host metrics, see the <>. diff --git a/docs/en/metrics/index.asciidoc b/docs/en/metrics/index.asciidoc index 1eacbabee..62d55c10a 100644 --- a/docs/en/metrics/index.asciidoc +++ b/docs/en/metrics/index.asciidoc @@ -14,8 +14,6 @@ include::metrics-overview.asciidoc[] include::metrics-installation.asciidoc[] -include::infrastructure-metrics.asciidoc[] - include::metrics-app-overview.asciidoc[] include::configuring-metrics-source.asciidoc[] @@ -25,3 +23,8 @@ include::explore-metrics-data.asciidoc[] include::metrics-explorer.asciidoc[] include::metrics-alerting.asciidoc[] + +include::metrics-fields-reference.asciidoc[] + +include::infrastructure-metrics.asciidoc[] + diff --git a/docs/en/metrics/infrastructure-metrics.asciidoc b/docs/en/metrics/infrastructure-metrics.asciidoc index fd5e57d04..c668195b3 100644 --- a/docs/en/metrics/infrastructure-metrics.asciidoc +++ b/docs/en/metrics/infrastructure-metrics.asciidoc @@ -1,24 +1,20 @@ [[infrastructure-metrics]] [role="xpack"] -== Infrastructure metrics += Infrastructure metrics -This section contains detailed information about each of the metricsets the {metrics-app} supports. The metrics listed below are provided by the {beats} shippers. +This section contains detailed information about each of the metricsets the {metrics-app} supports. -* <> -* <> -* <> -* <> -* <> -* <> -* <> - -[float] -=== Additional field details +The metrics listed below are provided by the {beats} shippers. -To display data properly in some views, the `event.dataset` field is required. This field is a combination of `metricset.module`, which is the Metricbeat module name, and `metricset.name`, which is the metricset name. +* <> +* <> +* <> +* <> +* <> +* <> +* <> -To determine the optimal time interval for each metric, all of the charts use the `metricset.period`. If `metricset.period` is not available then it falls back to 1 minute intervals. include::host-metricset.asciidoc[] diff --git a/docs/en/metrics/kubernetes-metricset.asciidoc b/docs/en/metrics/kubernetes-metricset.asciidoc index d253fde8b..cc2097b0c 100644 --- a/docs/en/metrics/kubernetes-metricset.asciidoc +++ b/docs/en/metrics/kubernetes-metricset.asciidoc @@ -1,14 +1,7 @@ [[kubernetes-metricset]] [role="xpack"] -=== Kubernetes Pod Fields - -*ID*:: `kubernetes.pod.uid` -*Name*:: `kubernetes.pod.name` -*IP Address*:: `kubernetes.pod.ip` - -[float] -=== Kubernetes Pod Metrics +== Kubernetes pod metrics *CPU Usage*:: Average of `kubernetes.pod.cpu.usage.node.pct` @@ -17,3 +10,6 @@ *Inbound Traffic*:: Derivative of the maximum of `kubernetes.pod.network.rx.bytes` scaled to a 1 second rate *Outbound Traffic*:: Derivative of the maximum of `kubernetes.pod.network.tx.bytes` scaled to a 1 second rate + + +For information about which required fields the {metrics-app} uses to display Kubernetes metrics, see the <>. \ No newline at end of file diff --git a/docs/en/metrics/metrics-fields-reference.asciidoc b/docs/en/metrics/metrics-fields-reference.asciidoc new file mode 100644 index 000000000..4e8227939 --- /dev/null +++ b/docs/en/metrics/metrics-fields-reference.asciidoc @@ -0,0 +1,403 @@ +[[metrics-fields-reference]] +[role="xpack"] += Metrics fields reference + +The following sections list the required fields the {metrics-app} uses to display data. +Some of the fields listed are https://www.elastic.co/guide/en/ecs/current/ecs-reference.html#_what_is_ecs[ECS fields]. + +The fields are grouped in the following categories: + +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> + +[float] +== Additional field details + +To display data properly in some views, the `event.dataset` field is required. This field is a combination of `metricset.module`, which is the Metricbeat module name, and `metricset.name`, which is the metricset name. + +To determine the optimal time interval for each metric, all of the charts use the `metricset.period`. If `metricset.period` is not available then it falls back to 1 minute intervals. + +[[base-fields]] +== Base fields + +The `base` field set contains all fields which are on the top level. These fields are common across all types of events. + +`@timestamp`:: + +Date/time when the event originated. ++ +This is the date/time extracted from the event, typically representing when the event was generated by the source. +If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. +Required field for all events. ++ +type: date ++ +required: True ++ +ECS field: True ++ +example: `May 27, 2020 @ 15:22:27.982` + +`message`:: + +For log events the message field contains the log message, optimized for viewing in a log viewer. ++ +For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. ++ +If multiple messages exist, they can be combined into one message. ++ +type: text ++ +required: True ++ +ECS field: True ++ +example: `Hello World` + + +[[host-fields]] +== Hosts fields + +These fields must be mapped to display host data in the {metrics-app}. + +`host.name`:: + +Name of the host. ++ +It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. ++ +type: keyword ++ +required: True ++ +ECS field: True ++ +example: `MacBook-Elastic.local` + +`host.ip`:: + +IP of the host that records the event. ++ +type: ip ++ +required: True ++ +ECS field: True + +[[docker-fields]] +== Docker container fields + +These fields must be mapped to display Docker container data in the {metrics-app}. + +`container.id`:: + +Unique container id. ++ +type: keyword ++ +required: True ++ +ECS field: True ++ +example: `data` + +`container.name`:: + +Container name. ++ +type: keyword ++ +required: True ++ +ECS field: True + +`container.ip_address`:: + +IP of the container. ++ +type: ip ++ +required: True ++ +ECS field: False + +[[kubernetes-fields]] +== Kubernetes pod fields + +These fields must be mapped to display Kubernetes pod data in the {metrics-app}. + +`kubernetes.pod.uid`:: + +Kubernetes Pod UID. ++ +type: keyword ++ +required: True ++ +ECS field: False ++ +example: `8454328b-673d-11ea-7d80-21010a840123` + +`kubernetes.pod.name`:: + +Kubernetes pod name. ++ +type: keyword ++ +required: True ++ +ECS field: False ++ +example: `nginx-demo` + +`kubernetes.pod.ip`:: + +IP of the Kubernetes pod. ++ +type: keyword ++ +required: True ++ +ECS field: False + +[[aws-ec2-fields]] +== AWS EC2 instance fields + +These fields must be mapped to display EC2 instance data in the {metrics-app}. + +`cloud.instance.id`:: + +Instance ID of the host machine. ++ +type: keyword ++ +required: True ++ +ECS field: True ++ +example: `i-1234567890abcdef0` + +`cloud.instance.name`:: + +Instance name of the host machine. ++ +type: keyword ++ +required: True ++ +ECS field: True + +`aws.ec2.instance.public.ip`:: + +Instance public IP of the host machine. ++ +type: keyword ++ +required: True ++ +ECS field: False + +[[aws-s3-fields]] +== AWS S3 bucket fields + +These fields must be mapped to display S3 bucket data in the {metrics-app}. + +`aws.s3.bucket.name`:: + +The name or ID of the AWS S3 bucket. ++ +type: keyword ++ +required: True ++ +ECS field: False + +[[aws-sqs-fields]] +== AWS SQS queue fields + +These fields must be mapped to display SQS queue data in the {metrics-app}. + +`aws.sqs.queue.name`:: + +The name or ID of the AWS SQS queue. ++ +type: keyword ++ +required: True ++ +ECS field: False + +[[aws-rds-fields]] +== AWS RDS database fields + +These fields must be mapped to display RDS database data in the {metrics-app}. + +`aws.rds.db_instance.arn`:: + +Amazon Resource Name(ARN) for each rds. ++ +type: keyword ++ +required: True ++ +ECS field: False + +`aws.rds.db_instance.identifier`:: + +Contains a user-supplied database identifier. This identifier is the unique key that identifies a DB instance. ++ +type: keyword ++ +required: True ++ +ECS field: False + +[[group-inventory-fields]] +== Additional grouping fields + +Depending on which entity you select in the *Inventory* view, these additional fields can be mapped to group entities by. + +`cloud.availability_zone`:: + +Availability zone in which this host is running. ++ +type: keyword ++ +required: True ++ +ECS field: True ++ +example: `us-east-1c` + +`cloud.machine.type`:: + +Machine type of the host machine. ++ +type: keyword ++ +required: True ++ +ECS field: True ++ +example: `t2.medium` + +`cloud.region`:: + +Region in which this host is running. ++ +type: keyword ++ +required: True ++ +ECS field: True ++ +example: `us-east-1` + +`cloud.instance.id`:: + +Instance ID of the host machine. ++ +type: keyword ++ +required: True ++ +ECS field: True ++ +example: `i-1234567890abcdef0` + +`cloud.provider`:: + +Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. ++ +type: keyword ++ +required: True ++ +ECS field: True ++ +example: `aws` + +`cloud.instance.name`:: + +Instance name of the host machine. ++ +type: keyword ++ +required: True ++ +ECS field: True + +`cloud.project.id`:: + +Name of the project in Google Cloud. ++ +type: keyword ++ +required: True ++ +ECS field: False + +`service.type`:: + +The type of the service data is collected from. ++ +The type can be used to group and correlate logs and metrics from one service type. ++ +Example: If metrics are collected from Elasticsearch, service.type would be elasticsearch. ++ +type: keyword ++ +required: True ++ +ECS field: False ++ +example: `elasticsearch` + +`host.hostname`:: + +Hostname of the host. ++ +It normally contains what the `hostname` command returns on the host machine. ++ +type: keyword ++ +required: True, if you want to use the {ml-features}. ++ +ECS field: True ++ +example: `Elastic.local` + +`host.os.name`:: + +Operating system name, without the version. ++ +Multi-fields: ++ +* os.name.text (type: text) ++ +type: keyword ++ +required: True ++ +ECS field: True ++ +example: `Mac OS X` + +`host.os.kernel`:: + +Operating system kernel version as a raw string. ++ +type: keyword ++ +required: True ++ +ECS field: True ++ +example: `4.4.0-112-generic` \ No newline at end of file diff --git a/docs/en/metrics/metrics-installation.asciidoc b/docs/en/metrics/metrics-installation.asciidoc index 8af990f7d..5d5a8a86c 100644 --- a/docs/en/metrics/metrics-installation.asciidoc +++ b/docs/en/metrics/metrics-installation.asciidoc @@ -42,31 +42,32 @@ Install {kib}, start it up, and open up the web interface: . {stack-gs}/get-started-elastic-stack.html#_launch_the_kibana_web_interface[Launch the Kibana Web Interface]. [[install-beats-for-metrics]] -=== Step 3: Install and enable {beats} shippers +=== Step 3: Set up and run {metricbeat} -To start collecting metrics data, you need to install and configure the {metricbeat} {beats} shipper. +To start collecting metrics data, you need to install {metricbeat} and configure the {metricbeat} modules directly from {kib}. -You can install and configure {beats} shippers for most kinds of data directly from {kib}, or you can install {beats} yourself. +Alternatively, you can install {metricbeat} and configure the {metricbeat} modules yourself. [float] -==== Install {beats} from {kib} +==== Install {metricbeat} from {kib} -To install {beats} from {kib}, on the machine where you want to collect the data, open a {kib} browser window. +To install {metricbeat} from {kib}, on the machine where you want to collect the data, open a {kib} browser window. In the *Observability* section displayed on the home page of {kib}, click *Add metric data*. Now follow the instructions for the type of data you want to collect. -The instructions include the steps required to download, install, and configure the appropriate Beats modules for your data. +The instructions include how to install and configure {metricbeat}, and enable the appropriate {metricbeat} module for your data. [role="screenshot"] image::images/add-data.png[Add metrics data] [float] -==== Install {beats} yourself +==== Install {metricbeat} yourself -If your data source doesn't have a {beats} module, or if you want to install {beats} the old fashioned way, follow the instructions in {metricbeat-ref}/metricbeat-getting-started.html[{metricbeat} getting started] and enable modules for the metrics you want to collect. +If your data source doesn't have a {metricbeat} module, or if you want to install one the old fashioned way, follow the instructions in {metricbeat-ref}/metricbeat-getting-started.html[{metricbeat} getting started] and enable modules for the metrics you want to collect. [float] -=== Enable modules -However you install {beats}, you need to enable the appropriate modules in {metricbeat} to start collecting metrics data. +=== Enable {metricbeat} modules + +To start collecting metrics data, to enable the appropriate modules in {metricbeat}. To populate the *Hosts* view with metrics data, enable: