diff --git a/.gitignore b/.gitignore index 0ce436e0b..c28faba1a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ .gradle .local-* build - +.DS_Store .project .classpath .settings @@ -10,5 +10,5 @@ bin stack-docs.iml stack-docs.ipr stack-docs.iws - +/html_docs .vscode diff --git a/docs/en/infraops/add-data.png b/docs/en/infraops/images/add-data.png similarity index 100% rename from docs/en/infraops/add-data.png rename to docs/en/infraops/images/add-data.png diff --git a/docs/en/infraops/images/infra-sysmon.png b/docs/en/infraops/images/infra-sysmon.png new file mode 100644 index 000000000..5b82d8c9b Binary files /dev/null and b/docs/en/infraops/images/infra-sysmon.png differ diff --git a/docs/en/infraops/images/logs-console.png b/docs/en/infraops/images/logs-console.png new file mode 100644 index 000000000..5feb3d960 Binary files /dev/null and b/docs/en/infraops/images/logs-console.png differ diff --git a/docs/en/infraops/monitoring-architecture.png b/docs/en/infraops/images/monitoring-architecture.png similarity index 100% rename from docs/en/infraops/monitoring-architecture.png rename to docs/en/infraops/images/monitoring-architecture.png diff --git a/docs/en/infraops/index.asciidoc b/docs/en/infraops/index.asciidoc index 44c051d1c..ac59de6ad 100644 --- a/docs/en/infraops/index.asciidoc +++ b/docs/en/infraops/index.asciidoc @@ -13,6 +13,8 @@ include::overview.asciidoc[] include::installation.asciidoc[] +include::infrastructure-metrics.asciidoc[] + include::infra-ui-intro.asciidoc[] include::logs-ui-intro.asciidoc[] diff --git a/docs/en/infraops/infra-ui-intro.asciidoc b/docs/en/infraops/infra-ui-intro.asciidoc index c58284c7d..a486820be 100644 --- a/docs/en/infraops/infra-ui-intro.asciidoc +++ b/docs/en/infraops/infra-ui-intro.asciidoc @@ -1,49 +1,10 @@ [[infrastructure-ui-overview]] [role="xpack"] -== {infra-ui} UI +== {infra-ui} app -beta[] +After you have <> and data is streaming to {es}, you can use the {infra-ui} app to monitor your infrastructure and identify problems in real time. -After you have infrastructure monitoring <> -and data is streaming to {es}, use the {infra-ui} UI in {kib} to monitor your -infrastructure and identify problems in real time. - -For more information about working with the {infra-ui} UI, see the -{kibana-ref}/xpack-infra.html[{kib} documentation]. - -[float] -=== Monitor your hosts and containers - -You start with an overview of the hosts and containers running in your -infrastructure. The overview provides a summary of high-level metrics, like CPU -usage, system load, memory usage, and network traffic, to help you assess the -overall health of your systems and services. - -You can search for specific hosts to filter the overview, or enter -{kibana-ref}/kuery-query.html[Kibana Query Language] for more sophisticated -searches. To see data about related hosts or containers, you can group by important -characteristics, such as availability zones for cloud infrastructure, or -namespaces for containers. - -When you see a potential problem, you can drill down into individual nodes to -view related metrics and logs. +For more information about the {infra-ui} app, see the {kibana-ref}/xpack-infra.html[{infra-ui} app in the {kib} documentation]. [role="screenshot"] -image::screenshot-systems-monitoring.jpg[] - -[float] -=== View detailed metrics - -After drilling down into the metrics for a specific node, you see details like -CPU usage, system load, memory usage, and network traffic over time. You can -place your cursor over a point in the timeline to see detailed metrics captured -at that moment in the timeline. - -[role="screenshot"] -image::screenshot-metrics-infrastructure-monitoring.jpg[] - -[float] -=== View related logs - -You can drill down into the logs for a specific node and explore the log data -in the <>. +image::images/infra-sysmon.png[] \ No newline at end of file diff --git a/docs/en/infraops/infrastructure-metrics.asciidoc b/docs/en/infraops/infrastructure-metrics.asciidoc new file mode 100644 index 000000000..e988d256a --- /dev/null +++ b/docs/en/infraops/infrastructure-metrics.asciidoc @@ -0,0 +1,52 @@ +[[infrastructure-metrics]] +[role="xpack"] + +== Infrastructure metrics + +// ++ More explanation needed. Beats provides these metrics automatically, but other solutions for collecting metrics or logs will need to provide these values. +The metrics listed below are provided by the {beats} shippers. +Each system type requires their corresponding identity field to be in the same event document: + +* Hosts require `host.name` +* Docker containers require `container.id` +* Kubernetes pods require `kubernetes.pod.uid` + +The `event.dataset` field is required to display data properly in some views. This field is a combination of `metricset.module`, which is the Metricbeat module name, and `metricset.name`, which is the metricset name. + +[float] +=== Host Metrics + +*CPU Usage*:: Average of `system.cpu.user.pct` added to the average of `system.cpu.system.pct` divided by `system.cpu.cores` + +*Memory Usage*:: Average of `system.memory.actual.used.pct` + +*Load*:: Average of `system.load.5` + +*Inbound Traffic*:: Derivative of the maximum of `system.network.in.bytes` scaled to a 1 second rate + +*Outbound Traffic*:: Derivative of the maximum of `system.network.out.bytes` scaled to a 1 second rate + +*Log Rate*:: Derivative of the cumulative sum of the document count scaled to a 1 second rate. +This metric relies on the same indices as the logs. + +[float] +=== Docker Container Metrics + +*CPU Usage*:: Average of `docker.cpu.total.pct` + +*Memory Usage*:: Average of `docker.memory.usage.pct` + +*Inbound Traffic*:: Derivative of the maximum of `docker.network.in.bytes` scaled to a 1 second rate + +*Outbound Traffic*:: Derivative of the maximum of `docker.network.out.bytes` scaled to a 1 second rate + +[float] +=== Kubernetes Pod Metrics + +*CPU Usage*:: Average of `kubernetes.pod.cpu.usage.node.pct` + +*Memory Usage*:: Average of `kubernetes.pod.memory.usage.node.pct` + +*Inbound Traffic*:: Derivative of the maximum of `kubernetes.pod.network.rx.bytes` scaled to a 1 second rate + +*Outbound Traffic*:: Derivative of the maximum of `kubernetes.pod.network.tx.bytes` scaled to a 1 second rate \ No newline at end of file diff --git a/docs/en/infraops/installation.asciidoc b/docs/en/infraops/installation.asciidoc index 83bd6bbc6..c3a1862a1 100644 --- a/docs/en/infraops/installation.asciidoc +++ b/docs/en/infraops/installation.asciidoc @@ -1,165 +1,108 @@ [[install-infrastructure-monitoring]] [role="xpack"] -== Get up and running - -beta[] - -To get up and running with infrastructure monitoring, you need: - -* An Elasticsearch cluster and Kibana (version 6.5 or later) with a basic -license. To learn how to get started quickly, see -{stack-gs}/get-started-elastic-stack.html[Getting started with the {stack}]. -+ -[TIP] -============== -You can skip having to install {es} and {kib} by using our -https://www.elastic.co/cloud/elasticsearch-service[hosted {es} Service] on -Elastic Cloud. The {es} Service is available on both AWS and GCP. -https://www.elastic.co/cloud/elasticsearch-service/signup[Try out the {es} -Service for free]. -============== - -* {beats} shippers (version 6.5 or later) installed on each system you want to +== Getting started with infrastructure monitoring + +To get started with infrastructure monitoring, you need: + +* An Elasticsearch cluster and Kibana (version 6.5 or later) with a basic license +// Add a link to what constitutes a basic license. And is this any different for the cloud? + +* Appropriate {beats} shippers (version 6.5 or later) installed and enabled on each system you want to monitor -You might also need to modify UI settings in {kib} to change default behaviors, -such as the index pattern used to query the data, and the timestamp field used -for sorting. For more information, see -{kibana-ref}/infrastructure-ui-settings-kb.html[{infra-ui} UI Settings] and -{kibana-ref}/logs-ui-settings-kb.html[{logs-ui} UI Settings]. +If your data uses nonstandard fields, you may also need to modify some default configuration settings. + +[float] +=== Get Elasticsearch and Kibana + +To get started, you can use our hosted {es} Service on Elastic Cloud (recommended for new users), or you can install {es} and {kib} locally. + +[float] +==== Use our hosted service + +The hosted {es} Service is available on both AWS and GCP. +https://www.elastic.co/cloud/elasticsearch-service/signup[Try out the {es} Service for free]. + +[float] +==== Install {es} and {kib} locally + +Alternatively, you can {stack-gs}/get-started-elastic-stack.html[install {es} and {kib} locally]. +Follow the instructions to install {es}, and to install and start {kib}. [float] [[install-beats-for-infra-UI]] === Install {beats} shippers -To populate the <> with metrics and -log data, you need to install and configure the following shippers: +To start collecting metrics and log data, you need to install and configure the following {beats} shippers: -* https://www.elastic.co/products/beats/metricbeat[{metricbeat}] for metrics -* https://www.elastic.co/products/beats/filebeat[{filebeat}] for log data +* {metricbeat} for metrics +* {filebeat} for log data -To learn how to install and configure the shippers, do one of the following: +You can install and configure {beats} shippers for most kinds of data directly from {kib}, or you can install {beats} yourself. -* Follow the instructions in the Add Data section of the {kib} home page. Click -*Add log data* or *Add metrics*, and follow the links for the types of data you -want to collect. -+ -[role="screenshot"] -image::add-data.png[] +[float] +==== Install {beats} from {kib} + +To install {beats} from {kib}, on the machine where you want to collect the data, open a {kib} browser window. +In the *Add Data to Kibana* section, click *Add metric data* or *Add log data*. +Now follow the instructions for the type of data you want to collect. +The instructions walk you through the steps required to download, install and configure the appropriate Beats modules for your data. -* If your data source isn't in the list, or you want to install {beats} the old -fashioned way: +[role="screenshot"] +image::images/add-data.png[] -** Follow the -{metricbeat-ref}/metricbeat-getting-started.html[{metricbeat} getting started] -and enable modules for the metrics you want to collect. +[float] +==== Install {beats} yourself -** Follow the -{filebeat-ref}/filebeat-modules-quickstart.html[{filebeat} modules quick start] -and enable modules for the logs you want to collect. If there is no module -for the logs you want to collect, see the -{filebeat-ref}/filebeat-getting-started.html[{filebeat} getting started] to -learn how to configure inputs. +If your data source doesn't have a {beats} module, or if you want to install {beats} the old fashioned way: -For either approach, you need to enable modules in {filebeat} and {metricbeat} -to populate the {infra-ui} UI with data. +** For metrics data, follow the instructions in {metricbeat-ref}/metricbeat-getting-started.html[{metricbeat} getting started] and enable modules for the metrics you want to collect. +** For logs data, follow the instructions in {filebeat-ref}/filebeat-modules-quickstart.html[{filebeat} modules quick start] and enable modules for the logs you want to collect. +If there is no module for the logs you want to collect, see the {filebeat-ref}/filebeat-getting-started.html[{filebeat} getting started] to learn how to configure inputs. [float] -==== Which modules and configuration options do I enable? +=== Enable modules +However you install {beats}, you need to enable the appropriate modules in {filebeat} and {metricbeat} to populate the Infrastructure and Logs views with data. + +// ++ I think some of this is still necessary even if you've followed the instructions to install Beats from Kibana. +// ++ The instructions there explain how to enable the module. Below, we enable more stuff. +// ++ What about if you are using Cloud? Is anything different? -To populate the *Hosts* view and add logs, enable: +To populate the *Hosts* view in the Infrastructure app and add logs, enable: * {metricbeat-ref}/metricbeat-module-system.html[{metricbeat} `system` module] (enabled by default) * {filebeat-ref}/filebeat-module-system.html[{filebeat} `system` module] -* {filebeat-ref}/filebeat-modules.html[Other {filebeat} modules] needed for -your environment, such as `apache2`, `redis`, and so on +* {filebeat-ref}/filebeat-modules.html[Other {filebeat} modules] needed for your environment, such as `apache2`, `redis`, and so on * {metricbeat-ref}/add-host-metadata.html[{metricbeat} `add_host_metadata` processor] (enabled by default) * {metricbeat-ref}/add-cloud-metadata.html[{metricbeat} `add_cloud_metadata` processor] (enabled by default) -To populate the *Docker* view and add logs, enable: +To populate the *Docker* view in the Infrastructure app and add logs, enable: * {metricbeat-ref}/metricbeat-module-docker.html[{metricbeat} `docker` module] * {metricbeat-ref}/add-docker-metadata.html[{metricbeat} `add_docker_metadata` processor] * {filebeat-ref}/filebeat-input-docker.html[{filebeat} `docker` input] * {filebeat-ref}/add-docker-metadata.html[{filebeat} `add_docker_metadata` processor] -To populate the *Kubernetes* view and add logs, enable: +To populate the *Kubernetes* view in the Infrastructure app and add logs, enable: -* {metricbeat-ref}/metricbeat-module-kubernetes.html[{metricbeat} `kubernetes` -module] +* {metricbeat-ref}/metricbeat-module-kubernetes.html[{metricbeat} `kubernetes` module] * {metricbeat-ref}/add-kubernetes-metadata.html[{metricbeat} `add_kubernetes_metadata` processor] * {filebeat-ref}/filebeat-input-docker.html[{filebeat} `docker` input] * {filebeat-ref}/add-kubernetes-metadata.html[{filebeat} `add_kubernetes_metadata` processor] [float] -==== Which fields are used for the metrics on the Infrastructure home page? - -The metrics listed below are provided by the Beats Shippers. Each system type requires their corresponding identity field to be in the same event document: - -* Hosts require `host.name` -* Docker containers require `container.id` -* Kubernetes pods require `kubernetes.pod.uid` - -For the metrics detail page, `event.dataset` is a required field. This field is a combination of `metricset.module`, which is the Metricbeat module name, and `metricset.name`, which is the sub module name. - - -[float] -===== Host Metrics - -*CPU Usage*:: Average of `system.cpu.user.pct` added to the average of `system.cpu.system.pct` divided by `system.cpu.cores` - -*Memory Usage*:: Average of `system.memory.actual.used.pct` - -*Load*:: Average of `system.load.5` - -*Inbound Traffic*:: Derivative of the max of `system.netowrk.in.bytes` scaled to a 1 second rate - -*Outbound Traffic*:: Derivative of the max of `system.netowrk.out.bytes` scaled to a 1 second rate - -*Log Rate*:: Derivative of the cumulative sum of the document count scaled to a 1 second rate. -This metric relies on the same indices as the logs. - - -[float] -===== Docker Container Metrics - -*CPU Usage*:: Average of `docker.cpu.total.pct` - -*Memory Usage*:: Average of `docker.memory.usage.pct` - -*Inbound Traffic*:: Derivative of the max of `docker.network.in.bytes` scaled to a 1 second rate - -*Outbound Traffic*:: Derivative of the max of `docker.network.out.bytes` scaled to a 1 second rate - +=== Configure your data sources +// ++ This should probably link directly to the Kibana Source config tab (to be), rather than the detailed list of settings as it currently does? +If your metrics data or logs data has non-standard fields, you may need to modify some configuration settings in {kib} to change the default behaviors, such as the index pattern used to query the data, and the timestamp field used for sorting. +For more information, see {kibana-ref}/infrastructure-ui-settings-kb.html[{infra-ui} UI Settings] and {kibana-ref}/logs-ui-settings-kb.html[{logs-ui} UI Settings]. [float] -===== Kubernetes Pod Metrics +=== More about container monitoring -*CPU Usage*:: Average of `kubernetes.pod.cpu.usage.node.pct` - -*Memory Usage*:: Average of `kubernetes.pod.memory.usage.node.pct` - -*Inbound Traffic*:: Derivative of the max of `kubernetes.pod.network.rx.bytes` scaled to a 1 second rate - -*Outbound Traffic*:: Derivative of the max of `kubernetes.pod.network.tx.bytes` scaled to a 1 second rate - - - -[float] -==== More about container monitoring - -If you're monitoring containers, you can use autodiscover to automatically apply -configuration changes in response to changes in your containers. To learn how, -see: - -* {filebeat-ref}/configuration-autodiscover.html[{filebeat} autodiscover -configuration] -* {metricbeat-ref}/configuration-autodiscover.html[{metricbeat} autodiscover -configuration] - - -[float] -==== Known Workarounds +If you're monitoring Docker containers or Kubernetes pods, you can use autodiscover to automatically change the configuration settings in response to changes in your containers. +This ensures you don't stop collecting data when your container configuration changes. +To learn how to do this, see: -*Running Metricbeat <=6.5 with Kibana 6.6+*:: For Kubernetes, you will need to change the "Pod ID" field in the "Configure Source" panel (`xpack.infra.sources.default.fields.pod` in `config/kibana.yml` for Kibana 6.6) to `kubernetes.pod.name`. There is a caveat for this workaround; if you have two pods with the same name, only one will be visible in the UI. +* {filebeat-ref}/configuration-autodiscover.html[{filebeat} autodiscover configuration] +* {metricbeat-ref}/configuration-autodiscover.html[{metricbeat} autodiscover configuration] \ No newline at end of file diff --git a/docs/en/infraops/logs-ui-intro.asciidoc b/docs/en/infraops/logs-ui-intro.asciidoc index e0032768f..46f6dc117 100644 --- a/docs/en/infraops/logs-ui-intro.asciidoc +++ b/docs/en/infraops/logs-ui-intro.asciidoc @@ -1,20 +1,16 @@ [[logs-ui-overview]] [role="xpack"] -== {logs-ui} UI +== {logs-ui} app -beta[] +After you have set up logs streaming as part of <>, you can view real-time and historical logs in a compact, customizable display. +The log data is correlated with metric data in the <>, making it easier for you to diagnose problems. -The {logs-ui} UI provides real-time log tailing in a compact, customizable -display. The log data is correlated with metrics in the -<>, making it easier for you to diagnose -problems. You can stream the logs in real time, or load the new data manually. -Logs are streamed from top to bottom, making it easier for you to explore the -logs in real time and see a historical view of your log data. The search bar in -the log viewer supports {kibana-ref}/kuery-query.html[Kibana Query Language]. +You can stream the logs in real time, or view historical logs from a specified time range. + +The search bar in the log viewer supports {kibana-ref}/kuery-query.html[Kibana Query Language]. You can enter ad hoc or structured queries. -For more information about working with the {logs-ui} UI, see the -{kibana-ref}/xpack-logs.html[{kib} documentation]. +For more information about using the {logs-ui} app, see the {kibana-ref}/xpack-logs.html[{logs-ui} app in the {kib} documentation]. [role="screenshot"] -image::screenshot-logs-ui.jpg[] +image::images/logs-console.png[] diff --git a/docs/en/infraops/overview.asciidoc b/docs/en/infraops/overview.asciidoc index ee6113c76..493cfe0ba 100644 --- a/docs/en/infraops/overview.asciidoc +++ b/docs/en/infraops/overview.asciidoc @@ -2,41 +2,39 @@ [role="xpack"] == Overview -beta[] - You can use the infrastructure monitoring capabilities of the {stack} to monitor your infrastructure in real time and check the health of your servers, containers, and services. -The <> in {kib} brings together data +The Infrastructure app in {kib} brings together data from a variety of sources, including logs and metrics, making it easier for you to identify and resolve infrastructure issues quickly. [float] === Infrastructure monitoring components +image::images/monitoring-architecture.png[] + Infrastructure monitoring requires the following {stack} components. -image::monitoring-architecture.png[] +*https://www.elastic.co/products/elasticsearch[{es}]* is a real-time, +distributed storage, search, and analytics engine. {es} excels at indexing +streams of semi-structured data, such as logs or metrics. *https://www.elastic.co/products/beats[{beats}]* are open source data shippers that you install as agents on your servers to send operational data to {es}. -*https://www.elastic.co/products/elasticsearch[{es}]* is a real-time, -distributed storage, search, and analytics engine. {es} excels at indexing -streams of semi-structured data, such as logs or metrics. - *https://www.elastic.co/products/kibana[{kib}]* is an open source analytics and -visualization platform designed to work with {es}. You use {kib} to search, -view, and interact with data stored in {es} indices. You can easily perform -advanced data analysis and visualize your data in a variety of charts, tables, +visualization platform designed to work with {es}. +You use {kib} to search, view, and interact with data stored in {es}. +You can easily perform advanced data analysis and visualize your data in a variety of charts, tables, and maps. -<> provides a +The <> in {kib} provides a dedicated user interface for visualizing the health of your servers, -containers, and services. +containers, and services in your infrastructure. -<> provides a compact, -customizable display for real-time log tailing. +The <> in {kib} provides a compact, +customizable display for viewing and analyzing real-time and historic log data. diff --git a/docs/en/infraops/screenshot-logs-ui.jpg b/docs/en/infraops/screenshot-logs-ui.jpg deleted file mode 100644 index 567c2ee81..000000000 Binary files a/docs/en/infraops/screenshot-logs-ui.jpg and /dev/null differ diff --git a/docs/en/infraops/screenshot-metrics-infrastructure-monitoring.jpg b/docs/en/infraops/screenshot-metrics-infrastructure-monitoring.jpg deleted file mode 100755 index fc35af10e..000000000 Binary files a/docs/en/infraops/screenshot-metrics-infrastructure-monitoring.jpg and /dev/null differ diff --git a/docs/en/infraops/screenshot-systems-monitoring.jpg b/docs/en/infraops/screenshot-systems-monitoring.jpg deleted file mode 100755 index 36afcb820..000000000 Binary files a/docs/en/infraops/screenshot-systems-monitoring.jpg and /dev/null differ