Merge pull request elastic#1416 from ruflin/mb-topbeat

Add System module with CPU and Memory stats

metricbeat/TODO.md

@@ -22,6 +22,9 @@ For most metricset, setup creates the connections to the remote hosts. One poten
 # Topbeat
 Topbeat should be added to metricbeat (see https://github.com/elastic/beats/pull/1081). To make the two better integrated some refactoring on the Topbeat side is needed so MapStr can be consumed directly.
 
+# Collection of fields
+Currently the fields.yml is combined from all fields.yml fields in the metricsets. This leads to the problem that if a module has two metricsets, the module is defined twice in the global fields.yml file. The module part should be moved to a fields.yml in the module.
+
 # More
 * Add service host as default event informartion. See https://github.com/elastic/beats/issues/619#issuecomment-185242407
 * Add version number of service. See https://github.com/elastic/beats/issues/619#issuecomment-185242407

metricbeat/docs/fields.asciidoc

@@ -13,6 +13,7 @@ grouped in the following categories:
 * <<exported-fields-apache>>
 * <<exported-fields-redis>>
 * <<exported-fields-mysql>>
+* <<exported-fields-system>>
 
 [[exported-fields-common]]
 === Common Fields
@@ -44,6 +45,16 @@ required: True
 The timestamp when the log line was read. The precision is in milliseconds. The timezone is UTC.
 
 
+==== module
+
+The name of the module that generated the event.
+
+
+==== metricset
+
+The name of the metricset that generated the event.
+
+
 ==== metricset-host
 
 Hostname of the machine from which the metricset was collected. This field may not be present when the data was collected locally.
@@ -58,6 +69,15 @@ required: True
 Event round trip time in microseconds
 
 
+==== type
+
+example: metricsets
+
+required: True
+
+The document type. Always set to "metricsets".
+
+
 [[exported-fields-apache]]
 === Apache Status Fields
 
@@ -486,3 +506,105 @@ type: integer
 The number of bytes sent to all clients.
 
 
+[[exported-fields-system]]
+=== System Status Fields
+
+System status metrics, like CPU and memory usage, that are collected from the operating system.
+
+
+
+=== system-memory Fields
+
+`system-memory` contains local memory stats.
+
+
+
+[float]
+=== mem Fields
+
+This group contains statistics related to the memory usage on the system.
+
+
+==== system-memory.mem.total
+
+type: int
+
+Total memory.
+
+
+==== system-memory.mem.used
+
+type: int
+
+Used memory.
+
+
+==== system-memory.mem.free
+
+type: int
+
+Available memory.
+
+
+==== system-memory.mem.used_p
+
+type: float
+
+The percentage of used memory.
+
+
+==== system-memory.mem.actual_used
+
+type: int
+
+Actual used memory. This value is the "used" memory minus the memory used for disk caches and buffers. Available only on Unix.
+
+
+==== system-memory.mem.actual_free
+
+type: int
+
+Actual available memory. This value is the "free" memory plus the memory used for disk caches and buffers. Available only on Unix.
+
+
+==== system-memory.mem.actual_used_p
+
+type: float
+
+The percentage of actual used memory.
+
+
+[float]
+=== swap Fields
+
+This group contains statistics related to the swap memory usage on the system.
+
+
+==== system-memory.swap.total
+
+type: int
+
+Total swap memory.
+
+
+==== system-memory.swap.used
+
+type: int
+
+Used swap memory.
+
+
+==== system-memory.swap.free
+
+type: int
+
+Available swap memory.
+
+
+==== system-memory.swap.used_p
+
+type: float
+
+The percentage of used swap memory.
+
+

metricbeat/etc/beat.yml

@@ -69,3 +69,9 @@ metricbeat:
 
       # Password of hosts. Empty by default
       #password: test
+
+    # System module
+    - module: system
+      metricsets: ["cpu", "memory"]
+      enabled: true
+      period: 2s

metricbeat/etc/fields.yml

@@ -12,6 +12,7 @@ sections:
   - ["apache", "Apache Status"]
   - ["redis", "Redis Info"]
   - ["mysql", "MySQL Status"]
+  - ["system", "System Status"]
 
 common:
   type: group
@@ -344,3 +345,142 @@ redis:
               type: float
               description: >
                 User CPU consumed by the background processes
+system:
+  type: group
+  description: >
+    System status metrics, like CPU and memory usage, that are collected from the operating system.
+  fields:
+    - name: system-cpu
+      type: group
+      description: >
+        `system-cpu` contains local cpu stats.
+      fields:
+        - name: user
+          type: int
+          description: >
+           The amount of CPU time spent in user space.
+
+        - name: user_p
+          type: float
+          description: >
+            The percentage of CPU time spent in user space. On multi-core systems, you can have percentages that are greater than 100%.
+            For example, if 3 cores are at 60% use, then the `cpu.user_p` will be 180%.
+
+        - name: nice
+          type: int
+          description: >
+            The amount of CPU time spent on low-priority processes.
+
+        - name: system
+          type: int
+          description: >
+            The amount of CPU time spent in kernel space.
+
+        - name: system_p
+          type: float
+          description: >
+            The percentage of CPU time spent in kernel space.
+
+        - name: idle
+          type: int
+          description: >
+            The amount of CPU time spent idle.
+
+        - name: iowait
+          type: int
+          description: >
+            The amount of CPU time spent in wait (on disk).
+
+        - name: irq
+          type: int
+          description: >
+            The amount of CPU time spent servicing and handling hardware interrupts.
+
+        - name: softirq
+          type: int
+          description:
+            The amount of CPU time spent servicing and handling software interrupts.
+
+        - name: steal
+          type: int
+          description: >
+            The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor
+            was servicing another processor.
+            Available only on Unix.
+
+system:
+  type: group
+  description: >
+    System status metrics, like CPU and memory usage, that are collected from the operating system.
+  fields:
+    - name: system-memory
+      type: group
+      description: >
+        `system-memory` contains local memory stats.
+      fields:
+        - name: mem
+          type: group
+          prefix: "[float]"
+          description: This group contains statistics related to the memory usage on the system.
+          fields:
+            - name: total
+              type: int
+              description: >
+                Total memory.
+
+            - name: used
+              type: int
+              description: >
+                Used memory.
+
+            - name: free
+              type: int
+              description: >
+                Available memory.
+
+            - name: used_p
+              type: float
+              description: >
+                The percentage of used memory.
+
+            - name: actual_used
+              type: int
+              description: >
+                Actual used memory. This value is the "used" memory minus the memory used for disk caches and buffers.
+                Available only on Unix.
+
+            - name: actual_free
+              type: int
+              description: >
+                Actual available memory. This value is the "free" memory plus the memory used for disk caches and
+                buffers. Available only on Unix.
+
+            - name: actual_used_p
+              type: float
+              description: >
+                The percentage of actual used memory.
+
+        - name: swap
+          type: group
+          prefix: "[float]"
+          description: This group contains statistics related to the swap memory usage on the system.
+          fields:
+            - name: total
+              type: int
+              description: >
+                Total swap memory.
+
+            - name: used
+              type: int
+              description: >
+                Used swap memory.
+
+            - name: free
+              type: int
+              description: >
+                Available swap memory.
+
+            - name: used_p
+              type: float
+              description: >
+                The percentage of used swap memory.

metricbeat/etc/fields_base.yml

@@ -12,6 +12,7 @@ sections:
   - ["apache", "Apache Status"]
   - ["redis", "Redis Info"]
   - ["mysql", "MySQL Status"]
+  - ["system", "System Status"]
 
 common:
   type: group

metricbeat/include/list.go

@@ -15,4 +15,9 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/mysql/status"
 	_ "github.com/elastic/beats/metricbeat/module/redis"
 	_ "github.com/elastic/beats/metricbeat/module/redis/info"
+
+	// System module and metricsets
+	_ "github.com/elastic/beats/metricbeat/module/system"
+	_ "github.com/elastic/beats/metricbeat/module/system/cpu"
+	_ "github.com/elastic/beats/metricbeat/module/system/memory"
 )

metricbeat/metricbeat.template.json

@@ -206,6 +206,27 @@
         },
         "rtt": {
           "type": "long"
+        },
+        "system-memory": {
+          "properties": {
+            "mem": {
+              "properties": {
+                "actual_used_p": {
+                  "type": "float"
+                },
+                "used_p": {
+                  "type": "float"
+                }
+              }
+            },
+            "swap": {
+              "properties": {
+                "used_p": {
+                  "type": "float"
+                }
+              }
+            }
+          }
         }
       }
     }

metricbeat/metricbeat.yml

@@ -69,6 +69,12 @@ metricbeat:
 
       # Password of hosts. Empty by default
       #password: test
+
+    # System module
+    - module: system
+      metricsets: ["cpu", "memory"]
+      enabled: true
+      period: 2s
 ###############################################################################
 ############################# Libbeat Config ##################################
 # Base config file used by all other beats for using libbeat features