Introduce event.duration and event.dataset for backward compatiblity (#…

…9393)

The field event.duration is introduced as a replacement for metricset.rtt. event.duration is in nano seconds, metricset.rtt in nano seconds so the two are not compatible. `metricset.rtt` will be removed in 7.0.

`event.dataset` is introduced as `{module}.{metricset}` for better backward compatiblity in 7.0.

Further changes

* Fix tests

CHANGELOG.asciidoc

@@ -114,6 +114,7 @@ https://github.com/elastic/beats/compare/v6.5.0...6.x[Check the HEAD diff]
 - Add setting to disable docker cpu metrics per core. {pull}9194[9194]
 - The `elasticsearch/node` metricset now reports the Elasticsearch cluster UUID. {pull}8771[8771]
 - Add `host.os.name` field to add_host_metadata processor. {issue}8948[8948] {pull}9405[9405]
+- Add field `event.dataset` which is `{module}.{metricset).
 
 *Packetbeat*
 
@@ -133,6 +134,8 @@ https://github.com/elastic/beats/compare/v6.5.0...6.x[Check the HEAD diff]
 
 *Metricbeat*
 
+- Deprecate field `metricset.rtt`. Replaced by `event.duration` which is in nano instead of micro seconds.
+
 *Packetbeat*
 
 - Support new TLS version negotiation introduced in TLS 1.3. {issue}8647[8647].

metricbeat/_meta/fields.common.yml

@@ -22,6 +22,7 @@
       required: true
       description: >
         Event round trip time in microseconds.
+      deprecated: true
 
     - name: metricset.namespace
       type: keyword
@@ -38,3 +39,13 @@
       example: elasticsearch
       description: >
         Name of the service metricbeat fetches the data from.
+
+    - name: event.duration
+      type: long
+      description: >
+        Duration of the event in nano seconds.
+
+    - name: event.dataset
+      type: keyword
+      description: >
+        Event dataset name.

metricbeat/docs/fields.asciidoc

@@ -1838,6 +1838,9 @@ Hostname of the machine from which the metricset was collected. This field may n
 *`metricset.rtt`*::
 +
 --
+
+deprecated[True]
+
 type: long
 
 required: True
@@ -1877,6 +1880,26 @@ example: elasticsearch
 Name of the service metricbeat fetches the data from.
 
 
+--
+
+*`event.duration`*::
++
+--
+type: long
+
+Duration of the event in nano seconds.
+
+
+--
+
+*`event.dataset`*::
++
+--
+type: keyword
+
+Event dataset name.
+
+
 --
 
 [[exported-fields-couchbase]]

metricbeat/mb/event.go

@@ -18,6 +18,7 @@
 package mb
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/elastic/beats/libbeat/beat"
@@ -116,14 +117,21 @@ func AddMetricSetInfo(module, metricset string, event *Event) {
 	if event.Host != "" {
 		info["host"] = event.Host
 	}
-	if event.Took > 0 {
-		info["rtt"] = event.Took / time.Microsecond
-	}
+
 	if event.Namespace != "" {
 		info["namespace"] = event.Namespace
 	}
 	info = common.MapStr{
 		"metricset": info,
+		"event": common.MapStr{
+			"dataset": fmt.Sprintf("%s.%s", module, metricset),
+		},
+	}
+
+	if event.Took > 0 {
+		// rtt is deprecated and will be removed in 7.0. Replaced by event.duration.
+		info.Put("metricset.rtt", event.Took/time.Microsecond)
+		info.Put("event.duration", event.Took/time.Nanosecond)
 	}
 
 	if event.RootFields == nil {

metricbeat/mb/event_test.go

@@ -180,6 +180,10 @@ func TestAddMetricSetInfo(t *testing.T) {
 				"name":   metricSetName,
 				"rtt":    time.Duration(500000),
 			},
+			"event": common.MapStr{
+				"duration": time.Duration(500000000),
+				"dataset":  moduleName + "." + metricSetName,
+			},
 		}, e.RootFields)
 	})
 
@@ -193,6 +197,9 @@ func TestAddMetricSetInfo(t *testing.T) {
 				"module": moduleName,
 				"name":   metricSetName,
 			},
+			"event": common.MapStr{
+				"dataset": moduleName + "." + metricSetName,
+			},
 		}, e.RootFields)
 	})
 }

metricbeat/mb/module/example_test.go

@@ -65,6 +65,7 @@ func ExampleWrapper() {
 		defer wg.Done()
 		for event := range output {
 			event.Fields.Put("metricset.rtt", 111)
+			event.Fields.Put("event.duration", 111000)
 
 			output, err := encodeEvent(event)
 			if err == nil {
@@ -90,6 +91,10 @@ func ExampleWrapper() {
 	//     "version": "1.2.3"
 	//   },
 	//   "@timestamp": "2016-05-10T23:27:58.485Z",
+	//   "event": {
+	//     "dataset": "fake.eventfetcher",
+	//     "duration": 111000
+	//   },
 	//   "fake": {
 	//     "eventfetcher": {
 	//       "metric": 1

metricbeat/tests/system/metricbeat.py

@@ -7,7 +7,7 @@
 from beat.beat import TestCase
 
 COMMON_FIELDS = ["@timestamp", "beat", "metricset.name", "metricset.host",
-                 "metricset.module", "metricset.rtt", "host.name"]
+                 "metricset.module", "metricset.rtt", "host.name", "event"]
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 

metricbeat/tests/system/test_processors.py

@@ -35,7 +35,7 @@ def test_drop_fields(self):
         print(evt.keys())
         self.assertItemsEqual(self.de_dot([
             'beat', '@timestamp', 'system', 'metricset.module',
-            'metricset.rtt', 'metricset.name', 'host'
+            'metricset.rtt', 'metricset.name', 'host', 'event'
         ]), evt.keys())
         cpu = evt["system"]["cpu"]
         print(cpu.keys())

metricbeat/tests/system/test_redis.py

@@ -92,7 +92,7 @@ def test_module_processors(self):
         Test local processors for Redis info event.
         """
         fields = ["clients", "cpu"]
-        eventFields = ['beat', 'metricset']
+        eventFields = ['beat', 'metricset', 'event']
         eventFields += ['redis.info.' + f for f in fields]
         self.render_config_template(modules=[{
             "name": "redis",