allow/merge fields.yml overrides

libbeat/generator/fields/fields.go

@@ -32,12 +32,7 @@ type YmlFile struct {
 }
 
 func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
-	commonFields := []string{
-		// Fields for custom beats
-		filepath.Join(beatPath, "_meta/fields.yml"),
-		filepath.Join(beatPath, "_meta/fields.common.yml"),
-	}
-
+	var commonFields []string
 	var libbeatFieldFiles []*YmlFile
 	var err error
 	if !isLibbeat(beatPath) {
@@ -53,6 +48,12 @@ func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]
 		}
 	}
 
+	// Fields for custom beats last, to enable overriding more generically defined fields
+	commonFields = append(commonFields,
+		filepath.Join(beatPath, "_meta/fields.common.yml"),
+		filepath.Join(beatPath, "_meta/fields.yml"),
+	)
+
 	var files []*YmlFile
 	for _, cf := range commonFields {
 		_, err := os.Stat(cf)

libbeat/kibana/fields_transformer.go

@@ -29,7 +29,7 @@ type fieldsTransformer struct {
 	transformedFields         []common.MapStr
 	transformedFieldFormatMap common.MapStr
 	version                   *common.Version
-	keys                      common.MapStr
+	keys                      map[string]int
 }
 
 func newFieldsTransformer(version *common.Version, fields common.Fields) (*fieldsTransformer, error) {
@@ -41,7 +41,7 @@ func newFieldsTransformer(version *common.Version, fields common.Fields) (*field
 		version:                   version,
 		transformedFields:         []common.MapStr{},
 		transformedFieldFormatMap: common.MapStr{},
-		keys: common.MapStr{},
+		keys: map[string]int{},
 	}, nil
 }
 
@@ -79,17 +79,11 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
 			f.Path = path + "." + f.Name
 		}
 
-		if t.keys[f.Path] != nil {
-			msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.\n", f.Path)
-			panic(errors.New(msg))
-		}
-
 		if f.Type == "group" {
 			if f.Enabled == nil || *f.Enabled {
 				t.transformFields(f.Fields, f.Path)
 			}
 		} else {
-			t.keys[f.Path] = true
 			t.add(f)
 
 			if f.MultiFields != nil {
@@ -104,13 +98,30 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
 	}
 }
 
+func (t *fieldsTransformer) update(target *common.MapStr, override common.Field) error {
+	field, _ := transformField(t.version, override)
+	if override.Type == "" || (*target)["type"] == field["type"] {
+		target.Update(field)
+		return nil
+	}
+	return fmt.Errorf("ERROR: Field <%s> is duplicated. Please update and try again.\n", override.Path)
+}
+
 func (t *fieldsTransformer) add(f common.Field) {
+	if idx := t.keys[f.Path]; idx > 0 {
+		target := &t.transformedFields[t.keys[f.Path]-1] // 1-indexed
+		if err := t.update(target, f); err != nil {
+			panic(err)
+		}
+		return
+	}
+
 	field, fieldFormat := transformField(t.version, f)
 	t.transformedFields = append(t.transformedFields, field)
+	t.keys[f.Path] = len(t.transformedFields) // 1-index
 	if fieldFormat != nil {
 		t.transformedFieldFormatMap[field["name"].(string)] = fieldFormat
 	}
-
 }
 
 func transformField(version *common.Version, f common.Field) (common.MapStr, common.MapStr) {

libbeat/kibana/fields_transformer_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/libbeat/common"
 )
@@ -99,14 +100,59 @@ func TestMissingVersion(t *testing.T) {
 func TestDuplicateField(t *testing.T) {
 	commonFields := common.Fields{
 		common.Field{Name: "context", Path: "something"},
-		common.Field{Name: "context", Path: "something", Type: "keyword"},
+		common.Field{Name: "context", Path: "something", Type: "date"},
 	}
 	trans, err := newFieldsTransformer(version, commonFields)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	_, err = trans.transform()
 	assert.Error(t, err)
 }
 
+func TestValidDuplicateField(t *testing.T) {
+	commonFields := common.Fields{
+		common.Field{Name: "context", Path: "something", Type: "keyword", Description: "original description"},
+		common.Field{Name: "context", Path: "something", Description: "updated description",
+			Aggregatable: &falsy,
+			Analyzed:     &truthy,
+			Count:        2,
+			DocValues:    &falsy,
+			Index:        &falsy,
+			Searchable:   &falsy,
+		},
+		common.Field{
+			Name: "context",
+			Type: "group",
+			Fields: common.Fields{
+				common.Field{Name: "another", Type: "date"},
+			},
+		},
+		common.Field{
+			Name: "context",
+			Type: "group",
+			Fields: common.Fields{
+				common.Field{Name: "another"},
+			},
+		},
+	}
+	trans, err := newFieldsTransformer(version, commonFields)
+	require.NoError(t, err)
+	transformed, err := trans.transform()
+	t.Log(transformed)
+	require.NoError(t, err)
+	out := transformed["fields"].([]common.MapStr)[0]
+	assert.Equal(t, out, common.MapStr{
+		"aggregatable": false,
+		"analyzed":     true,
+		"count":        2,
+		"doc_values":   false,
+		"indexed":      false,
+		"name":         "context",
+		"scripted":     false,
+		"searchable":   false,
+		"type":         "string",
+	})
+}
+
 func TestInvalidVersion(t *testing.T) {
 	commonFields := common.Fields{
 		common.Field{

libbeat/scripts/generate_fields_docs.py

@@ -1,6 +1,8 @@
-import yaml
-import os
 import argparse
+from collections import OrderedDict
+import os
+
+import yaml
 
 
 def document_fields(output, section, sections, path):
@@ -102,6 +104,21 @@ def fields_to_asciidoc(input, output, beat):
         print("fields.yml file is empty. fields.asciidoc cannot be generated.")
         return
 
+    # deduplicate fields, last one wins
+    for section in docs:
+        if not section.get("fields"):
+            continue
+        fields = OrderedDict()
+        for field in section["fields"]:
+            name = field["name"]
+            if name in fields:
+                assert field["type"] == fields[name]["type"], 'field "{}" redefined with different type "{}"'.format(
+                    name, field["type"])
+                fields[name].update(field)
+            else:
+                fields[name] = field
+        section["fields"] = list(fields.values())
+
     # Create sections from available fields
     sections = {}
     for v in docs: