allow/merge fields.yml overrides

dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go

@@ -79,7 +79,7 @@ func main() {
 
 		file, err := indexPattern.Generate()
 		if err != nil {
-			log.Fatal(err)
+			log.Fatalf("ERROR: %s", err)
 		}
 
 		// Log output file location.

libbeat/common/field.go

@@ -69,7 +69,8 @@ type Field struct {
 	UrlTemplate          []VersionizedString `config:"url_template"`
 	OpenLinkInCurrentTab *bool               `config:"open_link_in_current_tab"`
 
-	Path string
+	Overwrite bool `config:"overwrite"`
+	Path      string
 }
 
 type VersionizedString struct {

libbeat/generator/fields/fields.go

@@ -32,12 +32,7 @@ type YmlFile struct {
 }
 
 func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
-	commonFields := []string{
-		// Fields for custom beats
-		filepath.Join(beatPath, "_meta/fields.yml"),
-		filepath.Join(beatPath, "_meta/fields.common.yml"),
-	}
-
+	var commonFields []string
 	var libbeatFieldFiles []*YmlFile
 	var err error
 	if !isLibbeat(beatPath) {
@@ -53,6 +48,12 @@ func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]
 		}
 	}
 
+	// Fields for custom beats last, to enable overriding more generically defined fields
+	commonFields = append(commonFields,
+		filepath.Join(beatPath, "_meta/fields.common.yml"),
+		filepath.Join(beatPath, "_meta/fields.yml"),
+	)
+
 	var files []*YmlFile
 	for _, cf := range commonFields {
 		_, err := os.Stat(cf)

libbeat/kibana/fields_transformer.go

@@ -29,7 +29,7 @@ type fieldsTransformer struct {
 	transformedFields         []common.MapStr
 	transformedFieldFormatMap common.MapStr
 	version                   *common.Version
-	keys                      common.MapStr
+	keys                      map[string]int
 }
 
 func newFieldsTransformer(version *common.Version, fields common.Fields) (*fieldsTransformer, error) {
@@ -41,7 +41,7 @@ func newFieldsTransformer(version *common.Version, fields common.Fields) (*field
 		version:                   version,
 		transformedFields:         []common.MapStr{},
 		transformedFieldFormatMap: common.MapStr{},
-		keys:                      common.MapStr{},
+		keys:                      map[string]int{},
 	}, nil
 }
 
@@ -79,17 +79,11 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
 			f.Path = path + "." + f.Name
 		}
 
-		if t.keys[f.Path] != nil {
-			msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.\n", f.Path)
-			panic(errors.New(msg))
-		}
-
 		if f.Type == "group" {
 			if f.Enabled == nil || *f.Enabled {
 				t.transformFields(f.Fields, f.Path)
 			}
 		} else {
-			t.keys[f.Path] = true
 			t.add(f)
 
 			if f.MultiFields != nil {
@@ -104,13 +98,35 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
 	}
 }
 
+func (t *fieldsTransformer) update(target *common.MapStr, override common.Field) error {
+	field, _ := transformField(t.version, override)
+	if override.Type == "" || (*target)["type"] == field["type"] {
+		target.Update(field)
+		if !override.Overwrite {
+			// compatible duplication
+			return fmt.Errorf("field <%s> is duplicated, remove it or set 'overwrite: true'", override.Path)
+		}
+		return nil
+	}
+	// incompatible duplication
+	return fmt.Errorf("field <%s> is duplicated", override.Path)
+}
+
 func (t *fieldsTransformer) add(f common.Field) {
+	if idx := t.keys[f.Path]; idx > 0 {
+		target := &t.transformedFields[idx-1] // 1-indexed
+		if err := t.update(target, f); err != nil {
+			panic(err)
+		}
+		return
+	}
+
 	field, fieldFormat := transformField(t.version, f)
 	t.transformedFields = append(t.transformedFields, field)
+	t.keys[f.Path] = len(t.transformedFields) // 1-index
 	if fieldFormat != nil {
 		t.transformedFieldFormatMap[field["name"].(string)] = fieldFormat
 	}
-
 }
 
 func transformField(version *common.Version, f common.Field) (common.MapStr, common.MapStr) {

libbeat/kibana/fields_transformer_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/libbeat/common"
 )
@@ -97,14 +98,75 @@ func TestMissingVersion(t *testing.T) {
 }
 
 func TestDuplicateField(t *testing.T) {
+	testCases := []struct {
+		commonFields []common.Field
+	}{
+		// type change
+		{commonFields: []common.Field{
+			{Name: "context", Path: "something"},
+			{Name: "context", Path: "something", Type: "date"},
+		}},
+		// missing overwrite
+		{commonFields: []common.Field{
+			{Name: "context", Path: "something"},
+			{Name: "context", Path: "something"},
+		}},
+		// missing overwrite in source
+		{commonFields: []common.Field{
+			{Name: "context", Path: "something", Overwrite: true},
+			{Name: "context", Path: "something"},
+		}},
+	}
+	for _, testCase := range testCases {
+		trans, err := newFieldsTransformer(version, testCase.commonFields)
+		require.NoError(t, err)
+		_, err = trans.transform()
+		assert.Error(t, err)
+	}
+}
+
+func TestValidDuplicateField(t *testing.T) {
 	commonFields := common.Fields{
-		common.Field{Name: "context", Path: "something"},
-		common.Field{Name: "context", Path: "something", Type: "keyword"},
+		common.Field{Name: "context", Path: "something", Type: "keyword", Description: "original description"},
+		common.Field{Name: "context", Path: "something", Overwrite: true, Description: "updated description",
+			Aggregatable: &falsy,
+			Analyzed:     &truthy,
+			Count:        2,
+			DocValues:    &falsy,
+			Index:        &falsy,
+			Searchable:   &falsy,
+		},
+		common.Field{
+			Name: "context",
+			Type: "group",
+			Fields: common.Fields{
+				common.Field{Name: "another", Type: "date"},
+			},
+		},
+		common.Field{
+			Name: "context",
+			Type: "group",
+			Fields: common.Fields{
+				common.Field{Name: "another", Overwrite: true},
+			},
+		},
 	}
 	trans, err := newFieldsTransformer(version, commonFields)
-	assert.NoError(t, err)
-	_, err = trans.transform()
-	assert.Error(t, err)
+	require.NoError(t, err)
+	transformed, err := trans.transform()
+	require.NoError(t, err)
+	out := transformed["fields"].([]common.MapStr)[0]
+	assert.Equal(t, out, common.MapStr{
+		"aggregatable": false,
+		"analyzed":     true,
+		"count":        2,
+		"doc_values":   false,
+		"indexed":      false,
+		"name":         "context",
+		"scripted":     false,
+		"searchable":   false,
+		"type":         "string",
+	})
 }
 
 func TestInvalidVersion(t *testing.T) {

libbeat/kibana/try1.patch

@@ -0,0 +1,101 @@
+diff --git a/libbeat/kibana/fields_transformer.go b/libbeat/kibana/fields_transformer.go
+index 8f4f7005a..72827eb64 100644
+--- a/libbeat/kibana/fields_transformer.go
++++ b/libbeat/kibana/fields_transformer.go
+@@ -20,59 +20,37 @@ package kibana
+ import (
+ 	"errors"
+ 	"fmt"
++	"github.com/elastic/beats/libbeat/logp"
+
+ 	"github.com/elastic/beats/libbeat/common"
+ )
+
+-type fieldsTransformer struct {
+-	fields                    common.Fields
+-	transformedFields         []common.MapStr
+-	transformedFieldFormatMap common.MapStr
+-	version                   *common.Version
+-	keys                      common.MapStr
+-}
++var (
++	truthy = true
++	falsy = false
++)
+
+-func newFieldsTransformer(version *common.Version, fields common.Fields) (*fieldsTransformer, error) {
++func transform(version *common.Version, fields common.Fields) (common.MapStr, error) {
+ 	if version == nil {
+-		return nil, errors.New("Version must be given")
+-	}
+-	return &fieldsTransformer{
+-		fields:                    fields,
+-		version:                   version,
+-		transformedFields:         []common.MapStr{},
+-		transformedFieldFormatMap: common.MapStr{},
+-		keys: common.MapStr{},
+-	}, nil
+-}
+-
+-func (t *fieldsTransformer) transform() (transformed common.MapStr, err error) {
+-	defer func() {
+-		if r := recover(); r != nil {
+-			var ok bool
+-			if err, ok = r.(error); !ok {
+-				err = fmt.Errorf("Unrecoverable Error %v", r)
+-			}
+-		}
+-	}()
+-
+-	t.transformFields(t.fields, "")
++		return nil, errors.New("version is required")
++	}
++	transformedFields := map[string]common.Field{
++		// some meta fields
++		"_id":    {Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy},
++		"_type":  {Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy},
++		"_index": {Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy},
++		"_score": {Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy},
++	}
+
+-	// add some meta fields
+-	truthy := true
+-	falsy := false
+-	t.add(common.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy})
+-	t.add(common.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy})
+-	t.add(common.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy})
+-	t.add(common.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy})
++	var transformedFieldFormatMap common.MapStr
+
+-	transformed = common.MapStr{
+-		"fields":         t.transformedFields,
+-		"fieldFormatMap": t.transformedFieldFormatMap,
+-	}
+-	return
++	return common.MapStr{
++		"fields":         transformedFields,
++		"fieldFormatMap": transformedFieldFormatMap,
++	}, nil
+ }
+
+-func (t *fieldsTransformer) transformFields(commonFields common.Fields, path string) {
++func transformFields(commonFields common.Fields, path string) {
+ 	for _, f := range commonFields {
+ 		f.Path = f.Name
+ 		if path != "" {
+@@ -104,15 +82,6 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
+ 	}
+ }
+
+-func (t *fieldsTransformer) add(f common.Field) {
+-	field, fieldFormat := transformField(t.version, f)
+-	t.transformedFields = append(t.transformedFields, field)
+-	if fieldFormat != nil {
+-		t.transformedFieldFormatMap[field["name"].(string)] = fieldFormat
+-	}
+-
+-}
+-
+ func transformField(version *common.Version, f common.Field) (common.MapStr, common.MapStr) {
+ 	field := common.MapStr{
+ 		"name":         f.Path,

libbeat/scripts/generate_fields_docs.py

@@ -1,6 +1,8 @@
-import yaml
-import os
 import argparse
+from collections import OrderedDict
+import os
+
+import yaml
 
 
 def document_fields(output, section, sections, path):
@@ -102,6 +104,21 @@ def fields_to_asciidoc(input, output, beat):
         print("fields.yml file is empty. fields.asciidoc cannot be generated.")
         return
 
+    # deduplicate fields, last one wins
+    for section in docs:
+        if not section.get("fields"):
+            continue
+        fields = OrderedDict()
+        for field in section["fields"]:
+            name = field["name"]
+            if name in fields:
+                assert field["type"] == fields[name]["type"], 'field "{}" redefined with different type "{}"'.format(
+                    name, field["type"])
+                fields[name].update(field)
+            else:
+                fields[name] = field
+        section["fields"] = list(fields.values())
+
     # Create sections from available fields
     sections = {}
     for v in docs: