enable module fields overrides from custom beats (#10060)

libbeat/generator/fields/fields.go

@@ -52,39 +52,45 @@ func NewYmlFile(path string, indent int) (*YmlFile, error) {
 	}, nil
 }
 
-func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
-	var libbeatFieldFiles []*YmlFile
-	var err error
-	commonFields := []string{filepath.Join(esBeatsPath, "libbeat/_meta/fields.ecs.yml")}
-	if !isLibbeat(beatPath) {
-		commonFields = append(commonFields,
-			filepath.Join(esBeatsPath, "libbeat/_meta/fields.common.yml"),
-		)
-		libbeatModulesPath := filepath.Join(esBeatsPath, "libbeat/processors")
-		libbeatFieldFiles, err = CollectModuleFiles(libbeatModulesPath)
-		if err != nil {
+func makeYml(indent int, paths ...string) ([]*YmlFile, error) {
+	var files []*YmlFile
+	for _, path := range paths {
+		if ymlFile, err := NewYmlFile(path, indent); err != nil {
 			return nil, err
+		} else if ymlFile != nil {
+			files = append(files, ymlFile)
 		}
 	}
+	return files, nil
+}
 
-	// Fields for custom beats last, to enable overriding more generically defined fields
-	commonFields = append(commonFields,
-		filepath.Join(beatPath, "_meta/fields.common.yml"),
-		filepath.Join(beatPath, "_meta/fields.yml"),
-	)
-
+func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
 	var files []*YmlFile
-	for _, cf := range commonFields {
-		ymlFile, err := NewYmlFile(cf, 0)
+	var ymls []*YmlFile
+	var err error
+	if ymls, err = makeYml(0, filepath.Join(esBeatsPath, "libbeat/_meta/fields.ecs.yml")); err != nil {
+		return nil, err
+	}
+	files = append(files, ymls...)
 
+	if !isLibbeat(beatPath) {
+		if ymls, err = makeYml(0, filepath.Join(esBeatsPath, "libbeat/_meta/fields.common.yml")); err != nil {
+			return nil, err
+		}
+		files = append(files, ymls...)
+		libbeatModulesPath := filepath.Join(esBeatsPath, "libbeat/processors")
+		libbeatFieldFiles, err := CollectModuleFiles(libbeatModulesPath)
 		if err != nil {
 			return nil, err
-		} else if ymlFile != nil {
-			files = append(files, ymlFile)
 		}
+		files = append(files, libbeatFieldFiles...)
 	}
 
-	files = append(files, libbeatFieldFiles...)
+	// Fields for custom beats last, to enable overriding more generically defined fields
+	if ymls, err = makeYml(0, filepath.Join(beatPath, "_meta/fields.common.yml"), filepath.Join(beatPath, "_meta/fields.yml")); err != nil {
+		return nil, err
+	}
+	files = append(files, ymls...)
 
 	return append(files, fieldFiles...), nil
 }

libbeat/generator/fields/module_fields_collector.go

@@ -69,15 +69,13 @@ func CollectModuleFiles(modulesDir string) ([]*YmlFile, error) {
 // CollectFiles collects all files for the given module including filesets
 func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) {
 	var files []*YmlFile
+	var ymls []*YmlFile
+	var err error
 
-	fieldsYmlPath := filepath.Join(modulesPath, module, "_meta/fields.yml")
-	ymlFile, err := NewYmlFile(fieldsYmlPath, 0)
-
-	if err != nil {
+	if ymls, err = makeYml(0, filepath.Join(modulesPath, module, "_meta/fields.yml")); err != nil {
 		return nil, err
-	} else if ymlFile != nil {
-		files = append(files, ymlFile)
 	}
+	files = append(files, ymls...)
 
 	modulesRoot := filepath.Base(modulesPath)
 	sets, err := ioutil.ReadDir(filepath.Join(modulesPath, module))
@@ -90,14 +88,11 @@ func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) {
 			continue
 		}
 
-		fieldsYmlPath = filepath.Join(modulesPath, module, s.Name(), "_meta/fields.yml")
-		ymlFile, err := NewYmlFile(fieldsYmlPath, indentByModule[modulesRoot])
-
-		if err != nil {
+		fieldsYmlPath := filepath.Join(modulesPath, module, s.Name(), "_meta/fields.yml")
+		if ymls, err = makeYml(indentByModule[modulesRoot], fieldsYmlPath); err != nil {
 			return nil, err
-		} else if ymlFile != nil {
-			files = append(files, ymlFile)
 		}
+		files = append(files, ymls...)
 	}
 	return files, nil
 }