From 26292de72a917a5c4d2d506aabef95d583f78375 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 20 Oct 2021 18:07:04 +0200 Subject: [PATCH] Split validator for dimensions --- code/go/internal/validator/semantic/types.go | 97 ++++++++++++++ .../validator/semantic/validate_dimensions.go | 46 +++++++ .../semantic/validate_dimensions_test.go | 84 ++++++++++++ .../semantic/validate_field_groups.go | 121 +----------------- code/go/internal/validator/spec.go | 1 + 5 files changed, 235 insertions(+), 114 deletions(-) create mode 100644 code/go/internal/validator/semantic/types.go create mode 100644 code/go/internal/validator/semantic/validate_dimensions.go create mode 100644 code/go/internal/validator/semantic/validate_dimensions_test.go diff --git a/code/go/internal/validator/semantic/types.go b/code/go/internal/validator/semantic/types.go new file mode 100644 index 000000000..088355805 --- /dev/null +++ b/code/go/internal/validator/semantic/types.go @@ -0,0 +1,97 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "gopkg.in/yaml.v3" + "io/ioutil" + "os" + "path/filepath" + + "github.com/pkg/errors" + + ve "github.com/elastic/package-spec/code/go/internal/errors" +) + +type fields []field + +type field struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + Unit string `yaml:"unit"` + MetricType string `yaml:"metric_type"` + Dimension bool `yaml:"dimension"` + + Fields fields `yaml:"fields"` +} + +type validateFunc func(fieldsFile string, f field) ve.ValidationErrors + +func validateFields(pkgRoot string, validate validateFunc) ve.ValidationErrors { + fieldsFiles, err := listFieldsFiles(pkgRoot) + if err != nil { + return ve.ValidationErrors{errors.Wrap(err, "can't list fields files")} + } + + var vErrs ve.ValidationErrors + for _, fieldsFile := range fieldsFiles { + unmarshaled, err := unmarshalFields(fieldsFile) + if err != nil { + vErrs = append(vErrs, errors.Wrapf(err, `file "%s" is invalid: can't unmarshal fields`, fieldsFile)) + } + + for _, u := range unmarshaled { + errs := validate(fieldsFile, u) + if len(errs) > 0 { + vErrs = append(vErrs, errs...) + } + } + } + return vErrs +} + +func listFieldsFiles(pkgRoot string) ([]string, error) { + var fieldsFiles []string + + dataStreamDir := filepath.Join(pkgRoot, "data_stream") + dataStreams, err := ioutil.ReadDir(dataStreamDir) + if errors.Is(err, os.ErrNotExist) { + return fieldsFiles, nil + } + if err != nil { + return nil, errors.Wrap(err, "can't list data streams directory") + } + + for _, dataStream := range dataStreams { + fieldsDir := filepath.Join(dataStreamDir, dataStream.Name(), "fields") + fs, err := ioutil.ReadDir(fieldsDir) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return nil, errors.Wrapf(err, "can't list fields directory (path: %s)", fieldsDir) + } + + for _, f := range fs { + fieldsFiles = append(fieldsFiles, filepath.Join(fieldsDir, f.Name())) + } + } + + return fieldsFiles, nil +} + +func unmarshalFields(fieldsPath string) (fields, error) { + content, err := ioutil.ReadFile(fieldsPath) + if err != nil { + return nil, errors.Wrapf(err, "can't read file (path: %s)", fieldsPath) + } + + var f fields + err = yaml.Unmarshal(content, &f) + if err != nil { + return nil, errors.Wrapf(err, "yaml.Unmarshal failed (path: %s)", fieldsPath) + } + return f, nil +} diff --git a/code/go/internal/validator/semantic/validate_dimensions.go b/code/go/internal/validator/semantic/validate_dimensions.go new file mode 100644 index 000000000..7f1250030 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_dimensions.go @@ -0,0 +1,46 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "fmt" + "strings" + + "github.com/elastic/package-spec/code/go/internal/errors" +) + +// ValidateDimensionFields verifies if dimension fields are of one of the expected types. +func ValidateDimensionFields(pkgRoot string) errors.ValidationErrors { + return validateFields(pkgRoot, validateDimensionField) +} + +func validateDimensionField(fieldsFile string, f field) errors.ValidationErrors { + if f.Dimension && !isAllowedDimensionType(f.Type) { + return errors.ValidationErrors{fmt.Errorf(`file "%s" is invalid: field "%s" of type %s can't be a dimension, allowed types for dimensions: %s`, fieldsFile, f.Name, f.Type, strings.Join(allowedDimensionTypes, ", "))} + } + + return nil +} + +var allowedDimensionTypes = []string{ + // Keywords + "constant_keyword", "keyword", + + // Numeric types + "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long", + + // IPs + "ip", +} + +func isAllowedDimensionType(fieldType string) bool { + for _, allowedType := range allowedDimensionTypes { + if fieldType == allowedType { + return true + } + } + + return false +} diff --git a/code/go/internal/validator/semantic/validate_dimensions_test.go b/code/go/internal/validator/semantic/validate_dimensions_test.go new file mode 100644 index 000000000..8bc349fe9 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_dimensions_test.go @@ -0,0 +1,84 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateDimensionFields(t *testing.T) { + cases := []struct { + title string + field field + valid bool + }{ + { + title: "usual keyword dimension", + field: field{ + Name: "host.id", + Type: "keyword", + Dimension: true, + }, + valid: true, + }, + { + title: "not a dimension", + field: field{ + Name: "host.id", + Type: "histogram", + }, + valid: true, + }, + { + title: "ip dimension", + field: field{ + Name: "source.ip", + Type: "ip", + Dimension: true, + }, + valid: true, + }, + { + title: "numeric dimension", + field: field{ + Name: "http.body.size", + Type: "long", + Dimension: true, + }, + valid: true, + }, + { + title: "histogram dimension is not supported", + field: field{ + Name: "http.response.time", + Type: "histogram", + Dimension: true, + }, + valid: false, + }, + { + title: "nested field as dimension is not supported", + field: field{ + Name: "process.child", + Type: "nested", + Dimension: true, + }, + valid: false, + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + errs := validateDimensionField("fields.yml", c.field) + if c.valid { + assert.Empty(t, errs) + } else { + assert.NotEmpty(t, errs) + } + }) + } +} diff --git a/code/go/internal/validator/semantic/validate_field_groups.go b/code/go/internal/validator/semantic/validate_field_groups.go index 576cfc152..026cbd115 100644 --- a/code/go/internal/validator/semantic/validate_field_groups.go +++ b/code/go/internal/validator/semantic/validate_field_groups.go @@ -6,111 +6,25 @@ package semantic import ( "fmt" - "gopkg.in/yaml.v3" - "io/ioutil" - "os" - "path/filepath" - "strings" - "github.com/pkg/errors" - - ve "github.com/elastic/package-spec/code/go/internal/errors" + "github.com/elastic/package-spec/code/go/internal/errors" ) -type fields []field - -type field struct { - Name string `yaml:"name"` - Type string `yaml:"type"` - Unit string `yaml:"unit"` - MetricType string `yaml:"metric_type"` - Dimension bool `yaml:"dimension"` - - Fields fields `yaml:"fields"` -} - // ValidateFieldGroups verifies if field groups don't have units and metric types defined. -func ValidateFieldGroups(pkgRoot string) ve.ValidationErrors { - fieldsFiles, err := listFieldsFiles(pkgRoot) - if err != nil { - return ve.ValidationErrors{errors.Wrap(err, "can't list fields files")} - } - - var vErrs ve.ValidationErrors - for _, fieldsFile := range fieldsFiles { - unmarshaled, err := unmarshalFields(fieldsFile) - if err != nil { - vErrs = append(vErrs, errors.Wrapf(err, `file "%s" is invalid: can't unmarshal fields`, fieldsFile)) - } - - for _, u := range unmarshaled { - errs := validateFieldUnit(fieldsFile, u) - if len(errs) > 0 { - vErrs = append(vErrs, errs...) - } - } - } - return vErrs -} - -func listFieldsFiles(pkgRoot string) ([]string, error) { - var fieldsFiles []string - - dataStreamDir := filepath.Join(pkgRoot, "data_stream") - dataStreams, err := ioutil.ReadDir(dataStreamDir) - if errors.Is(err, os.ErrNotExist) { - return fieldsFiles, nil - } - if err != nil { - return nil, errors.Wrap(err, "can't list data streams directory") - } - - for _, dataStream := range dataStreams { - fieldsDir := filepath.Join(dataStreamDir, dataStream.Name(), "fields") - fs, err := ioutil.ReadDir(fieldsDir) - if errors.Is(err, os.ErrNotExist) { - continue - } - if err != nil { - return nil, errors.Wrapf(err, "can't list fields directory (path: %s)", fieldsDir) - } - - for _, f := range fs { - fieldsFiles = append(fieldsFiles, filepath.Join(fieldsDir, f.Name())) - } - } - - return fieldsFiles, nil -} - -func unmarshalFields(fieldsPath string) (fields, error) { - content, err := ioutil.ReadFile(fieldsPath) - if err != nil { - return nil, errors.Wrapf(err, "can't read file (path: %s)", fieldsPath) - } - - var f fields - err = yaml.Unmarshal(content, &f) - if err != nil { - return nil, errors.Wrapf(err, "yaml.Unmarshal failed (path: %s)", fieldsPath) - } - return f, nil +func ValidateFieldGroups(pkgRoot string) errors.ValidationErrors { + return validateFields(pkgRoot, validateFieldUnit) } -func validateFieldUnit(fieldsFile string, f field) ve.ValidationErrors { +func validateFieldUnit(fieldsFile string, f field) errors.ValidationErrors { if f.Type == "group" && f.Unit != "" { - return ve.ValidationErrors{fmt.Errorf(`file "%s" is invalid: field "%s" can't have unit property'`, fieldsFile, f.Name)} + return errors.ValidationErrors{fmt.Errorf(`file "%s" is invalid: field "%s" can't have unit property'`, fieldsFile, f.Name)} } if f.Type == "group" && f.MetricType != "" { - return ve.ValidationErrors{fmt.Errorf(`file "%s" is invalid: field "%s" can't have metric type property'`, fieldsFile, f.Name)} - } - - if f.Dimension && !isAllowedDimensionType(f.Type) { - return ve.ValidationErrors{fmt.Errorf(`file "%s" is invalid: field "%s" of type %s can't be a dimension, allowed types for dimensions: %s`, fieldsFile, f.Name, f.Type, strings.Join(allowedDimensionTypes, ", "))} + return errors.ValidationErrors{fmt.Errorf(`file "%s" is invalid: field "%s" can't have metric type property'`, fieldsFile, f.Name)} } - var vErrs ve.ValidationErrors + var vErrs errors.ValidationErrors for _, aField := range f.Fields { errs := validateFieldUnit(fieldsFile, aField) if len(errs) > 0 { @@ -119,24 +33,3 @@ func validateFieldUnit(fieldsFile string, f field) ve.ValidationErrors { } return vErrs } - -var allowedDimensionTypes = []string{ - // Keywords - "constant_keyword", "keyword", - - // Numeric types - "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long", - - // IPs - "ip", -} - -func isAllowedDimensionType(fieldType string) bool { - for _, allowedType := range allowedDimensionTypes { - if fieldType == allowedType { - return true - } - } - - return false -} diff --git a/code/go/internal/validator/spec.go b/code/go/internal/validator/spec.go index 32e41774e..74e319af6 100644 --- a/code/go/internal/validator/spec.go +++ b/code/go/internal/validator/spec.go @@ -68,6 +68,7 @@ func (s Spec) ValidatePackage(pkg Package) ve.ValidationErrors { semantic.ValidateKibanaObjectIDs, semantic.ValidateVersionIntegrity, semantic.ValidateFieldGroups, + semantic.ValidateDimensionFields, } return rules.validate(pkg.RootPath) }