Warnings are reported as errors

code/go/internal/validator/folder_spec.go

@@ -5,6 +5,7 @@
 package validator
 
 import (
+	"errors"
 	"fmt"
 	"io/fs"
 	"log"
@@ -68,8 +69,9 @@ func (v *validator) Validate() specerrors.ValidationErrors {
 			)
 		} else {
 			message := fmt.Sprintf("Warning: package with non-stable semantic version and active beta features (enabled in [%s]) can't be released as stable version.", v.pkg.Path(v.folderPath))
-			if common.IsDefinedWarningsAsErrors() {
-				errs = append(errs, specerrors.NewStructuredErrorf(message))
+			if common.IsDefinedWarningsAsErrors() || v.pkg.SpecVersion.Major() >= 3 {
+				err = errors.New(message)
+				errs = append(errs, specerrors.NewStructuredError(err, specerrors.CodePrereleaseFeatureOnGAPackage))
 			} else {
 				log.Printf(message)
 			}

code/go/internal/validator/semantic/validate_minimum_kibana_version.go

@@ -6,15 +6,13 @@ package semantic
 
 import (
 	"fmt"
-	"log"
 	"regexp"
 
 	"github.com/Masterminds/semver/v3"
 
 	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
 	"github.com/elastic/package-spec/v3/code/go/internal/packages"
 	"github.com/elastic/package-spec/v3/code/go/internal/pkgpath"
-	"github.com/elastic/package-spec/v3/code/go/internal/validator/common"
 	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
 )
 
@@ -38,23 +36,17 @@ func ValidateMinimumKibanaVersion(fsys fspath.FS) specerrors.ValidationErrors {
 	var errs specerrors.ValidationErrors
 	err = validateMinimumKibanaVersionInputPackages(pkg.Type, *pkg.Version, kibanaVersionCondition)
 	if err != nil {
-		errs.Append(specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)})
+		errs = append(errs, specerrors.NewStructuredError(err, specerrors.UnassignedCode))
 	}
 
 	err = validateMinimumKibanaVersionRuntimeFields(fsys, *pkg.Version, kibanaVersionCondition)
 	if err != nil {
-		errs.Append(specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)})
+		errs = append(errs, specerrors.NewStructuredError(err, specerrors.UnassignedCode))
 	}
 
-	warningsAsErrors := common.IsDefinedWarningsAsErrors()
 	err = validateMinimumKibanaVersionSavedObjectTags(fsys, pkg.Type, *pkg.Version, kibanaVersionCondition)
 	if err != nil {
-		err = fmt.Errorf("Warning: %v", err)
-		if warningsAsErrors {
-			errs.Append(specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)})
-		} else {
-			log.Println(err)
-		}
+		errs = append(errs, specerrors.NewStructuredError(err, specerrors.CodeMinimumKibanaVersion))
 	}
 
 	if errs != nil {

code/go/internal/validator/semantic/validate_visualizations_used_by_value.go

@@ -7,12 +7,10 @@ package semantic
 import (
 	"errors"
 	"fmt"
-	"log"
 	"path"
 
 	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
 	"github.com/elastic/package-spec/v3/code/go/internal/pkgpath"
-	"github.com/elastic/package-spec/v3/code/go/internal/validator/common"
 	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
 )
 
@@ -28,7 +26,6 @@ type reference struct {
 // defines some visualization using reference (containing an element of
 // "visualization" type inside references key).
 func ValidateVisualizationsUsedByValue(fsys fspath.FS) specerrors.ValidationErrors {
-	warningsAsErrors := common.IsDefinedWarningsAsErrors()
 	var errs specerrors.ValidationErrors
 
 	filePaths := path.Join("kibana", "dashboard", "*.json")
@@ -57,13 +54,8 @@ func ValidateVisualizationsUsedByValue(fsys fspath.FS) specerrors.ValidationErro
 				s = fmt.Sprintf("%s, %s (%s)", s, ref.ID, ref.Type)
 			}
 
-			message := fmt.Sprintf("Warning: references found in dashboard %s: %s", filePath, s)
-			if warningsAsErrors {
-				errs = append(errs, specerrors.NewStructuredErrorf(message))
-			} else {
-				log.Printf(message)
-			}
-
+			err = fmt.Errorf("references found in dashboard %s: %s", filePath, s)
+			errs = append(errs, specerrors.NewStructuredError(err, specerrors.CodeVisualizationByValue))
 		}
 	}
 

code/go/internal/validator/semantic/warning.go

@@ -0,0 +1,38 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"log"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/internal/validator/common"
+	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
+)
+
+// WarnOn returns a validation function that wraps another one. Errors returned by the
+// wrapped validation that have a filtering code are printed as warnings. Other errors
+// are directly returned.
+func WarnOn(validation func(fsys fspath.FS) specerrors.ValidationErrors) func(fspath.FS) specerrors.ValidationErrors {
+	return func(fsys fspath.FS) specerrors.ValidationErrors {
+		errs := validation(fsys)
+		if common.IsDefinedWarningsAsErrors() {
+			return errs
+		}
+
+		k := 0
+		for i := range errs {
+			if err := errs[i]; err.Code() != specerrors.UnassignedCode {
+				log.Printf("Warning: %s", err.Error())
+				continue
+			}
+
+			errs[k] = errs[i]
+			k++
+		}
+
+		return errs[:k]
+	}
+}

code/go/internal/validator/spec.go

@@ -145,15 +145,17 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules
 		{fn: semantic.ValidateVersionIntegrity},
 		{fn: semantic.ValidateChangelogLinks},
 		{fn: semantic.ValidatePrerelease},
-		{fn: semantic.ValidateMinimumKibanaVersion},
+		{fn: semantic.WarnOn(semantic.ValidateMinimumKibanaVersion), until: semver.MustParse("3.0.0")},
+		{fn: semantic.ValidateMinimumKibanaVersion, since: semver.MustParse("3.0.0")},
 		{fn: semantic.ValidateFieldGroups},
 		{fn: semantic.ValidateFieldsLimits(rootSpec.MaxFieldsPerDataStream())},
 		{fn: semantic.ValidateUniqueFields, since: semver.MustParse("2.0.0")},
 		{fn: semantic.ValidateDimensionFields},
 		{fn: semantic.ValidateDateFields},
 		{fn: semantic.ValidateRequiredFields},
 		{fn: semantic.ValidateExternalFieldsWithDevFolder},
-		{fn: semantic.ValidateVisualizationsUsedByValue, types: []string{"integration"}},
+		{fn: semantic.WarnOn(semantic.ValidateVisualizationsUsedByValue), types: []string{"integration"}, until: semver.MustParse("3.0.0")},
+		{fn: semantic.ValidateVisualizationsUsedByValue, types: []string{"integration"}, since: semver.MustParse("3.0.0")},
 		{fn: semantic.ValidateILMPolicyPresent, since: semver.MustParse("2.0.0"), types: []string{"integration"}},
 		{fn: semantic.ValidateProfilingNonGA, types: []string{"integration"}},
 		{fn: semantic.ValidateKibanaObjectIDs, types: []string{"integration"}},

code/go/pkg/specerrors/constants.go

@@ -9,10 +9,13 @@ const (
 	UnassignedCode = ""
 
 	// PSR - Package Spec [General] Rule
-	CodeNonGASpecOnGAPackage = "PSR00001"
+	CodeNonGASpecOnGAPackage         = "PSR00001"
+	CodePrereleaseFeatureOnGAPackage = "PSR00002"
 
 	// SVR - Semantic Validation Rules
 	CodeKibanaDashboardWithQueryButNoFilter = "SVR00001"
 	CodeKibanaDashboardWithoutFilter        = "SVR00002"
 	CodeKibanaDanglingObjectsIDs            = "SVR00003"
+	CodeVisualizationByValue                = "SVR00004"
+	CodeMinimumKibanaVersion                = "SVR00005"
 )

code/go/pkg/validator/validator_test.go

@@ -532,10 +532,10 @@ func TestValidateWarnings(t *testing.T) {
 		"good":    []string{},
 		"good_v2": []string{},
 		"visualizations_by_reference": []string{
-			"references found in dashboard kibana/dashboard/visualizations_by_reference-82273ffe-6acc-4f2f-bbee-c1004abba63d.json: visualizations_by_reference-5e1a01ff-6f9a-41c1-b7ad-326472db42b6 (visualization), visualizations_by_reference-8287a5d5-1576-4f3a-83c4-444e9058439b (lens)",
+			"references found in dashboard kibana/dashboard/visualizations_by_reference-82273ffe-6acc-4f2f-bbee-c1004abba63d.json: visualizations_by_reference-5e1a01ff-6f9a-41c1-b7ad-326472db42b6 (visualization), visualizations_by_reference-8287a5d5-1576-4f3a-83c4-444e9058439b (lens) (SVR00004)",
 		},
 		"bad_saved_object_tags_kibana_version": []string{
-			"conditions.kibana.version must be ^8.10.0 or greater to include saved object tags file: kibana/tags.yml",
+			"conditions.kibana.version must be ^8.10.0 or greater to include saved object tags file: kibana/tags.yml (SVR00005)",
 		},
 	}
 	if err := common.EnableWarningsAsErrors(); err != nil {
@@ -545,8 +545,6 @@ func TestValidateWarnings(t *testing.T) {
 
 	for pkgName, expectedWarnContains := range tests {
 		t.Run(pkgName, func(t *testing.T) {
-			warnPrefix := fmt.Sprintf("Warning: ")
-
 			pkgRootPath := path.Join("..", "..", "..", "..", "test", "packages", pkgName)
 			errs := ValidateFromPath(pkgRootPath)
 			if len(expectedWarnContains) == 0 {
@@ -562,8 +560,7 @@ func TestValidateWarnings(t *testing.T) {
 					}
 
 					for _, expectedWarnMessage := range expectedWarnContains {
-						expectedWarn := warnPrefix + expectedWarnMessage
-						require.Contains(t, warnMessages, expectedWarn)
+						require.Contains(t, warnMessages, expectedWarnMessage)
 					}
 					return
 				}

spec/changelog.yml

@@ -45,6 +45,9 @@
   - description: Require group or nested types for fields with subobjects
     type: bugfix
     link: https://github.com/elastic/package-spec/pull/629
+  - description: Warnings are reported as errors that can be filtered
+    type: breaking-change
+    link: https://github.com/elastic/package-spec/issues/636
 - version: 2.13.0
   changes:
   - description: Allow to define expected values in fields definitions.