Validate that custom ilm_policies are defined in the package

code/go/internal/validator/semantic/validate_ilmpolicypresent.go

@@ -0,0 +1,108 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"fmt"
+	"io/fs"
+	"path"
+	"strings"
+
+	ve "github.com/elastic/package-spec/code/go/internal/errors"
+	"github.com/elastic/package-spec/code/go/internal/fspath"
+	"gopkg.in/yaml.v3"
+)
+
+// ValidateILMPolicyPresent produces an error if the indicated ILM policy
+// is not defined in the data stream.
+func ValidateILMPolicyPresent(fsys fspath.FS) ve.ValidationErrors {
+	dataStreams, err := listDataStreams(fsys)
+	if err != nil {
+		return ve.ValidationErrors{err}
+	}
+
+	var errs ve.ValidationErrors
+	for _, dataStream := range dataStreams {
+		err = validateILMPolicyInDataStream(fsys, dataStream)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+	return errs
+}
+
+func validateILMPolicyInDataStream(fsys fspath.FS, dataStream string) error {
+	dsType, ilmPolicy, err := readILMPolicyInfoInDataStream(fsys, dataStream)
+	if err != nil {
+		return err
+	}
+
+	if ilmPolicy == "" {
+		return nil
+	}
+
+	packageName, err := readPackageName(fsys)
+	if err != nil {
+		return err
+	}
+
+	manifestPath := path.Join("data_stream", dataStream, "manifest.yml")
+	policyPrefix := expectedILMPolicyPrefix(dsType, packageName, dataStream)
+	if !strings.HasPrefix(ilmPolicy, policyPrefix) {
+		return fmt.Errorf("file %q is invalid: field ilm_policy must start with %q, found %q", fsys.Path(manifestPath), policyPrefix, ilmPolicy)
+	}
+
+	ilmFileName := ilmPolicy[len(policyPrefix):] + ".json"
+	ilmFilePath := path.Join("data_stream", dataStream, "elasticsearch", "ilm", ilmFileName)
+	_, err = fs.Stat(fsys, ilmFilePath)
+	if err != nil {
+		return fmt.Errorf("file %q is invalid: field ilm_policy: ILM policy %q not found in package, expected definition in %q", fsys.Path(manifestPath), ilmPolicy, fsys.Path(ilmFilePath))
+	}
+
+	return nil
+}
+
+func readILMPolicyInfoInDataStream(fsys fspath.FS, dataStream string) (dsType string, ilmPolicy string, err error) {
+	manifestPath := path.Join("data_stream", dataStream, "manifest.yml")
+
+	d, err := fs.ReadFile(fsys, manifestPath)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to read data stream manifest in %q: %w", fsys.Path(manifestPath), err)
+	}
+
+	var manifest struct {
+		Type      string `yaml:"type"`
+		ILMPolicy string `yaml:"ilm_policy"`
+	}
+	err = yaml.Unmarshal(d, &manifest)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to parse data stream manifest in %q: %w", fsys.Path(manifestPath), err)
+	}
+
+	return manifest.Type, manifest.ILMPolicy, nil
+}
+
+func readPackageName(fsys fspath.FS) (string, error) {
+	manifestPath := "manifest.yml"
+
+	d, err := fs.ReadFile(fsys, manifestPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to manifest in %q: %w", fsys.Path(manifestPath), err)
+	}
+
+	var manifest struct {
+		Name string `yaml:"name"`
+	}
+	err = yaml.Unmarshal(d, &manifest)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse manifest in %q: %w", fsys.Path(manifestPath), err)
+	}
+
+	return manifest.Name, nil
+}
+
+func expectedILMPolicyPrefix(dsType, packageName, dataStream string) string {
+	return fmt.Sprintf("%s-%s.%s-", dsType, packageName, dataStream)
+}

code/go/internal/validator/spec.go

@@ -84,6 +84,7 @@ func (s Spec) rules(rootSpec spectypes.ItemSpec) validationRules {
 		{fn: semantic.ValidateDimensionFields},
 		{fn: semantic.ValidateRequiredFields},
 		{fn: semantic.ValidateVisualizationsUsedByValue},
+		{fn: semantic.ValidateILMPolicyPresent, since: semver.MustParse("2.0.0")},
 	}
 
 	var validationRules validationRules

code/go/pkg/validator/validator_test.go

@@ -19,6 +19,9 @@ import (
 )
 
 func TestValidateFile(t *testing.T) {
+	// Workaround for error messages that contain OS-dependant base paths.
+	osTestBasePath := filepath.Join("..", "..", "..", "..", "test", "packages") + string(filepath.Separator)
+
 	tests := map[string]struct {
 		invalidPkgFilePath  string
 		expectedErrContains []string
@@ -30,6 +33,7 @@ func TestValidateFile(t *testing.T) {
 		"time_series":         {},
 		"missing_data_stream": {},
 		"icons_dark_mode":     {},
+		"custom_ilm_policy":   {},
 		"bad_additional_content": {
 			"bad-bad",
 			[]string{
@@ -100,6 +104,12 @@ func TestValidateFile(t *testing.T) {
 				"field (root): Additional property license is not allowed",
 			},
 		},
+		"bad_custom_ilm_policy": {
+			"data_stream/test/manifest.yml",
+			[]string{
+				fmt.Sprintf("field ilm_policy: ILM policy \"logs-bad_custom_ilm_policy.test-notexists\" not found in package, expected definition in \"%sbad_custom_ilm_policy/data_stream/test/elasticsearch/ilm/notexists.json\"", osTestBasePath),
+			},
+		},
 	}
 
 	for pkgName, test := range tests {

spec/changelog.yml

@@ -13,6 +13,9 @@
   - description: Remove `array` as supported field data type.
     type: breaking-change
     link: https://github.com/elastic/package-spec/pull/412
+  - description: Check that referenced ILM policies are defined for the data stream.
+    type: breaking-change
+    link: https://github.com/elastic/package-spec/pull/413
 - version: 1.18.1-next
   changes:
   - description: Prepare for next patch release

test/packages/bad_custom_ilm_policy/LICENSE.txt

@@ -0,0 +1,93 @@
+Elastic License 2.0
+
+URL: https://www.elastic.co/licensing/elastic-license
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensor’s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.

test/packages/bad_custom_ilm_policy/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link

test/packages/bad_custom_ilm_policy/data_stream/test/agent/stream/stream.yml.hbs

@@ -0,0 +1,7 @@
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~

test/packages/bad_custom_ilm_policy/data_stream/test/elasticsearch/ilm/diagnostics.json

@@ -0,0 +1,15 @@
+{
+    "policy": {
+        "phases": {
+            "hot": {
+                "min_age": "0ms",
+                "actions": {
+                    "rollover": {
+                        "max_size": "10gb",
+                        "max_age": "1d"
+                    }
+                }
+            }
+        }
+    }
+}

test/packages/bad_custom_ilm_policy/data_stream/test/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,10 @@
+---
+description: Pipeline for processing sample logs
+processors:
+- set:
+    field: sample_field
+    value: "1"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

test/packages/bad_custom_ilm_policy/data_stream/test/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/bad_custom_ilm_policy/data_stream/test/manifest.yml

@@ -0,0 +1,14 @@
+title: "Test"
+type: logs
+ilm_policy: "logs-bad_custom_ilm_policy.test-notexists"
+streams:
+  - input: logfile
+    title: Sample logs
+    description: Collect sample logs
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        default:
+          - /var/log/*.log