From d3d03fc1c65918175c923bc70ef5210f2fe662df Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 1 Jul 2020 09:32:32 -0600 Subject: [PATCH] [7.x] Add default composable templates for new indexing strategy (#57629) (#58757) Backports the following commits to 7.x: Add default composable templates for new indexing strategy (#57629) --- docs/reference/indices/templates.asciidoc | 8 +- .../test/cat.aliases/10_basic.yml | 6 +- .../test/cat.aliases/40_hidden.yml | 3 +- .../test/cluster.stats/10_basic.yml | 54 --- .../indices.get_index_template/10_basic.yml | 5 +- .../test/indices.put_template/10_basic.yml | 10 +- .../admin/cluster/stats/ClusterStatsIT.java | 30 ++ .../MetadataIndexTemplateService.java | 3 +- .../test/rest/ESRestTestCase.java | 33 +- .../rest/yaml/ESClientYamlSuiteTestCase.java | 9 + .../xpack/core/ClientHelper.java | 1 + .../SnapshotLifecycleTemplateRegistry.java | 4 +- .../core/template/IndexTemplateRegistry.java | 204 +++++++- .../src/main/resources/logs-mappings.json | 60 +++ .../core/src/main/resources/logs-policy.json | 12 + .../src/main/resources/logs-settings.json | 20 + .../src/main/resources/logs-template.json | 16 + .../src/main/resources/metrics-mappings.json | 57 +++ .../src/main/resources/metrics-policy.json | 12 + .../src/main/resources/metrics-settings.json | 20 + .../src/main/resources/metrics-template.json | 16 + ...napshotLifecycleTemplateRegistryTests.java | 8 +- .../xpack/ilm/CCRIndexLifecycleIT.java | 11 +- .../xpack/ilm/IndexLifecycle.java | 2 +- .../history/ILMHistoryTemplateRegistry.java | 2 +- .../xpack/ml/MlIndexTemplateRegistry.java | 2 +- .../security/authz/AuthorizationUtils.java | 2 + x-pack/plugin/stack/build.gradle | 29 ++ x-pack/plugin/stack/qa/build.gradle | 0 x-pack/plugin/stack/qa/rest/build.gradle | 38 ++ .../xpack/stack/StackYamlIT.java | 43 ++ .../rest-api-spec/test/stack/10_basic.yml | 105 ++++ .../xpack/stack/StackPlugin.java | 66 +++ .../xpack/stack/StackTemplateRegistry.java | 140 ++++++ .../stack/StackTemplateRegistryTests.java | 453 ++++++++++++++++++ .../support/WatcherIndexTemplateRegistry.java | 2 +- .../build.gradle | 1 + 37 files changed, 1389 insertions(+), 98 deletions(-) create mode 100644 x-pack/plugin/core/src/main/resources/logs-mappings.json create mode 100644 x-pack/plugin/core/src/main/resources/logs-policy.json create mode 100644 x-pack/plugin/core/src/main/resources/logs-settings.json create mode 100644 x-pack/plugin/core/src/main/resources/logs-template.json create mode 100644 x-pack/plugin/core/src/main/resources/metrics-mappings.json create mode 100644 x-pack/plugin/core/src/main/resources/metrics-policy.json create mode 100644 x-pack/plugin/core/src/main/resources/metrics-settings.json create mode 100644 x-pack/plugin/core/src/main/resources/metrics-template.json create mode 100644 x-pack/plugin/stack/build.gradle create mode 100644 x-pack/plugin/stack/qa/build.gradle create mode 100644 x-pack/plugin/stack/qa/rest/build.gradle create mode 100644 x-pack/plugin/stack/qa/rest/src/test/java/org/elasticsearch/xpack/stack/StackYamlIT.java create mode 100644 x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml create mode 100644 x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java create mode 100644 x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java create mode 100644 x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java diff --git a/docs/reference/indices/templates.asciidoc b/docs/reference/indices/templates.asciidoc index 0c4ed20d3b01d..154dec62de3a6 100644 --- a/docs/reference/indices/templates.asciidoc +++ b/docs/reference/indices/templates.asciidoc @@ -180,7 +180,7 @@ orders overriding them. For example: -------------------------------------------------- PUT /_template/template_1 { - "index_patterns" : ["*"], + "index_patterns" : ["te*"], "order" : 0, "settings" : { "number_of_shards" : 1 @@ -192,7 +192,7 @@ PUT /_template/template_1 PUT /_template/template_2 { - "index_patterns" : ["te*"], + "index_patterns" : ["tes*"], "order" : 1, "settings" : { "number_of_shards" : 1 @@ -204,7 +204,7 @@ PUT /_template/template_2 -------------------------------------------------- The above will disable storing the `_source`, but -for indices that start with `te*`, `_source` will still be enabled. +for indices that start with `tes*`, `_source` will still be enabled. Note, for mappings, the merging is "deep", meaning that specific object/property based mappings can easily be added/overridden on higher order templates, with lower order templates providing the basis. @@ -231,7 +231,7 @@ replace the template without specifying one. -------------------------------------------------- PUT /_template/template_1 { - "index_patterns" : ["*"], + "index_patterns" : ["myindex-*"], "order" : 0, "settings" : { "number_of_shards" : 1 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml index 5669206ee87ad..3263a3d999052 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml @@ -294,6 +294,7 @@ - do: cat.aliases: v: true + name: test* - match: $body: | @@ -364,6 +365,7 @@ - do: cat.aliases: h: [index, alias] + name: test* - match: $body: /^ test \s+ test_1 \s+ $/ @@ -373,6 +375,7 @@ cat.aliases: h: [index, alias] v: true + name: test* - match: $body: | /^ @@ -398,7 +401,8 @@ index: test_index - do: - cat.aliases: {} + cat.aliases: + name: test* - match: $body: | diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml index 3aa7fdbb1f760..3fe58d7ea08f4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml @@ -73,7 +73,8 @@ aliases: test_alias: {} - do: - cat.aliases: {} + cat.aliases: + name: test* - match: $body: | diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml index 84bce35a07327..f97eef77b4e9d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml @@ -89,57 +89,3 @@ cluster.stats: {} - is_true: nodes.packaging_types - ---- -"get cluster stats returns mapping stats": - - - skip: - version: " - 7.6.99" - reason: "mapping stats are added for v7.7.0" - - - do: - cluster.stats: {} - - - length: { indices.mappings.field_types: 0 } - - - do: - indices.create: - index: test-index1 - body: - mappings: - properties: - foo: - type: keyword - - - do: - indices.create: - index: test-index2 - body: - mappings: - properties: - foo: - type: keyword - bar: - properties: - quux: - type: integer - baz: - type: keyword - - - do: - cluster.stats: {} - - - length: { indices.mappings.field_types: 3 } - - - match: { indices.mappings.field_types.0.name: integer } - - match: { indices.mappings.field_types.0.count: 1 } - - match: { indices.mappings.field_types.0.index_count: 1 } - - - match: { indices.mappings.field_types.1.name: keyword } - - match: { indices.mappings.field_types.1.count: 3 } - - match: { indices.mappings.field_types.1.index_count: 2 } - - - match: { indices.mappings.field_types.2.name: object } - - match: { indices.mappings.field_types.2.count: 1 } - - match: { indices.mappings.field_types.2.index_count: 1 } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml index 9dfab9f263394..435b09db70d76 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml @@ -36,7 +36,7 @@ setup: - match: {index_templates.0.index_template.template.mappings: {properties: {field: {type: keyword}}}} --- -"Get all tindex emplates": +"Get all index templates": - skip: version: " - 7.7.99" reason: "index template v2 API unavailable before 7.8" @@ -56,7 +56,8 @@ setup: - do: indices.get_index_template: {} - - length: {index_templates: 2} + - is_true: index_templates.0.name + - is_true: index_templates.1.name --- "Get index template with local flag": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml index 3850ba4150b4f..05bca61811a96 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml @@ -142,7 +142,7 @@ body: > { "version": 10, - "index_patterns": "*", + "index_patterns": "foo*", "settings": { "number_of_shards": 1 } } - match: { acknowledged: true } @@ -159,7 +159,7 @@ body: > { "version": 9, - "index_patterns": "*", + "index_patterns": "foo*", "settings": { "number_of_shards": 1 } } - match: { acknowledged: true } @@ -176,7 +176,7 @@ body: > { "version": 6789, - "index_patterns": "*", + "index_patterns": "foo*", "settings": { "number_of_shards": 1 } } - match: { acknowledged: true } @@ -192,7 +192,7 @@ name: "my_template" body: > { - "index_patterns": "*", + "index_patterns": "foo*", "settings": { "number_of_shards": 1 } } - match: { acknowledged: true } @@ -209,7 +209,7 @@ body: > { "version": 5385, - "index_patterns": "*", + "index_patterns": "foo*", "settings": { "number_of_shards": 1 } } - match: { acknowledged: true } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java index 2b740136df5dc..7975863f96d89 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -29,6 +29,8 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.monitor.os.OsStats; import org.elasticsearch.node.NodeRoleSettings; import org.elasticsearch.test.ESIntegTestCase; @@ -46,6 +48,7 @@ import static org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.OS; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) @@ -250,4 +253,31 @@ public void testClusterStatusWhenStateNotRecovered() throws Exception { response = client().admin().cluster().prepareClusterStats().get(); assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); } + + public void testFieldTypes() { + internalCluster().startNode(); + ensureGreen(); + ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); + assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); + assertTrue(response.getIndicesStats().getMappings().getFieldTypeStats().isEmpty()); + + client().admin().indices().prepareCreate("test1").addMapping(MapperService.SINGLE_MAPPING_NAME, + "{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}", XContentType.JSON).get(); + client().admin().indices().prepareCreate("test2") + .addMapping(MapperService.SINGLE_MAPPING_NAME, + "{\"properties\":{\"foo\":{\"type\": \"keyword\"},\"bar\":{\"properties\":{\"baz\":{\"type\":\"keyword\"}," + + "\"eggplant\":{\"type\":\"integer\"}}}}}", XContentType.JSON).get(); + response = client().admin().cluster().prepareClusterStats().get(); + assertThat(response.getIndicesStats().getMappings().getFieldTypeStats().size(), equalTo(3)); + Set stats = response.getIndicesStats().getMappings().getFieldTypeStats(); + for (IndexFeatureStats stat : stats) { + if (stat.getName().equals("integer")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(1)); + } else if (stat.getName().equals("keyword")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(3)); + } else if (stat.getName().equals("object")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(1)); + } + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 2c7d531b8b369..e6e9a5f79a8d6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -498,7 +498,8 @@ public ClusterState addIndexTemplateV2(final ClusterState currentState, final bo (finalIndexTemplate.composedOf().size() > 0 ? "with component templates " + finalIndexTemplate.composedOf() + " " : "") + "is invalid", e); } - logger.info("{} index template [{}]", existing == null ? "adding" : "updating", name); + logger.info("{} index template [{}] for index patterns {}", existing == null ? "adding" : "updating", name, + template.indexPatterns()); return ClusterState.builder(currentState) .metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate)) .build(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 3f79d48bb3d59..e4dcca79cd14c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -27,6 +27,7 @@ import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; @@ -506,7 +507,8 @@ protected boolean preserveSLMPoliciesUponCompletion() { * A set of ILM policies that should be preserved between runs. */ protected Set preserveILMPolicyIds() { - return Sets.newHashSet("ilm-history-ilm-policy", "slm-history-ilm-policy", "watch-history-ilm-policy", "ml-size-based-ilm-policy"); + return Sets.newHashSet("ilm-history-ilm-policy", "slm-history-ilm-policy", + "watch-history-ilm-policy", "ml-size-based-ilm-policy", "logs", "metrics"); } /** @@ -604,8 +606,25 @@ private void wipeCluster() throws Exception { } } try { - adminClient().performRequest(new Request("DELETE", "_component_template/*")); - } catch (ResponseException e) { + Request compReq = new Request("GET", "_component_template"); + String componentTemplates = EntityUtils.toString(adminClient().performRequest(compReq).getEntity()); + Map cTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent, componentTemplates, false); + @SuppressWarnings("unchecked") + List names = ((List>) cTemplates.get("component_templates")).stream() + .map(ct -> (String) ct.get("name")) + .collect(Collectors.toList()); + for (String componentTemplate : names) { + try { + if (isXPackTemplate(componentTemplate)) { + continue; + } + adminClient().performRequest(new Request("DELETE", "_component_template/" + componentTemplate)); + } catch (ResponseException e) { + logger.debug(new ParameterizedMessage("unable to remove component template {}", componentTemplate), e); + } + } + } catch (Exception e) { + logger.info("ignoring exception removing all component templates", e); // We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore } } else { @@ -1212,7 +1231,6 @@ protected static boolean isXPackTemplate(String name) { return true; } switch (name) { - case ".triggered_watches": case ".watches": case "logstash-index-template": case ".logstash-management": @@ -1220,6 +1238,13 @@ protected static boolean isXPackTemplate(String name) { case ".slm-history": case ".async-search": case "saml-service-provider": + case "ilm-history": + case "logs": + case "logs-settings": + case "logs-mappings": + case "metrics": + case "metrics-settings": + case "metrics-mappings": return true; default: return false; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 5dbfbd0325329..7dc3a4161104c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -30,6 +30,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.client.sniff.ElasticsearchNodesSniffer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -364,6 +365,14 @@ public void test() throws IOException { && testCandidate.getTestSection().getSkipSection().getFeatures().contains("default_shards") == false) { final Request request = new Request("PUT", "/_template/global"); request.setJsonEntity("{\"index_patterns\":[\"*\"],\"settings\":{\"index.number_of_shards\":2}}"); + // Because this has not yet transitioned to a composable template, it's possible that + // this can overlap an installed composable template since this is a global (*) + // template. In order to avoid this failing the test, we override the warnings handler + // to be permissive in this case. This can be removed once all tests use composable + // templates instead of legacy templates + RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.setWarningsHandler(WarningsHandler.PERMISSIVE); + request.setOptions(builder.build()); adminClient().performRequest(request); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java index 2832fd634dd96..701822aea7d2f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java @@ -69,6 +69,7 @@ public static Map filterSecurityHeaders(Map head public static final String TRANSFORM_ORIGIN = "transform"; public static final String ASYNC_SEARCH_ORIGIN = "async_search"; public static final String IDP_ORIGIN = "idp"; + public static final String STACK_ORIGIN = "stack"; private ClientHelper() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java index da35b2b1745cb..908d469d0eba8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java @@ -69,7 +69,7 @@ public SnapshotLifecycleTemplateRegistry(Settings nodeSettings, ClusterService c } @Override - protected List getTemplateConfigs() { + protected List getLegacyTemplateConfigs() { if (slmHistoryEnabled == false) { return Collections.emptyList(); } @@ -90,7 +90,7 @@ protected String getOrigin() { } public boolean validate(ClusterState state) { - boolean allTemplatesPresent = getTemplateConfigs().stream() + boolean allTemplatesPresent = getLegacyTemplateConfigs().stream() .map(IndexTemplateConfig::getTemplateName) .allMatch(name -> state.metadata().getTemplates().containsKey(name)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java index 33ad3aca2e80f..2200ffc8ca05b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java @@ -9,20 +9,27 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.XPackClient; @@ -30,9 +37,12 @@ import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; @@ -70,14 +80,38 @@ public IndexTemplateRegistry(Settings nodeSettings, ClusterService clusterServic * the index templates that should be installed and managed. * @return The configurations for the templates that should be installed. */ - protected abstract List getTemplateConfigs(); + protected List getLegacyTemplateConfigs() { + return Collections.emptyList(); + } + + /** + * Retrieves return a list of {@link IndexTemplateConfig} that represents + * the component templates that should be installed and managed. Component + * templates are always installed prior composable templates, so they may + * be referenced by a composable template. + * @return The configurations for the templates that should be installed. + */ + protected List getComponentTemplateConfigs() { + return Collections.emptyList(); + } + + /** + * Retrieves return a list of {@link IndexTemplateConfig} that represents + * the composable templates that should be installed and managed. + * @return The configurations for the templates that should be installed. + */ + protected List getComposableTemplateConfigs() { + return Collections.emptyList(); + } /** * Retrieves a list of {@link LifecyclePolicyConfig} that represents the ILM * policies that should be installed and managed. Only called if ILM is enabled. * @return The configurations for the lifecycle policies that should be installed. */ - protected abstract List getPolicyConfigs(); + protected List getPolicyConfigs() { + return Collections.emptyList(); + } /** * Retrieves an identifier that is used to identify which plugin is asking for this. @@ -147,34 +181,116 @@ protected boolean requiresMasterNode() { } private void addTemplatesIfMissing(ClusterState state) { - final List indexTemplates = getTemplateConfigs(); + addLegacyTemplatesIfMissing(state); + addComponentTemplatesIfMissing(state); + addComposableTemplatesIfMissing(state); + } + + private void addLegacyTemplatesIfMissing(ClusterState state) { + final List indexTemplates = getLegacyTemplateConfigs(); for (IndexTemplateConfig newTemplate : indexTemplates) { final String templateName = newTemplate.getTemplateName(); final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false)); if (creationCheck.compareAndSet(false, true)) { IndexTemplateMetadata currentTemplate = state.metadata().getTemplates().get(templateName); if (Objects.isNull(currentTemplate)) { - logger.info("adding index template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); - putTemplate(newTemplate, creationCheck); + logger.info("adding legacy template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); + putLegacyTemplate(newTemplate, creationCheck); } else if (Objects.isNull(currentTemplate.getVersion()) || newTemplate.getVersion() > currentTemplate.getVersion()) { // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can // safely assume it's an old version of the template. - logger.info("upgrading index template [{}] for [{}] from version [{}] to version [{}]", + logger.info("upgrading legacy template [{}] for [{}] from version [{}] to version [{}]", templateName, getOrigin(), currentTemplate.getVersion(), newTemplate.getVersion()); - putTemplate(newTemplate, creationCheck); + putLegacyTemplate(newTemplate, creationCheck); } else { creationCheck.set(false); - logger.trace("not adding index template [{}] for [{}], because it already exists at version [{}]", + logger.trace("not adding legacy template [{}] for [{}], because it already exists at version [{}]", templateName, getOrigin(), currentTemplate.getVersion()); } } else { - logger.trace("skipping the creation of index template [{}] for [{}], because its creation is in progress", + logger.trace("skipping the creation of legacy template [{}] for [{}], because its creation is in progress", templateName, getOrigin()); } } } - private void putTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) { + private void addComponentTemplatesIfMissing(ClusterState state) { + final List indexTemplates = getComponentTemplateConfigs(); + for (IndexTemplateConfig newTemplate : indexTemplates) { + final String templateName = newTemplate.getTemplateName(); + final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false)); + if (creationCheck.compareAndSet(false, true)) { + ComponentTemplate currentTemplate = state.metadata().componentTemplates().get(templateName); + if (Objects.isNull(currentTemplate)) { + logger.debug("adding component template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); + putComponentTemplate(newTemplate, creationCheck); + } else if (Objects.isNull(currentTemplate.version()) || newTemplate.getVersion() > currentTemplate.version()) { + // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can + // safely assume it's an old version of the template. + logger.info("upgrading component template [{}] for [{}] from version [{}] to version [{}]", + templateName, getOrigin(), currentTemplate.version(), newTemplate.getVersion()); + putComponentTemplate(newTemplate, creationCheck); + } else { + creationCheck.set(false); + logger.trace("not adding component template [{}] for [{}], because it already exists at version [{}]", + templateName, getOrigin(), currentTemplate.version()); + } + } else { + logger.trace("skipping the creation of component template [{}] for [{}], because its creation is in progress", + templateName, getOrigin()); + } + } + } + + private void addComposableTemplatesIfMissing(ClusterState state) { + final List indexTemplates = getComposableTemplateConfigs(); + for (IndexTemplateConfig newTemplate : indexTemplates) { + final String templateName = newTemplate.getTemplateName(); + final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false)); + if (creationCheck.compareAndSet(false, true)) { + ComposableIndexTemplate currentTemplate = state.metadata().templatesV2().get(templateName); + boolean componentTemplatesAvailable = componentTemplatesExist(state, newTemplate); + if (componentTemplatesAvailable == false) { + creationCheck.set(false); + logger.trace("not adding composable template [{}] for [{}] because its required component templates do not exist", + templateName, getOrigin()); + } else if (Objects.isNull(currentTemplate)) { + logger.debug("adding composable template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); + putComposableTemplate(newTemplate, creationCheck); + } else if (Objects.isNull(currentTemplate.version()) || newTemplate.getVersion() > currentTemplate.version()) { + // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can + // safely assume it's an old version of the template. + logger.info("upgrading composable template [{}] for [{}] from version [{}] to version [{}]", + templateName, getOrigin(), currentTemplate.version(), newTemplate.getVersion()); + putComposableTemplate(newTemplate, creationCheck); + } else { + creationCheck.set(false); + logger.trace("not adding composable template [{}] for [{}], because it already exists at version [{}]", + templateName, getOrigin(), currentTemplate.version()); + } + } else { + logger.trace("skipping the creation of composable template [{}] for [{}], because its creation is in progress", + templateName, getOrigin()); + } + } + } + + /** + * Returns true if the cluster state contains all of the component templates needed by the composable template + */ + private static boolean componentTemplatesExist(ClusterState state, IndexTemplateConfig composableTemplate) { + final ComposableIndexTemplate indexTemplate; + try { + indexTemplate = ComposableIndexTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, composableTemplate.loadBytes())); + } catch (Exception e) { + throw new ElasticsearchParseException("unable to parse composable template " + composableTemplate.getTemplateName(), e); + } + Set neededComponents = new HashSet<>(indexTemplate.composedOf()); + return state.metadata().componentTemplates().keySet().containsAll(neededComponents); + } + + private void putLegacyTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) { final Executor executor = threadPool.generic(); executor.execute(() -> { final String templateName = config.getTemplateName(); @@ -187,7 +303,7 @@ private void putTemplate(final IndexTemplateConfig config, final AtomicBoolean c public void onResponse(AcknowledgedResponse response) { creationCheck.set(false); if (response.isAcknowledged() == false) { - logger.error("error adding index template [{}] for [{}], request was not acknowledged", + logger.error("error adding legacy template [{}] for [{}], request was not acknowledged", templateName, getOrigin()); } } @@ -201,6 +317,72 @@ public void onFailure(Exception e) { }); } + private void putComponentTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) { + final Executor executor = threadPool.generic(); + executor.execute(() -> { + final String templateName = config.getTemplateName(); + + PutComponentTemplateAction.Request request = new PutComponentTemplateAction.Request(templateName); + try { + request.componentTemplate(ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, config.loadBytes()))); + } catch (Exception e) { + throw new ElasticsearchParseException("unable to parse component template " + config.getTemplateName(), e); + } + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); + executeAsyncWithOrigin(client.threadPool().getThreadContext(), getOrigin(), request, + new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse response) { + creationCheck.set(false); + if (response.isAcknowledged() == false) { + logger.error("error adding component template [{}] for [{}], request was not acknowledged", + templateName, getOrigin()); + } + } + + @Override + public void onFailure(Exception e) { + creationCheck.set(false); + onPutTemplateFailure(config, e); + } + }, (req, listener) -> client.execute(PutComponentTemplateAction.INSTANCE, req, listener)); + }); + } + + private void putComposableTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) { + final Executor executor = threadPool.generic(); + executor.execute(() -> { + final String templateName = config.getTemplateName(); + + PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(templateName); + try { + request.indexTemplate(ComposableIndexTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, config.loadBytes()))); + } catch (Exception e) { + throw new ElasticsearchParseException("unable to parse composable template " + config.getTemplateName(), e); + } + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); + executeAsyncWithOrigin(client.threadPool().getThreadContext(), getOrigin(), request, + new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse response) { + creationCheck.set(false); + if (response.isAcknowledged() == false) { + logger.error("error adding composable template [{}] for [{}], request was not acknowledged", + templateName, getOrigin()); + } + } + + @Override + public void onFailure(Exception e) { + creationCheck.set(false); + onPutTemplateFailure(config, e); + } + }, (req, listener) -> client.execute(PutComposableIndexTemplateAction.INSTANCE, req, listener)); + }); + } + private void addIndexLifecyclePoliciesIfMissing(ClusterState state) { Optional maybeMeta = Optional.ofNullable(state.metadata().custom(IndexLifecycleMetadata.TYPE)); diff --git a/x-pack/plugin/core/src/main/resources/logs-mappings.json b/x-pack/plugin/core/src/main/resources/logs-mappings.json new file mode 100644 index 0000000000000..5e978ea56885c --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/logs-mappings.json @@ -0,0 +1,60 @@ +{ + "template": { + "mappings": { + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "dataset": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "logs" + }, + "name": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "message": { + "type": "text" + } + } + } + }, + "_meta": { + "description": "default mappings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/logs-policy.json b/x-pack/plugin/core/src/main/resources/logs-policy.json new file mode 100644 index 0000000000000..936bfe6c0e2e3 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/logs-policy.json @@ -0,0 +1,12 @@ +{ + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "50gb", + "max_age": "30d" + } + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/logs-settings.json b/x-pack/plugin/core/src/main/resources/logs-settings.json new file mode 100644 index 0000000000000..7370e40accc79 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/logs-settings.json @@ -0,0 +1,20 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs" + }, + "codec": "best_compression", + "query": { + "default_field": ["message"] + } + } + } + }, + "_meta": { + "description": "default settings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/logs-template.json b/x-pack/plugin/core/src/main/resources/logs-template.json new file mode 100644 index 0000000000000..84546a9c772e2 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/logs-template.json @@ -0,0 +1,16 @@ +{ + "index_patterns": ["logs-*-*"], + "priority": 100, + "data_stream": { + "timestamp_field": "@timestamp" + }, + "composed_of": [ + "logs-mappings", + "logs-settings" + ], + "_meta": { + "description": "default logs template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/metrics-mappings.json b/x-pack/plugin/core/src/main/resources/metrics-mappings.json new file mode 100644 index 0000000000000..6ef2f6b07118f --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/metrics-mappings.json @@ -0,0 +1,57 @@ +{ + "template": { + "mappings": { + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "dataset": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "metrics" + }, + "name": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "ip": { + "type": "ip" + } + } + } + } + } + }, + "_meta": { + "description": "default mappings for the metrics index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/metrics-policy.json b/x-pack/plugin/core/src/main/resources/metrics-policy.json new file mode 100644 index 0000000000000..936bfe6c0e2e3 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/metrics-policy.json @@ -0,0 +1,12 @@ +{ + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "50gb", + "max_age": "30d" + } + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/metrics-settings.json b/x-pack/plugin/core/src/main/resources/metrics-settings.json new file mode 100644 index 0000000000000..1a13139bb18a4 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/metrics-settings.json @@ -0,0 +1,20 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "metrics" + }, + "codec": "best_compression", + "query": { + "default_field": ["message"] + } + } + } + }, + "_meta": { + "description": "default settings for the metrics index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/metrics-template.json b/x-pack/plugin/core/src/main/resources/metrics-template.json new file mode 100644 index 0000000000000..b13857c1c70fa --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/metrics-template.json @@ -0,0 +1,16 @@ +{ + "index_patterns": ["metrics-*-*"], + "priority": 100, + "data_stream": { + "timestamp_field": "@timestamp" + }, + "composed_of": [ + "metrics-mappings", + "metrics-settings" + ], + "_meta": { + "description": "default metrics template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java index 5c36bfc86410d..e97bb4cbde256 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java @@ -105,7 +105,7 @@ public void testDisabledDoesNotAddTemplates() { Settings settings = Settings.builder().put(SLM_HISTORY_INDEX_ENABLED_SETTING.getKey(), false).build(); SnapshotLifecycleTemplateRegistry disabledRegistry = new SnapshotLifecycleTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry); - assertThat(disabledRegistry.getTemplateConfigs(), hasSize(0)); + assertThat(disabledRegistry.getLegacyTemplateConfigs(), hasSize(0)); assertThat(disabledRegistry.getPolicyConfigs(), hasSize(0)); } @@ -119,7 +119,7 @@ public void testThatNonExistingTemplatesAreAddedImmediately() throws Exception { AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size()))); calledTimes.set(0); @@ -233,7 +233,7 @@ public void testThatVersionedOldTemplatesAreUpgraded() throws Exception { AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size()))); } public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception { @@ -244,7 +244,7 @@ public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception { AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size()))); } diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java index 80910b848e0a3..814fab25f06c2 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java @@ -182,9 +182,9 @@ public void testCCRUnfollowDuringSnapshot() throws Exception { } public void testCcrAndIlmWithRollover() throws Exception { - String alias = "metrics"; - String indexName = "metrics-000001"; - String nextIndexName = "metrics-000002"; + String alias = "mymetrics"; + String indexName = "mymetrics-000001"; + String nextIndexName = "mymetrics-000002"; String policyName = "rollover-test"; if ("leader".equals(targetCluster)) { @@ -197,7 +197,8 @@ public void testCcrAndIlmWithRollover() throws Exception { .put("index.lifecycle.name", policyName) .put("index.lifecycle.rollover_alias", alias) .build(); - templateRequest.setJsonEntity("{\"index_patterns\": [\"metrics-*\"], \"settings\": " + Strings.toString(indexSettings) + "}"); + templateRequest.setJsonEntity("{\"index_patterns\": [\"mymetrics-*\"], \"settings\": " + + Strings.toString(indexSettings) + "}"); assertOK(client().performRequest(templateRequest)); } else if ("follow".equals(targetCluster)) { // Policy with the same name must exist in follower cluster too: @@ -205,7 +206,7 @@ public void testCcrAndIlmWithRollover() throws Exception { // Set up an auto-follow pattern Request createAutoFollowRequest = new Request("PUT", "/_ccr/auto_follow/my_auto_follow_pattern"); - createAutoFollowRequest.setJsonEntity("{\"leader_index_patterns\": [\"metrics-*\"], " + + createAutoFollowRequest.setJsonEntity("{\"leader_index_patterns\": [\"mymetrics-*\"], " + "\"remote_cluster\": \"leader_cluster\", \"read_poll_timeout\": \"1000ms\"}"); assertOK(client().performRequest(createAutoFollowRequest)); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 7e624d5f9d101..82f9b0413fa73 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -139,7 +139,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin { private final SetOnce snapshotLifecycleService = new SetOnce<>(); private final SetOnce snapshotRetentionService = new SetOnce<>(); private final SetOnce snapshotHistoryStore = new SetOnce<>(); - private Settings settings; + private final Settings settings; private boolean transportClientMode; public IndexLifecycle(Settings settings) { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index 838da9d04045b..323828e6e8a1c 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -62,7 +62,7 @@ public ILMHistoryTemplateRegistry(Settings nodeSettings, ClusterService clusterS } @Override - protected List getTemplateConfigs() { + protected List getLegacyTemplateConfigs() { if (this.ilmHistoryEnabled) { return Collections.singletonList(TEMPLATE_ILM_HISTORY); } else { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java index 1fdc1bfbb9277..714816e0eb6ef 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java @@ -132,7 +132,7 @@ protected boolean requiresMasterNode() { } @Override - protected List getTemplateConfigs() { + protected List getLegacyTemplateConfigs() { return templatesToUse; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java index 77d8c9d9737bb..b804237c6d531 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java @@ -23,6 +23,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ENRICH_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.IDP_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.STACK_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; @@ -117,6 +118,7 @@ public static void switchUserBasedOnActionOriginAndExecute(ThreadContext threadC case INDEX_LIFECYCLE_ORIGIN: case ENRICH_ORIGIN: case IDP_ORIGIN: + case STACK_ORIGIN: case TASKS_ORIGIN: // TODO use a more limited user for tasks securityContext.executeAsUser(XPackUser.INSTANCE, consumer, Version.CURRENT); break; diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle new file mode 100644 index 0000000000000..bee72631e3892 --- /dev/null +++ b/x-pack/plugin/stack/build.gradle @@ -0,0 +1,29 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'x-pack-stack' + description 'Elasticsearch Expanded Pack Plugin - Stack' + classname 'org.elasticsearch.xpack.stack.StackPlugin' + extendedPlugins = ['x-pack-core'] + hasNativeController false + requiresKeystore true +} +archivesBaseName = 'x-pack-stack' + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +// add all sub-projects of the qa sub-project +gradle.projectsEvaluated { + project.subprojects + .find { it.path == project.path + ":qa" } + .subprojects + .findAll { it.path.startsWith(project.path + ":qa") } + .each { check.dependsOn it.check } +} + +integTest.enabled = false diff --git a/x-pack/plugin/stack/qa/build.gradle b/x-pack/plugin/stack/qa/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/stack/qa/rest/build.gradle b/x-pack/plugin/stack/qa/rest/build.gradle new file mode 100644 index 0000000000000..38c6a121b7d3e --- /dev/null +++ b/x-pack/plugin/stack/qa/rest/build.gradle @@ -0,0 +1,38 @@ +import org.elasticsearch.gradle.test.RestIntegTestTask + +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-test' + +dependencies { + testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') + testImplementation project(path: xpackModule('stack'), configuration: 'runtime') +} + +restResources { + restApi { + includeCore '_common', 'cluster', 'indices', 'index', 'snapshot' + includeXpack 'ilm', 'slm', 'stack' + } +} + +def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'), + password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')] + +task restTest(type: RestIntegTestTask) { + mustRunAfter(precommit) + runner { + systemProperty 'tests.rest.cluster.username', clusterCredentials.username + systemProperty 'tests.rest.cluster.password', clusterCredentials.password + } +} + +testClusters.restTest { + testDistribution = 'DEFAULT' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.security.enabled', 'true' + setting 'xpack.license.self_generated.type', 'trial' + user clusterCredentials +} + +check.dependsOn restTest +test.enabled = false diff --git a/x-pack/plugin/stack/qa/rest/src/test/java/org/elasticsearch/xpack/stack/StackYamlIT.java b/x-pack/plugin/stack/qa/rest/src/test/java/org/elasticsearch/xpack/stack/StackYamlIT.java new file mode 100644 index 0000000000000..2b7acf360e4a5 --- /dev/null +++ b/x-pack/plugin/stack/qa/rest/src/test/java/org/elasticsearch/xpack/stack/StackYamlIT.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.stack; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; +import org.apache.lucene.util.TimeUnits; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +import java.util.Objects; + +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; + +@TimeoutSuite(millis = 30 * TimeUnits.MINUTE) // as default timeout seems not enough on the jenkins VMs +public class StackYamlIT extends ESClientYamlSuiteTestCase { + + private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username")); + private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password")); + + public StackYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())); + return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build(); + } +} diff --git a/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml new file mode 100644 index 0000000000000..8a420a1022ccb --- /dev/null +++ b/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml @@ -0,0 +1,105 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + +--- +"Test stack template installation": + - do: + ilm.get_lifecycle: + policy: "logs" + + - do: + ilm.get_lifecycle: + policy: "metrics" + + - do: + cluster.get_component_template: + name: logs-mappings + + - do: + cluster.get_component_template: + name: logs-settings + + - do: + cluster.get_component_template: + name: metrics-mappings + + - do: + cluster.get_component_template: + name: metrics-settings + + - do: + indices.get_index_template: + name: logs + + - do: + indices.get_index_template: + name: metrics + +--- +"Test logs index auto creation": + - do: + index: + index: logs-foo-bar + body: + "@timestamp": "2020-01-01" + message: "test-log-message" + + - do: + indices.get_data_stream: + name: logs-foo-bar + + - match: { 0.name: logs-foo-bar } + - match: { 0.timestamp_field.name: '@timestamp' } + - match: { 0.generation: 1 } + - length: { 0.indices: 1 } + - match: { 0.indices.0.index_name: '.ds-logs-foo-bar-000001' } + + - do: + indices.get: + index: .ds-logs-foo-bar-000001 + + - is_true: \.ds-logs-foo-bar-000001.settings + - is_true: \.ds-logs-foo-bar-000001.mappings + - match: { \.ds-logs-foo-bar-000001.settings.index.lifecycle.name: "logs" } + - is_true: \.ds-logs-foo-bar-000001.mappings.properties.message + - match: { \.ds-logs-foo-bar-000001.data_stream: "logs-foo-bar" } + + - do: + indices.delete_data_stream: + name: logs-foo-bar + +--- +"Test metrics index auto creation": + - do: + index: + index: metrics-foo-bar + body: + "@timestamp": "2020-01-01" + message: "test-log-message" + + - do: + indices.get_data_stream: + name: metrics-foo-bar + + - match: { 0.name: metrics-foo-bar } + - match: { 0.timestamp_field.name: '@timestamp' } + - match: { 0.generation: 1 } + - length: { 0.indices: 1 } + - match: { 0.indices.0.index_name: '.ds-metrics-foo-bar-000001' } + + - do: + indices.get: + index: .ds-metrics-foo-bar-000001 + + - is_true: \.ds-metrics-foo-bar-000001.settings + - is_true: \.ds-metrics-foo-bar-000001.mappings + - match: { \.ds-metrics-foo-bar-000001.settings.index.lifecycle.name: "metrics" } + - is_true: \.ds-metrics-foo-bar-000001.mappings.properties.message + - match: { \.ds-metrics-foo-bar-000001.data_stream: "metrics-foo-bar" } + + - do: + indices.delete_data_stream: + name: metrics-foo-bar diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java new file mode 100644 index 0000000000000..3e491aced83dd --- /dev/null +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.stack; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +public class StackPlugin extends Plugin implements ActionPlugin { + private final Settings settings; + + public static final Setting STACK_TEMPLATES_ENABLED = Setting.boolSetting( + "stack.templates.enabled", + true, + Setting.Property.NodeScope + ); + + public StackPlugin(Settings settings) { + this.settings = settings; + } + + @Override + public List> getSettings() { + return Collections.singletonList(STACK_TEMPLATES_ENABLED); + } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + final List components = new ArrayList<>(); + StackTemplateRegistry templateRegistry = new StackTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry); + return components; + } +} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java new file mode 100644 index 0000000000000..fbdbaa1a9cd46 --- /dev/null +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.stack; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.template.IndexTemplateConfig; +import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class StackTemplateRegistry extends IndexTemplateRegistry { + // The stack template registry should remain at version 0. This is because templates and + // policies will be changed by the ingest manager once they exist, and ES should only ever put + // the template in place if it does not exist. If this were incremented we could accidentally + // overwrite a template or policy changed by the ingest manager. + public static final int REGISTRY_VERSION = 0; + + public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; + + private final boolean stackTemplateEnabled; + + ////////////////////////////////////////////////////////// + // Logs components (for matching logs-*-* indices) + ////////////////////////////////////////////////////////// + public static final String LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "logs-mappings"; + public static final String LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME = "logs-settings"; + public static final String LOGS_ILM_POLICY_NAME = "logs"; + public static final String LOGS_INDEX_TEMPLATE_NAME = "logs"; + + public static final IndexTemplateConfig LOGS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/logs-mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public static final IndexTemplateConfig LOGS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/logs-settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public static final LifecyclePolicyConfig LOGS_ILM_POLICY = new LifecyclePolicyConfig(LOGS_ILM_POLICY_NAME, "/logs-policy.json"); + public static final IndexTemplateConfig LOGS_INDEX_TEMPLATE = new IndexTemplateConfig( + LOGS_INDEX_TEMPLATE_NAME, + "/logs-template.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + + ////////////////////////////////////////////////////////// + // Metrics components (for matching metric-*-* indices) + ////////////////////////////////////////////////////////// + public static final String METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "metrics-mappings"; + public static final String METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME = "metrics-settings"; + public static final String METRICS_ILM_POLICY_NAME = "metrics"; + public static final String METRICS_INDEX_TEMPLATE_NAME = "metrics"; + + public static final IndexTemplateConfig METRICS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/metrics-mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public static final IndexTemplateConfig METRICS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/metrics-settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public static final LifecyclePolicyConfig METRICS_ILM_POLICY = new LifecyclePolicyConfig( + METRICS_ILM_POLICY_NAME, + "/metrics-policy.json" + ); + public static final IndexTemplateConfig METRICS_INDEX_TEMPLATE = new IndexTemplateConfig( + METRICS_INDEX_TEMPLATE_NAME, + "/metrics-template.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + + public StackTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + this.stackTemplateEnabled = StackPlugin.STACK_TEMPLATES_ENABLED.get(nodeSettings); + } + + @Override + protected List getPolicyConfigs() { + if (stackTemplateEnabled) { + return Arrays.asList(LOGS_ILM_POLICY, METRICS_ILM_POLICY); + } else { + return Collections.emptyList(); + } + } + + @Override + protected List getComponentTemplateConfigs() { + if (stackTemplateEnabled) { + return Arrays.asList( + LOGS_MAPPINGS_COMPONENT_TEMPLATE, + LOGS_SETTINGS_COMPONENT_TEMPLATE, + METRICS_MAPPINGS_COMPONENT_TEMPLATE, + METRICS_SETTINGS_COMPONENT_TEMPLATE + ); + } else { + return Collections.emptyList(); + } + } + + @Override + protected List getComposableTemplateConfigs() { + if (stackTemplateEnabled) { + return Arrays.asList(LOGS_INDEX_TEMPLATE, METRICS_INDEX_TEMPLATE); + } else { + return Collections.emptyList(); + } + } + + @Override + protected String getOrigin() { + return ClientHelper.STACK_ORIGIN; + } +} diff --git a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java new file mode 100644 index 0000000000000..a30ebbf7222f5 --- /dev/null +++ b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java @@ -0,0 +1,453 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.stack; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.ilm.LifecycleAction; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.ilm.LifecycleType; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.elasticsearch.mock.orig.Mockito.when; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +public class StackTemplateRegistryTests extends ESTestCase { + private StackTemplateRegistry registry; + private NamedXContentRegistry xContentRegistry; + private ClusterService clusterService; + private ThreadPool threadPool; + private VerifyingClient client; + + @Before + public void createRegistryAndClient() { + threadPool = new TestThreadPool(this.getClass().getName()); + client = new VerifyingClient(threadPool); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + List entries = new ArrayList<>(ClusterModule.getNamedXWriteables()); + entries.addAll( + Arrays.asList( + new NamedXContentRegistry.Entry( + LifecycleType.class, + new ParseField(TimeseriesLifecycleType.TYPE), + (p) -> TimeseriesLifecycleType.INSTANCE + ), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse) + ) + ); + xContentRegistry = new NamedXContentRegistry(entries); + registry = new StackTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, xContentRegistry); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testDisabledDoesNotAddTemplates() { + Settings settings = Settings.builder().put(StackPlugin.STACK_TEMPLATES_ENABLED.getKey(), false).build(); + StackTemplateRegistry disabledRegistry = new StackTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry); + assertThat(disabledRegistry.getComponentTemplateConfigs(), hasSize(0)); + assertThat(disabledRegistry.getComposableTemplateConfigs(), hasSize(0)); + assertThat(disabledRegistry.getPolicyConfigs(), hasSize(0)); + } + + public void testThatNonExistingTemplatesAreAddedImmediately() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); + + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); + + calledTimes.set(0); + + // attempting to register the event multiple times as a race condition can yield this test flaky, namely: + // when calling registry.clusterChanged(newEvent) the templateCreationsInProgress state that the IndexTemplateRegistry maintains + // might've not yet been updated to reflect that the first template registration was complete, so a second template registration + // will not be issued anymore, leaving calledTimes to 0 + assertBusy(() -> { + // now delete one template from the cluster state and lets retry + ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), nodes); + registry.clusterChanged(newEvent); + assertThat(calledTimes.get(), greaterThan(1)); + }); + } + + public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof PutLifecycleAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutLifecycleAction.class)); + assertThat(request, instanceOf(PutLifecycleAction.Request.class)); + final PutLifecycleAction.Request putRequest = (PutLifecycleAction.Request) request; + assertThat( + putRequest.getPolicy().getName(), + anyOf(equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME), equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME)) + ); + assertNotNull(listener); + return new PutLifecycleAction.Response(true); + } else if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return new StackTemplateRegistryTests.TestPutIndexTemplateResponse(true); + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return new AcknowledgedResponse(true); + } else { + fail("client called with unexpected request: " + request.toString()); + return null; + } + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(2))); + } + + public void testPolicyAlreadyExists() { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map policyMap = new HashMap<>(); + List policies = registry.getPolicyConfigs() + .stream() + .map(policyConfig -> policyConfig.load(xContentRegistry)) + .collect(Collectors.toList()); + assertThat(policies, hasSize(2)); + policies.forEach(p -> policyMap.put(p.getName(), p.get())); + + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return new AcknowledgedResponse(true); + } else if (action instanceof PutLifecycleAction) { + fail("if the policy already exists it should be re-put"); + } else { + fail("client called with unexpected request: " + request.toString()); + } + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); + registry.clusterChanged(event); + } + + public void testPolicyAlreadyExistsButDiffers() throws IOException { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map policyMap = new HashMap<>(); + String policyStr = "{\"phases\":{\"delete\":{\"min_age\":\"1m\",\"actions\":{\"delete\":{}}}}}"; + List policies = registry.getPolicyConfigs() + .stream() + .map(policyConfig -> policyConfig.load(xContentRegistry)) + .collect(Collectors.toList()); + assertThat(policies, hasSize(2)); + policies.forEach(p -> policyMap.put(p.getName(), p.get())); + + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return new AcknowledgedResponse(true); + } else if (action instanceof PutLifecycleAction) { + fail("if the policy already exists it should be re-put"); + } else { + fail("client called with unexpected request: " + request.toString()); + } + return null; + }); + + try ( + XContentParser parser = XContentType.JSON.xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.THROW_UNSUPPORTED_OPERATION, policyStr) + ) { + LifecyclePolicy different = LifecyclePolicy.parse(parser, policies.get(0).getName()); + policyMap.put(policies.get(0).getName(), different); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); + registry.clusterChanged(event); + } + } + + public void testThatVersionedOldTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.singletonMap( + StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION - 1 + ), + nodes + ); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); + } + + public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.singletonMap(StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, null), + nodes + ); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); + } + + @TestLogging(value = "org.elasticsearch.xpack.core.template:DEBUG", reason = "test") + public void testSameOrHigherVersionTemplateNotUpgraded() { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map versions = new HashMap<>(); + versions.put(StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + versions.put(StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + versions.put(StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + versions.put(StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes); + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + fail("template should not have been re-installed"); + return null; + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return new PutLifecycleAction.Response(true); + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return new AcknowledgedResponse(true); + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + }); + registry.clusterChanged(sameVersionEvent); + + versions.clear(); + versions.put( + StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); + versions.put( + StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); + versions.put( + StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); + versions.put( + StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); + ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(versions, nodes); + registry.clusterChanged(higherVersionEvent); + } + + public void testThatMissingMasterNodeDoesNothing() { + DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); + + client.setVerifier((a, r, l) -> { + fail("if the master is missing nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.singletonMap(StackTemplateRegistry.LOGS_INDEX_TEMPLATE_NAME, null), + nodes + ); + registry.clusterChanged(event); + } + + // ------------- + + /** + * A client that delegates to a verifying function for action/request/listener + */ + public static class VerifyingClient extends NoOpClient { + + private TriFunction, ActionRequest, ActionListener, ActionResponse> verifier = (a, r, l) -> { + fail("verifier not set"); + return null; + }; + + VerifyingClient(ThreadPool threadPool) { + super(threadPool); + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + try { + listener.onResponse((Response) verifier.apply(action, request, listener)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public VerifyingClient setVerifier(TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { + this.verifier = verifier; + return this; + } + } + + private ActionResponse verifyComponentTemplateInstalled( + AtomicInteger calledTimes, + ActionType action, + ActionRequest request, + ActionListener listener + ) { + if (action instanceof PutComponentTemplateAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutComponentTemplateAction.class)); + assertThat(request, instanceOf(PutComponentTemplateAction.Request.class)); + final PutComponentTemplateAction.Request putRequest = (PutComponentTemplateAction.Request) request; + assertThat(putRequest.componentTemplate().version(), equalTo((long) StackTemplateRegistry.REGISTRY_VERSION)); + assertNotNull(listener); + return new TestPutIndexTemplateResponse(true); + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return new PutLifecycleAction.Response(true); + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return new AcknowledgedResponse(true); + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + } + + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplates, DiscoveryNodes nodes) { + return createClusterChangedEvent(existingTemplates, Collections.emptyMap(), nodes); + } + + private ClusterChangedEvent createClusterChangedEvent( + Map existingTemplates, + Map existingPolicies, + DiscoveryNodes nodes + ) { + ClusterState cs = createClusterState(Settings.EMPTY, existingTemplates, existingPolicies, nodes); + ClusterChangedEvent realEvent = new ClusterChangedEvent( + "created-from-test", + cs, + ClusterState.builder(new ClusterName("test")).build() + ); + ClusterChangedEvent event = spy(realEvent); + when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster()); + + return event; + } + + private ClusterState createClusterState( + Settings nodeSettings, + Map existingComponentTemplates, + Map existingPolicies, + DiscoveryNodes nodes + ) { + Map componentTemplates = new HashMap<>(); + for (Map.Entry template : existingComponentTemplates.entrySet()) { + ComponentTemplate mockTemplate = mock(ComponentTemplate.class); + when(mockTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue()); + componentTemplates.put(template.getKey(), mockTemplate); + } + + Map existingILMMeta = existingPolicies.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new LifecyclePolicyMetadata(e.getValue(), Collections.emptyMap(), 1, 1))); + IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(existingILMMeta, OperationMode.RUNNING); + + return ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .componentTemplates(componentTemplates) + .transientSettings(nodeSettings) + .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta) + .build() + ) + .blocks(new ClusterBlocks.Builder().build()) + .nodes(nodes) + .build(); + } + + private static class TestPutIndexTemplateResponse extends AcknowledgedResponse { + TestPutIndexTemplateResponse(boolean acknowledged) { + super(acknowledged); + } + } +} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java index 1ac3f9939ca8f..10f65524a25f7 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java @@ -70,7 +70,7 @@ public WatcherIndexTemplateRegistry(Settings nodeSettings, ClusterService cluste } @Override - protected List getTemplateConfigs() { + protected List getLegacyTemplateConfigs() { if (clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0)) { return Arrays.asList( ilmManagementEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM, diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index 4ec8de9d290ce..ce415a817c81b 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -31,6 +31,7 @@ testClusters.integTest { setting 'xpack.watcher.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' + setting 'indices.lifecycle.history_index_enabled', 'false' user username: System.getProperty('tests.rest.cluster.username', 'test_user'), password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')