From 9f3af21d17bbcae7216e9d02789734f48b20283d Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 28 May 2020 15:04:37 -0600 Subject: [PATCH 01/20] Add default composable templates for new indexing strategy This commit adds the component and composable templates, as well as ILM policies, for the new default indexing strategy. It installs: - logs-default-mappings (component) - logs-default-settings (component) - logs-default-policy (ilm policy) - logs-default-template (composable template) - metrics-default-mappings (component) - metrics-default-settings (component) - metrics-default-policy (ilm policy) - metrics-default-template (composable template) These templates and policies are managed by a new x-pack module, `stack`, and can be disabled by setting `stack.templates.enabled` to `false`. These ensure that patterns for the `logs-*-*` and `metrics-*-*` indices are set up to create data streams with the proper mappings and settings. This also makes changes to the `IndexTemplateRegistry` to support installing component and composable templates (previously it supported only legacy templates). Resolves #56709 --- .../indices/PutComponentTemplateRequest.java | 4 + .../test/cat.aliases/10_basic.yml | 6 +- .../test/cat.aliases/40_hidden.yml | 3 +- .../test/cluster.stats/10_basic.yml | 54 --- .../indices.get_index_template/10_basic.yml | 5 +- .../test/indices.put_template/10_basic.yml | 10 +- .../admin/cluster/stats/ClusterStatsIT.java | 27 ++ .../MetadataIndexTemplateService.java | 2 +- .../rest/yaml/ESClientYamlSuiteTestCase.java | 9 + .../xpack/core/ClientHelper.java | 1 + .../SnapshotLifecycleTemplateRegistry.java | 4 +- .../core/template/IndexTemplateRegistry.java | 204 +++++++- .../main/resources/logs-default-mappings.json | 152 ++++++ .../main/resources/logs-default-policy.json | 13 + .../main/resources/logs-default-settings.json | 21 + .../main/resources/logs-default-template.json | 15 + .../resources/metrics-default-mappings.json | 149 ++++++ .../resources/metrics-default-policy.json | 13 + .../resources/metrics-default-settings.json | 21 + .../resources/metrics-default-template.json | 15 + ...napshotLifecycleTemplateRegistryTests.java | 8 +- .../xpack/ilm/CCRIndexLifecycleIT.java | 11 +- .../xpack/ilm/IndexLifecycle.java | 2 +- .../history/ILMHistoryTemplateRegistry.java | 2 +- .../xpack/ml/MlIndexTemplateRegistry.java | 2 +- .../security/authz/AuthorizationUtils.java | 2 + x-pack/plugin/stack/build.gradle | 29 ++ x-pack/plugin/stack/qa/build.gradle | 0 x-pack/plugin/stack/qa/rest/build.gradle | 38 ++ .../xpack/stack/StackYamlIT.java | 43 ++ .../rest-api-spec/test/stack/10_basic.yml | 107 +++++ .../xpack/stack/StackPlugin.java | 66 +++ .../xpack/stack/StackTemplateRegistry.java | 141 ++++++ .../stack/StackTemplateRegistryTests.java | 453 ++++++++++++++++++ .../support/WatcherIndexTemplateRegistry.java | 2 +- .../build.gradle | 1 + 36 files changed, 1545 insertions(+), 90 deletions(-) create mode 100644 x-pack/plugin/core/src/main/resources/logs-default-mappings.json create mode 100644 x-pack/plugin/core/src/main/resources/logs-default-policy.json create mode 100644 x-pack/plugin/core/src/main/resources/logs-default-settings.json create mode 100644 x-pack/plugin/core/src/main/resources/logs-default-template.json create mode 100644 x-pack/plugin/core/src/main/resources/metrics-default-mappings.json create mode 100644 x-pack/plugin/core/src/main/resources/metrics-default-policy.json create mode 100644 x-pack/plugin/core/src/main/resources/metrics-default-settings.json create mode 100644 x-pack/plugin/core/src/main/resources/metrics-default-template.json create mode 100644 x-pack/plugin/stack/build.gradle create mode 100644 x-pack/plugin/stack/qa/build.gradle create mode 100644 x-pack/plugin/stack/qa/rest/build.gradle create mode 100644 x-pack/plugin/stack/qa/rest/src/test/java/org/elasticsearch/xpack/stack/StackYamlIT.java create mode 100644 x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml create mode 100644 x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java create mode 100644 x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java create mode 100644 x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutComponentTemplateRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutComponentTemplateRequest.java index e37d44baee8d8..9dbf090807038 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutComponentTemplateRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutComponentTemplateRequest.java @@ -78,6 +78,10 @@ public PutComponentTemplateRequest componentTemplate(ComponentTemplate component return this; } + public ComponentTemplate componentTemplate() { + return this.componentTemplate; + } + /** * The cause for this component template creation. */ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml index 5669206ee87ad..3263a3d999052 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml @@ -294,6 +294,7 @@ - do: cat.aliases: v: true + name: test* - match: $body: | @@ -364,6 +365,7 @@ - do: cat.aliases: h: [index, alias] + name: test* - match: $body: /^ test \s+ test_1 \s+ $/ @@ -373,6 +375,7 @@ cat.aliases: h: [index, alias] v: true + name: test* - match: $body: | /^ @@ -398,7 +401,8 @@ index: test_index - do: - cat.aliases: {} + cat.aliases: + name: test* - match: $body: | diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml index 3aa7fdbb1f760..3fe58d7ea08f4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml @@ -73,7 +73,8 @@ aliases: test_alias: {} - do: - cat.aliases: {} + cat.aliases: + name: test* - match: $body: | diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml index 236053236da85..c58d959c934ca 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml @@ -83,57 +83,3 @@ cluster.stats: {} - is_true: nodes.packaging_types - ---- -"get cluster stats returns mapping stats": - - - skip: - version: " - 7.6.99" - reason: "mapping stats are added for v7.7.0" - - - do: - cluster.stats: {} - - - length: { indices.mappings.field_types: 0 } - - - do: - indices.create: - index: test-index1 - body: - mappings: - properties: - foo: - type: keyword - - - do: - indices.create: - index: test-index2 - body: - mappings: - properties: - foo: - type: keyword - bar: - properties: - quux: - type: integer - baz: - type: keyword - - - do: - cluster.stats: {} - - - length: { indices.mappings.field_types: 3 } - - - match: { indices.mappings.field_types.0.name: integer } - - match: { indices.mappings.field_types.0.count: 1 } - - match: { indices.mappings.field_types.0.index_count: 1 } - - - match: { indices.mappings.field_types.1.name: keyword } - - match: { indices.mappings.field_types.1.count: 3 } - - match: { indices.mappings.field_types.1.index_count: 2 } - - - match: { indices.mappings.field_types.2.name: object } - - match: { indices.mappings.field_types.2.count: 1 } - - match: { indices.mappings.field_types.2.index_count: 1 } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml index 29eec542535ab..bd17f3143bf24 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml @@ -36,7 +36,7 @@ setup: - match: {index_templates.0.index_template.template.mappings: {properties: {field: {type: keyword}}}} --- -"Get all tindex emplates": +"Get all index templates": - skip: version: " - 7.7.99" reason: "index template v2 API unavailable before 7.8" @@ -56,7 +56,8 @@ setup: - do: indices.get_index_template: {} - - length: {index_templates: 2} + - is_true: index_templates.0.name + - is_true: index_templates.1.name --- "Get index template with local flag": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml index e5e298ee83d56..ff35707f1058f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml @@ -133,7 +133,7 @@ body: > { "version": 10, - "index_patterns": "*", + "index_patterns": "foo*", "settings": { "number_of_shards": 1 } } - match: { acknowledged: true } @@ -150,7 +150,7 @@ body: > { "version": 9, - "index_patterns": "*", + "index_patterns": "foo*", "settings": { "number_of_shards": 1 } } - match: { acknowledged: true } @@ -167,7 +167,7 @@ body: > { "version": 6789, - "index_patterns": "*", + "index_patterns": "foo*", "settings": { "number_of_shards": 1 } } - match: { acknowledged: true } @@ -183,7 +183,7 @@ name: "my_template" body: > { - "index_patterns": "*", + "index_patterns": "foo*", "settings": { "number_of_shards": 1 } } - match: { acknowledged: true } @@ -200,7 +200,7 @@ body: > { "version": 5385, - "index_patterns": "*", + "index_patterns": "foo*", "settings": { "number_of_shards": 1 } } - match: { acknowledged: true } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java index 6e7454cd44b83..cb2f97743d4ef 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -39,9 +39,11 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) @@ -234,4 +236,29 @@ public void testClusterStatusWhenStateNotRecovered() throws Exception { response = client().admin().cluster().prepareClusterStats().get(); assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); } + + public void testFieldTypes() { + internalCluster().startNode(); + ensureGreen(); + ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); + assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); + assertTrue(response.getIndicesStats().getMappings().getFieldTypeStats().isEmpty()); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + client().admin().indices().prepareCreate("test2") + .setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"},\"bar\":{\"properties\":{\"baz\":{\"type\":\"keyword\"}," + + "\"eggplant\":{\"type\":\"integer\"}}}}}").get(); + response = client().admin().cluster().prepareClusterStats().get(); + assertThat(response.getIndicesStats().getMappings().getFieldTypeStats().size(), equalTo(3)); + Set stats = response.getIndicesStats().getMappings().getFieldTypeStats(); + for (IndexFeatureStats stat : stats) { + if (stat.getName().equals("integer")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(1)); + } else if (stat.getName().equals("keyword")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(3)); + } else if (stat.getName().equals("object")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(1)); + } + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index caa15eb560830..a1898f44812f5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -441,7 +441,7 @@ public ClusterState addIndexTemplateV2(final ClusterState currentState, final bo } validate(name, finalIndexTemplate); - logger.info("adding index template [{}]", name); + logger.info("adding index template [{}] for index patterns {}", name, template.indexPatterns()); return ClusterState.builder(currentState) .metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate)) .build(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 5dbfbd0325329..7dc3a4161104c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -30,6 +30,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.client.sniff.ElasticsearchNodesSniffer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -364,6 +365,14 @@ public void test() throws IOException { && testCandidate.getTestSection().getSkipSection().getFeatures().contains("default_shards") == false) { final Request request = new Request("PUT", "/_template/global"); request.setJsonEntity("{\"index_patterns\":[\"*\"],\"settings\":{\"index.number_of_shards\":2}}"); + // Because this has not yet transitioned to a composable template, it's possible that + // this can overlap an installed composable template since this is a global (*) + // template. In order to avoid this failing the test, we override the warnings handler + // to be permissive in this case. This can be removed once all tests use composable + // templates instead of legacy templates + RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.setWarningsHandler(WarningsHandler.PERMISSIVE); + request.setOptions(builder.build()); adminClient().performRequest(request); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java index 49a457f0a7f55..e5fa39aac8e62 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java @@ -54,6 +54,7 @@ public final class ClientHelper { public static final String TRANSFORM_ORIGIN = "transform"; public static final String ASYNC_SEARCH_ORIGIN = "async_search"; public static final String IDP_ORIGIN = "idp"; + public static final String STACK_ORIGIN = "stack"; private ClientHelper() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java index da35b2b1745cb..908d469d0eba8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java @@ -69,7 +69,7 @@ public SnapshotLifecycleTemplateRegistry(Settings nodeSettings, ClusterService c } @Override - protected List getTemplateConfigs() { + protected List getLegacyTemplateConfigs() { if (slmHistoryEnabled == false) { return Collections.emptyList(); } @@ -90,7 +90,7 @@ protected String getOrigin() { } public boolean validate(ClusterState state) { - boolean allTemplatesPresent = getTemplateConfigs().stream() + boolean allTemplatesPresent = getLegacyTemplateConfigs().stream() .map(IndexTemplateConfig::getTemplateName) .allMatch(name -> state.metadata().getTemplates().containsKey(name)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java index 1d53c5d43b8c1..41f6559721f4e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java @@ -9,29 +9,39 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; @@ -67,14 +77,38 @@ public IndexTemplateRegistry(Settings nodeSettings, ClusterService clusterServic * the index templates that should be installed and managed. * @return The configurations for the templates that should be installed. */ - protected abstract List getTemplateConfigs(); + protected List getLegacyTemplateConfigs() { + return Collections.emptyList(); + } + + /** + * Retrieves return a list of {@link IndexTemplateConfig} that represents + * the component templates that should be installed and managed. Component + * templates are always installed prior composable templates, so they may + * be referenced by a composable template. + * @return The configurations for the templates that should be installed. + */ + protected List getComponentTemplateConfigs() { + return Collections.emptyList(); + } + + /** + * Retrieves return a list of {@link IndexTemplateConfig} that represents + * the composable templates that should be installed and managed. + * @return The configurations for the templates that should be installed. + */ + protected List getComposableTemplateConfigs() { + return Collections.emptyList(); + } /** * Retrieves a list of {@link LifecyclePolicyConfig} that represents the ILM * policies that should be installed and managed. Only called if ILM is enabled. * @return The configurations for the lifecycle policies that should be installed. */ - protected abstract List getPolicyConfigs(); + protected List getPolicyConfigs() { + return Collections.emptyList(); + } /** * Retrieves an identifier that is used to identify which plugin is asking for this. @@ -144,34 +178,116 @@ protected boolean requiresMasterNode() { } private void addTemplatesIfMissing(ClusterState state) { - final List indexTemplates = getTemplateConfigs(); + addLegacyTemplatesIfMissing(state); + addComponentTemplatesIfMissing(state); + addComposableTemplatesIfMissing(state); + } + + private void addLegacyTemplatesIfMissing(ClusterState state) { + final List indexTemplates = getLegacyTemplateConfigs(); for (IndexTemplateConfig newTemplate : indexTemplates) { final String templateName = newTemplate.getTemplateName(); final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false)); if (creationCheck.compareAndSet(false, true)) { IndexTemplateMetadata currentTemplate = state.metadata().getTemplates().get(templateName); if (Objects.isNull(currentTemplate)) { - logger.debug("adding index template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); - putTemplate(newTemplate, creationCheck); + logger.debug("adding legacy template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); + putLegacyTemplate(newTemplate, creationCheck); } else if (Objects.isNull(currentTemplate.getVersion()) || newTemplate.getVersion() > currentTemplate.getVersion()) { // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can // safely assume it's an old version of the template. - logger.info("upgrading index template [{}] for [{}] from version [{}] to version [{}]", + logger.info("upgrading legacy template [{}] for [{}] from version [{}] to version [{}]", templateName, getOrigin(), currentTemplate.getVersion(), newTemplate.getVersion()); - putTemplate(newTemplate, creationCheck); + putLegacyTemplate(newTemplate, creationCheck); } else { creationCheck.set(false); - logger.trace("not adding index template [{}] for [{}], because it already exists at version [{}]", + logger.trace("not adding legacy template [{}] for [{}], because it already exists at version [{}]", templateName, getOrigin(), currentTemplate.getVersion()); } } else { - logger.trace("skipping the creation of index template [{}] for [{}], because its creation is in progress", + logger.trace("skipping the creation of legacy template [{}] for [{}], because its creation is in progress", templateName, getOrigin()); } } } - private void putTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) { + private void addComponentTemplatesIfMissing(ClusterState state) { + final List indexTemplates = getComponentTemplateConfigs(); + for (IndexTemplateConfig newTemplate : indexTemplates) { + final String templateName = newTemplate.getTemplateName(); + final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false)); + if (creationCheck.compareAndSet(false, true)) { + ComponentTemplate currentTemplate = state.metadata().componentTemplates().get(templateName); + if (Objects.isNull(currentTemplate)) { + logger.debug("adding component template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); + putComponentTemplate(newTemplate, creationCheck); + } else if (Objects.isNull(currentTemplate.version()) || newTemplate.getVersion() > currentTemplate.version()) { + // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can + // safely assume it's an old version of the template. + logger.info("upgrading component template [{}] for [{}] from version [{}] to version [{}]", + templateName, getOrigin(), currentTemplate.version(), newTemplate.getVersion()); + putComponentTemplate(newTemplate, creationCheck); + } else { + creationCheck.set(false); + logger.trace("not adding component template [{}] for [{}], because it already exists at version [{}]", + templateName, getOrigin(), currentTemplate.version()); + } + } else { + logger.trace("skipping the creation of component template [{}] for [{}], because its creation is in progress", + templateName, getOrigin()); + } + } + } + + private void addComposableTemplatesIfMissing(ClusterState state) { + final List indexTemplates = getComposableTemplateConfigs(); + for (IndexTemplateConfig newTemplate : indexTemplates) { + final String templateName = newTemplate.getTemplateName(); + final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false)); + if (creationCheck.compareAndSet(false, true)) { + ComposableIndexTemplate currentTemplate = state.metadata().templatesV2().get(templateName); + boolean componentTemplatesAvailable = componentTemplatesExist(state, newTemplate); + if (componentTemplatesAvailable == false) { + creationCheck.set(false); + logger.trace("not adding composable template [{}] for [{}] because its required component templates do not exist", + templateName, getOrigin()); + } else if (Objects.isNull(currentTemplate)) { + logger.debug("adding composable template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); + putComposableTemplate(newTemplate, creationCheck); + } else if (Objects.isNull(currentTemplate.version()) || newTemplate.getVersion() > currentTemplate.version()) { + // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can + // safely assume it's an old version of the template. + logger.info("upgrading composable template [{}] for [{}] from version [{}] to version [{}]", + templateName, getOrigin(), currentTemplate.version(), newTemplate.getVersion()); + putComposableTemplate(newTemplate, creationCheck); + } else { + creationCheck.set(false); + logger.trace("not adding composable template [{}] for [{}], because it already exists at version [{}]", + templateName, getOrigin(), currentTemplate.version()); + } + } else { + logger.trace("skipping the creation of composable template [{}] for [{}], because its creation is in progress", + templateName, getOrigin()); + } + } + } + + /** + * Returns true if the cluster state contains all of the component templates needed by the composable template + */ + private static boolean componentTemplatesExist(ClusterState state, IndexTemplateConfig composableTemplate) { + final ComposableIndexTemplate indexTemplate; + try { + indexTemplate = ComposableIndexTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, composableTemplate.loadBytes())); + } catch (Exception e) { + throw new ElasticsearchParseException("unable to parse composable template " + composableTemplate.getTemplateName(), e); + } + Set neededComponents = new HashSet<>(indexTemplate.composedOf()); + return state.metadata().componentTemplates().keySet().containsAll(neededComponents); + } + + private void putLegacyTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) { final Executor executor = threadPool.generic(); executor.execute(() -> { final String templateName = config.getTemplateName(); @@ -184,7 +300,7 @@ private void putTemplate(final IndexTemplateConfig config, final AtomicBoolean c public void onResponse(AcknowledgedResponse response) { creationCheck.set(false); if (response.isAcknowledged() == false) { - logger.error("error adding index template [{}] for [{}], request was not acknowledged", + logger.error("error adding legacy template [{}] for [{}], request was not acknowledged", templateName, getOrigin()); } } @@ -198,6 +314,72 @@ public void onFailure(Exception e) { }); } + private void putComponentTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) { + final Executor executor = threadPool.generic(); + executor.execute(() -> { + final String templateName = config.getTemplateName(); + + PutComponentTemplateAction.Request request = new PutComponentTemplateAction.Request(templateName); + try { + request.componentTemplate(ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, config.loadBytes()))); + } catch (Exception e) { + throw new ElasticsearchParseException("unable to parse component template " + config.getTemplateName(), e); + } + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); + executeAsyncWithOrigin(client.threadPool().getThreadContext(), getOrigin(), request, + new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse response) { + creationCheck.set(false); + if (response.isAcknowledged() == false) { + logger.error("error adding component template [{}] for [{}], request was not acknowledged", + templateName, getOrigin()); + } + } + + @Override + public void onFailure(Exception e) { + creationCheck.set(false); + onPutTemplateFailure(config, e); + } + }, (req, listener) -> client.execute(PutComponentTemplateAction.INSTANCE, req, listener)); + }); + } + + private void putComposableTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) { + final Executor executor = threadPool.generic(); + executor.execute(() -> { + final String templateName = config.getTemplateName(); + + PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(templateName); + try { + request.indexTemplate(ComposableIndexTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, config.loadBytes()))); + } catch (Exception e) { + throw new ElasticsearchParseException("unable to parse composable template " + config.getTemplateName(), e); + } + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); + executeAsyncWithOrigin(client.threadPool().getThreadContext(), getOrigin(), request, + new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse response) { + creationCheck.set(false); + if (response.isAcknowledged() == false) { + logger.error("error adding composable template [{}] for [{}], request was not acknowledged", + templateName, getOrigin()); + } + } + + @Override + public void onFailure(Exception e) { + creationCheck.set(false); + onPutTemplateFailure(config, e); + } + }, (req, listener) -> client.execute(PutComposableIndexTemplateAction.INSTANCE, req, listener)); + }); + } + private void addIndexLifecyclePoliciesIfMissing(ClusterState state) { Optional maybeMeta = Optional.ofNullable(state.metadata().custom(IndexLifecycleMetadata.TYPE)); diff --git a/x-pack/plugin/core/src/main/resources/logs-default-mappings.json b/x-pack/plugin/core/src/main/resources/logs-default-mappings.json new file mode 100644 index 0000000000000..bbf0a8ff8f1c6 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/logs-default-mappings.json @@ -0,0 +1,152 @@ +{ + "template": { + "mappings": { + "_meta": { + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "stream": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "logs" + }, + "dataset": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + } + } + }, + "agent": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "containerized": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "architecture": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "message": { + "type": "text" + } + } + } + }, + "_meta": { + "description": "default mappings for the logs index template installed by x-pack" + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/logs-default-policy.json b/x-pack/plugin/core/src/main/resources/logs-default-policy.json new file mode 100644 index 0000000000000..06fb976bfffa3 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/logs-default-policy.json @@ -0,0 +1,13 @@ +{ + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50gb", + "max_age": "30d" + } + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/logs-default-settings.json b/x-pack/plugin/core/src/main/resources/logs-default-settings.json new file mode 100644 index 0000000000000..98be32f544753 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/logs-default-settings.json @@ -0,0 +1,21 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs-default-policy" + }, + "codec": "best_compression", + "refresh_interval": "5s", + "query": { + "default_field": ["message"] + }, + "number_of_routing_shards": "30" + } + } + }, + "_meta": { + "description": "default settings for the logs index template installed by x-pack" + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/logs-default-template.json b/x-pack/plugin/core/src/main/resources/logs-default-template.json new file mode 100644 index 0000000000000..72810f24b30e1 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/logs-default-template.json @@ -0,0 +1,15 @@ +{ + "index_patterns": ["logs-*-*"], + "priority": 100, + "data_stream": { + "timestamp_field": "@timestamp" + }, + "composed_of": [ + "logs-default-mappings", + "logs-default-settings" + ], + "_meta": { + "description": "default logs template installed by x-pack" + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-mappings.json b/x-pack/plugin/core/src/main/resources/metrics-default-mappings.json new file mode 100644 index 0000000000000..0b04211565ff4 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/metrics-default-mappings.json @@ -0,0 +1,149 @@ +{ + "template": { + "mappings": { + "_meta": { + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "stream": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "metrics" + }, + "dataset": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + } + } + }, + "agent": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "containerized": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "architecture": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + }, + "_meta": { + "description": "default mappings for the metrics index template installed by x-pack" + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-policy.json b/x-pack/plugin/core/src/main/resources/metrics-default-policy.json new file mode 100644 index 0000000000000..06fb976bfffa3 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/metrics-default-policy.json @@ -0,0 +1,13 @@ +{ + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50gb", + "max_age": "30d" + } + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-settings.json b/x-pack/plugin/core/src/main/resources/metrics-default-settings.json new file mode 100644 index 0000000000000..8082c4a6a0ef2 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/metrics-default-settings.json @@ -0,0 +1,21 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "metrics-default-policy" + }, + "codec": "best_compression", + "refresh_interval": "5s", + "query": { + "default_field": ["message"] + }, + "number_of_routing_shards": "30" + } + } + }, + "_meta": { + "description": "default settings for the metrics index template installed by x-pack" + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-template.json b/x-pack/plugin/core/src/main/resources/metrics-default-template.json new file mode 100644 index 0000000000000..5a29d77e826c1 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/metrics-default-template.json @@ -0,0 +1,15 @@ +{ + "index_patterns": ["metrics-*-*"], + "priority": 100, + "data_stream": { + "timestamp_field": "@timestamp" + }, + "composed_of": [ + "metrics-default-mappings", + "metrics-default-settings" + ], + "_meta": { + "description": "default metrics template installed by x-pack" + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java index 8b8cbffb02d26..7e9e3c900a871 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java @@ -105,7 +105,7 @@ public void testDisabledDoesNotAddTemplates() { Settings settings = Settings.builder().put(SLM_HISTORY_INDEX_ENABLED_SETTING.getKey(), false).build(); SnapshotLifecycleTemplateRegistry disabledRegistry = new SnapshotLifecycleTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry); - assertThat(disabledRegistry.getTemplateConfigs(), hasSize(0)); + assertThat(disabledRegistry.getLegacyTemplateConfigs(), hasSize(0)); assertThat(disabledRegistry.getPolicyConfigs(), hasSize(0)); } @@ -118,7 +118,7 @@ public void testThatNonExistingTemplatesAreAddedImmediately() throws Exception { AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size()))); calledTimes.set(0); @@ -232,7 +232,7 @@ public void testThatVersionedOldTemplatesAreUpgraded() throws Exception { AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size()))); } public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception { @@ -243,7 +243,7 @@ public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception { AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size()))); } diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java index acc5c12fbf0b0..7dd8dea4cba93 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java @@ -179,9 +179,9 @@ public void testCCRUnfollowDuringSnapshot() throws Exception { } public void testCcrAndIlmWithRollover() throws Exception { - String alias = "metrics"; - String indexName = "metrics-000001"; - String nextIndexName = "metrics-000002"; + String alias = "mymetrics"; + String indexName = "mymetrics-000001"; + String nextIndexName = "mymetrics-000002"; String policyName = "rollover-test"; if ("leader".equals(targetCluster)) { @@ -194,7 +194,8 @@ public void testCcrAndIlmWithRollover() throws Exception { .put("index.lifecycle.name", policyName) .put("index.lifecycle.rollover_alias", alias) .build(); - templateRequest.setJsonEntity("{\"index_patterns\": [\"metrics-*\"], \"settings\": " + Strings.toString(indexSettings) + "}"); + templateRequest.setJsonEntity("{\"index_patterns\": [\"mymetrics-*\"], \"settings\": " + + Strings.toString(indexSettings) + "}"); assertOK(client().performRequest(templateRequest)); } else if ("follow".equals(targetCluster)) { // Policy with the same name must exist in follower cluster too: @@ -202,7 +203,7 @@ public void testCcrAndIlmWithRollover() throws Exception { // Set up an auto-follow pattern Request createAutoFollowRequest = new Request("PUT", "/_ccr/auto_follow/my_auto_follow_pattern"); - createAutoFollowRequest.setJsonEntity("{\"leader_index_patterns\": [\"metrics-*\"], " + + createAutoFollowRequest.setJsonEntity("{\"leader_index_patterns\": [\"mymetrics-*\"], " + "\"remote_cluster\": \"leader_cluster\", \"read_poll_timeout\": \"1000ms\"}"); assertOK(client().performRequest(createAutoFollowRequest)); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 699155639e0ca..7e73af21fcbfb 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -140,7 +140,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin { private final SetOnce snapshotLifecycleService = new SetOnce<>(); private final SetOnce snapshotRetentionService = new SetOnce<>(); private final SetOnce snapshotHistoryStore = new SetOnce<>(); - private Settings settings; + private final Settings settings; public IndexLifecycle(Settings settings) { this.settings = settings; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index 838da9d04045b..323828e6e8a1c 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -62,7 +62,7 @@ public ILMHistoryTemplateRegistry(Settings nodeSettings, ClusterService clusterS } @Override - protected List getTemplateConfigs() { + protected List getLegacyTemplateConfigs() { if (this.ilmHistoryEnabled) { return Collections.singletonList(TEMPLATE_ILM_HISTORY); } else { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java index 840034b7b2ba9..7a57b776b51a8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java @@ -132,7 +132,7 @@ protected boolean requiresMasterNode() { } @Override - protected List getTemplateConfigs() { + protected List getLegacyTemplateConfigs() { return templatesToUse; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java index 77d8c9d9737bb..b804237c6d531 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java @@ -23,6 +23,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ENRICH_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.IDP_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.STACK_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; @@ -117,6 +118,7 @@ public static void switchUserBasedOnActionOriginAndExecute(ThreadContext threadC case INDEX_LIFECYCLE_ORIGIN: case ENRICH_ORIGIN: case IDP_ORIGIN: + case STACK_ORIGIN: case TASKS_ORIGIN: // TODO use a more limited user for tasks securityContext.executeAsUser(XPackUser.INSTANCE, consumer, Version.CURRENT); break; diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle new file mode 100644 index 0000000000000..ccda33f8b7cab --- /dev/null +++ b/x-pack/plugin/stack/build.gradle @@ -0,0 +1,29 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'x-pack-stack' + description 'Elasticsearch Expanded Pack Plugin - Stack' + classname 'org.elasticsearch.xpack.stack.StackPlugin' + extendedPlugins = ['x-pack-core'] + hasNativeController false + requiresKeystore true +} +archivesBaseName = 'x-pack-stack' + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +// add all sub-projects of the qa sub-project +// gradle.projectsEvaluated { +// project.subprojects +// .find { it.path == project.path + ":qa" } +// .subprojects +// .findAll { it.path.startsWith(project.path + ":qa") } +// .each { check.dependsOn it.check } +// } + +integTest.enabled = false diff --git a/x-pack/plugin/stack/qa/build.gradle b/x-pack/plugin/stack/qa/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/stack/qa/rest/build.gradle b/x-pack/plugin/stack/qa/rest/build.gradle new file mode 100644 index 0000000000000..5c337853cfc4e --- /dev/null +++ b/x-pack/plugin/stack/qa/rest/build.gradle @@ -0,0 +1,38 @@ +import org.elasticsearch.gradle.test.RestIntegTestTask + +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-test' + +dependencies { + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + testCompile project(path: xpackModule('stack'), configuration: 'runtime') +} + +restResources { + restApi { + includeCore '_common', 'cluster', 'indices', 'index', 'snapshot' + includeXpack 'ilm', 'slm', 'stack' + } +} + +def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'), + password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')] + +task restTest(type: RestIntegTestTask) { + mustRunAfter(precommit) + runner { + systemProperty 'tests.rest.cluster.username', clusterCredentials.username + systemProperty 'tests.rest.cluster.password', clusterCredentials.password + } +} + +testClusters.restTest { + testDistribution = 'DEFAULT' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.security.enabled', 'true' + setting 'xpack.license.self_generated.type', 'trial' + user clusterCredentials +} + +check.dependsOn restTest +test.enabled = false diff --git a/x-pack/plugin/stack/qa/rest/src/test/java/org/elasticsearch/xpack/stack/StackYamlIT.java b/x-pack/plugin/stack/qa/rest/src/test/java/org/elasticsearch/xpack/stack/StackYamlIT.java new file mode 100644 index 0000000000000..2b7acf360e4a5 --- /dev/null +++ b/x-pack/plugin/stack/qa/rest/src/test/java/org/elasticsearch/xpack/stack/StackYamlIT.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.stack; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; +import org.apache.lucene.util.TimeUnits; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +import java.util.Objects; + +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; + +@TimeoutSuite(millis = 30 * TimeUnits.MINUTE) // as default timeout seems not enough on the jenkins VMs +public class StackYamlIT extends ESClientYamlSuiteTestCase { + + private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username")); + private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password")); + + public StackYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())); + return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build(); + } +} diff --git a/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml new file mode 100644 index 0000000000000..45df4ef3df255 --- /dev/null +++ b/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml @@ -0,0 +1,107 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + +--- +"Test stack template installation": + - do: + ilm.get_lifecycle: + policy: "logs-default-policy" + + - do: + ilm.get_lifecycle: + policy: "metrics-default-policy" + + - do: + cluster.get_component_template: + name: logs-default-mappings + + - do: + cluster.get_component_template: + name: logs-default-settings + + - do: + cluster.get_component_template: + name: metrics-default-mappings + + - do: + cluster.get_component_template: + name: metrics-default-settings + + - do: + indices.get_index_template: + name: logs-default-template + + - do: + indices.get_index_template: + name: metrics-default-template + +--- +"Test logs index auto creation": + - do: + index: + index: logs-foo-bar + body: + "@timestamp": "2020-01-01" + message: "test-log-message" + + - do: + indices.get_data_stream: + name: logs-foo-bar + + - match: { 0.name: logs-foo-bar } + - match: { 0.timestamp_field: '@timestamp' } + - match: { 0.generation: 1 } + - length: { 0.indices: 1 } + - match: { 0.indices.0.index_name: 'logs-foo-bar-000001' } + + - do: + indices.get: + index: logs-foo-bar-000001 + + - is_true: logs-foo-bar-000001.aliases + - is_true: logs-foo-bar-000001.settings + - is_true: logs-foo-bar-000001.mappings + - match: { logs-foo-bar-000001.settings.index.lifecycle.name: "logs-default-policy" } + - is_true: logs-foo-bar-000001.mappings.properties.message + - match: { logs-foo-bar-000001.data_stream: "logs-foo-bar" } + + - do: + indices.delete_data_stream: + name: logs-foo-bar + +--- +"Test metrics index auto creation": + - do: + index: + index: metrics-foo-bar + body: + "@timestamp": "2020-01-01" + message: "test-log-message" + + - do: + indices.get_data_stream: + name: metrics-foo-bar + + - match: { 0.name: metrics-foo-bar } + - match: { 0.timestamp_field: '@timestamp' } + - match: { 0.generation: 1 } + - length: { 0.indices: 1 } + - match: { 0.indices.0.index_name: 'metrics-foo-bar-000001' } + + - do: + indices.get: + index: metrics-foo-bar-000001 + + - is_true: metrics-foo-bar-000001.aliases + - is_true: metrics-foo-bar-000001.settings + - is_true: metrics-foo-bar-000001.mappings + - match: { metrics-foo-bar-000001.settings.index.lifecycle.name: "metrics-default-policy" } + - is_true: metrics-foo-bar-000001.mappings.properties.message + - match: { metrics-foo-bar-000001.data_stream: "metrics-foo-bar" } + + - do: + indices.delete_data_stream: + name: metrics-foo-bar diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java new file mode 100644 index 0000000000000..3e491aced83dd --- /dev/null +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.stack; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +public class StackPlugin extends Plugin implements ActionPlugin { + private final Settings settings; + + public static final Setting STACK_TEMPLATES_ENABLED = Setting.boolSetting( + "stack.templates.enabled", + true, + Setting.Property.NodeScope + ); + + public StackPlugin(Settings settings) { + this.settings = settings; + } + + @Override + public List> getSettings() { + return Collections.singletonList(STACK_TEMPLATES_ENABLED); + } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + final List components = new ArrayList<>(); + StackTemplateRegistry templateRegistry = new StackTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry); + return components; + } +} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java new file mode 100644 index 0000000000000..30a74568b0f84 --- /dev/null +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.stack; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.template.IndexTemplateConfig; +import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class StackTemplateRegistry extends IndexTemplateRegistry { + // history (please add a comment why you increased the version here) + // version 1: initial + public static final int REGISTRY_VERSION = 1; + + public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; + + private final boolean stackTemplateEnabled; + + ////////////////////////////////////////////////////////// + // Logs components (for matching logs-*-* indices) + ////////////////////////////////////////////////////////// + public static final String LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "logs-default-mappings"; + public static final String LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME = "logs-default-settings"; + public static final String LOGS_ILM_POLICY_NAME = "logs-default-policy"; + public static final String LOGS_INDEX_TEMPLATE_NAME = "logs-default-template"; + + public static final IndexTemplateConfig LOGS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/logs-default-mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public static final IndexTemplateConfig LOGS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/logs-default-settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public static final LifecyclePolicyConfig LOGS_ILM_POLICY = new LifecyclePolicyConfig( + LOGS_ILM_POLICY_NAME, + "/logs-default-policy.json" + ); + public static final IndexTemplateConfig LOGS_INDEX_TEMPLATE = new IndexTemplateConfig( + LOGS_INDEX_TEMPLATE_NAME, + "/logs-default-template.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + + ////////////////////////////////////////////////////////// + // Metrics components (for matching metric-*-* indices) + ////////////////////////////////////////////////////////// + public static final String METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "metrics-default-mappings"; + public static final String METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME = "metrics-default-settings"; + public static final String METRICS_ILM_POLICY_NAME = "metrics-default-policy"; + public static final String METRICS_INDEX_TEMPLATE_NAME = "metrics-default-template"; + + public static final IndexTemplateConfig METRICS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/metrics-default-mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public static final IndexTemplateConfig METRICS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/metrics-default-settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public static final LifecyclePolicyConfig METRICS_ILM_POLICY = new LifecyclePolicyConfig( + METRICS_ILM_POLICY_NAME, + "/metrics-default-policy.json" + ); + public static final IndexTemplateConfig METRICS_INDEX_TEMPLATE = new IndexTemplateConfig( + METRICS_INDEX_TEMPLATE_NAME, + "/metrics-default-template.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + + public StackTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + this.stackTemplateEnabled = StackPlugin.STACK_TEMPLATES_ENABLED.get(nodeSettings); + } + + @Override + protected List getPolicyConfigs() { + if (stackTemplateEnabled) { + return Arrays.asList(LOGS_ILM_POLICY, METRICS_ILM_POLICY); + } else { + return Collections.emptyList(); + } + } + + @Override + protected List getComponentTemplateConfigs() { + if (stackTemplateEnabled) { + return Arrays.asList( + LOGS_MAPPINGS_COMPONENT_TEMPLATE, + LOGS_SETTINGS_COMPONENT_TEMPLATE, + METRICS_MAPPINGS_COMPONENT_TEMPLATE, + METRICS_SETTINGS_COMPONENT_TEMPLATE + ); + } else { + return Collections.emptyList(); + } + } + + @Override + protected List getComposableTemplateConfigs() { + if (stackTemplateEnabled) { + return Arrays.asList(LOGS_INDEX_TEMPLATE, METRICS_INDEX_TEMPLATE); + } else { + return Collections.emptyList(); + } + } + + @Override + protected String getOrigin() { + return ClientHelper.STACK_ORIGIN; + } +} diff --git a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java new file mode 100644 index 0000000000000..a30ebbf7222f5 --- /dev/null +++ b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java @@ -0,0 +1,453 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.stack; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.ilm.LifecycleAction; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.ilm.LifecycleType; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.elasticsearch.mock.orig.Mockito.when; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +public class StackTemplateRegistryTests extends ESTestCase { + private StackTemplateRegistry registry; + private NamedXContentRegistry xContentRegistry; + private ClusterService clusterService; + private ThreadPool threadPool; + private VerifyingClient client; + + @Before + public void createRegistryAndClient() { + threadPool = new TestThreadPool(this.getClass().getName()); + client = new VerifyingClient(threadPool); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + List entries = new ArrayList<>(ClusterModule.getNamedXWriteables()); + entries.addAll( + Arrays.asList( + new NamedXContentRegistry.Entry( + LifecycleType.class, + new ParseField(TimeseriesLifecycleType.TYPE), + (p) -> TimeseriesLifecycleType.INSTANCE + ), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse) + ) + ); + xContentRegistry = new NamedXContentRegistry(entries); + registry = new StackTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, xContentRegistry); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testDisabledDoesNotAddTemplates() { + Settings settings = Settings.builder().put(StackPlugin.STACK_TEMPLATES_ENABLED.getKey(), false).build(); + StackTemplateRegistry disabledRegistry = new StackTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry); + assertThat(disabledRegistry.getComponentTemplateConfigs(), hasSize(0)); + assertThat(disabledRegistry.getComposableTemplateConfigs(), hasSize(0)); + assertThat(disabledRegistry.getPolicyConfigs(), hasSize(0)); + } + + public void testThatNonExistingTemplatesAreAddedImmediately() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); + + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); + + calledTimes.set(0); + + // attempting to register the event multiple times as a race condition can yield this test flaky, namely: + // when calling registry.clusterChanged(newEvent) the templateCreationsInProgress state that the IndexTemplateRegistry maintains + // might've not yet been updated to reflect that the first template registration was complete, so a second template registration + // will not be issued anymore, leaving calledTimes to 0 + assertBusy(() -> { + // now delete one template from the cluster state and lets retry + ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), nodes); + registry.clusterChanged(newEvent); + assertThat(calledTimes.get(), greaterThan(1)); + }); + } + + public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof PutLifecycleAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutLifecycleAction.class)); + assertThat(request, instanceOf(PutLifecycleAction.Request.class)); + final PutLifecycleAction.Request putRequest = (PutLifecycleAction.Request) request; + assertThat( + putRequest.getPolicy().getName(), + anyOf(equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME), equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME)) + ); + assertNotNull(listener); + return new PutLifecycleAction.Response(true); + } else if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return new StackTemplateRegistryTests.TestPutIndexTemplateResponse(true); + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return new AcknowledgedResponse(true); + } else { + fail("client called with unexpected request: " + request.toString()); + return null; + } + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(2))); + } + + public void testPolicyAlreadyExists() { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map policyMap = new HashMap<>(); + List policies = registry.getPolicyConfigs() + .stream() + .map(policyConfig -> policyConfig.load(xContentRegistry)) + .collect(Collectors.toList()); + assertThat(policies, hasSize(2)); + policies.forEach(p -> policyMap.put(p.getName(), p.get())); + + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return new AcknowledgedResponse(true); + } else if (action instanceof PutLifecycleAction) { + fail("if the policy already exists it should be re-put"); + } else { + fail("client called with unexpected request: " + request.toString()); + } + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); + registry.clusterChanged(event); + } + + public void testPolicyAlreadyExistsButDiffers() throws IOException { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map policyMap = new HashMap<>(); + String policyStr = "{\"phases\":{\"delete\":{\"min_age\":\"1m\",\"actions\":{\"delete\":{}}}}}"; + List policies = registry.getPolicyConfigs() + .stream() + .map(policyConfig -> policyConfig.load(xContentRegistry)) + .collect(Collectors.toList()); + assertThat(policies, hasSize(2)); + policies.forEach(p -> policyMap.put(p.getName(), p.get())); + + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return new AcknowledgedResponse(true); + } else if (action instanceof PutLifecycleAction) { + fail("if the policy already exists it should be re-put"); + } else { + fail("client called with unexpected request: " + request.toString()); + } + return null; + }); + + try ( + XContentParser parser = XContentType.JSON.xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.THROW_UNSUPPORTED_OPERATION, policyStr) + ) { + LifecyclePolicy different = LifecyclePolicy.parse(parser, policies.get(0).getName()); + policyMap.put(policies.get(0).getName(), different); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); + registry.clusterChanged(event); + } + } + + public void testThatVersionedOldTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.singletonMap( + StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION - 1 + ), + nodes + ); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); + } + + public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.singletonMap(StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, null), + nodes + ); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); + } + + @TestLogging(value = "org.elasticsearch.xpack.core.template:DEBUG", reason = "test") + public void testSameOrHigherVersionTemplateNotUpgraded() { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map versions = new HashMap<>(); + versions.put(StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + versions.put(StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + versions.put(StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + versions.put(StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes); + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + fail("template should not have been re-installed"); + return null; + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return new PutLifecycleAction.Response(true); + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return new AcknowledgedResponse(true); + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + }); + registry.clusterChanged(sameVersionEvent); + + versions.clear(); + versions.put( + StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); + versions.put( + StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); + versions.put( + StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); + versions.put( + StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); + ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(versions, nodes); + registry.clusterChanged(higherVersionEvent); + } + + public void testThatMissingMasterNodeDoesNothing() { + DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); + + client.setVerifier((a, r, l) -> { + fail("if the master is missing nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.singletonMap(StackTemplateRegistry.LOGS_INDEX_TEMPLATE_NAME, null), + nodes + ); + registry.clusterChanged(event); + } + + // ------------- + + /** + * A client that delegates to a verifying function for action/request/listener + */ + public static class VerifyingClient extends NoOpClient { + + private TriFunction, ActionRequest, ActionListener, ActionResponse> verifier = (a, r, l) -> { + fail("verifier not set"); + return null; + }; + + VerifyingClient(ThreadPool threadPool) { + super(threadPool); + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + try { + listener.onResponse((Response) verifier.apply(action, request, listener)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public VerifyingClient setVerifier(TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { + this.verifier = verifier; + return this; + } + } + + private ActionResponse verifyComponentTemplateInstalled( + AtomicInteger calledTimes, + ActionType action, + ActionRequest request, + ActionListener listener + ) { + if (action instanceof PutComponentTemplateAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutComponentTemplateAction.class)); + assertThat(request, instanceOf(PutComponentTemplateAction.Request.class)); + final PutComponentTemplateAction.Request putRequest = (PutComponentTemplateAction.Request) request; + assertThat(putRequest.componentTemplate().version(), equalTo((long) StackTemplateRegistry.REGISTRY_VERSION)); + assertNotNull(listener); + return new TestPutIndexTemplateResponse(true); + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return new PutLifecycleAction.Response(true); + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return new AcknowledgedResponse(true); + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + } + + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplates, DiscoveryNodes nodes) { + return createClusterChangedEvent(existingTemplates, Collections.emptyMap(), nodes); + } + + private ClusterChangedEvent createClusterChangedEvent( + Map existingTemplates, + Map existingPolicies, + DiscoveryNodes nodes + ) { + ClusterState cs = createClusterState(Settings.EMPTY, existingTemplates, existingPolicies, nodes); + ClusterChangedEvent realEvent = new ClusterChangedEvent( + "created-from-test", + cs, + ClusterState.builder(new ClusterName("test")).build() + ); + ClusterChangedEvent event = spy(realEvent); + when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster()); + + return event; + } + + private ClusterState createClusterState( + Settings nodeSettings, + Map existingComponentTemplates, + Map existingPolicies, + DiscoveryNodes nodes + ) { + Map componentTemplates = new HashMap<>(); + for (Map.Entry template : existingComponentTemplates.entrySet()) { + ComponentTemplate mockTemplate = mock(ComponentTemplate.class); + when(mockTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue()); + componentTemplates.put(template.getKey(), mockTemplate); + } + + Map existingILMMeta = existingPolicies.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new LifecyclePolicyMetadata(e.getValue(), Collections.emptyMap(), 1, 1))); + IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(existingILMMeta, OperationMode.RUNNING); + + return ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .componentTemplates(componentTemplates) + .transientSettings(nodeSettings) + .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta) + .build() + ) + .blocks(new ClusterBlocks.Builder().build()) + .nodes(nodes) + .build(); + } + + private static class TestPutIndexTemplateResponse extends AcknowledgedResponse { + TestPutIndexTemplateResponse(boolean acknowledged) { + super(acknowledged); + } + } +} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java index 74f1db9b2113e..5c5237be631b5 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java @@ -64,7 +64,7 @@ public WatcherIndexTemplateRegistry(Settings nodeSettings, ClusterService cluste } @Override - protected List getTemplateConfigs() { + protected List getLegacyTemplateConfigs() { return templatesToUse; } diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index 456d22b5e2c0f..ca5f360fe8632 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -31,6 +31,7 @@ testClusters.integTest { setting 'xpack.watcher.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' + setting 'indices.lifecycle.history_index_enabled', 'false' user username: System.getProperty('tests.rest.cluster.username', 'test_user'), password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password') From 099c782ced2c9619ffcc3a48f88b39c9ce1a8efc Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 3 Jun 2020 16:41:55 -0600 Subject: [PATCH 02/20] Add logs and metrics policies to list of expected x-pack policies --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 167c350792b13..27f8a2707aeb0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1183,6 +1183,8 @@ protected static boolean isXPackTemplate(String name) { case ".slm-history": case ".async-search": case "saml-service-provider": + case "logs-default-template": + case "metrics-default-template": return true; default: return false; From a74934a70f389a2de703307fa8bd1dd932ceb375 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 4 Jun 2020 10:38:33 -0600 Subject: [PATCH 03/20] Fix docs check --- docs/reference/indices/templates.asciidoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/indices/templates.asciidoc b/docs/reference/indices/templates.asciidoc index 0c4ed20d3b01d..154dec62de3a6 100644 --- a/docs/reference/indices/templates.asciidoc +++ b/docs/reference/indices/templates.asciidoc @@ -180,7 +180,7 @@ orders overriding them. For example: -------------------------------------------------- PUT /_template/template_1 { - "index_patterns" : ["*"], + "index_patterns" : ["te*"], "order" : 0, "settings" : { "number_of_shards" : 1 @@ -192,7 +192,7 @@ PUT /_template/template_1 PUT /_template/template_2 { - "index_patterns" : ["te*"], + "index_patterns" : ["tes*"], "order" : 1, "settings" : { "number_of_shards" : 1 @@ -204,7 +204,7 @@ PUT /_template/template_2 -------------------------------------------------- The above will disable storing the `_source`, but -for indices that start with `te*`, `_source` will still be enabled. +for indices that start with `tes*`, `_source` will still be enabled. Note, for mappings, the merging is "deep", meaning that specific object/property based mappings can easily be added/overridden on higher order templates, with lower order templates providing the basis. @@ -231,7 +231,7 @@ replace the template without specifying one. -------------------------------------------------- PUT /_template/template_1 { - "index_patterns" : ["*"], + "index_patterns" : ["myindex-*"], "order" : 0, "settings" : { "number_of_shards" : 1 From 73101067d2c89891100fbc8da043e1e959028f46 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 4 Jun 2020 10:50:24 -0600 Subject: [PATCH 04/20] Rename stream.* -> dataset.* --- .../plugin/core/src/main/resources/logs-default-mappings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/resources/logs-default-mappings.json b/x-pack/plugin/core/src/main/resources/logs-default-mappings.json index bbf0a8ff8f1c6..a732986274311 100644 --- a/x-pack/plugin/core/src/main/resources/logs-default-mappings.json +++ b/x-pack/plugin/core/src/main/resources/logs-default-mappings.json @@ -19,7 +19,7 @@ "@timestamp": { "type": "date" }, - "stream": { + "dataset": { "properties": { "type": { "type": "constant_keyword", From 7e891126a5665a07cbe55a8b2adbd84cbf84451c Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 4 Jun 2020 10:50:46 -0600 Subject: [PATCH 05/20] Rename dataset.dataset -> dataset.name --- .../plugin/core/src/main/resources/logs-default-mappings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/resources/logs-default-mappings.json b/x-pack/plugin/core/src/main/resources/logs-default-mappings.json index a732986274311..33bcb4bdabd16 100644 --- a/x-pack/plugin/core/src/main/resources/logs-default-mappings.json +++ b/x-pack/plugin/core/src/main/resources/logs-default-mappings.json @@ -25,7 +25,7 @@ "type": "constant_keyword", "value": "logs" }, - "dataset": { + "name": { "type": "constant_keyword" }, "namespace": { From f249d7059d1fd5d47fbee3235f4b644cd6221345 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 4 Jun 2020 10:53:37 -0600 Subject: [PATCH 06/20] Remove non-"ip" fields under "host" --- .../main/resources/logs-default-mappings.json | 63 ------------------- 1 file changed, 63 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/logs-default-mappings.json b/x-pack/plugin/core/src/main/resources/logs-default-mappings.json index 33bcb4bdabd16..f703efe43a90a 100644 --- a/x-pack/plugin/core/src/main/resources/logs-default-mappings.json +++ b/x-pack/plugin/core/src/main/resources/logs-default-mappings.json @@ -71,71 +71,8 @@ }, "host": { "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, "ip": { "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" } } }, From dcdaa033752704e7942c7be56980cdce2ef5e127 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 4 Jun 2020 11:01:36 -0600 Subject: [PATCH 07/20] Use default refresh interval for logs/metrics --- x-pack/plugin/core/src/main/resources/logs-default-settings.json | 1 - .../plugin/core/src/main/resources/metrics-default-settings.json | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/logs-default-settings.json b/x-pack/plugin/core/src/main/resources/logs-default-settings.json index 98be32f544753..5e967e0c0c031 100644 --- a/x-pack/plugin/core/src/main/resources/logs-default-settings.json +++ b/x-pack/plugin/core/src/main/resources/logs-default-settings.json @@ -6,7 +6,6 @@ "name": "logs-default-policy" }, "codec": "best_compression", - "refresh_interval": "5s", "query": { "default_field": ["message"] }, diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-settings.json b/x-pack/plugin/core/src/main/resources/metrics-default-settings.json index 8082c4a6a0ef2..cceb525480aec 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-default-settings.json +++ b/x-pack/plugin/core/src/main/resources/metrics-default-settings.json @@ -6,7 +6,6 @@ "name": "metrics-default-policy" }, "codec": "best_compression", - "refresh_interval": "5s", "query": { "default_field": ["message"] }, From 42f9f8c8468f297dabbb5d825969b5fe207a6667 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 4 Jun 2020 11:03:03 -0600 Subject: [PATCH 08/20] Use default number_of_routing_shards --- .../plugin/core/src/main/resources/logs-default-settings.json | 3 +-- .../core/src/main/resources/metrics-default-settings.json | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/logs-default-settings.json b/x-pack/plugin/core/src/main/resources/logs-default-settings.json index 5e967e0c0c031..6b9312428d11c 100644 --- a/x-pack/plugin/core/src/main/resources/logs-default-settings.json +++ b/x-pack/plugin/core/src/main/resources/logs-default-settings.json @@ -8,8 +8,7 @@ "codec": "best_compression", "query": { "default_field": ["message"] - }, - "number_of_routing_shards": "30" + } } } }, diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-settings.json b/x-pack/plugin/core/src/main/resources/metrics-default-settings.json index cceb525480aec..f27669b3f167f 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-default-settings.json +++ b/x-pack/plugin/core/src/main/resources/metrics-default-settings.json @@ -8,8 +8,7 @@ "codec": "best_compression", "query": { "default_field": ["message"] - }, - "number_of_routing_shards": "30" + } } } }, From f2f074134371471fcb92158fed5ff5d133316181 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 17 Jun 2020 16:11:34 -0600 Subject: [PATCH 09/20] Update mappings after discussion --- .../main/resources/logs-default-mappings.json | 33 +----- .../resources/metrics-default-mappings.json | 100 +----------------- 2 files changed, 6 insertions(+), 127 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/logs-default-mappings.json b/x-pack/plugin/core/src/main/resources/logs-default-mappings.json index f703efe43a90a..5e978ea56885c 100644 --- a/x-pack/plugin/core/src/main/resources/logs-default-mappings.json +++ b/x-pack/plugin/core/src/main/resources/logs-default-mappings.json @@ -1,8 +1,6 @@ { "template": { "mappings": { - "_meta": { - }, "dynamic_templates": [ { "strings_as_keyword": { @@ -33,34 +31,6 @@ } } }, - "agent": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, "ecs": { "properties": { "version": { @@ -83,7 +53,8 @@ } }, "_meta": { - "description": "default mappings for the logs index template installed by x-pack" + "description": "default mappings for the logs index template installed by x-pack", + "managed": true }, "version": ${xpack.stack.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-mappings.json b/x-pack/plugin/core/src/main/resources/metrics-default-mappings.json index 0b04211565ff4..6ef2f6b07118f 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-default-mappings.json +++ b/x-pack/plugin/core/src/main/resources/metrics-default-mappings.json @@ -1,8 +1,6 @@ { "template": { "mappings": { - "_meta": { - }, "dynamic_templates": [ { "strings_as_keyword": { @@ -19,13 +17,13 @@ "@timestamp": { "type": "date" }, - "stream": { + "dataset": { "properties": { "type": { "type": "constant_keyword", "value": "metrics" }, - "dataset": { + "name": { "type": "constant_keyword" }, "namespace": { @@ -33,34 +31,6 @@ } } }, - "agent": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, "ecs": { "properties": { "version": { @@ -71,71 +41,8 @@ }, "host": { "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, "ip": { "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" } } } @@ -143,7 +50,8 @@ } }, "_meta": { - "description": "default mappings for the metrics index template installed by x-pack" + "description": "default mappings for the metrics index template installed by x-pack", + "managed": true }, "version": ${xpack.stack.template.version} } From a6eeb48dc7e8512baae2b6a730a0f3bd3cf1a4fc Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 17 Jun 2020 16:18:26 -0600 Subject: [PATCH 10/20] Update version constant and comment --- .../elasticsearch/xpack/stack/StackTemplateRegistry.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 30a74568b0f84..0d311ba0a1af1 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -21,9 +21,11 @@ import java.util.List; public class StackTemplateRegistry extends IndexTemplateRegistry { - // history (please add a comment why you increased the version here) - // version 1: initial - public static final int REGISTRY_VERSION = 1; + // The stack template registry should remain at version 0. This is because templates and + // policies will be changed by the ingest manager once they exist, and ES should only ever put + // the template in place if it does not exist. If this were incremented we could accidentally + // overwrite a template or policy changed by the ingest manager. + public static final int REGISTRY_VERSION = 0; public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; From 53feca5a6a186900d08ce826f616ab143793ece6 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 17 Jun 2020 17:04:21 -0600 Subject: [PATCH 11/20] Fix index name in yaml test for master merge --- .../rest-api-spec/test/stack/10_basic.yml | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml index 45df4ef3df255..e69d0ffea84b5 100644 --- a/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml +++ b/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml @@ -55,18 +55,17 @@ setup: - match: { 0.timestamp_field: '@timestamp' } - match: { 0.generation: 1 } - length: { 0.indices: 1 } - - match: { 0.indices.0.index_name: 'logs-foo-bar-000001' } + - match: { 0.indices.0.index_name: '.ds-logs-foo-bar-000001' } - do: indices.get: - index: logs-foo-bar-000001 + index: .ds-logs-foo-bar-000001 - - is_true: logs-foo-bar-000001.aliases - - is_true: logs-foo-bar-000001.settings - - is_true: logs-foo-bar-000001.mappings - - match: { logs-foo-bar-000001.settings.index.lifecycle.name: "logs-default-policy" } - - is_true: logs-foo-bar-000001.mappings.properties.message - - match: { logs-foo-bar-000001.data_stream: "logs-foo-bar" } + - is_true: \.ds-logs-foo-bar-000001.settings + - is_true: \.ds-logs-foo-bar-000001.mappings + - match: { \.ds-logs-foo-bar-000001.settings.index.lifecycle.name: "logs-default-policy" } + - is_true: \.ds-logs-foo-bar-000001.mappings.properties.message + - match: { \.ds-logs-foo-bar-000001.data_stream: "logs-foo-bar" } - do: indices.delete_data_stream: @@ -89,18 +88,17 @@ setup: - match: { 0.timestamp_field: '@timestamp' } - match: { 0.generation: 1 } - length: { 0.indices: 1 } - - match: { 0.indices.0.index_name: 'metrics-foo-bar-000001' } + - match: { 0.indices.0.index_name: '.ds-metrics-foo-bar-000001' } - do: indices.get: - index: metrics-foo-bar-000001 - - - is_true: metrics-foo-bar-000001.aliases - - is_true: metrics-foo-bar-000001.settings - - is_true: metrics-foo-bar-000001.mappings - - match: { metrics-foo-bar-000001.settings.index.lifecycle.name: "metrics-default-policy" } - - is_true: metrics-foo-bar-000001.mappings.properties.message - - match: { metrics-foo-bar-000001.data_stream: "metrics-foo-bar" } + index: .ds-metrics-foo-bar-000001 + + - is_true: \.ds-metrics-foo-bar-000001.settings + - is_true: \.ds-metrics-foo-bar-000001.mappings + - match: { \.ds-metrics-foo-bar-000001.settings.index.lifecycle.name: "metrics-default-policy" } + - is_true: \.ds-metrics-foo-bar-000001.mappings.properties.message + - match: { \.ds-metrics-foo-bar-000001.data_stream: "metrics-foo-bar" } - do: indices.delete_data_stream: From 2b1b0186220c062d050afb9f107a02166137133c Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 24 Jun 2020 10:55:20 -0600 Subject: [PATCH 12/20] Remove unnecessary min_age from policies --- x-pack/plugin/core/src/main/resources/logs-default-policy.json | 1 - .../plugin/core/src/main/resources/metrics-default-policy.json | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/logs-default-policy.json b/x-pack/plugin/core/src/main/resources/logs-default-policy.json index 06fb976bfffa3..936bfe6c0e2e3 100644 --- a/x-pack/plugin/core/src/main/resources/logs-default-policy.json +++ b/x-pack/plugin/core/src/main/resources/logs-default-policy.json @@ -1,7 +1,6 @@ { "phases": { "hot": { - "min_age": "0ms", "actions": { "rollover": { "max_size": "50gb", diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-policy.json b/x-pack/plugin/core/src/main/resources/metrics-default-policy.json index 06fb976bfffa3..936bfe6c0e2e3 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-default-policy.json +++ b/x-pack/plugin/core/src/main/resources/metrics-default-policy.json @@ -1,7 +1,6 @@ { "phases": { "hot": { - "min_age": "0ms", "actions": { "rollover": { "max_size": "50gb", From ce81e39a5f6d20e452193a5cdfd21cbf7397df05 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 24 Jun 2020 11:14:03 -0600 Subject: [PATCH 13/20] Fix gradle for master merge --- x-pack/plugin/stack/build.gradle | 2 +- x-pack/plugin/stack/qa/rest/build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index ccda33f8b7cab..3bc30b8071135 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -14,7 +14,7 @@ archivesBaseName = 'x-pack-stack' dependencies { compileOnly project(path: xpackModule('core'), configuration: 'default') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') } // add all sub-projects of the qa sub-project diff --git a/x-pack/plugin/stack/qa/rest/build.gradle b/x-pack/plugin/stack/qa/rest/build.gradle index 5c337853cfc4e..38c6a121b7d3e 100644 --- a/x-pack/plugin/stack/qa/rest/build.gradle +++ b/x-pack/plugin/stack/qa/rest/build.gradle @@ -4,8 +4,8 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' dependencies { - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') - testCompile project(path: xpackModule('stack'), configuration: 'runtime') + testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') + testImplementation project(path: xpackModule('stack'), configuration: 'runtime') } restResources { From 455fdfb20853a913893d1c678ac650d082fc4ce6 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 24 Jun 2020 12:26:21 -0600 Subject: [PATCH 14/20] Reference changed name in YML test --- .../src/test/resources/rest-api-spec/test/stack/10_basic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml index e69d0ffea84b5..444e250ae2eba 100644 --- a/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml +++ b/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml @@ -52,7 +52,7 @@ setup: name: logs-foo-bar - match: { 0.name: logs-foo-bar } - - match: { 0.timestamp_field: '@timestamp' } + - match: { 0.timestamp_field.name: '@timestamp' } - match: { 0.generation: 1 } - length: { 0.indices: 1 } - match: { 0.indices.0.index_name: '.ds-logs-foo-bar-000001' } @@ -85,7 +85,7 @@ setup: name: metrics-foo-bar - match: { 0.name: metrics-foo-bar } - - match: { 0.timestamp_field: '@timestamp' } + - match: { 0.timestamp_field.name: '@timestamp' } - match: { 0.generation: 1 } - length: { 0.indices: 1 } - match: { 0.indices.0.index_name: '.ds-metrics-foo-bar-000001' } From fd0247646c45c325974f9276027e34ef0439d675 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 24 Jun 2020 13:10:48 -0600 Subject: [PATCH 15/20] Rename templates and files --- ...fault-mappings.json => logs-mappings.json} | 0 ...s-default-policy.json => logs-policy.json} | 0 ...fault-settings.json => logs-settings.json} | 2 +- ...fault-template.json => logs-template.json} | 7 ++-- ...lt-mappings.json => metrics-mappings.json} | 0 ...efault-policy.json => metrics-policy.json} | 0 ...lt-settings.json => metrics-settings.json} | 2 +- ...lt-template.json => metrics-template.json} | 7 ++-- .../rest-api-spec/test/stack/10_basic.yml | 20 ++++++------ .../xpack/stack/StackTemplateRegistry.java | 32 +++++++++---------- 10 files changed, 36 insertions(+), 34 deletions(-) rename x-pack/plugin/core/src/main/resources/{logs-default-mappings.json => logs-mappings.json} (100%) rename x-pack/plugin/core/src/main/resources/{logs-default-policy.json => logs-policy.json} (100%) rename x-pack/plugin/core/src/main/resources/{logs-default-settings.json => logs-settings.json} (90%) rename x-pack/plugin/core/src/main/resources/{logs-default-template.json => logs-template.json} (62%) rename x-pack/plugin/core/src/main/resources/{metrics-default-mappings.json => metrics-mappings.json} (100%) rename x-pack/plugin/core/src/main/resources/{metrics-default-policy.json => metrics-policy.json} (100%) rename x-pack/plugin/core/src/main/resources/{metrics-default-settings.json => metrics-settings.json} (89%) rename x-pack/plugin/core/src/main/resources/{metrics-default-template.json => metrics-template.json} (79%) diff --git a/x-pack/plugin/core/src/main/resources/logs-default-mappings.json b/x-pack/plugin/core/src/main/resources/logs-mappings.json similarity index 100% rename from x-pack/plugin/core/src/main/resources/logs-default-mappings.json rename to x-pack/plugin/core/src/main/resources/logs-mappings.json diff --git a/x-pack/plugin/core/src/main/resources/logs-default-policy.json b/x-pack/plugin/core/src/main/resources/logs-policy.json similarity index 100% rename from x-pack/plugin/core/src/main/resources/logs-default-policy.json rename to x-pack/plugin/core/src/main/resources/logs-policy.json diff --git a/x-pack/plugin/core/src/main/resources/logs-default-settings.json b/x-pack/plugin/core/src/main/resources/logs-settings.json similarity index 90% rename from x-pack/plugin/core/src/main/resources/logs-default-settings.json rename to x-pack/plugin/core/src/main/resources/logs-settings.json index 6b9312428d11c..fb6c684265ba3 100644 --- a/x-pack/plugin/core/src/main/resources/logs-default-settings.json +++ b/x-pack/plugin/core/src/main/resources/logs-settings.json @@ -3,7 +3,7 @@ "settings": { "index": { "lifecycle": { - "name": "logs-default-policy" + "name": "logs" }, "codec": "best_compression", "query": { diff --git a/x-pack/plugin/core/src/main/resources/logs-default-template.json b/x-pack/plugin/core/src/main/resources/logs-template.json similarity index 62% rename from x-pack/plugin/core/src/main/resources/logs-default-template.json rename to x-pack/plugin/core/src/main/resources/logs-template.json index 72810f24b30e1..84546a9c772e2 100644 --- a/x-pack/plugin/core/src/main/resources/logs-default-template.json +++ b/x-pack/plugin/core/src/main/resources/logs-template.json @@ -5,11 +5,12 @@ "timestamp_field": "@timestamp" }, "composed_of": [ - "logs-default-mappings", - "logs-default-settings" + "logs-mappings", + "logs-settings" ], "_meta": { - "description": "default logs template installed by x-pack" + "description": "default logs template installed by x-pack", + "managed": true }, "version": ${xpack.stack.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-mappings.json b/x-pack/plugin/core/src/main/resources/metrics-mappings.json similarity index 100% rename from x-pack/plugin/core/src/main/resources/metrics-default-mappings.json rename to x-pack/plugin/core/src/main/resources/metrics-mappings.json diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-policy.json b/x-pack/plugin/core/src/main/resources/metrics-policy.json similarity index 100% rename from x-pack/plugin/core/src/main/resources/metrics-default-policy.json rename to x-pack/plugin/core/src/main/resources/metrics-policy.json diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-settings.json b/x-pack/plugin/core/src/main/resources/metrics-settings.json similarity index 89% rename from x-pack/plugin/core/src/main/resources/metrics-default-settings.json rename to x-pack/plugin/core/src/main/resources/metrics-settings.json index f27669b3f167f..6601c55ba3871 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-default-settings.json +++ b/x-pack/plugin/core/src/main/resources/metrics-settings.json @@ -3,7 +3,7 @@ "settings": { "index": { "lifecycle": { - "name": "metrics-default-policy" + "name": "metrics" }, "codec": "best_compression", "query": { diff --git a/x-pack/plugin/core/src/main/resources/metrics-default-template.json b/x-pack/plugin/core/src/main/resources/metrics-template.json similarity index 79% rename from x-pack/plugin/core/src/main/resources/metrics-default-template.json rename to x-pack/plugin/core/src/main/resources/metrics-template.json index 5a29d77e826c1..b13857c1c70fa 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-default-template.json +++ b/x-pack/plugin/core/src/main/resources/metrics-template.json @@ -5,11 +5,12 @@ "timestamp_field": "@timestamp" }, "composed_of": [ - "metrics-default-mappings", - "metrics-default-settings" + "metrics-mappings", + "metrics-settings" ], "_meta": { - "description": "default metrics template installed by x-pack" + "description": "default metrics template installed by x-pack", + "managed": true }, "version": ${xpack.stack.template.version} } diff --git a/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml index 444e250ae2eba..8a420a1022ccb 100644 --- a/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml +++ b/x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml @@ -8,35 +8,35 @@ setup: "Test stack template installation": - do: ilm.get_lifecycle: - policy: "logs-default-policy" + policy: "logs" - do: ilm.get_lifecycle: - policy: "metrics-default-policy" + policy: "metrics" - do: cluster.get_component_template: - name: logs-default-mappings + name: logs-mappings - do: cluster.get_component_template: - name: logs-default-settings + name: logs-settings - do: cluster.get_component_template: - name: metrics-default-mappings + name: metrics-mappings - do: cluster.get_component_template: - name: metrics-default-settings + name: metrics-settings - do: indices.get_index_template: - name: logs-default-template + name: logs - do: indices.get_index_template: - name: metrics-default-template + name: metrics --- "Test logs index auto creation": @@ -63,7 +63,7 @@ setup: - is_true: \.ds-logs-foo-bar-000001.settings - is_true: \.ds-logs-foo-bar-000001.mappings - - match: { \.ds-logs-foo-bar-000001.settings.index.lifecycle.name: "logs-default-policy" } + - match: { \.ds-logs-foo-bar-000001.settings.index.lifecycle.name: "logs" } - is_true: \.ds-logs-foo-bar-000001.mappings.properties.message - match: { \.ds-logs-foo-bar-000001.data_stream: "logs-foo-bar" } @@ -96,7 +96,7 @@ setup: - is_true: \.ds-metrics-foo-bar-000001.settings - is_true: \.ds-metrics-foo-bar-000001.mappings - - match: { \.ds-metrics-foo-bar-000001.settings.index.lifecycle.name: "metrics-default-policy" } + - match: { \.ds-metrics-foo-bar-000001.settings.index.lifecycle.name: "metrics" } - is_true: \.ds-metrics-foo-bar-000001.mappings.properties.message - match: { \.ds-metrics-foo-bar-000001.data_stream: "metrics-foo-bar" } diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 0d311ba0a1af1..6a43ec334b0d3 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -34,30 +34,30 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { ////////////////////////////////////////////////////////// // Logs components (for matching logs-*-* indices) ////////////////////////////////////////////////////////// - public static final String LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "logs-default-mappings"; - public static final String LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME = "logs-default-settings"; - public static final String LOGS_ILM_POLICY_NAME = "logs-default-policy"; - public static final String LOGS_INDEX_TEMPLATE_NAME = "logs-default-template"; + public static final String LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "logs-mappings"; + public static final String LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME = "logs-settings"; + public static final String LOGS_ILM_POLICY_NAME = "logs"; + public static final String LOGS_INDEX_TEMPLATE_NAME = "logs"; public static final IndexTemplateConfig LOGS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, - "/logs-default-mappings.json", + "/logs-mappings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE ); public static final IndexTemplateConfig LOGS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, - "/logs-default-settings.json", + "/logs-settings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE ); public static final LifecyclePolicyConfig LOGS_ILM_POLICY = new LifecyclePolicyConfig( LOGS_ILM_POLICY_NAME, - "/logs-default-policy.json" + "/logs-policy.json" ); public static final IndexTemplateConfig LOGS_INDEX_TEMPLATE = new IndexTemplateConfig( LOGS_INDEX_TEMPLATE_NAME, - "/logs-default-template.json", + "/logs-template.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE ); @@ -65,30 +65,30 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { ////////////////////////////////////////////////////////// // Metrics components (for matching metric-*-* indices) ////////////////////////////////////////////////////////// - public static final String METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "metrics-default-mappings"; - public static final String METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME = "metrics-default-settings"; - public static final String METRICS_ILM_POLICY_NAME = "metrics-default-policy"; - public static final String METRICS_INDEX_TEMPLATE_NAME = "metrics-default-template"; + public static final String METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "metrics-mappings"; + public static final String METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME = "metrics-settings"; + public static final String METRICS_ILM_POLICY_NAME = "metrics"; + public static final String METRICS_INDEX_TEMPLATE_NAME = "metrics"; public static final IndexTemplateConfig METRICS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, - "/metrics-default-mappings.json", + "/metrics-mappings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE ); public static final IndexTemplateConfig METRICS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, - "/metrics-default-settings.json", + "/metrics-settings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE ); public static final LifecyclePolicyConfig METRICS_ILM_POLICY = new LifecyclePolicyConfig( METRICS_ILM_POLICY_NAME, - "/metrics-default-policy.json" + "/metrics-policy.json" ); public static final IndexTemplateConfig METRICS_INDEX_TEMPLATE = new IndexTemplateConfig( METRICS_INDEX_TEMPLATE_NAME, - "/metrics-default-template.json", + "/metrics-template.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE ); From 897c645a3d593a7939c872b249a9c8f8aaa5c8f0 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 24 Jun 2020 13:18:30 -0600 Subject: [PATCH 16/20] Ugh spotless :| --- .../org/elasticsearch/xpack/stack/StackTemplateRegistry.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 6a43ec334b0d3..fbdbaa1a9cd46 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -51,10 +51,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE ); - public static final LifecyclePolicyConfig LOGS_ILM_POLICY = new LifecyclePolicyConfig( - LOGS_ILM_POLICY_NAME, - "/logs-policy.json" - ); + public static final LifecyclePolicyConfig LOGS_ILM_POLICY = new LifecyclePolicyConfig(LOGS_ILM_POLICY_NAME, "/logs-policy.json"); public static final IndexTemplateConfig LOGS_INDEX_TEMPLATE = new IndexTemplateConfig( LOGS_INDEX_TEMPLATE_NAME, "/logs-template.json", From 68f7898c96927cb7fbec7758ed8ccac7d36102b0 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 24 Jun 2020 15:10:58 -0600 Subject: [PATCH 17/20] Add managed: true to settings component templates --- x-pack/plugin/core/src/main/resources/logs-settings.json | 3 ++- x-pack/plugin/core/src/main/resources/metrics-settings.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/logs-settings.json b/x-pack/plugin/core/src/main/resources/logs-settings.json index fb6c684265ba3..7370e40accc79 100644 --- a/x-pack/plugin/core/src/main/resources/logs-settings.json +++ b/x-pack/plugin/core/src/main/resources/logs-settings.json @@ -13,7 +13,8 @@ } }, "_meta": { - "description": "default settings for the logs index template installed by x-pack" + "description": "default settings for the logs index template installed by x-pack", + "managed": true }, "version": ${xpack.stack.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/metrics-settings.json b/x-pack/plugin/core/src/main/resources/metrics-settings.json index 6601c55ba3871..1a13139bb18a4 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-settings.json +++ b/x-pack/plugin/core/src/main/resources/metrics-settings.json @@ -13,7 +13,8 @@ } }, "_meta": { - "description": "default settings for the metrics index template installed by x-pack" + "description": "default settings for the metrics index template installed by x-pack", + "managed": true }, "version": ${xpack.stack.template.version} } From 7a555eaaaaf963135d0d3f8544a347baa8ce942c Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 24 Jun 2020 15:24:48 -0600 Subject: [PATCH 18/20] Don't clean up stack-installed templates --- .../test/rest/ESRestTestCase.java | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 27f8a2707aeb0..df86372bb3d3d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -27,6 +27,7 @@ import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; @@ -471,7 +472,8 @@ protected boolean preserveILMPoliciesUponCompletion() { * A set of ILM policies that should be preserved between runs. */ protected Set preserveILMPolicyIds() { - return Sets.newHashSet("ilm-history-ilm-policy", "slm-history-ilm-policy", "watch-history-ilm-policy", "ml-size-based-ilm-policy"); + return Sets.newHashSet("ilm-history-ilm-policy", "slm-history-ilm-policy", + "watch-history-ilm-policy", "ml-size-based-ilm-policy", "logs", "metrics"); } /** @@ -575,8 +577,25 @@ private void wipeCluster() throws Exception { } } try { - adminClient().performRequest(new Request("DELETE", "_component_template/*")); - } catch (ResponseException e) { + Request compReq = new Request("GET", "_component_templates"); + String componentTemplates = EntityUtils.toString(adminClient().performRequest(compReq).getEntity()); + Map cTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent, componentTemplates, false); + @SuppressWarnings("unchecked") + List names = ((List>) cTemplates.get("component_templates")).stream() + .map(ct -> (String) ct.get("name")) + .collect(Collectors.toList()); + for (String componentTemplate : names) { + try { + if (isXPackTemplate(componentTemplate)) { + continue; + } + adminClient().performRequest(new Request("DELETE", "_component_template/" + componentTemplate)); + } catch (ResponseException e) { + logger.debug(new ParameterizedMessage("unable to remove component template {}", componentTemplate), e); + } + } + } catch (Exception e) { + logger.info("ignoring exception removing all component templates", e); // We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore } } else { @@ -1175,7 +1194,6 @@ protected static boolean isXPackTemplate(String name) { return true; } switch (name) { - case ".triggered_watches": case ".watches": case "logstash-index-template": case ".logstash-management": @@ -1183,8 +1201,12 @@ protected static boolean isXPackTemplate(String name) { case ".slm-history": case ".async-search": case "saml-service-provider": - case "logs-default-template": - case "metrics-default-template": + case "logs": + case "logs-settings": + case "logs-mappings": + case "metrics": + case "metrics-settings": + case "metrics-mappings": return true; default: return false; From 9991bed6b418ea0e3a803dd8d0a7170c07370572 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 30 Jun 2020 08:10:29 -0600 Subject: [PATCH 19/20] Remove unused componentTemplate getter --- .../client/indices/PutComponentTemplateRequest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutComponentTemplateRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutComponentTemplateRequest.java index 9dbf090807038..e37d44baee8d8 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutComponentTemplateRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutComponentTemplateRequest.java @@ -78,10 +78,6 @@ public PutComponentTemplateRequest componentTemplate(ComponentTemplate component return this; } - public ComponentTemplate componentTemplate() { - return this.componentTemplate; - } - /** * The cause for this component template creation. */ From 36285abb957bf18848d7efb07257d4c110a73c10 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 30 Jun 2020 08:12:26 -0600 Subject: [PATCH 20/20] Enable sub project in gradle --- x-pack/plugin/stack/build.gradle | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index 3bc30b8071135..bee72631e3892 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -18,12 +18,12 @@ dependencies { } // add all sub-projects of the qa sub-project -// gradle.projectsEvaluated { -// project.subprojects -// .find { it.path == project.path + ":qa" } -// .subprojects -// .findAll { it.path.startsWith(project.path + ":qa") } -// .each { check.dependsOn it.check } -// } +gradle.projectsEvaluated { + project.subprojects + .find { it.path == project.path + ":qa" } + .subprojects + .findAll { it.path.startsWith(project.path + ":qa") } + .each { check.dependsOn it.check } +} integTest.enabled = false