[7.x] Add default composable templates for new indexing strategy (#57629

) (#58757)

Backports the following commits to 7.x:

    Add default composable templates for new indexing strategy (#57629)

docs/reference/indices/templates.asciidoc

@@ -180,7 +180,7 @@ orders overriding them. For example:
 --------------------------------------------------
 PUT /_template/template_1
 {
-    "index_patterns" : ["*"],
+    "index_patterns" : ["te*"],
     "order" : 0,
     "settings" : {
         "number_of_shards" : 1
@@ -192,7 +192,7 @@ PUT /_template/template_1
 
 PUT /_template/template_2
 {
-    "index_patterns" : ["te*"],
+    "index_patterns" : ["tes*"],
     "order" : 1,
     "settings" : {
         "number_of_shards" : 1
@@ -204,7 +204,7 @@ PUT /_template/template_2
 --------------------------------------------------
 
 The above will disable storing the `_source`, but
-for indices that start with `te*`, `_source` will still be enabled.
+for indices that start with `tes*`, `_source` will still be enabled.
 Note, for mappings, the merging is "deep", meaning that specific
 object/property based mappings can easily be added/overridden on higher
 order templates, with lower order templates providing the basis.
@@ -231,7 +231,7 @@ replace the template without specifying	one.
 --------------------------------------------------
 PUT /_template/template_1
 {
-    "index_patterns" : ["*"],
+    "index_patterns" : ["myindex-*"],
     "order" : 0,
     "settings" : {
         "number_of_shards" : 1

rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml

@@ -294,6 +294,7 @@
   - do:
         cat.aliases:
             v: true
+            name: test*
 
   - match:
       $body: |
@@ -364,6 +365,7 @@
   - do:
         cat.aliases:
             h:      [index, alias]
+            name: test*
 
   - match:
       $body: /^ test \s+ test_1 \s+ $/
@@ -373,6 +375,7 @@
         cat.aliases:
             h:      [index, alias]
             v:      true
+            name: test*
   - match:
       $body: |
             /^
@@ -398,7 +401,8 @@
         index: test_index
 
   - do:
-      cat.aliases: {}
+      cat.aliases:
+        name: test*
 
   - match:
       $body: |

rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml

@@ -73,7 +73,8 @@
           aliases:
             test_alias: {}
   - do:
-      cat.aliases: {}
+      cat.aliases:
+        name: test*
 
   - match:
       $body: |

rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml

@@ -89,57 +89,3 @@
       cluster.stats: {}
 
   - is_true: nodes.packaging_types
-
----
-"get cluster stats returns mapping stats":
-
-  - skip:
-      version: " - 7.6.99"
-      reason:  "mapping stats are added for v7.7.0"
-
-  - do:
-      cluster.stats: {}
-
-  - length: { indices.mappings.field_types: 0 }
-
-  - do:
-      indices.create:
-        index: test-index1
-        body:
-          mappings:
-            properties:
-              foo:
-                type: keyword
-
-  - do:
-      indices.create:
-        index: test-index2
-        body:
-          mappings:
-            properties:
-              foo:
-                type: keyword
-              bar:
-                properties:
-                  quux:
-                    type: integer
-                  baz:
-                    type: keyword
-
-  - do:
-      cluster.stats: {}
-
-  - length: { indices.mappings.field_types: 3 }
-
-  - match: { indices.mappings.field_types.0.name: integer }
-  - match: { indices.mappings.field_types.0.count: 1 }
-  - match: { indices.mappings.field_types.0.index_count: 1 }
-
-  - match: { indices.mappings.field_types.1.name: keyword }
-  - match: { indices.mappings.field_types.1.count: 3 }
-  - match: { indices.mappings.field_types.1.index_count: 2 }
-
-  - match: { indices.mappings.field_types.2.name: object }
-  - match: { indices.mappings.field_types.2.count: 1 }
-  - match: { indices.mappings.field_types.2.index_count: 1 }
-

rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml

@@ -36,7 +36,7 @@ setup:
   - match: {index_templates.0.index_template.template.mappings: {properties: {field: {type: keyword}}}}
 
 ---
-"Get all tindex emplates":
+"Get all index templates":
   - skip:
       version: " - 7.7.99"
       reason: "index template v2 API unavailable before 7.8"
@@ -56,7 +56,8 @@ setup:
   - do:
       indices.get_index_template: {}
 
-  - length: {index_templates: 2}
+  - is_true: index_templates.0.name
+  - is_true: index_templates.1.name
 
 ---
 "Get index template with local flag":

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml

@@ -142,7 +142,7 @@
         body: >
           {
             "version": 10,
-            "index_patterns": "*",
+            "index_patterns": "foo*",
             "settings": { "number_of_shards": 1 }
           }
   - match: { acknowledged: true }
@@ -159,7 +159,7 @@
         body: >
           {
             "version": 9,
-            "index_patterns": "*",
+            "index_patterns": "foo*",
             "settings": { "number_of_shards": 1 }
           }
   - match: { acknowledged: true }
@@ -176,7 +176,7 @@
         body: >
           {
             "version": 6789,
-            "index_patterns": "*",
+            "index_patterns": "foo*",
             "settings": { "number_of_shards": 1 }
           }
   - match: { acknowledged: true }
@@ -192,7 +192,7 @@
         name: "my_template"
         body: >
           {
-            "index_patterns": "*",
+            "index_patterns": "foo*",
             "settings": { "number_of_shards": 1 }
           }
   - match: { acknowledged: true }
@@ -209,7 +209,7 @@
         body: >
           {
             "version": 5385,
-            "index_patterns": "*",
+            "index_patterns": "foo*",
             "settings": { "number_of_shards": 1 }
           }
   - match: { acknowledged: true }

server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java

@@ -29,6 +29,8 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.monitor.os.OsStats;
 import org.elasticsearch.node.NodeRoleSettings;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -46,6 +48,7 @@
 
 import static org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.OS;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
@@ -250,4 +253,31 @@ public void testClusterStatusWhenStateNotRecovered() throws Exception {
         response = client().admin().cluster().prepareClusterStats().get();
         assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN));
     }
+
+    public void testFieldTypes() {
+        internalCluster().startNode();
+        ensureGreen();
+        ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
+        assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN));
+        assertTrue(response.getIndicesStats().getMappings().getFieldTypeStats().isEmpty());
+
+        client().admin().indices().prepareCreate("test1").addMapping(MapperService.SINGLE_MAPPING_NAME,
+            "{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}", XContentType.JSON).get();
+        client().admin().indices().prepareCreate("test2")
+            .addMapping(MapperService.SINGLE_MAPPING_NAME,
+                "{\"properties\":{\"foo\":{\"type\": \"keyword\"},\"bar\":{\"properties\":{\"baz\":{\"type\":\"keyword\"}," +
+                "\"eggplant\":{\"type\":\"integer\"}}}}}", XContentType.JSON).get();
+        response = client().admin().cluster().prepareClusterStats().get();
+        assertThat(response.getIndicesStats().getMappings().getFieldTypeStats().size(), equalTo(3));
+        Set<IndexFeatureStats> stats = response.getIndicesStats().getMappings().getFieldTypeStats();
+        for (IndexFeatureStats stat : stats) {
+            if (stat.getName().equals("integer")) {
+                assertThat(stat.getCount(), greaterThanOrEqualTo(1));
+            } else if (stat.getName().equals("keyword")) {
+                assertThat(stat.getCount(), greaterThanOrEqualTo(3));
+            } else if (stat.getName().equals("object")) {
+                assertThat(stat.getCount(), greaterThanOrEqualTo(1));
+            }
+        }
+    }
 }

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -498,7 +498,8 @@ public ClusterState addIndexTemplateV2(final ClusterState currentState, final bo
                 (finalIndexTemplate.composedOf().size() > 0 ? "with component templates " + finalIndexTemplate.composedOf() + " " : "") +
                 "is invalid", e);
         }
-        logger.info("{} index template [{}]", existing == null ? "adding" : "updating", name);
+        logger.info("{} index template [{}] for index patterns {}", existing == null ? "adding" : "updating", name,
+            template.indexPatterns());
         return ClusterState.builder(currentState)
             .metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate))
             .build();

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -27,6 +27,7 @@
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
@@ -506,7 +507,8 @@ protected boolean preserveSLMPoliciesUponCompletion() {
      * A set of ILM policies that should be preserved between runs.
      */
     protected Set<String> preserveILMPolicyIds() {
-        return Sets.newHashSet("ilm-history-ilm-policy", "slm-history-ilm-policy", "watch-history-ilm-policy", "ml-size-based-ilm-policy");
+        return Sets.newHashSet("ilm-history-ilm-policy", "slm-history-ilm-policy",
+            "watch-history-ilm-policy", "ml-size-based-ilm-policy", "logs", "metrics");
     }
 
     /**
@@ -604,8 +606,25 @@ private void wipeCluster() throws Exception {
                     }
                 }
                 try {
-                    adminClient().performRequest(new Request("DELETE", "_component_template/*"));
-                } catch (ResponseException e) {
+                    Request compReq = new Request("GET", "_component_template");
+                    String componentTemplates = EntityUtils.toString(adminClient().performRequest(compReq).getEntity());
+                    Map<String, Object> cTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent, componentTemplates, false);
+                    @SuppressWarnings("unchecked")
+                    List<String> names = ((List<Map<String, Object>>) cTemplates.get("component_templates")).stream()
+                        .map(ct -> (String) ct.get("name"))
+                        .collect(Collectors.toList());
+                    for (String componentTemplate : names) {
+                        try {
+                            if (isXPackTemplate(componentTemplate)) {
+                                continue;
+                            }
+                            adminClient().performRequest(new Request("DELETE", "_component_template/" + componentTemplate));
+                        } catch (ResponseException e) {
+                                logger.debug(new ParameterizedMessage("unable to remove component template {}", componentTemplate), e);
+                        }
+                    }
+                } catch (Exception e) {
+                    logger.info("ignoring exception removing all component templates", e);
                     // We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore
                 }
             } else {
@@ -1212,14 +1231,20 @@ protected static boolean isXPackTemplate(String name) {
             return true;
         }
         switch (name) {
-            case ".triggered_watches":
             case ".watches":
             case "logstash-index-template":
             case ".logstash-management":
             case "security_audit_log":
             case ".slm-history":
             case ".async-search":
             case "saml-service-provider":
+            case "ilm-history":
+            case "logs":
+            case "logs-settings":
+            case "logs-mappings":
+            case "metrics":
+            case "metrics-settings":
+            case "metrics-mappings":
                 return true;
             default:
                 return false;

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.WarningsHandler;
 import org.elasticsearch.client.sniff.ElasticsearchNodesSniffer;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
@@ -364,6 +365,14 @@ public void test() throws IOException {
                 && testCandidate.getTestSection().getSkipSection().getFeatures().contains("default_shards") == false) {
             final Request request = new Request("PUT", "/_template/global");
             request.setJsonEntity("{\"index_patterns\":[\"*\"],\"settings\":{\"index.number_of_shards\":2}}");
+            // Because this has not yet transitioned to a composable template, it's possible that
+            // this can overlap an installed composable template since this is a global (*)
+            // template. In order to avoid this failing the test, we override the warnings handler
+            // to be permissive in this case. This can be removed once all tests use composable
+            // templates instead of legacy templates
+            RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
+            builder.setWarningsHandler(WarningsHandler.PERMISSIVE);
+            request.setOptions(builder.build());
             adminClient().performRequest(request);
         }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java

@@ -69,6 +69,7 @@ public static Map<String, String> filterSecurityHeaders(Map<String, String> head
     public static final String TRANSFORM_ORIGIN = "transform";
     public static final String ASYNC_SEARCH_ORIGIN = "async_search";
     public static final String IDP_ORIGIN = "idp";
+    public static final String STACK_ORIGIN = "stack";
 
     private ClientHelper() {}
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java

@@ -69,7 +69,7 @@ public SnapshotLifecycleTemplateRegistry(Settings nodeSettings, ClusterService c
     }
 
     @Override
-    protected List<IndexTemplateConfig> getTemplateConfigs() {
+    protected List<IndexTemplateConfig> getLegacyTemplateConfigs() {
         if (slmHistoryEnabled == false) {
             return Collections.emptyList();
         }
@@ -90,7 +90,7 @@ protected String getOrigin() {
     }
 
     public boolean validate(ClusterState state) {
-        boolean allTemplatesPresent = getTemplateConfigs().stream()
+        boolean allTemplatesPresent = getLegacyTemplateConfigs().stream()
             .map(IndexTemplateConfig::getTemplateName)
             .allMatch(name -> state.metadata().getTemplates().containsKey(name));