From deed8c74b109bd2aa348c38a9739ff4d1a1685cb Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 18 May 2020 11:44:15 +0200 Subject: [PATCH 01/11] Ensure template exists when creating data stream Limit the creation of data streams only for namespaces that have a composable template with a data stream definition. This way we ensure that mappings/settings have been specified and will be used at data stream creation and data stream rollover. Relates to #53100 --- .../indices/create-data-stream.asciidoc | 16 +++++++++ .../indices/delete-data-stream.asciidoc | 15 ++++++++ .../indices/get-data-stream.asciidoc | 9 +++++ .../reference/indices/rollover-index.asciidoc | 16 +++++++++ .../test/indices.data_stream/10_basic.yml | 13 +++++++ .../20_unsupported_apis.yml | 11 ++++++ .../indices.delete/20_backing_indices.yml | 13 +++++++ .../test/indices.get/20_backing_indices.yml | 11 ++++++ .../test/indices.rollover/50_data_streams.yml | 11 ++++++ .../action/bulk/BulkIntegrationIT.java | 5 ++- .../elasticsearch/indices/DataStreamIT.java | 29 ++++++++++++++- .../rollover/MetadataRolloverService.java | 3 ++ .../MetadataCreateDataStreamService.java | 19 ++++++++++ .../MetadataRolloverServiceTests.java | 35 +++++++++++++++++++ .../MetadataCreateDataStreamServiceTests.java | 33 ++++++++++++++++- .../10_data_stream_resolvability.yml | 11 ++++++ 16 files changed, 245 insertions(+), 5 deletions(-) diff --git a/docs/reference/indices/create-data-stream.asciidoc b/docs/reference/indices/create-data-stream.asciidoc index e3a3da3ee0ec5..b91dbb07e9ef1 100644 --- a/docs/reference/indices/create-data-stream.asciidoc +++ b/docs/reference/indices/create-data-stream.asciidoc @@ -23,6 +23,20 @@ addressed directly, data streams are integrated with the <> to facilitate the management of the time series data contained in their backing indices. +//// +[source,console] +----------------------------------- +PUT _index_template/template +{ + "index_patterns": ["my-data-stream*"], + "data_stream": { + "timestamp_field": "@timestamp" + } +} +----------------------------------- +// TEST +//// + [source,console] -------------------------------------------------- PUT _data_stream/my-data-stream @@ -30,11 +44,13 @@ PUT _data_stream/my-data-stream "timestamp_field": "@timestamp" } -------------------------------------------------- +// TEST[continued] //// [source,console] ----------------------------------- DELETE /_data_stream/my-data-stream +DELETE /_index_template/* ----------------------------------- // TEST[continued] //// diff --git a/docs/reference/indices/delete-data-stream.asciidoc b/docs/reference/indices/delete-data-stream.asciidoc index 2a926130999cc..144a006ead0fa 100644 --- a/docs/reference/indices/delete-data-stream.asciidoc +++ b/docs/reference/indices/delete-data-stream.asciidoc @@ -9,6 +9,14 @@ Deletes an existing data stream along with its backing indices. //// [source,console] ----------------------------------- +PUT _index_template/template +{ + "index_patterns": ["my-data-stream*"], + "data_stream": { + "timestamp_field": "@timestamp" + } +} + PUT /_data_stream/my-data-stream { "timestamp_field" : "@timestamp" @@ -22,6 +30,13 @@ PUT /_data_stream/my-data-stream DELETE _data_stream/my-data-stream -------------------------------------------------- +//// +[source,console] +----------------------------------- +DELETE /_index_template/* +----------------------------------- +// TEST[continued] +//// [[delete-data-stream-api-request]] ==== {api-request-title} diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index 813339b5b45a4..c0f0b34c7f053 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -9,6 +9,14 @@ Returns information about one or more data streams. //// [source,console] ----------------------------------- +PUT _index_template/template +{ + "index_patterns": ["my-data-stream*"], + "data_stream": { + "timestamp_field": "@timestamp" + } +} + PUT /_data_stream/my-data-stream { "timestamp_field" : "@timestamp" @@ -21,6 +29,7 @@ PUT /_data_stream/my-data-stream [source,console] ----------------------------------- DELETE /_data_stream/my-data-stream +DELETE /_index_template/* ----------------------------------- // TEARDOWN //// diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index 8a76bc832ef97..338df4dd3566a 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -225,6 +225,20 @@ The API returns the following response: [[rollover-data-stream-ex]] ===== Roll over a data stream +//// +[source,console] +----------------------------------- +PUT _index_template/template +{ + "index_patterns": ["my-data-stream*"], + "data_stream": { + "timestamp_field": "@timestamp" + } +} +----------------------------------- +// TEST +//// + [source,console] -------------------------------------------------- PUT /_data_stream/my-data-stream <1> @@ -243,6 +257,7 @@ POST /my-data-stream/_rollover <2> } } -------------------------------------------------- +// TEST[continued] // TEST[setup:huge_twitter] // TEST[s/# Add > 1000 documents to my-data-stream/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"my-data-stream-000001"}}/] <1> Creates a data stream called `my-data-stream` with one initial backing index @@ -286,6 +301,7 @@ The API returns the following response: [source,console] ----------------------------------- DELETE /_data_stream/my-data-stream +DELETE /_index_template/* ----------------------------------- // TEST[continued] //// diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml index 5ead5bdf2c3db..760ee574c2aa4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml @@ -1,3 +1,16 @@ +setup: + - skip: + features: allowed_warnings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [simple-*, delete*, get*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [simple-*, delete*, get*] + data_stream: + timestamp_field: @timestamp + --- "Create data stream": - skip: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml index 132dd8f4c47ce..40f5eab1c7a12 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml @@ -3,6 +3,17 @@ - skip: version: " - 7.8.99" reason: "data streams only supported in 7.9+" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [logs-*] + data_stream: + timestamp_field: @timestamp - do: indices.create_data_stream: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml index 021d36f91c7fa..04d88c38df375 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml @@ -1,3 +1,16 @@ +setup: + - skip: + features: allowed_warnings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [simple-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [simple-*] + data_stream: + timestamp_field: @timestamp + --- "Delete backing index on data stream": - skip: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml index 9e5320e844ab4..7c6c3317a890b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml @@ -3,6 +3,17 @@ - skip: version: " - 7.8.99" reason: "data streams only supported in 7.9+" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [data-*] + data_stream: + timestamp_field: @timestamp - do: indices.create_data_stream: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml index f21e8b22df6ba..d5025401b0bb9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml @@ -3,6 +3,17 @@ - skip: version: " - 7.8.99" reason: "data streams only supported in 7.9+" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [data-*] + data_stream: + timestamp_field: @timestamp - do: indices.create_data_stream: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 1af5b8b910ae3..0aed8c2aa6c67 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -39,7 +39,6 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateV2; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -218,14 +217,14 @@ public void testDeleteIndexWhileIndexing() throws Exception { } } - public void testMixedAutoCreate() { + public void testMixedAutoCreate() throws Exception { Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(); PutIndexTemplateV2Action.Request createTemplateRequest = new PutIndexTemplateV2Action.Request("logs-foo"); createTemplateRequest.indexTemplate( new IndexTemplateV2( List.of("logs-foo*"), - new Template(settings, null, null), + null, null, null, null, null, new IndexTemplateV2.DataStreamTemplate("@timestamp")) ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java index 2c2099276939d..3ffe6842e5c52 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java @@ -28,6 +28,8 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateV2Action; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -42,10 +44,13 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESIntegTestCase; +import org.junit.After; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -72,11 +77,19 @@ public class DataStreamIT extends ESIntegTestCase { + @After + public void deleteAllComposableTemplates() { + DeleteIndexTemplateV2Action.Request deleteTemplateRequest = new DeleteIndexTemplateV2Action.Request("*"); + client().execute(DeleteIndexTemplateV2Action.INSTANCE, deleteTemplateRequest).actionGet(); + } + public void testBasicScenario() throws Exception { + createIndexTemplate("id1", "metrics-foo*", "@timestamp1"); CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo"); createDataStreamRequest.setTimestampFieldName("@timestamp1"); client().admin().indices().createDataStream(createDataStreamRequest).get(); + createIndexTemplate("id2", "metrics-bar*", "@timestamp2"); createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar"); createDataStreamRequest.setTimestampFieldName("@timestamp2"); client().admin().indices().createDataStream(createDataStreamRequest).get(); @@ -151,6 +164,7 @@ public void testBasicScenario() throws Exception { } public void testOtherWriteOps() throws Exception { + createIndexTemplate("id", "metrics-foobar*", "@timestamp1"); String dataStreamName = "metrics-foobar"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); createDataStreamRequest.setTimestampFieldName("@timestamp1"); @@ -199,7 +213,8 @@ public void testOtherWriteOps() throws Exception { } } - public void testResolvabilityOfDataStreamsInAPIs() { + public void testResolvabilityOfDataStreamsInAPIs() throws Exception { + createIndexTemplate("id", "logs-*", "ts"); String dataStreamName = "logs-foobar"; CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStreamName); request.setTimestampFieldName("ts"); @@ -325,4 +340,16 @@ private static void expectFailure(String dataStreamName, ThrowingRunnable runnab "] matches a data stream, specify the corresponding concrete indices instead.")); } + static void createIndexTemplate(String id, String pattern, String timestampFieldName) throws IOException { + PutIndexTemplateV2Action.Request request = new PutIndexTemplateV2Action.Request(id); + request.indexTemplate( + new IndexTemplateV2( + List.of(pattern), + null, + null, null, null, null, + new IndexTemplateV2.DataStreamTemplate(timestampFieldName)) + ); + client().execute(PutIndexTemplateV2Action.INSTANCE, request).actionGet(); + } + } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 0c6e178274f1d..f59cd3645e43b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -48,6 +48,7 @@ import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.ALIAS; import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.DATA_STREAM; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTemplateForDataStream; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; @@ -140,6 +141,8 @@ private RolloverResult rolloverAlias(ClusterState currentState, IndexAbstraction private RolloverResult rolloverDataStream(ClusterState currentState, IndexAbstraction.DataStream dataStream, String dataStreamName, CreateIndexRequest createIndexRequest, List> metConditions, boolean silent) throws Exception { + validateTemplateForDataStream(dataStreamName, currentState.metadata()); + final DataStream ds = dataStream.getDataStream(); final IndexMetadata originalWriteIndex = dataStream.getWriteIndex(); final String newWriteIndexName = DataStream.getBackingIndexName(ds.getName(), ds.getGeneration() + 1); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 58ab35b247142..ce194605eeaea 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -33,10 +33,15 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ObjectPath; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.threadpool.ThreadPool; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; public class MetadataCreateDataStreamService { @@ -131,6 +136,8 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '.'"); } + validateTemplateForDataStream(request.name, currentState.metadata()); + String firstBackingIndexName = DataStream.getBackingIndexName(request.name, 1); CreateIndexClusterStateUpdateRequest createIndexRequest = new CreateIndexClusterStateUpdateRequest("initialize_data_stream", firstBackingIndexName, firstBackingIndexName) @@ -145,4 +152,16 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn return ClusterState.builder(currentState).metadata(builder).build(); } + public static void validateTemplateForDataStream(String dataStreamName, Metadata metadata) { + final String v2Template = MetadataIndexTemplateService.findV2Template(metadata, dataStreamName, false); + if (v2Template == null) { + throw new IllegalArgumentException("no matching index template found for data stream [" + dataStreamName + "]"); + } + IndexTemplateV2 indexTemplateV2 = metadata.templatesV2().get(v2Template); + if (indexTemplateV2.getDataStreamTemplate() == null) { + throw new IllegalArgumentException("matching index template [" + v2Template + "] for data stream [" + dataStreamName + + "] has no data stream template"); + } + } + } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index fc896e95f52f3..749970058dd3f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -515,7 +515,10 @@ public void testRolloverClusterState() throws Exception { public void testRolloverClusterStateForDataStream() throws Exception { final DataStream dataStream = DataStreamTests.randomInstance(); + IndexTemplateV2 template = new IndexTemplateV2(List.of(dataStream.getName() + "*"), null, null, null, null, null, + new IndexTemplateV2.DataStreamTemplate("@timestamp")); Metadata.Builder builder = Metadata.builder(); + builder.put("_id", template); for (Index index : dataStream.getIndices()) { builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); } @@ -575,6 +578,38 @@ public void testRolloverClusterStateForDataStream() throws Exception { } } + public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception { + final DataStream dataStream = DataStreamTests.randomInstance(); + Metadata.Builder builder = Metadata.builder(); + for (Index index : dataStream.getIndices()) { + builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); + } + builder.put(dataStream); + final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build(); + + ThreadPool testThreadPool = mock(ThreadPool.class); + ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); + Environment env = mock(Environment.class); + AllocationService allocationService = mock(AllocationService.class); + IndicesService indicesService = mockIndicesServices(); + IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class); + + MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY, + clusterService, indicesService, allocationService, null, env, null, testThreadPool, null, Collections.emptyList(), false); + MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, + new AliasValidator(), null, xContentRegistry()); + MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, + mockIndexNameExpressionResolver); + + MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); + List> metConditions = Collections.singletonList(condition); + CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_"); + + Exception e = expectThrows(IllegalArgumentException.class, () -> rolloverService.rolloverClusterState(clusterState, + dataStream.getName(), null, createIndexRequest, metConditions, false)); + assertThat(e.getMessage(), equalTo("no matching index template found for data stream [bnlgrnekwc]")); + } + private IndicesService mockIndicesServices() throws Exception { /* * Throws Exception because Eclipse uses the lower bound for diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index fc101c7d24b52..41b5f5c5e9349 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -44,7 +44,11 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase { public void testCreateDataStream() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; - ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); + IndexTemplateV2 template = new IndexTemplateV2(List.of(dataStreamName + "*"), null, null, null, null, null, + new IndexTemplateV2.DataStreamTemplate("@timestamp")); + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metadata(Metadata.builder().put("template", template).build()) + .build(); CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); ClusterState newState = MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); @@ -103,6 +107,33 @@ public void testCreateDataStreamStartingWithPeriod() throws Exception { assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must not start with '.'")); } + public void testCreateDataStreamNoTemplate() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = "my-data-stream"; + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .build(); + CreateDataStreamClusterStateUpdateRequest req = + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + Exception e = expectThrows(IllegalArgumentException.class, + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + assertThat(e.getMessage(), equalTo("no matching index template found for data stream [my-data-stream]")); + } + + public void testCreateDataStreamNoValidTemplate() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = "my-data-stream"; + IndexTemplateV2 template = new IndexTemplateV2(List.of(dataStreamName + "*"), null, null, null, null, null, null); + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metadata(Metadata.builder().put("template", template).build()) + .build(); + CreateDataStreamClusterStateUpdateRequest req = + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + Exception e = expectThrows(IllegalArgumentException.class, + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + assertThat(e.getMessage(), + equalTo("matching index template [template] for data stream [my-data-stream] has no data stream template")); + } + private static MetadataCreateIndexService getMetadataCreateIndexService() throws Exception { MetadataCreateIndexService s = mock(MetadataCreateIndexService.class); when(s.applyCreateIndexRequest(any(ClusterState.class), any(CreateIndexClusterStateUpdateRequest.class), anyBoolean())) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml index 81295f28d4ece..7f762a58ef441 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml @@ -3,6 +3,17 @@ - skip: version: " - 7.8.99" reason: "data streams only supported in 7.9+" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [logs-*] + data_stream: + timestamp_field: @timestamp - do: indices.create_data_stream: From b6bb38f16a240525fb809079b5ce6d3c0f91a600 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 18 May 2020 12:56:16 +0200 Subject: [PATCH 02/11] fixed yaml issues --- .../rest-api-spec/test/indices.data_stream/10_basic.yml | 2 +- .../test/indices.data_stream/20_unsupported_apis.yml | 2 +- .../rest-api-spec/test/indices.delete/20_backing_indices.yml | 2 +- .../rest-api-spec/test/indices.get/20_backing_indices.yml | 2 +- .../rest-api-spec/test/indices.rollover/50_data_streams.yml | 2 +- .../test/data_stream/10_data_stream_resolvability.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml index 760ee574c2aa4..a2acb31ab8321 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml @@ -9,7 +9,7 @@ setup: body: index_patterns: [simple-*, delete*, get*] data_stream: - timestamp_field: @timestamp + timestamp_field: '@timestamp' --- "Create data stream": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml index 40f5eab1c7a12..d523dc34e878f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml @@ -13,7 +13,7 @@ body: index_patterns: [logs-*] data_stream: - timestamp_field: @timestamp + timestamp_field: '@timestamp' - do: indices.create_data_stream: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml index 04d88c38df375..1ac837d4d22eb 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml @@ -9,7 +9,7 @@ setup: body: index_patterns: [simple-*] data_stream: - timestamp_field: @timestamp + timestamp_field: '@timestamp' --- "Delete backing index on data stream": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml index 7c6c3317a890b..6b83d4847a2f8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml @@ -13,7 +13,7 @@ body: index_patterns: [data-*] data_stream: - timestamp_field: @timestamp + timestamp_field: '@timestamp' - do: indices.create_data_stream: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml index d5025401b0bb9..41a3a265b1884 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml @@ -13,7 +13,7 @@ body: index_patterns: [data-*] data_stream: - timestamp_field: @timestamp + timestamp_field: '@timestamp' - do: indices.create_data_stream: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml index 7f762a58ef441..8f89a155f99c7 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml @@ -13,7 +13,7 @@ body: index_patterns: [logs-*] data_stream: - timestamp_field: @timestamp + timestamp_field: '@timestamp' - do: indices.create_data_stream: From e6d675cb6c13396443944a97a76abfc8b1b96d7f Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 18 May 2020 13:16:37 +0200 Subject: [PATCH 03/11] fixed checkstyle violation --- .../cluster/metadata/MetadataCreateDataStreamService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index ce194605eeaea..dc7faaac633a6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -33,15 +33,10 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ObjectPath; -import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.threadpool.ThreadPool; -import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; -import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicReference; public class MetadataCreateDataStreamService { From 04341ac5e4afe4f0da0acbddf66575ad24cf7b4e Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 18 May 2020 13:47:41 +0200 Subject: [PATCH 04/11] fixed test --- .../admin/indices/rollover/MetadataRolloverServiceTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 749970058dd3f..35393f3f640e0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -607,7 +607,7 @@ public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception { Exception e = expectThrows(IllegalArgumentException.class, () -> rolloverService.rolloverClusterState(clusterState, dataStream.getName(), null, createIndexRequest, metConditions, false)); - assertThat(e.getMessage(), equalTo("no matching index template found for data stream [bnlgrnekwc]")); + assertThat(e.getMessage(), equalTo("no matching index template found for data stream [" + dataStream.getName() + "]")); } private IndicesService mockIndicesServices() throws Exception { From fb2294dd6f54d3f25dbc7f8042b5fe3337004e9d Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 20 May 2020 16:54:04 +0200 Subject: [PATCH 05/11] Remove timestamp_field from create data stream request and let the create data stream api resolve the timestamp field from the data stream template snippet inside an itv2. --- .../api/indices.create_data_stream.json | 3 +- .../test/indices.data_stream/10_basic.yml | 63 +++++++++---------- .../20_unsupported_apis.yml | 2 - .../indices.delete/20_backing_indices.yml | 4 -- .../test/indices.get/20_backing_indices.yml | 2 - .../test/indices.rollover/50_data_streams.yml | 2 - .../elasticsearch/indices/DataStreamIT.java | 5 -- .../indices/create/AutoCreateAction.java | 2 +- .../datastream/CreateDataStreamAction.java | 25 ++++---- .../rollover/MetadataRolloverService.java | 4 +- .../MetadataCreateDataStreamService.java | 10 ++- .../indices/RestCreateDataStreamAction.java | 9 --- .../CreateDataStreamRequestTests.java | 11 ++-- .../MetadataCreateDataStreamServiceTests.java | 14 ++--- .../10_data_stream_resolvability.yml | 2 - 15 files changed, 60 insertions(+), 98 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json index ef8615a69b1ca..a96d5d32d5eb2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json @@ -24,8 +24,7 @@ "params":{ }, "body":{ - "description":"The data stream definition", - "required":true + "description":"The data stream definition" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml index a2acb31ab8321..509758ba68a2c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml @@ -3,13 +3,22 @@ setup: features: allowed_warnings - do: allowed_warnings: - - "index template [my-template] has index patterns [simple-*, delete*, get*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + - "index template [my-template1] has index patterns [simple-data-stream1*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" indices.put_index_template: - name: my-template + name: my-template1 body: - index_patterns: [simple-*, delete*, get*] + index_patterns: [simple-data-stream1] data_stream: timestamp_field: '@timestamp' + - do: + allowed_warnings: + - "index template [my-template2] has index patterns [simple-data-stream2*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" + indices.put_index_template: + name: my-template2 + body: + index_patterns: [simple-data-stream2] + data_stream: + timestamp_field: '@timestamp2' --- "Create data stream": @@ -20,15 +29,11 @@ setup: - do: indices.create_data_stream: name: simple-data-stream1 - body: - timestamp_field: "@timestamp" - is_true: acknowledged - do: indices.create_data_stream: name: simple-data-stream2 - body: - timestamp_field: "@timestamp2" - is_true: acknowledged - do: @@ -82,8 +87,6 @@ setup: catch: bad_request indices.create_data_stream: name: invalid-data-stream#-name - body: - timestamp_field: "@timestamp" - match: { status: 400 } - match: { error.root_cause.0.type: "illegal_argument_exception" } @@ -96,41 +99,37 @@ setup: - do: indices.create_data_stream: - name: get-data-stream1 - body: - timestamp_field: "@timestamp" + name: simple-data-stream1 - is_true: acknowledged - do: indices.create_data_stream: - name: get-data-stream2 - body: - timestamp_field: "@timestamp2" + name: simple-data-stream2 - is_true: acknowledged - do: indices.get_data_stream: {} - - match: { 0.name: get-data-stream1 } + - match: { 0.name: simple-data-stream1 } - match: { 0.timestamp_field: '@timestamp' } - match: { 0.generation: 1 } - - match: { 1.name: get-data-stream2 } + - match: { 1.name: simple-data-stream2 } - match: { 1.timestamp_field: '@timestamp2' } - match: { 1.generation: 1 } - do: indices.get_data_stream: - name: get-data-stream1 - - match: { 0.name: get-data-stream1 } + name: simple-data-stream1 + - match: { 0.name: simple-data-stream1 } - match: { 0.timestamp_field: '@timestamp' } - match: { 0.generation: 1 } - do: indices.get_data_stream: - name: get-data-* - - match: { 0.name: get-data-stream1 } + name: simple-data-stream* + - match: { 0.name: simple-data-stream1 } - match: { 0.timestamp_field: '@timestamp' } - match: { 0.generation: 1 } - - match: { 1.name: get-data-stream2 } + - match: { 1.name: simple-data-stream2 } - match: { 1.timestamp_field: '@timestamp2' } - match: { 1.generation: 1 } @@ -149,12 +148,12 @@ setup: - do: indices.delete_data_stream: - name: get-data-stream1 + name: simple-data-stream1 - is_true: acknowledged - do: indices.delete_data_stream: - name: get-data-stream2 + name: simple-data-stream2 - is_true: acknowledged --- @@ -165,9 +164,7 @@ setup: - do: indices.create_data_stream: - name: delete-data-stream1 - body: - timestamp_field: "@timestamp" + name: simple-data-stream1 - is_true: acknowledged - do: @@ -180,25 +177,25 @@ setup: - do: indices.get: - index: ['delete-data-stream1-000001', 'test_index'] + index: ['simple-data-stream1-000001', 'test_index'] - is_true: test_index.settings - - is_true: delete-data-stream1-000001.settings + - is_true: simple-data-stream1-000001.settings - do: indices.get_data_stream: {} - - match: { 0.name: delete-data-stream1 } + - match: { 0.name: simple-data-stream1 } - match: { 0.timestamp_field: '@timestamp' } - match: { 0.generation: 1 } - length: { 0.indices: 1 } - - match: { 0.indices.0.index_name: 'delete-data-stream1-000001' } + - match: { 0.indices.0.index_name: 'simple-data-stream1-000001' } - do: indices.delete_data_stream: - name: delete-data-stream1 + name: simple-data-stream1 - is_true: acknowledged - do: catch: missing indices.get: - index: "delete-data-stream1-000001" + index: "simple-data-stream1-000001" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml index d523dc34e878f..bc45c2501a087 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml @@ -18,8 +18,6 @@ - do: indices.create_data_stream: name: logs-foobar - body: - timestamp_field: "@timestamp" - is_true: acknowledged - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml index 1ac837d4d22eb..cfd86be5b4cac 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml @@ -20,8 +20,6 @@ setup: - do: indices.create_data_stream: name: simple-data-stream - body: - timestamp_field: "@timestamp" - is_true: acknowledged # rollover data stream to create new backing index @@ -74,8 +72,6 @@ setup: - do: indices.create_data_stream: name: simple-data-stream - body: - timestamp_field: "@timestamp" - is_true: acknowledged # rollover data stream to create new backing index diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml index 6b83d4847a2f8..1543f63609d88 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml @@ -18,8 +18,6 @@ - do: indices.create_data_stream: name: data-stream1 - body: - timestamp_field: "@timestamp" - is_true: acknowledged - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml index 41a3a265b1884..29a4e8138e1f8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml @@ -18,8 +18,6 @@ - do: indices.create_data_stream: name: data-stream-for-rollover - body: - timestamp_field: "@timestamp" - is_true: acknowledged # rollover data stream to create new backing index diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java index 3ffe6842e5c52..deb74f3daeed9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java @@ -86,12 +86,10 @@ public void deleteAllComposableTemplates() { public void testBasicScenario() throws Exception { createIndexTemplate("id1", "metrics-foo*", "@timestamp1"); CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo"); - createDataStreamRequest.setTimestampFieldName("@timestamp1"); client().admin().indices().createDataStream(createDataStreamRequest).get(); createIndexTemplate("id2", "metrics-bar*", "@timestamp2"); createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar"); - createDataStreamRequest.setTimestampFieldName("@timestamp2"); client().admin().indices().createDataStream(createDataStreamRequest).get(); GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request("*"); @@ -167,7 +165,6 @@ public void testOtherWriteOps() throws Exception { createIndexTemplate("id", "metrics-foobar*", "@timestamp1"); String dataStreamName = "metrics-foobar"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); - createDataStreamRequest.setTimestampFieldName("@timestamp1"); client().admin().indices().createDataStream(createDataStreamRequest).get(); { @@ -217,7 +214,6 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { createIndexTemplate("id", "logs-*", "ts"); String dataStreamName = "logs-foobar"; CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStreamName); - request.setTimestampFieldName("ts"); client().admin().indices().createDataStream(request).actionGet(); verifyResolvability(dataStreamName, client().prepareIndex(dataStreamName) @@ -245,7 +241,6 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { verifyResolvability(dataStreamName, client().prepareFieldCaps(dataStreamName).setFields("*"), false); request = new CreateDataStreamAction.Request("logs-barbaz"); - request.setTimestampFieldName("ts"); client().admin().indices().createDataStream(request).actionGet(); verifyResolvability("logs-barbaz", client().prepareIndex("logs-barbaz") .setSource("{}", XContentType.JSON) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index 46a562045a80b..aa058d3498c4b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -126,7 +126,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { DataStreamTemplate dataStreamTemplate = resolveAutoCreateDataStream(request, currentState.metadata()); if (dataStreamTemplate != null) { CreateDataStreamClusterStateUpdateRequest createRequest = new CreateDataStreamClusterStateUpdateRequest( - request.index(), dataStreamTemplate.getTimestampField(), request.masterNodeTimeout(), request.timeout()); + request.index(), request.masterNodeTimeout(), request.timeout()); ClusterState clusterState = metadataCreateDataStreamService.createDataStream(createRequest, currentState); indexNameRef.set(clusterState.metadata().dataStreams().get(request.index()).getIndices().get(0).getName()); return clusterState; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java index 1035e1e3873bd..a368410b2fcdd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.action.admin.indices.datastream; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; @@ -56,39 +57,37 @@ private CreateDataStreamAction() { public static class Request extends AcknowledgedRequest { private final String name; - private String timestampFieldName; public Request(String name) { this.name = name; } - public void setTimestampFieldName(String timestampFieldName) { - this.timestampFieldName = timestampFieldName; - } - @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (Strings.hasText(name) == false) { validationException = ValidateActions.addValidationError("name is missing", validationException); } - if (Strings.hasText(timestampFieldName) == false) { - validationException = ValidateActions.addValidationError("timestamp field name is missing", validationException); - } return validationException; } public Request(StreamInput in) throws IOException { super(in); this.name = in.readString(); - this.timestampFieldName = in.readString(); + // TODO: remove when backported + if (in.getVersion().before(Version.V_8_0_0)) { + in.readString(); + } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(name); - out.writeString(timestampFieldName); + // TODO: remove when backported + if (out.getVersion().before(Version.V_8_0_0)) { + out.writeString(""); + } } @Override @@ -96,13 +95,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return name.equals(request.name) && - timestampFieldName.equals(request.timestampFieldName); + return name.equals(request.name); } @Override public int hashCode() { - return Objects.hash(name, timestampFieldName); + return Objects.hash(name); } } @@ -133,7 +131,6 @@ protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) throws Exception { CreateDataStreamClusterStateUpdateRequest updateRequest = new CreateDataStreamClusterStateUpdateRequest( request.name, - request.timestampFieldName, request.masterNodeTimeout(), request.timeout() ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index f59cd3645e43b..00a5dff8c61d2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -48,7 +48,7 @@ import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.ALIAS; import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.DATA_STREAM; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTemplateForDataStream; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; @@ -141,7 +141,7 @@ private RolloverResult rolloverAlias(ClusterState currentState, IndexAbstraction private RolloverResult rolloverDataStream(ClusterState currentState, IndexAbstraction.DataStream dataStream, String dataStreamName, CreateIndexRequest createIndexRequest, List> metConditions, boolean silent) throws Exception { - validateTemplateForDataStream(dataStreamName, currentState.metadata()); + lookupTemplateForDataStream(dataStreamName, currentState.metadata()); final DataStream ds = dataStream.getDataStream(); final IndexMetadata originalWriteIndex = dataStream.getWriteIndex(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index dc7faaac633a6..dec483d8853cf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -101,14 +101,11 @@ public ClusterState createDataStream(CreateDataStreamClusterStateUpdateRequest r public static final class CreateDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest { private final String name; - private final String timestampFieldName; public CreateDataStreamClusterStateUpdateRequest(String name, - String timestampFieldName, TimeValue masterNodeTimeout, TimeValue timeout) { this.name = name; - this.timestampFieldName = timestampFieldName; masterNodeTimeout(masterNodeTimeout); ackTimeout(timeout); } @@ -131,7 +128,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '.'"); } - validateTemplateForDataStream(request.name, currentState.metadata()); + IndexTemplateV2 template = lookupTemplateForDataStream(request.name, currentState.metadata()); String firstBackingIndexName = DataStream.getBackingIndexName(request.name, 1); CreateIndexClusterStateUpdateRequest createIndexRequest = @@ -142,12 +139,12 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn assert firstBackingIndex != null; Metadata.Builder builder = Metadata.builder(currentState.metadata()).put( - new DataStream(request.name, request.timestampFieldName, List.of(firstBackingIndex.getIndex()))); + new DataStream(request.name, template.getDataStreamTemplate().getTimestampField(), List.of(firstBackingIndex.getIndex()))); logger.info("adding data stream [{}]", request.name); return ClusterState.builder(currentState).metadata(builder).build(); } - public static void validateTemplateForDataStream(String dataStreamName, Metadata metadata) { + public static IndexTemplateV2 lookupTemplateForDataStream(String dataStreamName, Metadata metadata) { final String v2Template = MetadataIndexTemplateService.findV2Template(metadata, dataStreamName, false); if (v2Template == null) { throw new IllegalArgumentException("no matching index template found for data stream [" + dataStreamName + "]"); @@ -157,6 +154,7 @@ public static void validateTemplateForDataStream(String dataStreamName, Metadata throw new IllegalArgumentException("matching index template [" + v2Template + "] for data stream [" + dataStreamName + "] has no data stream template"); } + return indexTemplateV2; } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java index 67b67677195dc..4091316caeafa 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java @@ -20,14 +20,12 @@ import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; import java.util.List; -import java.util.Map; public class RestCreateDataStreamAction extends BaseRestHandler { @@ -46,13 +44,6 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { CreateDataStreamAction.Request putDataStreamRequest = new CreateDataStreamAction.Request(request.param("name")); - request.withContentOrSourceParamParserOrNull(parser -> { - Map body = parser.map(); - String timeStampFieldName = (String) body.get(DataStream.TIMESTAMP_FIELD_FIELD.getPreferredName()); - if (timeStampFieldName != null) { - putDataStreamRequest.setTimestampFieldName(timeStampFieldName); - } - }); return channel -> client.admin().indices().createDataStream(putDataStreamRequest, new RestToXContentListener<>(channel)); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java index b4b7326a6b5e4..863c2f3245e06 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java @@ -35,24 +35,21 @@ protected Writeable.Reader instanceReader() { @Override protected Request createTestInstance() { - Request request = new Request(randomAlphaOfLength(8)); - request.setTimestampFieldName(randomAlphaOfLength(8)); - return request; + return new Request(randomAlphaOfLength(8)); } public void testValidateRequest() { CreateDataStreamAction.Request req = new CreateDataStreamAction.Request("my-data-stream"); - req.setTimestampFieldName("my-timestamp-field"); ActionRequestValidationException e = req.validate(); assertNull(e); } - public void testValidateRequestWithoutTimestampField() { - CreateDataStreamAction.Request req = new CreateDataStreamAction.Request("my-data-stream"); + public void testValidateRequestWithoutName() { + CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(""); ActionRequestValidationException e = req.validate(); assertNotNull(e); assertThat(e.validationErrors().size(), equalTo(1)); - assertThat(e.validationErrors().get(0), containsString("timestamp field name is missing")); + assertThat(e.validationErrors().get(0), containsString("name is missing")); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 41b5f5c5e9349..68adba551e47a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -50,7 +50,7 @@ public void testCreateDataStream() throws Exception { .metadata(Metadata.builder().put("template", template).build()) .build(); CreateDataStreamClusterStateUpdateRequest req = - new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); ClusterState newState = MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); @@ -67,7 +67,7 @@ public void testCreateDuplicateDataStream() throws Exception { ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build(); CreateDataStreamClusterStateUpdateRequest req = - new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); @@ -79,7 +79,7 @@ public void testCreateDataStreamWithInvalidName() throws Exception { final String dataStreamName = "_My-da#ta- ,stream-"; ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); CreateDataStreamClusterStateUpdateRequest req = - new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), containsString("must not contain the following characters")); @@ -90,7 +90,7 @@ public void testCreateDataStreamWithUppercaseCharacters() throws Exception { final String dataStreamName = "MAY_NOT_USE_UPPERCASE"; ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); CreateDataStreamClusterStateUpdateRequest req = - new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must be lowercase")); @@ -101,7 +101,7 @@ public void testCreateDataStreamStartingWithPeriod() throws Exception { final String dataStreamName = ".may_not_start_with_period"; ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); CreateDataStreamClusterStateUpdateRequest req = - new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must not start with '.'")); @@ -113,7 +113,7 @@ public void testCreateDataStreamNoTemplate() throws Exception { ClusterState cs = ClusterState.builder(new ClusterName("_name")) .build(); CreateDataStreamClusterStateUpdateRequest req = - new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); Exception e = expectThrows(IllegalArgumentException.class, () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), equalTo("no matching index template found for data stream [my-data-stream]")); @@ -127,7 +127,7 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { .metadata(Metadata.builder().put("template", template).build()) .build(); CreateDataStreamClusterStateUpdateRequest req = - new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); Exception e = expectThrows(IllegalArgumentException.class, () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml index 8f89a155f99c7..520e5e0b3e5b6 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml @@ -18,8 +18,6 @@ - do: indices.create_data_stream: name: logs-foobar - body: - timestamp_field: "@timestamp" - is_true: acknowledged - do: From 6d716508235ad2f08b787f6d63c9e18f7c63c35c Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 20 May 2020 19:04:35 +0200 Subject: [PATCH 06/11] adjusted docs tests and skip data stream bwc tests until backported --- .../indices/create-data-stream.asciidoc | 3 --- .../indices/delete-data-stream.asciidoc | 3 --- docs/reference/indices/get-data-stream.asciidoc | 3 --- .../test/indices.data_stream/10_basic.yml | 16 ++++++++-------- .../indices.data_stream/20_unsupported_apis.yml | 4 ++-- .../test/indices.delete/20_backing_indices.yml | 8 ++++---- .../test/indices.get/20_backing_indices.yml | 2 +- .../test/indices.rollover/50_data_streams.yml | 4 ++-- .../data_stream/10_data_stream_resolvability.yml | 4 ++-- 9 files changed, 19 insertions(+), 28 deletions(-) diff --git a/docs/reference/indices/create-data-stream.asciidoc b/docs/reference/indices/create-data-stream.asciidoc index b91dbb07e9ef1..7378d36c321d3 100644 --- a/docs/reference/indices/create-data-stream.asciidoc +++ b/docs/reference/indices/create-data-stream.asciidoc @@ -40,9 +40,6 @@ PUT _index_template/template [source,console] -------------------------------------------------- PUT _data_stream/my-data-stream -{ - "timestamp_field": "@timestamp" -} -------------------------------------------------- // TEST[continued] diff --git a/docs/reference/indices/delete-data-stream.asciidoc b/docs/reference/indices/delete-data-stream.asciidoc index 144a006ead0fa..0bcf9fcb3f43f 100644 --- a/docs/reference/indices/delete-data-stream.asciidoc +++ b/docs/reference/indices/delete-data-stream.asciidoc @@ -18,9 +18,6 @@ PUT _index_template/template } PUT /_data_stream/my-data-stream -{ - "timestamp_field" : "@timestamp" -} ----------------------------------- // TESTSETUP //// diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index c0f0b34c7f053..bb35736d459bf 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -18,9 +18,6 @@ PUT _index_template/template } PUT /_data_stream/my-data-stream -{ - "timestamp_field" : "@timestamp" -} ----------------------------------- // TESTSETUP //// diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml index 509758ba68a2c..0837b84eb7e1c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml @@ -23,8 +23,8 @@ setup: --- "Create data stream": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.99.99" + reason: "mute bwc until backported" - do: indices.create_data_stream: @@ -80,8 +80,8 @@ setup: --- "Create data stream with invalid name": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.99.99" + reason: "mute bwc until backported" - do: catch: bad_request @@ -94,8 +94,8 @@ setup: --- "Get data stream": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.99.99" + reason: "mute bwc until backported" - do: indices.create_data_stream: @@ -159,8 +159,8 @@ setup: --- "Delete data stream with backing indices": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.99.99" + reason: "mute bwc until backported" - do: indices.create_data_stream: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml index bc45c2501a087..33f2b15519168 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml @@ -1,8 +1,8 @@ --- "Test apis that do not supported data streams": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.99.99" + reason: "mute bwc until backported" features: allowed_warnings - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml index cfd86be5b4cac..d85a252a4cfe5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml @@ -14,8 +14,8 @@ setup: --- "Delete backing index on data stream": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.99.99" + reason: "mute bwc until backported" - do: indices.create_data_stream: @@ -66,8 +66,8 @@ setup: --- "Attempt to delete write index on data stream is rejected": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.99.99" + reason: "mute bwc until backported" - do: indices.create_data_stream: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml index 1543f63609d88..12c02310c0acc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml @@ -1,7 +1,7 @@ --- "Get backing indices for data stream": - skip: - version: " - 7.8.99" + version: " - 7.9.99" reason: "data streams only supported in 7.9+" features: allowed_warnings diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml index 29a4e8138e1f8..f9f4f17d638ea 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml @@ -1,8 +1,8 @@ --- "Roll over a data stream": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.99.99" + reason: "mute bwc until backported" features: allowed_warnings - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml index 520e5e0b3e5b6..3722a5e5fab52 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml @@ -1,8 +1,8 @@ --- "Verify data stream resolvability for xpack apis": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.99.99" + reason: "mute bwc until backported" features: allowed_warnings - do: From 825c787af936f1399dd7180c326f6fd2e31b2d70 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 20 May 2020 22:37:47 +0200 Subject: [PATCH 07/11] fixed docs test --- docs/reference/indices/rollover-index.asciidoc | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index 338df4dd3566a..f01ee5532fc49 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -242,9 +242,6 @@ PUT _index_template/template [source,console] -------------------------------------------------- PUT /_data_stream/my-data-stream <1> -{ - "timestamp_field": "date" -} # Add > 1000 documents to my-data-stream From 491a9c545bd19739a7cde2fce32a6ff7fca0fcd9 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 26 May 2020 14:35:49 +0200 Subject: [PATCH 08/11] applied feedback comments --- .../reference/indices/create-data-stream.asciidoc | 15 ++++----------- .../reference/indices/delete-data-stream.asciidoc | 2 +- docs/reference/indices/get-data-stream.asciidoc | 2 +- docs/reference/indices/rollover-index.asciidoc | 2 -- .../test/indices.data_stream/10_basic.yml | 1 + .../action/bulk/BulkIntegrationIT.java | 2 -- .../rollover/MetadataRolloverServiceTests.java | 2 +- 7 files changed, 8 insertions(+), 18 deletions(-) diff --git a/docs/reference/indices/create-data-stream.asciidoc b/docs/reference/indices/create-data-stream.asciidoc index 7378d36c321d3..e602c26c90ef0 100644 --- a/docs/reference/indices/create-data-stream.asciidoc +++ b/docs/reference/indices/create-data-stream.asciidoc @@ -23,7 +23,9 @@ addressed directly, data streams are integrated with the <> to facilitate the management of the time series data contained in their backing indices. -//// +A data stream can only be created if the namespace it targets has a component +template exists with a `data_stream` definition. + [source,console] ----------------------------------- PUT _index_template/template @@ -35,7 +37,6 @@ PUT _index_template/template } ----------------------------------- // TEST -//// [source,console] -------------------------------------------------- @@ -47,7 +48,7 @@ PUT _data_stream/my-data-stream [source,console] ----------------------------------- DELETE /_data_stream/my-data-stream -DELETE /_index_template/* +DELETE /_index_template/template ----------------------------------- // TEST[continued] //// @@ -84,11 +85,3 @@ Data stream names must meet the following criteria: will count towards the 255 limit faster) -- -[[indices-create-data-stream-api-request-body]] -==== {api-request-body-title} - -`timestamp_field`:: -(Required, string) The name of the timestamp field. This field must be present -in all documents indexed into the data stream and must be of type -<> or <>. - diff --git a/docs/reference/indices/delete-data-stream.asciidoc b/docs/reference/indices/delete-data-stream.asciidoc index 0bcf9fcb3f43f..5e6cf82e3e3c7 100644 --- a/docs/reference/indices/delete-data-stream.asciidoc +++ b/docs/reference/indices/delete-data-stream.asciidoc @@ -30,7 +30,7 @@ DELETE _data_stream/my-data-stream //// [source,console] ----------------------------------- -DELETE /_index_template/* +DELETE /_index_template/template ----------------------------------- // TEST[continued] //// diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index bb35736d459bf..ef533fc9f2e7b 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -26,7 +26,7 @@ PUT /_data_stream/my-data-stream [source,console] ----------------------------------- DELETE /_data_stream/my-data-stream -DELETE /_index_template/* +DELETE /_index_template/template ----------------------------------- // TEARDOWN //// diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index f01ee5532fc49..39a28c4b7a86d 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -225,7 +225,6 @@ The API returns the following response: [[rollover-data-stream-ex]] ===== Roll over a data stream -//// [source,console] ----------------------------------- PUT _index_template/template @@ -237,7 +236,6 @@ PUT _index_template/template } ----------------------------------- // TEST -//// [source,console] -------------------------------------------------- diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml index 0837b84eb7e1c..724a2891dae5f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml @@ -90,6 +90,7 @@ setup: - match: { status: 400 } - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.root_cause.0.reason: "data_stream [invalid-data-stream#-name] must not contain '#'" } --- "Get data stream": diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 0aed8c2aa6c67..a2669ef84131e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -218,8 +218,6 @@ public void testDeleteIndexWhileIndexing() throws Exception { } public void testMixedAutoCreate() throws Exception { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(); - PutIndexTemplateV2Action.Request createTemplateRequest = new PutIndexTemplateV2Action.Request("logs-foo"); createTemplateRequest.indexTemplate( new IndexTemplateV2( diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 35393f3f640e0..5635497902a59 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -518,7 +518,7 @@ public void testRolloverClusterStateForDataStream() throws Exception { IndexTemplateV2 template = new IndexTemplateV2(List.of(dataStream.getName() + "*"), null, null, null, null, null, new IndexTemplateV2.DataStreamTemplate("@timestamp")); Metadata.Builder builder = Metadata.builder(); - builder.put("_id", template); + builder.put("template", template); for (Index index : dataStream.getIndices()) { builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); } From 7d30aeed7441802ca82aa19f1d8d77295e9afe9a Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 27 May 2020 10:12:58 +0200 Subject: [PATCH 09/11] added docs about data_stream definition in a composable template. --- .../indices/index-templates.asciidoc | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index 23b265d1329af..86ad46f60ec69 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -26,7 +26,7 @@ specify settings, mappings, and aliases. If a new index matches more than one index template, the index template with the highest priority is used. -If an index is created with explicit settings and also matches an index template, +If an index is created with explicit settings and also matches an index template, the settings from the create index request take precedence over settings specified in the index template and its component templates. [source,console] @@ -112,7 +112,7 @@ DELETE _component_template/* [[put-index-template-api-desc]] ==== {api-description-title} -Creates or updates an index template. +Creates or updates an index template. // tag::index-template-def[] Index templates define <> and <> that you can @@ -555,3 +555,27 @@ PUT /_index_template/template_1 -------------------------------------------------- To check the `_meta`, you can use the <> API. + +[[data-stream-definition]] +===== Data stream definition + +If a composable template should auto create a data stream instead of an index then +a `data_stream` definition can be added to a composable template. + +[source,console] +-------------------------------------------------- +PUT /_index_template/template_1 +{ + "index_patterns": ["logs-"], + "data_stream": { + "timestamp_field": "@timestamp" + } +} +-------------------------------------------------- + +Required properties of a data stream definition: + +`timestamp_field`:: +(Required, string) The name of the timestamp field. This field must be present +in all documents indexed into the data stream and must be of type +<> or <>. From 78cf3882539da82a939d384237f62c88bcb9e2c5 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 27 May 2020 10:18:26 +0200 Subject: [PATCH 10/11] altered snippet --- docs/reference/indices/index-templates.asciidoc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index 86ad46f60ec69..1f717fd167f9d 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -566,7 +566,16 @@ a `data_stream` definition can be added to a composable template. -------------------------------------------------- PUT /_index_template/template_1 { - "index_patterns": ["logs-"], + "index_patterns": ["logs-*"], + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + } + } + } + }, "data_stream": { "timestamp_field": "@timestamp" } From 115c9cd08d93ec5810edeca2a7d41fa9c83b19f6 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 28 May 2020 11:26:52 +0200 Subject: [PATCH 11/11] fixed compile errors after merging in master --- .../elasticsearch/indices/DataStreamIT.java | 18 +++++++++--------- .../MetadataCreateDataStreamService.java | 10 +++++----- .../rollover/MetadataRolloverServiceTests.java | 4 ++-- .../MetadataCreateDataStreamServiceTests.java | 6 +++--- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java index deb74f3daeed9..728ec6293d4d9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java @@ -28,8 +28,8 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateV2Action; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; +import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -43,8 +43,8 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexTemplateV2; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESIntegTestCase; @@ -79,8 +79,8 @@ public class DataStreamIT extends ESIntegTestCase { @After public void deleteAllComposableTemplates() { - DeleteIndexTemplateV2Action.Request deleteTemplateRequest = new DeleteIndexTemplateV2Action.Request("*"); - client().execute(DeleteIndexTemplateV2Action.INSTANCE, deleteTemplateRequest).actionGet(); + DeleteComposableIndexTemplateAction.Request deleteTemplateRequest = new DeleteComposableIndexTemplateAction.Request("*"); + client().execute(DeleteComposableIndexTemplateAction.INSTANCE, deleteTemplateRequest).actionGet(); } public void testBasicScenario() throws Exception { @@ -336,15 +336,15 @@ private static void expectFailure(String dataStreamName, ThrowingRunnable runnab } static void createIndexTemplate(String id, String pattern, String timestampFieldName) throws IOException { - PutIndexTemplateV2Action.Request request = new PutIndexTemplateV2Action.Request(id); + PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(id); request.indexTemplate( - new IndexTemplateV2( + new ComposableIndexTemplate( List.of(pattern), null, null, null, null, null, - new IndexTemplateV2.DataStreamTemplate(timestampFieldName)) + new ComposableIndexTemplate.DataStreamTemplate(timestampFieldName)) ); - client().execute(PutIndexTemplateV2Action.INSTANCE, request).actionGet(); + client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index dec483d8853cf..87aaeeb36fdd0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -128,7 +128,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '.'"); } - IndexTemplateV2 template = lookupTemplateForDataStream(request.name, currentState.metadata()); + ComposableIndexTemplate template = lookupTemplateForDataStream(request.name, currentState.metadata()); String firstBackingIndexName = DataStream.getBackingIndexName(request.name, 1); CreateIndexClusterStateUpdateRequest createIndexRequest = @@ -144,17 +144,17 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn return ClusterState.builder(currentState).metadata(builder).build(); } - public static IndexTemplateV2 lookupTemplateForDataStream(String dataStreamName, Metadata metadata) { + public static ComposableIndexTemplate lookupTemplateForDataStream(String dataStreamName, Metadata metadata) { final String v2Template = MetadataIndexTemplateService.findV2Template(metadata, dataStreamName, false); if (v2Template == null) { throw new IllegalArgumentException("no matching index template found for data stream [" + dataStreamName + "]"); } - IndexTemplateV2 indexTemplateV2 = metadata.templatesV2().get(v2Template); - if (indexTemplateV2.getDataStreamTemplate() == null) { + ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(v2Template); + if (composableIndexTemplate.getDataStreamTemplate() == null) { throw new IllegalArgumentException("matching index template [" + v2Template + "] for data stream [" + dataStreamName + "] has no data stream template"); } - return indexTemplateV2; + return composableIndexTemplate; } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 295cad17c0f5d..39abd03f5b2aa 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -517,8 +517,8 @@ public void testRolloverClusterState() throws Exception { public void testRolloverClusterStateForDataStream() throws Exception { final DataStream dataStream = DataStreamTests.randomInstance(); - IndexTemplateV2 template = new IndexTemplateV2(List.of(dataStream.getName() + "*"), null, null, null, null, null, - new IndexTemplateV2.DataStreamTemplate("@timestamp")); + ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStream.getName() + "*"), null, null, null, null, null, + new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); Metadata.Builder builder = Metadata.builder(); builder.put("template", template); for (Index index : dataStream.getIndices()) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 68adba551e47a..367225bf4ce12 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -44,8 +44,8 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase { public void testCreateDataStream() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; - IndexTemplateV2 template = new IndexTemplateV2(List.of(dataStreamName + "*"), null, null, null, null, null, - new IndexTemplateV2.DataStreamTemplate("@timestamp")); + ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, + new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); @@ -122,7 +122,7 @@ public void testCreateDataStreamNoTemplate() throws Exception { public void testCreateDataStreamNoValidTemplate() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; - IndexTemplateV2 template = new IndexTemplateV2(List.of(dataStreamName + "*"), null, null, null, null, null, null); + ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, null); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build();