From 32519c04f8ec638f47aeaec857484fc31a818975 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Tue, 31 May 2022 22:13:41 +0300 Subject: [PATCH 01/28] Initial commit for Downsampling ILM Action --- .../xpack/core/ilm/RollupILMAction.java | 14 ++++++++++++-- .../xpack/core/ilm/RollupStep.java | 11 +++++++++-- .../xpack/core/ilm/TimeseriesLifecycleType.java | 9 +++++---- .../xpack/core/ilm/RollupILMActionTests.java | 17 ++++++++++------- .../xpack/ilm/actions/RollupActionIT.java | 3 +-- 5 files changed, 37 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 9d57e6d4a3ef0..747228b3f1386 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -104,10 +104,12 @@ public boolean isSafeAction() { @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); + StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME); StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyStep.NAME); StepKey generateRollupIndexNameKey = new StepKey(phase, NAME, GENERATE_ROLLUP_STEP_NAME); StepKey rollupKey = new StepKey(phase, NAME, NAME); CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, readOnlyKey); + WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, readOnlyKey, client); ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, generateRollupIndexNameKey, client); GenerateUniqueIndexNameStep generateRollupIndexNameStep = new GenerateUniqueIndexNameStep( generateRollupIndexNameKey, @@ -115,9 +117,10 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { ROLLUP_INDEX_PREFIX, (rollupIndexName, lifecycleStateBuilder) -> lifecycleStateBuilder.setRollupIndexName(rollupIndexName) ); + if (rollupPolicy == null) { Step rollupStep = new RollupStep(rollupKey, nextStepKey, client, config); - return List.of(checkNotWriteIndexStep, readOnlyStep, generateRollupIndexNameStep, rollupStep); + return List.of(checkNotWriteIndexStep, waitForNoFollowersStep, readOnlyStep, generateRollupIndexNameStep, rollupStep); } else { StepKey updateRollupIndexPolicyStepKey = new StepKey(phase, NAME, UpdateRollupIndexPolicyStep.NAME); Step rollupStep = new RollupStep(rollupKey, updateRollupIndexPolicyStepKey, client, config); @@ -127,7 +130,14 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { client, rollupPolicy ); - return List.of(checkNotWriteIndexStep, readOnlyStep, generateRollupIndexNameStep, rollupStep, updateRollupIndexPolicyStep); + return List.of( + checkNotWriteIndexStep, + waitForNoFollowersStep, + readOnlyStep, + generateRollupIndexNameStep, + rollupStep, + updateRollupIndexPolicyStep + ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index 3ad55a2b15d24..36a531345ffac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -20,11 +20,13 @@ import java.util.Objects; /** - * Rolls up index using a {@link RollupActionConfig} + * ILM step that invokes the rollup action for an index using a {@link RollupActionConfig}. The rollup + * action produces a rollup index using a prefix prepended to the original index name for the name of the rollup + * index. Also, the rollup action deletes the source index at the end, so no DeleteStep is required after this + * step. */ public class RollupStep extends AsyncActionStep { public static final String NAME = "rollup"; - public static final String ROLLUP_INDEX_NAME_PREFIX = "rollup-"; private final RollupActionConfig config; @@ -38,6 +40,11 @@ public boolean isRetryable() { return true; } + @Override + public boolean indexSurvives() { + return false; + } + @Override public void performAction( IndexMetadata indexMetadata, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index 8e295f99c81e0..006fcde7a131f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -61,24 +61,25 @@ public class TimeseriesLifecycleType implements LifecycleType { ForceMergeAction.NAME, SearchableSnapshotAction.NAME ).filter(Objects::nonNull).toList(); - public static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList( + public static final List ORDERED_VALID_WARM_ACTIONS = Stream.of( SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, + IndexSettings.isTimeSeriesModeEnabled() ? RollupILMAction.NAME : null, AllocateAction.NAME, MigrateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME - ); + ).filter(Objects::nonNull).toList(); public static final List ORDERED_VALID_COLD_ACTIONS = Stream.of( SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, + IndexSettings.isTimeSeriesModeEnabled() ? RollupILMAction.NAME : null, SearchableSnapshotAction.NAME, AllocateAction.NAME, MigrateAction.NAME, - FreezeAction.NAME, - IndexSettings.isTimeSeriesModeEnabled() ? RollupILMAction.NAME : null + FreezeAction.NAME ).filter(Objects::nonNull).toList(); public static final List ORDERED_VALID_FROZEN_ACTIONS = List.of(UnfollowAction.NAME, SearchableSnapshotAction.NAME); public static final List ORDERED_VALID_DELETE_ACTIONS = List.of(WaitForSnapshotAction.NAME, DeleteAction.NAME); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 42421ac761a82..8ad8cdd0668c4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -57,15 +57,18 @@ public void testToSteps() { ); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(4, steps.size()); + assertEquals(5, steps.size()); assertThat(steps.get(0).getKey().getName(), equalTo(CheckNotDataStreamWriteIndexStep.NAME)); assertThat(steps.get(0).getNextStepKey().getName(), equalTo(ReadOnlyStep.NAME)); - assertThat(steps.get(1).getKey().getName(), equalTo(ReadOnlyStep.NAME)); - assertThat(steps.get(1).getNextStepKey().getName(), equalTo(GENERATE_ROLLUP_STEP_NAME)); - assertThat(steps.get(2).getKey().getName(), equalTo(GENERATE_ROLLUP_STEP_NAME)); - assertThat(steps.get(2).getNextStepKey().getName(), equalTo(RollupStep.NAME)); - assertThat(steps.get(3).getKey().getName(), equalTo(RollupStep.NAME)); - assertThat(steps.get(3).getNextStepKey(), equalTo(nextStepKey)); + assertThat(steps.get(1).getKey().getName(), equalTo(WaitForNoFollowersStep.NAME)); + assertThat(steps.get(1).getNextStepKey().getName(), equalTo(ReadOnlyStep.NAME)); + + assertThat(steps.get(2).getKey().getName(), equalTo(ReadOnlyStep.NAME)); + assertThat(steps.get(2).getNextStepKey().getName(), equalTo(GENERATE_ROLLUP_STEP_NAME)); + assertThat(steps.get(3).getKey().getName(), equalTo(GENERATE_ROLLUP_STEP_NAME)); + assertThat(steps.get(3).getNextStepKey().getName(), equalTo(RollupStep.NAME)); + assertThat(steps.get(4).getKey().getName(), equalTo(RollupStep.NAME)); + assertThat(steps.get(4).getNextStepKey(), equalTo(nextStepKey)); } public void testEqualsAndHashCode() { diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index 85e8c557a7267..6f299194c4f29 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.ilm.actions; -import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -30,7 +29,7 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.updatePolicy; import static org.hamcrest.Matchers.equalTo; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/68609") +//@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/68609") public class RollupActionIT extends ESRestTestCase { private String index; From a34aa1cbe1fde82ca59833c3f17bfc6f3b54d134 Mon Sep 17 00:00:00 2001 From: Christos Soulios <1561376+csoulios@users.noreply.github.com> Date: Tue, 31 May 2022 22:21:32 +0300 Subject: [PATCH 02/28] Update docs/changelog/87269.yaml --- docs/changelog/87269.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/87269.yaml diff --git a/docs/changelog/87269.yaml b/docs/changelog/87269.yaml new file mode 100644 index 0000000000000..db76379a1b4cf --- /dev/null +++ b/docs/changelog/87269.yaml @@ -0,0 +1,6 @@ +pr: 87269 +summary: "TSDB: Implement Downsampling ILM Action for time-series indices" +area: "TSDB, ILM+SLM, Rollup" +type: feature +issues: + - 68609 From d0039854c9a2d002959334c31a66618c82a2b5ef Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Tue, 31 May 2022 22:35:07 +0300 Subject: [PATCH 03/28] Fixed changelog --- docs/changelog/87269.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog/87269.yaml b/docs/changelog/87269.yaml index db76379a1b4cf..1c401c7669ba6 100644 --- a/docs/changelog/87269.yaml +++ b/docs/changelog/87269.yaml @@ -1,6 +1,6 @@ pr: 87269 -summary: "TSDB: Implement Downsampling ILM Action for time-series indices" -area: "TSDB, ILM+SLM, Rollup" +summary: "TSDB: Implement downsampling ILM Action for time-series indices" +area: TSDB type: feature issues: - 68609 From f2c5471bb3c7f17a514179fbc152ec0dbd2dda8b Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Wed, 1 Jun 2022 16:09:34 +0300 Subject: [PATCH 04/28] Minor change to test --- .../rollup/v2/RollupActionSingleNodeTests.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/RollupActionSingleNodeTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/RollupActionSingleNodeTests.java index f633f23842065..18439ae99511f 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/RollupActionSingleNodeTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/RollupActionSingleNodeTests.java @@ -131,8 +131,8 @@ public void setup() { .prepareCreate(sourceIndex) .setSettings( Settings.builder() - .put("index.number_of_shards", numOfShards) - .put("index.number_of_replicas", numOfReplicas) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1)) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), Instant.ofEpochMilli(startTime).toString()) @@ -251,8 +251,8 @@ public void testCannotRollupIndexWithNoMetrics() { .prepareCreate(sourceIndex) .setSettings( Settings.builder() - .put("index.number_of_shards", numOfShards) - .put("index.number_of_replicas", numOfReplicas) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1)) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), Instant.ofEpochMilli(startTime).toString()) @@ -360,7 +360,10 @@ private void cloneSourceIndex(String sourceIndex, String sourceIndexClone) { .prepareResizeIndex(sourceIndex, sourceIndexClone) .setResizeType(ResizeType.CLONE) .setSettings( - Settings.builder().put("index.number_of_shards", numOfShards).put("index.number_of_replicas", numOfReplicas).build() + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) + .build() ) .get(); assertTrue(r.isAcknowledged()); @@ -556,8 +559,8 @@ private String createDataStream() throws Exception { String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault()); Template indexTemplate = new Template( Settings.builder() - .put("index.number_of_shards", numOfShards) - .put("index.number_of_replicas", numOfReplicas) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) .put("index.mode", "time_series") .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1)) .build(), From 45bc157cc8bfd4b3bd9319d999e832dc14b3146f Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Fri, 3 Jun 2022 19:48:18 +0300 Subject: [PATCH 05/28] Remove ilm policy for rollup index Fix tests --- .../xpack/core/ilm/RollupILMAction.java | 46 ++------- .../xpack/core/ilm/RollupILMActionTests.java | 26 ++--- .../xpack/core/ilm/RollupStepTests.java | 2 +- .../ilm/TimeseriesLifecycleTypeTests.java | 5 +- .../core/rollup/RollupActionConfigTests.java | 5 +- .../xpack/ilm/actions/RollupActionIT.java | 95 +++++++++++-------- 6 files changed, 75 insertions(+), 104 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 747228b3f1386..21eef9ea37af3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -30,18 +29,16 @@ public class RollupILMAction implements LifecycleAction { public static final String NAME = "rollup"; private static final ParseField CONFIG_FIELD = new ParseField("config"); - private static final ParseField POLICY_FIELD = new ParseField("rollup_policy"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, - a -> new RollupILMAction((RollupActionConfig) a[0], (String) a[1]) + a -> new RollupILMAction((RollupActionConfig) a[0]) ); public static final String ROLLUP_INDEX_PREFIX = "rollup-"; public static final String GENERATE_ROLLUP_STEP_NAME = "generate-rollup-name"; private final RollupActionConfig config; - private final String rollupPolicy; static { PARSER.declareField( @@ -50,20 +47,18 @@ public class RollupILMAction implements LifecycleAction { CONFIG_FIELD, ObjectParser.ValueType.OBJECT ); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), POLICY_FIELD); } public static RollupILMAction parse(XContentParser parser) { return PARSER.apply(parser, null); } - public RollupILMAction(RollupActionConfig config, @Nullable String rollupPolicy) { + public RollupILMAction(RollupActionConfig config) { this.config = config; - this.rollupPolicy = rollupPolicy; } public RollupILMAction(StreamInput in) throws IOException { - this(new RollupActionConfig(in), in.readOptionalString()); + this(new RollupActionConfig(in)); } @Override @@ -75,17 +70,10 @@ RollupActionConfig config() { return config; } - String rollupPolicy() { - return rollupPolicy; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(CONFIG_FIELD.getPreferredName(), config); - if (rollupPolicy != null) { - builder.field(POLICY_FIELD.getPreferredName(), rollupPolicy); - } builder.endObject(); return builder; } @@ -93,7 +81,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public void writeTo(StreamOutput out) throws IOException { config.writeTo(out); - out.writeOptionalString(rollupPolicy); } @Override @@ -118,27 +105,8 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { (rollupIndexName, lifecycleStateBuilder) -> lifecycleStateBuilder.setRollupIndexName(rollupIndexName) ); - if (rollupPolicy == null) { - Step rollupStep = new RollupStep(rollupKey, nextStepKey, client, config); - return List.of(checkNotWriteIndexStep, waitForNoFollowersStep, readOnlyStep, generateRollupIndexNameStep, rollupStep); - } else { - StepKey updateRollupIndexPolicyStepKey = new StepKey(phase, NAME, UpdateRollupIndexPolicyStep.NAME); - Step rollupStep = new RollupStep(rollupKey, updateRollupIndexPolicyStepKey, client, config); - Step updateRollupIndexPolicyStep = new UpdateRollupIndexPolicyStep( - updateRollupIndexPolicyStepKey, - nextStepKey, - client, - rollupPolicy - ); - return List.of( - checkNotWriteIndexStep, - waitForNoFollowersStep, - readOnlyStep, - generateRollupIndexNameStep, - rollupStep, - updateRollupIndexPolicyStep - ); - } + Step rollupStep = new RollupStep(rollupKey, nextStepKey, client, config); + return List.of(checkNotWriteIndexStep, waitForNoFollowersStep, readOnlyStep, generateRollupIndexNameStep, rollupStep); } @Override @@ -148,12 +116,12 @@ public boolean equals(Object o) { RollupILMAction that = (RollupILMAction) o; - return Objects.equals(this.config, that.config) && Objects.equals(this.rollupPolicy, that.rollupPolicy); + return Objects.equals(this.config, that.config); } @Override public int hashCode() { - return Objects.hash(config, rollupPolicy); + return Objects.hash(config); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 8ad8cdd0668c4..171eeecf3ccae 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -23,7 +23,7 @@ public class RollupILMActionTests extends AbstractActionTestCase { static RollupILMAction randomInstance() { - return new RollupILMAction(RollupActionConfigTests.randomConfig(random()), randomBoolean() ? randomAlphaOfLength(5) : null); + return new RollupILMAction(RollupActionConfigTests.randomConfig()); } @Override @@ -48,7 +48,7 @@ public boolean isSafeAction() { @Override public void testToSteps() { - RollupILMAction action = new RollupILMAction(RollupActionConfigTests.randomConfig(random()), null); + RollupILMAction action = new RollupILMAction(RollupActionConfigTests.randomConfig()); String phase = randomAlphaOfLengthBetween(1, 10); StepKey nextStepKey = new StepKey( randomAlphaOfLengthBetween(1, 10), @@ -76,23 +76,15 @@ public void testEqualsAndHashCode() { } RollupILMAction copy(RollupILMAction rollupILMAction) { - return new RollupILMAction(rollupILMAction.config(), rollupILMAction.rollupPolicy()); + return new RollupILMAction(rollupILMAction.config()); } RollupILMAction notCopy(RollupILMAction rollupILMAction) { - RollupActionConfig newConfig = rollupILMAction.config(); - String newRollupPolicy = rollupILMAction.rollupPolicy(); - switch (randomIntBetween(0, 1)) { - case 0 -> { - DateHistogramInterval fixedInterval = randomValueOtherThan( - rollupILMAction.config().getFixedInterval(), - ConfigTestHelpers::randomInterval - ); - newConfig = new RollupActionConfig(fixedInterval); - } - case 1 -> newRollupPolicy = randomAlphaOfLength(3); - default -> throw new IllegalStateException("unreachable branch"); - } - return new RollupILMAction(newConfig, newRollupPolicy); + DateHistogramInterval fixedInterval = randomValueOtherThan( + rollupILMAction.config().getFixedInterval(), + ConfigTestHelpers::randomInterval + ); + RollupActionConfig newConfig = new RollupActionConfig(fixedInterval); + return new RollupILMAction(newConfig); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java index 4e7cbc1bab050..2a6b625c7d53d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java @@ -37,7 +37,7 @@ public class RollupStepTests extends AbstractStepTestCase { public RollupStep createRandomInstance() { StepKey stepKey = randomStepKey(); StepKey nextStepKey = randomStepKey(); - RollupActionConfig config = RollupActionConfigTests.randomConfig(random()); + RollupActionConfig config = RollupActionConfigTests.randomConfig(); return new RollupStep(stepKey, nextStepKey, client, config); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index e1e817477b159..6101a842fe600 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -9,9 +9,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.rollup.RollupActionConfig; +import org.elasticsearch.xpack.core.rollup.RollupActionConfigTests; import java.util.ArrayList; import java.util.Arrays; @@ -72,7 +71,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase { // keeping the migrate action disabled as otherwise it could conflict with the allocate action if both are randomly selected for the // same phase private static final MigrateAction TEST_MIGRATE_ACTION = MigrateAction.DISABLED; - private static final RollupILMAction TEST_ROLLUP_ACTION = new RollupILMAction(new RollupActionConfig(DateHistogramInterval.DAY), null); + private static final RollupILMAction TEST_ROLLUP_ACTION = new RollupILMAction(RollupActionConfigTests.randomConfig()); public void testValidatePhases() { boolean invalid = randomBoolean(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupActionConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupActionConfigTests.java index 3c199cdc57564..4018cd25802da 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupActionConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupActionConfigTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; -import java.util.Random; import static org.hamcrest.Matchers.equalTo; @@ -20,10 +19,10 @@ public class RollupActionConfigTests extends AbstractSerializingTestCase running [{}] with index [{}], alias [{}] and policy [{}]", getTestName(), index, alias, policy); - } - - public void testRollupIndex() throws Exception { - createIndexWithSettings( - client(), - index, - alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - ); - index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); - RollupActionConfig rollupConfig = new RollupActionConfig(DateHistogramInterval.DAY); - createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, null)); - updatePolicy(client(), index, policy); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("dim_field1")) + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2006-01-08T23:40:53.384Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z") + .build(); - assertBusy(() -> assertNotNull(getRollupIndexName(index))); - String rollupIndex = getRollupIndexName(index); - assertBusy(() -> assertTrue(indexExists(rollupIndex))); - assertBusy(() -> assertFalse(getOnlyIndexSettings(client(), rollupIndex).containsKey(LifecycleSettings.LIFECYCLE_NAME))); - assertBusy(() -> assertTrue(indexExists(index))); + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("@timestamp") + .field("type", "date") + .endObject() + .startObject("dim_field1") + .field("type", "keyword") + .field("time_series_dimension", true) + .endObject() + .startObject("volume") + .field("type", "double") + .field("time_series_metric", "gauge") + .endObject() + .endObject() + .endObject(); + String mapping = Strings.toString(builder); + ESRestTestCase.createIndex(client(), index, settings, mapping, null); + index(client(), index, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "dim_field1", randomAlphaOfLength(5)); } - public void testRollupIndexAndSetNewRollupPolicy() throws Exception { - createIndexWithSettings( - client(), - index, - alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - ); - index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); - RollupActionConfig rollupConfig = new RollupActionConfig(DateHistogramInterval.DAY); - - createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, policy)); + public void testRollupIndex() throws Exception { + RollupActionConfig rollupConfig = RollupActionConfigTests.randomConfig(); + String phaseName = randomFrom("warm", "cold"); + createNewSingletonPolicy(client(), policy, phaseName, new RollupILMAction(rollupConfig)); updatePolicy(client(), index, policy); - assertBusy(() -> assertNotNull(getRollupIndexName(index))); + assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(index))); String rollupIndex = getRollupIndexName(index); - assertBusy(() -> assertTrue(indexExists(rollupIndex))); - assertBusy(() -> assertThat(getOnlyIndexSettings(client(), rollupIndex).get(LifecycleSettings.LIFECYCLE_NAME), equalTo(policy))); - assertBusy(() -> assertTrue(indexExists(index))); + assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex))); + assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(index))); } + // public void testRollupIndexInTheHotPhase() throws Exception { + // RollupActionConfig rollupConfig = RollupActionConfigTests.randomConfig(); + // createNewSingletonPolicy(client(), policy, "hot", new RollupILMAction(rollupConfig)); + // updatePolicy(client(), index, policy); + // + // assertBusy(() -> assertNotNull(getRollupIndexName(index))); + // String rollupIndex = getRollupIndexName(index); + // assertTrue(indexExists(rollupIndex)); + // assertFalse(indexExists(index)); + // } + /** * gets the generated rollup index name for a given index by looking at newly created indices that match the rollup index name pattern * @@ -92,7 +105,7 @@ public void testRollupIndexAndSetNewRollupPolicy() throws Exception { * @throws IOException if request fails */ private String getRollupIndexName(String index) throws IOException { - Response response = client().performRequest(new Request("GET", "/rollup-*-" + index)); + Response response = client().performRequest(new Request("GET", "/" + RollupILMAction.ROLLUP_INDEX_PREFIX + "*-" + index)); Map asMap = responseAsMap(response); if (asMap.size() == 1) { return (String) asMap.keySet().toArray()[0]; From 15650da7700def687c8ad282d3efdde673b95b38 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Fri, 3 Jun 2022 20:18:56 +0300 Subject: [PATCH 06/28] Fix test --- .../xpack/core/ilm/TimeseriesLifecycleTypeTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index 6101a842fe600..066a1582c8b6a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -9,8 +9,9 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.rollup.RollupActionConfigTests; +import org.elasticsearch.xpack.core.rollup.RollupActionConfig; import java.util.ArrayList; import java.util.Arrays; @@ -71,7 +72,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase { // keeping the migrate action disabled as otherwise it could conflict with the allocate action if both are randomly selected for the // same phase private static final MigrateAction TEST_MIGRATE_ACTION = MigrateAction.DISABLED; - private static final RollupILMAction TEST_ROLLUP_ACTION = new RollupILMAction(RollupActionConfigTests.randomConfig()); + private static final RollupILMAction TEST_ROLLUP_ACTION = new RollupILMAction(new RollupActionConfig(DateHistogramInterval.DAY)); public void testValidatePhases() { boolean invalid = randomBoolean(); From 1f478918668ec87badbdfd1183adee32bdc453b3 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Tue, 14 Jun 2022 19:05:17 +0300 Subject: [PATCH 07/28] Added more tests --- .../xpack/MigrateToDataTiersIT.java | 9 +- .../xpack/TimeSeriesRestDriver.java | 9 +- .../xpack/ilm/ChangePolicyForIndexIT.java | 3 +- .../xpack/ilm/ExplainLifecycleIT.java | 15 ++- .../ilm/TimeSeriesLifecycleActionsIT.java | 64 ++++++++----- .../xpack/ilm/TimeseriesMoveToStepIT.java | 25 +++-- .../xpack/ilm/actions/ReadonlyActionIT.java | 6 +- .../xpack/ilm/actions/RolloverActionIT.java | 17 +++- .../xpack/ilm/actions/RollupActionIT.java | 94 +++++++++++++++---- .../xpack/ilm/actions/ShrinkActionIT.java | 24 +++-- 10 files changed, 194 insertions(+), 72 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java index 336f5fe91bcf3..a8e287a2742fe 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java @@ -126,7 +126,8 @@ public void testMigrateToDataTiersAction() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .putNull(DataTier.TIER_PREFERENCE) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null ); // wait for the index to advance to the warm phase @@ -156,7 +157,8 @@ public void testMigrateToDataTiersAction() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .putNull(DataTier.TIER_PREFERENCE) // since we always enforce a tier preference, this will be ignored (i.e. // data_content) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias + i) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias + i), + null ); // the tier preference will have defaulted to data_content, set it back to null @@ -401,7 +403,8 @@ public void testMigrationDryRun() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .putNull(DataTier.TIER_PREFERENCE) // since we always enforce a tier preference, this will be ignored (i.e. data_content) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null ); // wait for the index to advance to the warm phase diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java index 48f21f23ccecb..1ae9d76020cde 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java @@ -287,9 +287,9 @@ public static Map getOnlyIndexSettings(RestClient client, String } } - public static void createIndexWithSettings(RestClient client, String index, String alias, Settings.Builder settings) + public static void createIndexWithSettings(RestClient client, String index, String alias, Settings.Builder settings, String mapping) throws IOException { - createIndexWithSettings(client, index, alias, settings, randomBoolean()); + createIndexWithSettings(client, index, alias, settings, mapping, randomBoolean()); } public static void createIndexWithSettings( @@ -297,6 +297,7 @@ public static void createIndexWithSettings( String index, String alias, Settings.Builder settings, + String mapping, boolean useWriteIndex ) throws IOException { Request request = new Request("PUT", "/" + index); @@ -305,11 +306,13 @@ public static void createIndexWithSettings( if (useWriteIndex) { writeIndexSnippet = "\"is_write_index\": true"; } + String m = mapping != null ? "\"mappings\": %s, ".formatted(mapping) : ""; request.setJsonEntity(""" { "settings": %s, + %s "aliases" : { "%s": { %s } } - }""".formatted(Strings.toString(settings.build()), alias, writeIndexSnippet)); + }""".formatted(Strings.toString(settings.build()), m, alias, writeIndexSnippet)); client.performRequest(request); // wait for the shards to initialize ensureGreen(index); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java index d23999449f9bf..030e2f80ea976 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java @@ -180,7 +180,8 @@ public void testILMHonoursTheCachedPhaseAfterPolicyUpdate() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) - .put(LifecycleSettings.LIFECYCLE_NAME, policyName) + .put(LifecycleSettings.LIFECYCLE_NAME, policyName), + null ); // Check the index is on the check-rollover-ready step diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java index c3e68610502ee..229b652043a4a 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java @@ -92,7 +92,8 @@ public void testExplainFilters() throws Exception { Settings.builder() .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null ); createIndexWithSettings( client(), @@ -159,7 +160,8 @@ public void testExplainIndicesWildcard() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null ); createIndexWithSettings( client(), @@ -168,13 +170,15 @@ public void testExplainIndicesWildcard() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null ); createIndexWithSettings( client(), unmanagedIndex, alias + unmanagedIndex, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); String missingPolicyName = "missing_policy_"; createIndexWithSettings( @@ -184,7 +188,8 @@ public void testExplainIndicesWildcard() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, missingPolicyName) + .put(LifecycleSettings.LIFECYCLE_NAME, missingPolicyName), + null ); assertBusy(() -> { diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 25ea0b1bf7995..464e69d6f49f3 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -105,7 +105,8 @@ public void testFullPolicy() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "javaRestTest-0") - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null ); // create policy @@ -139,7 +140,8 @@ public void testRetryFailedDeleteAction() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_READ_ONLY, true) - .put("index.lifecycle.name", policy) + .put("index.lifecycle.name", policy), + null ); assertBusy( @@ -166,7 +168,8 @@ public void testUpdatePolicyToNotContainFailedStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_READ_ONLY, true) - .put("index.lifecycle.name", policy) + .put("index.lifecycle.name", policy), + null ); assertBusy( @@ -197,7 +200,8 @@ public void testFreezeNoop() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.lifecycle.name", policy) + .put("index.lifecycle.name", policy), + null ); assertBusy( @@ -213,7 +217,8 @@ public void testAllocateOnlyAllocation() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); String allocateNodeName = "javaRestTest-0,javaRestTest-1,javaRestTest-2,javaRestTest-3"; AllocateAction allocateAction = new AllocateAction(null, null, singletonMap("_name", allocateNodeName), null, null); @@ -238,7 +243,8 @@ public void testAllocateActionOnlyReplicas() throws Exception { alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas), + null ); AllocateAction allocateAction = new AllocateAction(finalNumReplicas, null, null, null, null); String endPhase = randomFrom("warm", "cold"); @@ -256,7 +262,8 @@ public void testWaitForSnapshot() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); String slmPolicy = randomAlphaOfLengthBetween(4, 10); String snapshotRepo = randomAlphaOfLengthBetween(4, 10); @@ -301,7 +308,8 @@ public void testWaitForSnapshotFast() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); String slmPolicy = randomAlphaOfLengthBetween(4, 10); String snapshotRepo = randomAlphaOfLengthBetween(4, 10); @@ -327,7 +335,8 @@ public void testWaitForSnapshotSlmExecutedBefore() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); String slmPolicy = randomAlphaOfLengthBetween(4, 10); String snapshotRepo = randomAlphaOfLengthBetween(4, 10); @@ -406,7 +415,8 @@ public void testDelete() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); createNewSingletonPolicy(client(), policy, "delete", DeleteAction.WITH_SNAPSHOT_DELETE); updatePolicy(client(), index, policy); @@ -418,7 +428,8 @@ public void testDeleteOnlyShouldNotMakeIndexReadonly() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); createNewSingletonPolicy(client(), policy, "delete", DeleteAction.WITH_SNAPSHOT_DELETE, TimeValue.timeValueHours(1)); updatePolicy(client(), index, policy); @@ -454,7 +465,8 @@ public void testDeleteDuringSnapshot() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); // index document so snapshot actually does something indexDocument(client(), index); @@ -478,7 +490,8 @@ public void checkForceMergeAction(String codec) throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); for (int i = 0; i < randomIntBetween(2, 10); i++) { Request request = new Request("PUT", index + "/_doc/" + i); @@ -518,7 +531,8 @@ public void testSetPriority() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100) + .put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100), + null ); int priority = randomIntBetween(0, 99); createNewSingletonPolicy(client(), policy, "warm", new SetPriorityAction(priority)); @@ -538,7 +552,8 @@ public void testSetNullPriority() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100) + .put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100), + null ); createNewSingletonPolicy(client(), policy, "warm", new SetPriorityAction((Integer) null)); updatePolicy(client(), index, policy); @@ -703,7 +718,8 @@ public void testRemoveAndReaddPolicy() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null ); // Index a document @@ -741,7 +757,8 @@ public void testCanStopILMWithPolicyUsingNonexistentPolicy() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, randomAlphaOfLengthBetween(5, 15)) + .put(LifecycleSettings.LIFECYCLE_NAME, randomAlphaOfLengthBetween(5, 15)), + null ); Request stopILMRequest = new Request("POST", "_ilm/stop"); @@ -772,6 +789,7 @@ public void testWaitForActiveShardsStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null, true ); @@ -819,7 +837,7 @@ public void testHistoryIsWrittenWithSuccess() throws Exception { createIndexTemplate.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING)); client().performRequest(createIndexTemplate); - createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), true); + createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), null, true); // Index a document index(client(), index + "-1", "1", "foo", "bar"); @@ -837,7 +855,7 @@ public void testHistoryIsWrittenWithSuccess() throws Exception { } public void testHistoryIsWrittenWithFailure() throws Exception { - createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), false); + createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), null, false); createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, null, 1L, null)); updatePolicy(client(), index + "-1", policy); @@ -856,7 +874,7 @@ public void testHistoryIsWrittenWithFailure() throws Exception { public void testHistoryIsWrittenWithDeletion() throws Exception { // Index should be created and then deleted by ILM - createIndexWithSettings(client(), index, alias, Settings.builder(), false); + createIndexWithSettings(client(), index, alias, Settings.builder(), null, false); createNewSingletonPolicy(client(), policy, "delete", DeleteAction.WITH_SNAPSHOT_DELETE); updatePolicy(client(), index, policy); @@ -887,7 +905,8 @@ public void testRetryableInitializationStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, false) + .put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, false), + null ); updateIndexSettings(index, Settings.builder().put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, true)); @@ -933,6 +952,7 @@ public void testRefreshablePhaseJson() throws Exception { index + "-1", alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null, true ); @@ -963,6 +983,7 @@ public void testHaltAtEndOfPhase() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null, randomBoolean() ); @@ -1020,6 +1041,7 @@ public void testDeleteActionDoesntDeleteSearchableSnapshot() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null, randomBoolean() ); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java index c4f6e86e20421..70065a57670fa 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java @@ -66,7 +66,8 @@ public void testMoveToAllocateStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "javaRestTest-0") .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias") + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"), + null ); // move to a step @@ -104,7 +105,8 @@ public void testMoveToRolloverStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "javaRestTest-0") .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null ); // move to a step @@ -152,7 +154,8 @@ public void testMoveToInjectedStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null ); assertBusy(() -> assertThat(getStepKeyForIndex(client(), index), equalTo(new StepKey("new", "complete", "complete")))); @@ -202,6 +205,7 @@ public void testMoveToStepRereadsPolicy() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null, true ); @@ -248,7 +252,8 @@ public void testMoveToStepWithInvalidNextStep() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null ); // move to a step @@ -290,7 +295,8 @@ public void testMoveToStepWithoutStepName() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null ); // move to a step @@ -327,7 +333,8 @@ public void testMoveToStepWithoutAction() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null ); // move to a step @@ -363,7 +370,8 @@ public void testInvalidToMoveToStepWithoutActionButWithName() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null ); // move to a step with an invalid request @@ -399,7 +407,8 @@ public void testResolveToNonexistentStep() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null ); // move to a step with an invalid request diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ReadonlyActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ReadonlyActionIT.java index b2a39f2a2e1df..f0a4bee372637 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ReadonlyActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ReadonlyActionIT.java @@ -54,7 +54,8 @@ public void testReadOnly() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); String phaseName = randomFrom("warm", "cold"); createNewSingletonPolicy(client(), policy, phaseName, new ReadOnlyAction()); @@ -92,7 +93,8 @@ public void testReadOnlyInTheHotPhase() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) - .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + null ); index(client(), originalIndex, "_id", "foo", "bar"); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java index eab4b442868e0..ab15c8391ea44 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java @@ -63,7 +63,8 @@ public void testRolloverAction() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null ); // create policy @@ -91,7 +92,8 @@ public void testRolloverActionWithIndexingComplete() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null ); Request updateSettingsRequest = new Request("PUT", "/" + originalIndex + "/_settings"); @@ -142,7 +144,8 @@ public void testRolloverActionWithMaxPrimaryShardSize() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null ); index(client(), originalIndex, "_id", "foo", "bar"); @@ -170,7 +173,8 @@ public void testRolloverActionWithMaxPrimaryDocsSize() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null ); index(client(), originalIndex, "_id", "foo", "bar"); @@ -204,6 +208,7 @@ public void testILMRolloverRetriesOnReadOnlyBlock() throws Exception { .put(LifecycleSettings.LIFECYCLE_NAME, policy) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) .put("index.blocks.read_only", true), + null, true ); @@ -251,6 +256,7 @@ public void testILMRolloverOnManuallyRolledIndex() throws Exception { originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null, true ); @@ -300,6 +306,7 @@ public void testRolloverStepRetriesUntilRolledOverIndexIsDeleted() throws Except .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null, false ); @@ -312,6 +319,7 @@ public void testRolloverStepRetriesUntilRolledOverIndexIsDeleted() throws Except .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null, true ); @@ -382,6 +390,7 @@ public void testUpdateRolloverLifecycleDateStepRetriesWhenRolloverInfoIsMissing( .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), + null, true ); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index beae92aeff7fc..5acad73eab58b 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -9,14 +9,22 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xpack.core.ilm.LifecycleAction; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.RollupILMAction; import org.elasticsearch.xpack.core.rollup.RollupActionConfig; import org.elasticsearch.xpack.core.rollup.RollupActionConfigTests; @@ -26,7 +34,9 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.index; import static org.elasticsearch.xpack.TimeSeriesRestDriver.updatePolicy; @@ -43,15 +53,18 @@ public void refreshIndex() throws IOException { policy = "policy-" + randomAlphaOfLength(5); alias = "alias-" + randomAlphaOfLength(5); logger.info("--> running [{}] with index [{}], alias [{}] and policy [{}]", getTestName(), index, alias, policy); + } - Settings settings = Settings.builder() + private void createIndex(String index, String alias) throws IOException { + Settings.Builder settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("dim_field1")) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2006-01-08T23:40:53.384Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z") - .build(); + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(LifecycleSettings.LIFECYCLE_NAME, policy); XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() @@ -70,11 +83,14 @@ public void refreshIndex() throws IOException { .endObject() .endObject(); String mapping = Strings.toString(builder); - ESRestTestCase.createIndex(client(), index, settings, mapping, null); - index(client(), index, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "dim_field1", randomAlphaOfLength(5)); + + createIndexWithSettings(client(), index, alias, settings, mapping); } public void testRollupIndex() throws Exception { + createIndex(index, alias); + index(client(), index, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "dim_field1", randomAlphaOfLength(5)); + RollupActionConfig rollupConfig = RollupActionConfigTests.randomConfig(); String phaseName = randomFrom("warm", "cold"); createNewSingletonPolicy(client(), policy, phaseName, new RollupILMAction(rollupConfig)); @@ -82,20 +98,66 @@ public void testRollupIndex() throws Exception { assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(index))); String rollupIndex = getRollupIndexName(index); - assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex))); - assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(index))); + assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); + assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(index)), 30, TimeUnit.SECONDS); + } + + public void testRollupIndexInTheHotPhase() throws Exception { + createIndex(index, alias); + index(client(), index, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "dim_field1", randomAlphaOfLength(5)); + RollupActionConfig rollupConfig = RollupActionConfigTests.randomConfig(); + + ResponseException e = expectThrows( + ResponseException.class, + () -> createNewSingletonPolicy(client(), policy, "hot", new RollupILMAction(rollupConfig)) + ); + assertTrue( + e.getMessage().contains("the [rollup] action(s) may not be used in the [hot] phase without an accompanying [rollover] action") + ); } - // public void testRollupIndexInTheHotPhase() throws Exception { - // RollupActionConfig rollupConfig = RollupActionConfigTests.randomConfig(); - // createNewSingletonPolicy(client(), policy, "hot", new RollupILMAction(rollupConfig)); - // updatePolicy(client(), index, policy); - // - // assertBusy(() -> assertNotNull(getRollupIndexName(index))); - // String rollupIndex = getRollupIndexName(index); - // assertTrue(indexExists(rollupIndex)); - // assertFalse(indexExists(index)); - // } + public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { + String originalIndex = index + "-000001"; + + // add a policy + Map hotActions = Map.of( + RolloverAction.NAME, + new RolloverAction(null, null, null, 1L, null), + RollupILMAction.NAME, + new RollupILMAction(RollupActionConfigTests.randomConfig()) + ); + Map phases = Map.of("hot", new Phase("hot", TimeValue.ZERO, hotActions)); + LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); + Request createPolicyRequest = new Request("PUT", "_ilm/policy/" + policy); + createPolicyRequest.setJsonEntity("{ \"policy\":" + Strings.toString(lifecyclePolicy) + "}"); + client().performRequest(createPolicyRequest); + + // and a template + Request createTemplateRequest = new Request("PUT", "_template/" + index); + createTemplateRequest.setJsonEntity(""" + { + "index_patterns": ["%s-*"], + "settings": { + "number_of_shards": %s, + "number_of_replicas": 0, + "index.lifecycle.name": "%s", + "index.lifecycle.rollover_alias": "%s" + } + }""".formatted(index, 1, policy, alias)); + createTemplateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING)); + client().performRequest(createTemplateRequest); + + // then create the index and index a document to trigger rollover + createIndex(originalIndex, alias); + index(client(), originalIndex, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "dim_field1", randomAlphaOfLength(5)); + + assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(originalIndex)), 30, TimeUnit.SECONDS); + String rollupIndex = getRollupIndexName(originalIndex); + + assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); + assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(originalIndex)), 30, TimeUnit.SECONDS); + // assertBusy(() -> assertThat(getStepKeyForIndex(client(), rollupIndex), equalTo(PhaseCompleteStep.finalStep("hot").getKey()))); + } /** * gets the generated rollup index name for a given index by looking at newly created indices that match the rollup index name pattern diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java index 8c70150376267..2bfa5bd4ee9a7 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java @@ -9,7 +9,6 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -80,7 +79,8 @@ public void testShrinkAction() throws Exception { client(), index, alias, - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(expectedFinalShards, null)); updatePolicy(client(), index, policy); @@ -106,7 +106,8 @@ public void testSkipShrinkSameShardsWithNumberOfShards() throws Exception { client(), index, alias, - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(numberOfShards, null)); updatePolicy(client(), index, policy); @@ -127,7 +128,8 @@ public void testSkipShrinkSameShardsWithMaxShardSize() throws Exception { client(), index, alias, - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(null, ByteSizeValue.ofGb(50))); updatePolicy(client(), index, policy); @@ -170,7 +172,8 @@ public void testShrinkDuringSnapshot() throws Exception { .put(SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) // required so the shrink doesn't wait on SetSingleNodeAllocateStep - .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_name", "javaRestTest-0") + .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_name", "javaRestTest-0"), + null ); // index document so snapshot actually does something indexDocument(client(), index); @@ -232,7 +235,7 @@ public void testShrinkActionInTheHotPhase() throws Exception { client().performRequest(createTemplateRequest); // then create the index and index a document to trigger rollover - createIndexWithSettings(client(), originalIndex, alias, Settings.builder(), true); + createIndexWithSettings(client(), originalIndex, alias, Settings.builder(), null, true); index(client(), originalIndex, "_id", "foo", "bar"); String shrunkenIndex = waitAndGetShrinkIndexName(client(), originalIndex); @@ -254,7 +257,8 @@ public void testSetSingleNodeAllocationRetriesUntilItSucceeds() throws Exception Settings.builder() .put(SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .putNull(DataTier.TIER_PREFERENCE) + .putNull(DataTier.TIER_PREFERENCE), + null ); ensureGreen(index); @@ -330,7 +334,8 @@ public void testAutomaticRetryFailedShrinkAction() throws Exception { client(), index, alias, - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), + null ); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(numShards + randomIntBetween(1, numShards), null)); updatePolicy(client(), index, policy); @@ -376,7 +381,8 @@ public void testTotalShardsPerNodeTooLow() throws Exception { Settings.builder() .put(SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), numShards - 2) + .put(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), numShards - 2), + null ); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(expectedFinalShards, null)); updatePolicy(client(), index, policy); From c82bd6ee6f7282b411b9ae24fcb021963f961d27 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Thu, 16 Jun 2022 00:27:02 +0300 Subject: [PATCH 08/28] Added tests for datastreams --- .../xpack/core/ilm/RollupILMAction.java | 18 ++- .../xpack/TimeSeriesRestDriver.java | 10 ++ .../xpack/ilm/TimeSeriesDataStreamsIT.java | 12 +- .../xpack/ilm/actions/RollupActionIT.java | 105 +++++++++++++++--- 4 files changed, 111 insertions(+), 34 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 21eef9ea37af3..56317c6a52c4d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -62,12 +62,8 @@ public RollupILMAction(StreamInput in) throws IOException { } @Override - public String getWriteableName() { - return NAME; - } - - RollupActionConfig config() { - return config; + public void writeTo(StreamOutput out) throws IOException { + config.writeTo(out); } @Override @@ -79,8 +75,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public void writeTo(StreamOutput out) throws IOException { - config.writeTo(out); + public String getWriteableName() { + return NAME; + } + + public RollupActionConfig config() { + return config; } @Override @@ -104,7 +104,6 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { ROLLUP_INDEX_PREFIX, (rollupIndexName, lifecycleStateBuilder) -> lifecycleStateBuilder.setRollupIndexName(rollupIndexName) ); - Step rollupStep = new RollupStep(rollupKey, nextStepKey, client, config); return List.of(checkNotWriteIndexStep, waitForNoFollowersStep, readOnlyStep, generateRollupIndexNameStep, rollupStep); } @@ -115,7 +114,6 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; RollupILMAction that = (RollupILMAction) o; - return Objects.equals(this.config, that.config); } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java index 1ae9d76020cde..75fd1c878d5b3 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java @@ -14,6 +14,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -29,6 +30,7 @@ import org.elasticsearch.xpack.core.ilm.ForceMergeAction; import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.core.ilm.Phase; import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; @@ -421,4 +423,12 @@ public static String waitAndGetShrinkIndexName(RestClient client, String origina logger.info("--> original index name is [{}], shrunken index name is [{}]", originalIndex, shrunkenIndexName[0]); return shrunkenIndexName[0]; } + + public static Template getTemplate(String policyName) { + return new Template(getLifecycleSettings(policyName), null, null); + } + + public static Settings getLifecycleSettings(String policyName) { + return Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).build(); + } } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java index 65f4538a5b705..213db42b881b0 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java @@ -7,13 +7,10 @@ package org.elasticsearch.xpack.ilm; -import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.test.rest.ESRestTestCase; @@ -22,7 +19,6 @@ import org.elasticsearch.xpack.core.ilm.DeleteAction; import org.elasticsearch.xpack.core.ilm.ForceMergeAction; import org.elasticsearch.xpack.core.ilm.FreezeAction; -import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; import org.elasticsearch.xpack.core.ilm.ReadOnlyAction; import org.elasticsearch.xpack.core.ilm.RolloverAction; @@ -31,7 +27,6 @@ import org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep; import org.junit.Before; -import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Locale; @@ -44,6 +39,7 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getOnlyIndexSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getTemplate; import static org.elasticsearch.xpack.TimeSeriesRestDriver.indexDocument; import static org.elasticsearch.xpack.TimeSeriesRestDriver.rolloverMaxOneDocCondition; import static org.elasticsearch.xpack.TimeSeriesRestDriver.waitAndGetShrinkIndexName; @@ -307,11 +303,5 @@ public void testDeleteOnlyIndexInDataStreamDeletesDataStream() throws Exception }); } - private static Template getTemplate(String policyName) throws IOException { - return new Template(getLifecycleSettings(policyName), null, null); - } - private static Settings getLifecycleSettings(String policyName) { - return Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).build(); - } } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index 5acad73eab58b..99b8a9f604ac4 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -10,9 +10,12 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; @@ -20,17 +23,18 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xpack.core.ilm.CheckNotDataStreamWriteIndexStep; import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.core.ilm.Phase; import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.RollupILMAction; -import org.elasticsearch.xpack.core.rollup.RollupActionConfig; import org.elasticsearch.xpack.core.rollup.RollupActionConfigTests; import org.junit.Before; import java.io.IOException; +import java.time.Instant; import java.util.List; import java.util.Locale; import java.util.Map; @@ -38,8 +42,11 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.index; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.rolloverMaxOneDocCondition; import static org.elasticsearch.xpack.TimeSeriesRestDriver.updatePolicy; +import static org.hamcrest.Matchers.is; public class RollupActionIT extends ESRestTestCase { @@ -47,8 +54,39 @@ public class RollupActionIT extends ESRestTestCase { private String policy; private String alias; + private static final String TEMPLATE = """ + { + "index_patterns": ["%s*"], + "template": { + "settings":{ + "index": { + "number_of_replicas": 0, + "number_of_shards": 1, + "mode": "time_series" + }, + "index.lifecycle.name": "%s" + }, + "mappings":{ + "properties": { + "@timestamp" : { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + }, + "volume": { + "type": "double", + "time_series_metric": "gauge" + } + } + } + }, + "data_stream": { } + }"""; + @Before - public void refreshIndex() throws IOException { + public void refreshAbstractions() { index = "index-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); policy = "policy-" + randomAlphaOfLength(5); alias = "alias-" + randomAlphaOfLength(5); @@ -60,7 +98,7 @@ private void createIndex(String index, String alias) throws IOException { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) - .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("dim_field1")) + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("metricset")) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2006-01-08T23:40:53.384Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z") .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) @@ -72,7 +110,7 @@ private void createIndex(String index, String alias) throws IOException { .startObject("@timestamp") .field("type", "date") .endObject() - .startObject("dim_field1") + .startObject("metricset") .field("type", "keyword") .field("time_series_dimension", true) .endObject() @@ -83,17 +121,15 @@ private void createIndex(String index, String alias) throws IOException { .endObject() .endObject(); String mapping = Strings.toString(builder); - createIndexWithSettings(client(), index, alias, settings, mapping); } public void testRollupIndex() throws Exception { createIndex(index, alias); - index(client(), index, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "dim_field1", randomAlphaOfLength(5)); + index(client(), index, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); - RollupActionConfig rollupConfig = RollupActionConfigTests.randomConfig(); String phaseName = randomFrom("warm", "cold"); - createNewSingletonPolicy(client(), policy, phaseName, new RollupILMAction(rollupConfig)); + createNewSingletonPolicy(client(), policy, phaseName, new RollupILMAction(RollupActionConfigTests.randomConfig())); updatePolicy(client(), index, policy); assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(index))); @@ -104,12 +140,11 @@ public void testRollupIndex() throws Exception { public void testRollupIndexInTheHotPhase() throws Exception { createIndex(index, alias); - index(client(), index, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "dim_field1", randomAlphaOfLength(5)); - RollupActionConfig rollupConfig = RollupActionConfigTests.randomConfig(); + index(client(), index, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); ResponseException e = expectThrows( ResponseException.class, - () -> createNewSingletonPolicy(client(), policy, "hot", new RollupILMAction(rollupConfig)) + () -> createNewSingletonPolicy(client(), policy, "hot", new RollupILMAction(RollupActionConfigTests.randomConfig())) ); assertTrue( e.getMessage().contains("the [rollup] action(s) may not be used in the [hot] phase without an accompanying [rollover] action") @@ -149,7 +184,7 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { // then create the index and index a document to trigger rollover createIndex(originalIndex, alias); - index(client(), originalIndex, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "dim_field1", randomAlphaOfLength(5)); + index(client(), originalIndex, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(originalIndex)), 30, TimeUnit.SECONDS); String rollupIndex = getRollupIndexName(originalIndex); @@ -159,6 +194,48 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { // assertBusy(() -> assertThat(getStepKeyForIndex(client(), rollupIndex), equalTo(PhaseCompleteStep.finalStep("hot").getKey()))); } + public void testTsdbDataStreams() throws Exception { + final String dataStream = "k8s-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + + // Create the ILM policy + createNewSingletonPolicy(client(), policy, "warm", new RollupILMAction(RollupActionConfigTests.randomConfig())); + + // Create a template + Request createIndexTemplateRequest = new Request("POST", "/_index_template/" + dataStream); + createIndexTemplateRequest.setJsonEntity(TEMPLATE.formatted(dataStream, policy)); + assertOK(client().performRequest(createIndexTemplateRequest)); + + String now = DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(Instant.now()); + index(client(), dataStream, "", "@timestamp", now, "volume", 11.0, "metricset", randomAlphaOfLength(5)); + + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + assertBusy( + () -> assertThat( + "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", + explainIndex(client(), backingIndexName).get("step"), + is(CheckNotDataStreamWriteIndexStep.NAME) + ), + 30, + TimeUnit.SECONDS + ); + + // Manual rollover the original index such that it's not the write index in the data stream anymore + rolloverMaxOneDocCondition(client(), dataStream); +// assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(backingIndexName)), 30, TimeUnit.SECONDS); + waitUntil(() -> { + try { + String rollupIndex = getRollupIndexName(backingIndexName); + return rollupIndex != null; + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS); + String rollupIndex = getRollupIndexName(backingIndexName); + + assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); + assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(backingIndexName)), 30, TimeUnit.SECONDS); + } + /** * gets the generated rollup index name for a given index by looking at newly created indices that match the rollup index name pattern * @@ -167,7 +244,9 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { * @throws IOException if request fails */ private String getRollupIndexName(String index) throws IOException { - Response response = client().performRequest(new Request("GET", "/" + RollupILMAction.ROLLUP_INDEX_PREFIX + "*-" + index)); + Response response = client().performRequest( + new Request("GET", "/" + RollupILMAction.ROLLUP_INDEX_PREFIX + "*-" + index + "/?expand_wildcards=all") + ); Map asMap = responseAsMap(response); if (asMap.size() == 1) { return (String) asMap.keySet().toArray()[0]; From 9c0d42279cc0ea0985e36f48577ce56b757e236a Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Thu, 16 Jun 2022 11:09:26 +0300 Subject: [PATCH 09/28] Checkstyle --- .../org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java | 1 - .../org/elasticsearch/xpack/ilm/actions/RollupActionIT.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java index 213db42b881b0..78745c29c4d03 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java @@ -303,5 +303,4 @@ public void testDeleteOnlyIndexInDataStreamDeletesDataStream() throws Exception }); } - } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index 99b8a9f604ac4..471cbbb086080 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -221,7 +221,7 @@ public void testTsdbDataStreams() throws Exception { // Manual rollover the original index such that it's not the write index in the data stream anymore rolloverMaxOneDocCondition(client(), dataStream); -// assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(backingIndexName)), 30, TimeUnit.SECONDS); + // assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(backingIndexName)), 30, TimeUnit.SECONDS); waitUntil(() -> { try { String rollupIndex = getRollupIndexName(backingIndexName); From 6bc6b154a956768c4c92576edb718c16c1311adc Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Thu, 16 Jun 2022 13:25:17 +0300 Subject: [PATCH 10/28] Fix broken test --- .../xpack/MigrateToDataTiersIT.java | 9 ++--- .../xpack/TimeSeriesRestDriver.java | 21 +++++++++++- .../xpack/ilm/ChangePolicyForIndexIT.java | 3 +- .../xpack/ilm/ExplainLifecycleIT.java | 3 +- .../ilm/TimeSeriesLifecycleActionsIT.java | 10 ++---- .../xpack/ilm/TimeseriesMoveToStepIT.java | 1 - .../xpack/ilm/actions/RolloverActionIT.java | 5 --- .../xpack/ilm/actions/RollupActionIT.java | 33 ++++++++++--------- .../xpack/ilm/actions/ShrinkActionIT.java | 2 +- 9 files changed, 47 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java index a8e287a2742fe..336f5fe91bcf3 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java @@ -126,8 +126,7 @@ public void testMigrateToDataTiersAction() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .putNull(DataTier.TIER_PREFERENCE) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); // wait for the index to advance to the warm phase @@ -157,8 +156,7 @@ public void testMigrateToDataTiersAction() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .putNull(DataTier.TIER_PREFERENCE) // since we always enforce a tier preference, this will be ignored (i.e. // data_content) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias + i), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias + i) ); // the tier preference will have defaulted to data_content, set it back to null @@ -403,8 +401,7 @@ public void testMigrationDryRun() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .putNull(DataTier.TIER_PREFERENCE) // since we always enforce a tier preference, this will be ignored (i.e. data_content) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); // wait for the index to advance to the warm phase diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java index 75fd1c878d5b3..2271334a352b7 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java @@ -115,12 +115,16 @@ public static void indexDocument(RestClient client, String indexAbstractionName, } public static void index(RestClient client, String index, String id, Object... fields) throws IOException { + index(client, index, false, id, fields); + } + + public static void index(RestClient client, String index, boolean refresh, String id, Object... fields) throws IOException { XContentBuilder document = jsonBuilder().startObject(); for (int i = 0; i < fields.length; i += 2) { document.field((String) fields[i], fields[i + 1]); } document.endObject(); - final Request request = new Request("POST", "/" + index + "/_doc/" + id); + final Request request = new Request("POST", "/" + index + "/_doc/" + (id != null ? id : "") + (refresh ? "?refresh" : "")); request.setJsonEntity(Strings.toString(document)); assertThat(client.performRequest(request).getStatusLine().getStatusCode(), anyOf(equalTo(200), equalTo(201))); } @@ -289,11 +293,26 @@ public static Map getOnlyIndexSettings(RestClient client, String } } + public static void createIndexWithSettings(RestClient client, String index, String alias, Settings.Builder settings) + throws IOException { + createIndexWithSettings(client, index, alias, settings, null); + } + public static void createIndexWithSettings(RestClient client, String index, String alias, Settings.Builder settings, String mapping) throws IOException { createIndexWithSettings(client, index, alias, settings, mapping, randomBoolean()); } + public static void createIndexWithSettings( + RestClient client, + String index, + String alias, + Settings.Builder settings, + boolean useWriteIndex + ) throws IOException { + createIndexWithSettings(client, index, alias, settings, null, useWriteIndex); + } + public static void createIndexWithSettings( RestClient client, String index, diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java index 030e2f80ea976..d23999449f9bf 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java @@ -180,8 +180,7 @@ public void testILMHonoursTheCachedPhaseAfterPolicyUpdate() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) - .put(LifecycleSettings.LIFECYCLE_NAME, policyName), - null + .put(LifecycleSettings.LIFECYCLE_NAME, policyName) ); // Check the index is on the check-rollover-ready step diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java index 229b652043a4a..b6063c40013fe 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java @@ -92,8 +92,7 @@ public void testExplainFilters() throws Exception { Settings.builder() .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null + .put(LifecycleSettings.LIFECYCLE_NAME, policy) ); createIndexWithSettings( client(), diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 464e69d6f49f3..04161c2fc7551 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -789,7 +789,6 @@ public void testWaitForActiveShardsStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null, true ); @@ -837,7 +836,7 @@ public void testHistoryIsWrittenWithSuccess() throws Exception { createIndexTemplate.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING)); client().performRequest(createIndexTemplate); - createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), null, true); + createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), true); // Index a document index(client(), index + "-1", "1", "foo", "bar"); @@ -855,7 +854,7 @@ public void testHistoryIsWrittenWithSuccess() throws Exception { } public void testHistoryIsWrittenWithFailure() throws Exception { - createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), null, false); + createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), false); createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, null, 1L, null)); updatePolicy(client(), index + "-1", policy); @@ -874,7 +873,7 @@ public void testHistoryIsWrittenWithFailure() throws Exception { public void testHistoryIsWrittenWithDeletion() throws Exception { // Index should be created and then deleted by ILM - createIndexWithSettings(client(), index, alias, Settings.builder(), null, false); + createIndexWithSettings(client(), index, alias, Settings.builder(), false); createNewSingletonPolicy(client(), policy, "delete", DeleteAction.WITH_SNAPSHOT_DELETE); updatePolicy(client(), index, policy); @@ -952,7 +951,6 @@ public void testRefreshablePhaseJson() throws Exception { index + "-1", alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null, true ); @@ -983,7 +981,6 @@ public void testHaltAtEndOfPhase() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null, randomBoolean() ); @@ -1041,7 +1038,6 @@ public void testDeleteActionDoesntDeleteSearchableSnapshot() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null, randomBoolean() ); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java index 70065a57670fa..305f814b544af 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java @@ -205,7 +205,6 @@ public void testMoveToStepRereadsPolicy() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null, true ); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java index ab15c8391ea44..0d0ecabb6cacb 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java @@ -208,7 +208,6 @@ public void testILMRolloverRetriesOnReadOnlyBlock() throws Exception { .put(LifecycleSettings.LIFECYCLE_NAME, policy) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) .put("index.blocks.read_only", true), - null, true ); @@ -256,7 +255,6 @@ public void testILMRolloverOnManuallyRolledIndex() throws Exception { originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null, true ); @@ -306,7 +304,6 @@ public void testRolloverStepRetriesUntilRolledOverIndexIsDeleted() throws Except .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null, false ); @@ -319,7 +316,6 @@ public void testRolloverStepRetriesUntilRolledOverIndexIsDeleted() throws Except .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null, true ); @@ -390,7 +386,6 @@ public void testUpdateRolloverLifecycleDateStepRetriesWhenRolloverInfoIsMissing( .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null, true ); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index 471cbbb086080..f764de476bc54 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -126,13 +126,13 @@ private void createIndex(String index, String alias) throws IOException { public void testRollupIndex() throws Exception { createIndex(index, alias); - index(client(), index, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); + index(client(), index, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); String phaseName = randomFrom("warm", "cold"); createNewSingletonPolicy(client(), policy, phaseName, new RollupILMAction(RollupActionConfigTests.randomConfig())); updatePolicy(client(), index, policy); - assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(index))); + assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(index)), 30, TimeUnit.SECONDS); String rollupIndex = getRollupIndexName(index); assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(index)), 30, TimeUnit.SECONDS); @@ -140,7 +140,7 @@ public void testRollupIndex() throws Exception { public void testRollupIndexInTheHotPhase() throws Exception { createIndex(index, alias); - index(client(), index, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); + index(client(), index, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); ResponseException e = expectThrows( ResponseException.class, @@ -184,7 +184,18 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { // then create the index and index a document to trigger rollover createIndex(originalIndex, alias); - index(client(), originalIndex, "", "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); + index( + client(), + originalIndex, + true, + null, + "@timestamp", + "2020-01-01T05:10:00Z", + "volume", + 11.0, + "metricset", + randomAlphaOfLength(5) + ); assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(originalIndex)), 30, TimeUnit.SECONDS); String rollupIndex = getRollupIndexName(originalIndex); @@ -206,7 +217,7 @@ public void testTsdbDataStreams() throws Exception { assertOK(client().performRequest(createIndexTemplateRequest)); String now = DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(Instant.now()); - index(client(), dataStream, "", "@timestamp", now, "volume", 11.0, "metricset", randomAlphaOfLength(5)); + index(client(), dataStream, true, null, "@timestamp", now, "volume", 11.0, "metricset", randomAlphaOfLength(5)); String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); assertBusy( @@ -221,17 +232,9 @@ public void testTsdbDataStreams() throws Exception { // Manual rollover the original index such that it's not the write index in the data stream anymore rolloverMaxOneDocCondition(client(), dataStream); - // assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(backingIndexName)), 30, TimeUnit.SECONDS); - waitUntil(() -> { - try { - String rollupIndex = getRollupIndexName(backingIndexName); - return rollupIndex != null; - } catch (IOException e) { - return false; - } - }, 30, TimeUnit.SECONDS); - String rollupIndex = getRollupIndexName(backingIndexName); + assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(backingIndexName)), 30, TimeUnit.SECONDS); + String rollupIndex = getRollupIndexName(backingIndexName); assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(backingIndexName)), 30, TimeUnit.SECONDS); } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java index 2bfa5bd4ee9a7..9b0615940fe60 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java @@ -235,7 +235,7 @@ public void testShrinkActionInTheHotPhase() throws Exception { client().performRequest(createTemplateRequest); // then create the index and index a document to trigger rollover - createIndexWithSettings(client(), originalIndex, alias, Settings.builder(), null, true); + createIndexWithSettings(client(), originalIndex, alias, Settings.builder(), true); index(client(), originalIndex, "_id", "foo", "bar"); String shrunkenIndex = waitAndGetShrinkIndexName(client(), originalIndex); From 4114af627778509a64be5e311d11ff86b854fb45 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Thu, 16 Jun 2022 13:35:02 +0300 Subject: [PATCH 11/28] Cleanup code --- .../xpack/ilm/ExplainLifecycleIT.java | 12 ++--- .../ilm/TimeSeriesLifecycleActionsIT.java | 54 +++++++------------ .../xpack/ilm/TimeseriesMoveToStepIT.java | 24 +++------ .../xpack/ilm/actions/ReadonlyActionIT.java | 6 +-- .../xpack/ilm/actions/RolloverActionIT.java | 12 ++--- .../xpack/ilm/actions/ShrinkActionIT.java | 22 +++----- 6 files changed, 44 insertions(+), 86 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java index b6063c40013fe..c3e68610502ee 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java @@ -159,8 +159,7 @@ public void testExplainIndicesWildcard() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null + .put(LifecycleSettings.LIFECYCLE_NAME, policy) ); createIndexWithSettings( client(), @@ -169,15 +168,13 @@ public void testExplainIndicesWildcard() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null + .put(LifecycleSettings.LIFECYCLE_NAME, policy) ); createIndexWithSettings( client(), unmanagedIndex, alias + unmanagedIndex, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); String missingPolicyName = "missing_policy_"; createIndexWithSettings( @@ -187,8 +184,7 @@ public void testExplainIndicesWildcard() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, missingPolicyName), - null + .put(LifecycleSettings.LIFECYCLE_NAME, missingPolicyName) ); assertBusy(() -> { diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 04161c2fc7551..25ea0b1bf7995 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -105,8 +105,7 @@ public void testFullPolicy() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "javaRestTest-0") - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); // create policy @@ -140,8 +139,7 @@ public void testRetryFailedDeleteAction() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_READ_ONLY, true) - .put("index.lifecycle.name", policy), - null + .put("index.lifecycle.name", policy) ); assertBusy( @@ -168,8 +166,7 @@ public void testUpdatePolicyToNotContainFailedStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_READ_ONLY, true) - .put("index.lifecycle.name", policy), - null + .put("index.lifecycle.name", policy) ); assertBusy( @@ -200,8 +197,7 @@ public void testFreezeNoop() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.lifecycle.name", policy), - null + .put("index.lifecycle.name", policy) ); assertBusy( @@ -217,8 +213,7 @@ public void testAllocateOnlyAllocation() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); String allocateNodeName = "javaRestTest-0,javaRestTest-1,javaRestTest-2,javaRestTest-3"; AllocateAction allocateAction = new AllocateAction(null, null, singletonMap("_name", allocateNodeName), null, null); @@ -243,8 +238,7 @@ public void testAllocateActionOnlyReplicas() throws Exception { alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas), - null + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas) ); AllocateAction allocateAction = new AllocateAction(finalNumReplicas, null, null, null, null); String endPhase = randomFrom("warm", "cold"); @@ -262,8 +256,7 @@ public void testWaitForSnapshot() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); String slmPolicy = randomAlphaOfLengthBetween(4, 10); String snapshotRepo = randomAlphaOfLengthBetween(4, 10); @@ -308,8 +301,7 @@ public void testWaitForSnapshotFast() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); String slmPolicy = randomAlphaOfLengthBetween(4, 10); String snapshotRepo = randomAlphaOfLengthBetween(4, 10); @@ -335,8 +327,7 @@ public void testWaitForSnapshotSlmExecutedBefore() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); String slmPolicy = randomAlphaOfLengthBetween(4, 10); String snapshotRepo = randomAlphaOfLengthBetween(4, 10); @@ -415,8 +406,7 @@ public void testDelete() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); createNewSingletonPolicy(client(), policy, "delete", DeleteAction.WITH_SNAPSHOT_DELETE); updatePolicy(client(), index, policy); @@ -428,8 +418,7 @@ public void testDeleteOnlyShouldNotMakeIndexReadonly() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); createNewSingletonPolicy(client(), policy, "delete", DeleteAction.WITH_SNAPSHOT_DELETE, TimeValue.timeValueHours(1)); updatePolicy(client(), index, policy); @@ -465,8 +454,7 @@ public void testDeleteDuringSnapshot() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); // index document so snapshot actually does something indexDocument(client(), index); @@ -490,8 +478,7 @@ public void checkForceMergeAction(String codec) throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); for (int i = 0; i < randomIntBetween(2, 10); i++) { Request request = new Request("PUT", index + "/_doc/" + i); @@ -531,8 +518,7 @@ public void testSetPriority() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100), - null + .put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100) ); int priority = randomIntBetween(0, 99); createNewSingletonPolicy(client(), policy, "warm", new SetPriorityAction(priority)); @@ -552,8 +538,7 @@ public void testSetNullPriority() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100), - null + .put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100) ); createNewSingletonPolicy(client(), policy, "warm", new SetPriorityAction((Integer) null)); updatePolicy(client(), index, policy); @@ -718,8 +703,7 @@ public void testRemoveAndReaddPolicy() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); // Index a document @@ -757,8 +741,7 @@ public void testCanStopILMWithPolicyUsingNonexistentPolicy() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, randomAlphaOfLengthBetween(5, 15)), - null + .put(LifecycleSettings.LIFECYCLE_NAME, randomAlphaOfLengthBetween(5, 15)) ); Request stopILMRequest = new Request("POST", "_ilm/stop"); @@ -904,8 +887,7 @@ public void testRetryableInitializationStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, false), - null + .put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, false) ); updateIndexSettings(index, Settings.builder().put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, true)); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java index 305f814b544af..c4f6e86e20421 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java @@ -66,8 +66,7 @@ public void testMoveToAllocateStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "javaRestTest-0") .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias") ); // move to a step @@ -105,8 +104,7 @@ public void testMoveToRolloverStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "javaRestTest-0") .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); // move to a step @@ -154,8 +152,7 @@ public void testMoveToInjectedStep() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); assertBusy(() -> assertThat(getStepKeyForIndex(client(), index), equalTo(new StepKey("new", "complete", "complete")))); @@ -251,8 +248,7 @@ public void testMoveToStepWithInvalidNextStep() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null + .put(LifecycleSettings.LIFECYCLE_NAME, policy) ); // move to a step @@ -294,8 +290,7 @@ public void testMoveToStepWithoutStepName() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null + .put(LifecycleSettings.LIFECYCLE_NAME, policy) ); // move to a step @@ -332,8 +327,7 @@ public void testMoveToStepWithoutAction() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null + .put(LifecycleSettings.LIFECYCLE_NAME, policy) ); // move to a step @@ -369,8 +363,7 @@ public void testInvalidToMoveToStepWithoutActionButWithName() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null + .put(LifecycleSettings.LIFECYCLE_NAME, policy) ); // move to a step with an invalid request @@ -406,8 +399,7 @@ public void testResolveToNonexistentStep() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null + .put(LifecycleSettings.LIFECYCLE_NAME, policy) ); // move to a step with an invalid request diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ReadonlyActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ReadonlyActionIT.java index f0a4bee372637..b2a39f2a2e1df 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ReadonlyActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ReadonlyActionIT.java @@ -54,8 +54,7 @@ public void testReadOnly() throws Exception { client(), index, alias, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); String phaseName = randomFrom("warm", "cold"); createNewSingletonPolicy(client(), policy, phaseName, new ReadOnlyAction()); @@ -93,8 +92,7 @@ public void testReadOnlyInTheHotPhase() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - null + .put(LifecycleSettings.LIFECYCLE_NAME, policy) ); index(client(), originalIndex, "_id", "foo", "bar"); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java index 0d0ecabb6cacb..eab4b442868e0 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java @@ -63,8 +63,7 @@ public void testRolloverAction() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); // create policy @@ -92,8 +91,7 @@ public void testRolloverActionWithIndexingComplete() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); Request updateSettingsRequest = new Request("PUT", "/" + originalIndex + "/_settings"); @@ -144,8 +142,7 @@ public void testRolloverActionWithMaxPrimaryShardSize() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); index(client(), originalIndex, "_id", "foo", "bar"); @@ -173,8 +170,7 @@ public void testRolloverActionWithMaxPrimaryDocsSize() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), - null + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); index(client(), originalIndex, "_id", "foo", "bar"); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java index 9b0615940fe60..8c70150376267 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java @@ -9,6 +9,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -79,8 +80,7 @@ public void testShrinkAction() throws Exception { client(), index, alias, - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(expectedFinalShards, null)); updatePolicy(client(), index, policy); @@ -106,8 +106,7 @@ public void testSkipShrinkSameShardsWithNumberOfShards() throws Exception { client(), index, alias, - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(numberOfShards, null)); updatePolicy(client(), index, policy); @@ -128,8 +127,7 @@ public void testSkipShrinkSameShardsWithMaxShardSize() throws Exception { client(), index, alias, - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(null, ByteSizeValue.ofGb(50))); updatePolicy(client(), index, policy); @@ -172,8 +170,7 @@ public void testShrinkDuringSnapshot() throws Exception { .put(SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) // required so the shrink doesn't wait on SetSingleNodeAllocateStep - .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_name", "javaRestTest-0"), - null + .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_name", "javaRestTest-0") ); // index document so snapshot actually does something indexDocument(client(), index); @@ -257,8 +254,7 @@ public void testSetSingleNodeAllocationRetriesUntilItSucceeds() throws Exception Settings.builder() .put(SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .putNull(DataTier.TIER_PREFERENCE), - null + .putNull(DataTier.TIER_PREFERENCE) ); ensureGreen(index); @@ -334,8 +330,7 @@ public void testAutomaticRetryFailedShrinkAction() throws Exception { client(), index, alias, - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), - null + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) ); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(numShards + randomIntBetween(1, numShards), null)); updatePolicy(client(), index, policy); @@ -381,8 +376,7 @@ public void testTotalShardsPerNodeTooLow() throws Exception { Settings.builder() .put(SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), numShards - 2), - null + .put(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), numShards - 2) ); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(expectedFinalShards, null)); updatePolicy(client(), index, policy); From b46c4224a6ca11a945e73bc640752d7ac2192775 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Thu, 7 Jul 2022 16:39:20 +0300 Subject: [PATCH 12/28] Add more steps to swap indices in the data stream and delete the source index --- .../xpack/core/ilm/RollupILMAction.java | 52 +++++++++++++++++-- .../xpack/core/ilm/RollupStep.java | 5 -- .../rollup/v2/TransportRollupAction.java | 2 +- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 56317c6a52c4d..1b3231ee0e61b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.ilm; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -26,7 +27,10 @@ * A {@link LifecycleAction} which calls {@link org.elasticsearch.xpack.core.rollup.action.RollupAction} on an index */ public class RollupILMAction implements LifecycleAction { + public static final String NAME = "rollup"; + public static final String ROLLUP_INDEX_PREFIX = "rollup-"; + public static final String CONDITIONAL_DATASTREAM_CHECK_KEY = BranchingStep.NAME + "-on-datastream-check"; private static final ParseField CONFIG_FIELD = new ParseField("config"); @@ -35,7 +39,6 @@ public class RollupILMAction implements LifecycleAction { NAME, a -> new RollupILMAction((RollupActionConfig) a[0]) ); - public static final String ROLLUP_INDEX_PREFIX = "rollup-"; public static final String GENERATE_ROLLUP_STEP_NAME = "generate-rollup-name"; private final RollupActionConfig config; @@ -95,17 +98,60 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyStep.NAME); StepKey generateRollupIndexNameKey = new StepKey(phase, NAME, GENERATE_ROLLUP_STEP_NAME); StepKey rollupKey = new StepKey(phase, NAME, NAME); + StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME); + StepKey dataStreamCheckBranchingKey = new StepKey(phase, NAME, CONDITIONAL_DATASTREAM_CHECK_KEY); + StepKey replaceDataStreamIndexKey = new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME); + StepKey deleteIndexKey = new StepKey(phase, NAME, DeleteStep.NAME); + CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, readOnlyKey); WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, readOnlyKey, client); ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, generateRollupIndexNameKey, client); + + // Generate a unique rollup index name and store it in the ILM execution state GenerateUniqueIndexNameStep generateRollupIndexNameStep = new GenerateUniqueIndexNameStep( generateRollupIndexNameKey, rollupKey, ROLLUP_INDEX_PREFIX, (rollupIndexName, lifecycleStateBuilder) -> lifecycleStateBuilder.setRollupIndexName(rollupIndexName) ); - Step rollupStep = new RollupStep(rollupKey, nextStepKey, client, config); - return List.of(checkNotWriteIndexStep, waitForNoFollowersStep, readOnlyStep, generateRollupIndexNameStep, rollupStep); + RollupStep rollupStep = new RollupStep(rollupKey, copyMetadataKey, client, config); + + CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep( + copyMetadataKey, + dataStreamCheckBranchingKey, + (sourceIndexName, lifecycleState) -> lifecycleState.rollupIndexName(), + deleteIndexKey + ); + + BranchingStep isDataStreamBranchingStep = new BranchingStep( + dataStreamCheckBranchingKey, + deleteIndexKey, + replaceDataStreamIndexKey, + (index, clusterState) -> { + IndexAbstraction indexAbstraction = clusterState.metadata().getIndicesLookup().get(index.getName()); + assert indexAbstraction != null : "invalid cluster metadata. index [" + index.getName() + "] was not found"; + return indexAbstraction.getParentDataStream() != null; + } + ); + + ReplaceDataStreamBackingIndexStep replaceDataStreamBackingIndex = new ReplaceDataStreamBackingIndexStep( + replaceDataStreamIndexKey, + deleteIndexKey, + (sourceIndexName, lifecycleState) -> lifecycleState.rollupIndexName() + ); + DeleteStep deleteSourceIndexStep = new DeleteStep(deleteIndexKey, nextStepKey, client); + + return List.of( + checkNotWriteIndexStep, + waitForNoFollowersStep, + readOnlyStep, + generateRollupIndexNameStep, + rollupStep, + copyMetadata, + isDataStreamBranchingStep, + replaceDataStreamBackingIndex, + deleteSourceIndexStep + ); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index 36a531345ffac..042e18e6120dd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -40,11 +40,6 @@ public boolean isRetryable() { return true; } - @Override - public boolean indexSurvives() { - return false; - } - @Override public void performAction( IndexMetadata indexMetadata, diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/TransportRollupAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/TransportRollupAction.java index 748bbc5713d52..cc4e059adb983 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/TransportRollupAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/TransportRollupAction.java @@ -591,7 +591,7 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { @Override public void onFailure(Exception deleteException) { - listener.onFailure(new ElasticsearchException("Unable to delete the temporary rollup index [" + rollupIndex + "]", e)); + listener.onFailure(new ElasticsearchException("Unable to delete rollup index [" + rollupIndex + "]", e)); } }); } From 4d2adec0e5ee52b7ce55b759781a91e1fb5758c0 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Thu, 7 Jul 2022 18:24:18 +0300 Subject: [PATCH 13/28] Fix test failures --- .../xpack/core/ilm/RollupILMAction.java | 9 +++-- .../xpack/core/ilm/RollupILMActionTests.java | 34 +++++++++++++++++-- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 1b3231ee0e61b..c49e13cc4dc28 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -103,7 +103,10 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey replaceDataStreamIndexKey = new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME); StepKey deleteIndexKey = new StepKey(phase, NAME, DeleteStep.NAME); - CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, readOnlyKey); + CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep( + checkNotWriteIndex, + waitForNoFollowerStepKey + ); WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, readOnlyKey, client); ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, generateRollupIndexNameKey, client); @@ -116,7 +119,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { ); RollupStep rollupStep = new RollupStep(rollupKey, copyMetadataKey, client, config); - CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep( + CopyExecutionStateStep copyExecutionStateStep = new CopyExecutionStateStep( copyMetadataKey, dataStreamCheckBranchingKey, (sourceIndexName, lifecycleState) -> lifecycleState.rollupIndexName(), @@ -147,7 +150,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { readOnlyStep, generateRollupIndexNameStep, rollupStep, - copyMetadata, + copyExecutionStateStep, isDataStreamBranchingStep, replaceDataStreamBackingIndex, deleteSourceIndexStep diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 171eeecf3ccae..d4430e2301d0f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -17,6 +17,7 @@ import java.util.List; +import static org.elasticsearch.xpack.core.ilm.RollupILMAction.CONDITIONAL_DATASTREAM_CHECK_KEY; import static org.elasticsearch.xpack.core.ilm.RollupILMAction.GENERATE_ROLLUP_STEP_NAME; import static org.hamcrest.Matchers.equalTo; @@ -57,18 +58,45 @@ public void testToSteps() { ); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(5, steps.size()); + assertEquals(9, steps.size()); + + assertTrue(steps.get(0) instanceof CheckNotDataStreamWriteIndexStep); assertThat(steps.get(0).getKey().getName(), equalTo(CheckNotDataStreamWriteIndexStep.NAME)); - assertThat(steps.get(0).getNextStepKey().getName(), equalTo(ReadOnlyStep.NAME)); + assertThat(steps.get(0).getNextStepKey().getName(), equalTo(WaitForNoFollowersStep.NAME)); + + assertTrue(steps.get(1) instanceof WaitForNoFollowersStep); assertThat(steps.get(1).getKey().getName(), equalTo(WaitForNoFollowersStep.NAME)); assertThat(steps.get(1).getNextStepKey().getName(), equalTo(ReadOnlyStep.NAME)); + assertTrue(steps.get(2) instanceof ReadOnlyStep); assertThat(steps.get(2).getKey().getName(), equalTo(ReadOnlyStep.NAME)); assertThat(steps.get(2).getNextStepKey().getName(), equalTo(GENERATE_ROLLUP_STEP_NAME)); + + assertTrue(steps.get(3) instanceof GenerateUniqueIndexNameStep); assertThat(steps.get(3).getKey().getName(), equalTo(GENERATE_ROLLUP_STEP_NAME)); assertThat(steps.get(3).getNextStepKey().getName(), equalTo(RollupStep.NAME)); + + assertTrue(steps.get(4) instanceof RollupStep); assertThat(steps.get(4).getKey().getName(), equalTo(RollupStep.NAME)); - assertThat(steps.get(4).getNextStepKey(), equalTo(nextStepKey)); + assertThat(steps.get(4).getNextStepKey().getName(), equalTo(CopyExecutionStateStep.NAME)); + + assertTrue(steps.get(5) instanceof CopyExecutionStateStep); + assertThat(steps.get(5).getKey().getName(), equalTo(CopyExecutionStateStep.NAME)); + assertThat(steps.get(5).getNextStepKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); + + assertTrue(steps.get(6) instanceof BranchingStep); + assertThat(steps.get(6).getKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); + expectThrows(IllegalStateException.class, () -> steps.get(6).getNextStepKey()); + assertThat(((BranchingStep) steps.get(6)).getNextStepKeyOnFalse().getName(), equalTo(DeleteStep.NAME)); + assertThat(((BranchingStep) steps.get(6)).getNextStepKeyOnTrue().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); + + assertTrue(steps.get(7) instanceof ReplaceDataStreamBackingIndexStep); + assertThat(steps.get(7).getKey().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); + assertThat(steps.get(7).getNextStepKey().getName(), equalTo(DeleteStep.NAME)); + + assertTrue(steps.get(8) instanceof DeleteStep); + assertThat(steps.get(8).getKey().getName(), equalTo(DeleteStep.NAME)); + assertThat(steps.get(8).getNextStepKey(), equalTo(nextStepKey)); } public void testEqualsAndHashCode() { From 81a0638a0d9a2989c9dee7363ff3ed24d8c37053 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Fri, 8 Jul 2022 23:23:39 +0300 Subject: [PATCH 14/28] Added steps to RollupILMAction - CleanupTargetIndexStep - CopySettingsStep --- .../core/ilm/CleanupTargetIndexStep.java | 101 ++++++++++++++++++ .../xpack/core/ilm/CopySettingsStep.java | 38 ++++--- .../xpack/core/ilm/RollupILMAction.java | 38 ++++++- .../xpack/core/ilm/RollupStep.java | 45 +++++++- .../core/ilm/SearchableSnapshotAction.java | 2 +- .../core/ilm/CopyExecutionStateStepTests.java | 2 +- .../xpack/core/ilm/CopySettingsStepTests.java | 22 ++-- .../xpack/core/ilm/RollupILMActionTests.java | 58 +++++----- .../xpack/ilm/actions/RollupActionIT.java | 15 ++- 9 files changed, 265 insertions(+), 56 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStep.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStep.java new file mode 100644 index 0000000000000..609fc65835d26 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStep.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexNotFoundException; + +import java.util.function.Function; + +/** + * Deletes the target index created by an operation such as shrink or rollup and + * identified the target index name stored in the lifecycle state of the managed + * index (if any was generated) + */ +public class CleanupTargetIndexStep extends AsyncRetryDuringSnapshotActionStep { + public static final String NAME = "cleanup-target-index"; + private static final Logger logger = LogManager.getLogger(CleanupTargetIndexStep.class); + + private final Function sourceIndexNameSupplier; + private final Function targetIndexNameSupplier; + + public CleanupTargetIndexStep( + StepKey key, + StepKey nextStepKey, + Client client, + Function sourceIndexNameSupplier, + Function targetIndexNameSupplier + ) { + super(key, nextStepKey, client); + this.sourceIndexNameSupplier = sourceIndexNameSupplier; + this.targetIndexNameSupplier = targetIndexNameSupplier; + } + + @Override + public boolean isRetryable() { + return true; + } + + @Override + void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentClusterState, ActionListener listener) { + final String sourceIndexName = sourceIndexNameSupplier.apply(indexMetadata); + if (Strings.isNullOrEmpty(sourceIndexName) == false) { + // the current managed index is the target index + if (currentClusterState.metadata().index(sourceIndexName) == null) { + // if the source index does not exist, we'll skip deleting the + // (managed) target index as that will cause data loss + String policyName = indexMetadata.getLifecyclePolicyName(); + logger.warn( + "managed index [{}] has been created as part of policy [{}] and the source index [{}] does not exist " + + "anymore. will skip the [{}] step", + indexMetadata.getIndex().getName(), + policyName, + sourceIndexName, + NAME + ); + listener.onResponse(null); + return; + } + } + + final String targetIndexName = targetIndexNameSupplier.apply(indexMetadata); + // if the target index was not generated there is nothing to delete so we move on + if (Strings.hasText(targetIndexName) == false) { + listener.onResponse(null); + return; + } + getClient().admin() + .indices() + .delete(new DeleteIndexRequest(targetIndexName).masterNodeTimeout(TimeValue.MAX_VALUE), new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + // even if not all nodes acked the delete request yet we can consider this operation as successful as + // we'll generate a new index name and attempt to create an index with the newly generated name + listener.onResponse(null); + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + // we can move on if the index was deleted in the meantime + listener.onResponse(null); + } else { + listener.onFailure(e); + } + } + }); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java index e9cbf4c29408b..e2a625ae05daf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java @@ -10,19 +10,21 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import java.util.Locale; import java.util.Objects; +import java.util.function.BiFunction; /** * Copy the provided settings from the source to the target index. *

- * The target index is derived from the source index using the provided prefix. - * This is useful for actions like shrink or searchable snapshot that create a new index and migrate the ILM execution from the source - * to the target index. + * The target index is generated by a supplier function. + * This is useful for actions like shrink, rollup or searchable snapshot that create + * a new index and migrate the ILM execution from the source to the target index. */ public class CopySettingsStep extends ClusterStateActionStep { public static final String NAME = "copy-settings"; @@ -30,14 +32,19 @@ public class CopySettingsStep extends ClusterStateActionStep { private static final Logger logger = LogManager.getLogger(CopySettingsStep.class); private final String[] settingsKeys; - private final String indexPrefix; - public CopySettingsStep(StepKey key, StepKey nextStepKey, String indexPrefix, String... settingsKeys) { + private final BiFunction targetIndexNameSupplier; + + public CopySettingsStep( + StepKey key, + StepKey nextStepKey, + BiFunction targetIndexNameSupplier, + String... settingsKeys + ) { super(key, nextStepKey); - Objects.requireNonNull(indexPrefix); Objects.requireNonNull(settingsKeys); - this.indexPrefix = indexPrefix; this.settingsKeys = settingsKeys; + this.targetIndexNameSupplier = targetIndexNameSupplier; } @Override @@ -49,17 +56,14 @@ public String[] getSettingsKeys() { return settingsKeys; } - public String getIndexPrefix() { - return indexPrefix; - } + BiFunction getTargetIndexNameSupplier() { + return targetIndexNameSupplier; + }; @Override public ClusterState performAction(Index index, ClusterState clusterState) { String sourceIndexName = index.getName(); IndexMetadata sourceIndexMetadata = clusterState.metadata().index(sourceIndexName); - String targetIndexName = indexPrefix + sourceIndexName; - IndexMetadata targetIndexMetadata = clusterState.metadata().index(targetIndexName); - if (sourceIndexMetadata == null) { // Index must have been since deleted, ignore it logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), sourceIndexName); @@ -70,6 +74,8 @@ public ClusterState performAction(Index index, ClusterState clusterState) { return clusterState; } + String targetIndexName = targetIndexNameSupplier.apply(sourceIndexName, sourceIndexMetadata.getLifecycleExecutionState()); + IndexMetadata targetIndexMetadata = clusterState.metadata().index(targetIndexName); if (targetIndexMetadata == null) { String errorMessage = String.format( Locale.ROOT, @@ -107,11 +113,13 @@ public boolean equals(Object o) { return false; } CopySettingsStep that = (CopySettingsStep) o; - return Objects.equals(settingsKeys, that.settingsKeys) && Objects.equals(indexPrefix, that.indexPrefix); + return super.equals(o) + && Objects.equals(targetIndexNameSupplier, that.targetIndexNameSupplier) + && Objects.equals(settingsKeys, that.settingsKeys); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), settingsKeys, indexPrefix); + return Objects.hash(super.hashCode(), targetIndexNameSupplier, settingsKeys); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index c49e13cc4dc28..3ae3635e3f838 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -96,9 +97,11 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME); StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyStep.NAME); + StepKey cleanupRollupIndexKey = new StepKey(phase, NAME, CleanupTargetIndexStep.NAME); StepKey generateRollupIndexNameKey = new StepKey(phase, NAME, GENERATE_ROLLUP_STEP_NAME); StepKey rollupKey = new StepKey(phase, NAME, NAME); StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME); + StepKey copyLifecyclePolicySettingKey = new StepKey(phase, NAME, CopySettingsStep.NAME); StepKey dataStreamCheckBranchingKey = new StepKey(phase, NAME, CONDITIONAL_DATASTREAM_CHECK_KEY); StepKey replaceDataStreamIndexKey = new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME); StepKey deleteIndexKey = new StepKey(phase, NAME, DeleteStep.NAME); @@ -107,7 +110,19 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { checkNotWriteIndex, waitForNoFollowerStepKey ); - WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, readOnlyKey, client); + WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, cleanupRollupIndexKey, client); + + // We generate a unique rollup index name, but we also retry if the allocation of the rollup index is not possible, so we want to + // delete the "previously generated" rollup index (this is a no-op if it's the first run of the action, and we haven't generated a + // rollup index name) + CleanupTargetIndexStep cleanupRollupIndexStep = new CleanupTargetIndexStep( + cleanupRollupIndexKey, + readOnlyKey, + client, + (indexMetadata) -> IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.get(indexMetadata.getSettings()), + (indexMetadata) -> indexMetadata.getLifecycleExecutionState().rollupIndexName() + ); + // Mark source index as read-only ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, generateRollupIndexNameKey, client); // Generate a unique rollup index name and store it in the ILM execution state @@ -117,15 +132,28 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { ROLLUP_INDEX_PREFIX, (rollupIndexName, lifecycleStateBuilder) -> lifecycleStateBuilder.setRollupIndexName(rollupIndexName) ); - RollupStep rollupStep = new RollupStep(rollupKey, copyMetadataKey, client, config); + // Here is where the actual rollup action takes place + RollupStep rollupStep = new RollupStep(rollupKey, copyMetadataKey, client, config); CopyExecutionStateStep copyExecutionStateStep = new CopyExecutionStateStep( copyMetadataKey, + copyLifecyclePolicySettingKey, + (indexName, lifecycleState) -> lifecycleState.rollupIndexName(), + nextStepKey + ); + + // Copy the index.lifecycle.name setting to the rollup index settings + CopySettingsStep copySettingsStep = new CopySettingsStep( + copyLifecyclePolicySettingKey, dataStreamCheckBranchingKey, - (sourceIndexName, lifecycleState) -> lifecycleState.rollupIndexName(), - deleteIndexKey + (indexName, lifecycleState) -> lifecycleState.rollupIndexName(), + LifecycleSettings.LIFECYCLE_NAME ); + // By the time we get to this step we have 2 indices, the source and the rollup one. We now need to choose an index + // swapping strategy such that the rollup index takes the place of the source index (which will also be deleted). + // If the source index is part of a data stream it's a matter of replacing it with the rollup index one in the data stream and + // then deleting the source index. BranchingStep isDataStreamBranchingStep = new BranchingStep( dataStreamCheckBranchingKey, deleteIndexKey, @@ -147,10 +175,12 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { return List.of( checkNotWriteIndexStep, waitForNoFollowersStep, + cleanupRollupIndexStep, readOnlyStep, generateRollupIndexNameStep, rollupStep, copyExecutionStateStep, + copySettingsStep, isDataStreamBranchingStep, replaceDataStreamBackingIndex, deleteSourceIndexStep diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index 042e18e6120dd..6fd2c00d34999 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; @@ -28,6 +30,8 @@ public class RollupStep extends AsyncActionStep { public static final String NAME = "rollup"; + private static final Logger logger = LogManager.getLogger(RollupStep.class); + private final RollupActionConfig config; public RollupStep(StepKey key, StepKey nextStepKey, Client client, RollupActionConfig config) { @@ -47,9 +51,13 @@ public void performAction( ClusterStateObserver observer, ActionListener listener ) { + LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState(); + if (lifecycleState.lifecycleDate() == null) { + throw new IllegalStateException("source index [" + indexMetadata.getIndex().getName() + "] is missing lifecycle date"); + } + final String policyName = indexMetadata.getLifecyclePolicyName(); final String indexName = indexMetadata.getIndex().getName(); - final LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState(); final String rollupIndexName = lifecycleState.rollupIndexName(); if (Strings.hasText(rollupIndexName) == false) { listener.onFailure( @@ -59,8 +67,41 @@ public void performAction( ); return; } + + IndexMetadata rollupIndexMetadata = currentState.metadata().index(rollupIndexName); + if (rollupIndexMetadata != null) { + IndexMetadata.RollupTaskStatus rollupIndexStatus = IndexMetadata.INDEX_ROLLUP_STATUS.get(indexMetadata.getSettings()); + if (IndexMetadata.RollupTaskStatus.SUCCESS.equals(rollupIndexStatus)) { + logger.warn( + "skipping [{}] step for index [{}] as part of policy [{}] as the rollup index [{}] already exists", + RollupStep.NAME, + indexName, + policyName, + rollupIndexName + ); + listener.onResponse(null); + } else { + listener.onFailure( + new IllegalStateException( + "failing [" + + RollupStep.NAME + + "] step for index [" + + indexName + + "] as part of policy [" + + policyName + + "] because the rollup index [" + + rollupIndexName + + "] already exists with rollup status [" + + rollupIndexStatus + + "]" + ) + ); + } + return; + } + RollupAction.Request request = new RollupAction.Request(indexName, rollupIndexName, config).masterNodeTimeout(TimeValue.MAX_VALUE); - // currently RollupAction always acknowledges action was complete when no exceptions are thrown. + // Currently, RollupAction always acknowledges action was complete when no exceptions are thrown. getClient().execute( RollupAction.INSTANCE, request, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index c3af8d8b440f4..5a6103467b760 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -289,7 +289,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey, XPac CopySettingsStep copySettingsStep = new CopySettingsStep( copyLifecyclePolicySettingKey, dataStreamCheckBranchingKey, - getRestoredIndexPrefix(copyLifecyclePolicySettingKey), + (index, lifecycleState) -> getRestoredIndexPrefix(copyLifecyclePolicySettingKey) + index, LifecycleSettings.LIFECYCLE_NAME ); BranchingStep isDataStreamBranchingStep = new BranchingStep( diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CopyExecutionStateStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CopyExecutionStateStepTests.java index 82f9d624e9a3f..9ec64d9286b8d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CopyExecutionStateStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CopyExecutionStateStepTests.java @@ -40,7 +40,7 @@ protected CopyExecutionStateStep mutateInstance(CopyExecutionStateStep instance) BiFunction indexNameSupplier = instance.getTargetIndexNameSupplier(); StepKey targetNextStepKey = instance.getTargetNextStepKey(); - switch (between(0, 2)) { + switch (between(0, 3)) { case 0 -> key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); case 1 -> nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); case 2 -> indexNameSupplier = (index, state) -> randomAlphaOfLengthBetween(11, 15) + index; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CopySettingsStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CopySettingsStepTests.java index 85e617202fb8a..57a748ca93cd1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CopySettingsStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CopySettingsStepTests.java @@ -9,8 +9,11 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.metadata.Metadata; +import java.util.function.BiFunction; + import static org.hamcrest.Matchers.is; public class CopySettingsStepTests extends AbstractStepTestCase { @@ -20,7 +23,7 @@ protected CopySettingsStep createRandomInstance() { return new CopySettingsStep( randomStepKey(), randomStepKey(), - randomAlphaOfLengthBetween(1, 10), + (index, lifecycleState) -> randomAlphaOfLengthBetween(1, 10) + index, IndexMetadata.SETTING_NUMBER_OF_SHARDS ); } @@ -29,22 +32,27 @@ protected CopySettingsStep createRandomInstance() { protected CopySettingsStep mutateInstance(CopySettingsStep instance) { Step.StepKey key = instance.getKey(); Step.StepKey nextKey = instance.getNextStepKey(); - String indexPrefix = instance.getIndexPrefix(); + BiFunction targetIndexNameSupplier = instance.getTargetIndexNameSupplier(); String[] settingsKeys = instance.getSettingsKeys(); switch (between(0, 3)) { case 0 -> key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); case 1 -> nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); - case 2 -> indexPrefix = randomValueOtherThan(indexPrefix, () -> randomAlphaOfLengthBetween(1, 10)); - case 3 -> settingsKeys = new String[] { randomAlphaOfLengthBetween(1, 10) }; + case 2 -> settingsKeys = new String[] { randomAlphaOfLengthBetween(1, 10) }; + case 3 -> targetIndexNameSupplier = (index, state) -> randomAlphaOfLengthBetween(11, 15) + index; default -> throw new AssertionError("Illegal randomisation branch"); } - return new CopySettingsStep(key, nextKey, indexPrefix, settingsKeys); + return new CopySettingsStep(key, nextKey, targetIndexNameSupplier, settingsKeys); } @Override protected CopySettingsStep copyInstance(CopySettingsStep instance) { - return new CopySettingsStep(instance.getKey(), instance.getNextStepKey(), instance.getIndexPrefix(), instance.getSettingsKeys()); + return new CopySettingsStep( + instance.getKey(), + instance.getNextStepKey(), + instance.getTargetIndexNameSupplier(), + instance.getSettingsKeys() + ); } public void testPerformAction() { @@ -71,7 +79,7 @@ public void testPerformAction() { CopySettingsStep copySettingsStep = new CopySettingsStep( randomStepKey(), randomStepKey(), - indexPrefix, + (sourceIndexName, lifecycleState) -> indexPrefix + indexName, LifecycleSettings.LIFECYCLE_NAME ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index d4430e2301d0f..5f74cf8698117 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -58,7 +58,7 @@ public void testToSteps() { ); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(9, steps.size()); + assertEquals(11, steps.size()); assertTrue(steps.get(0) instanceof CheckNotDataStreamWriteIndexStep); assertThat(steps.get(0).getKey().getName(), equalTo(CheckNotDataStreamWriteIndexStep.NAME)); @@ -66,37 +66,45 @@ public void testToSteps() { assertTrue(steps.get(1) instanceof WaitForNoFollowersStep); assertThat(steps.get(1).getKey().getName(), equalTo(WaitForNoFollowersStep.NAME)); - assertThat(steps.get(1).getNextStepKey().getName(), equalTo(ReadOnlyStep.NAME)); + assertThat(steps.get(1).getNextStepKey().getName(), equalTo(CleanupTargetIndexStep.NAME)); - assertTrue(steps.get(2) instanceof ReadOnlyStep); - assertThat(steps.get(2).getKey().getName(), equalTo(ReadOnlyStep.NAME)); - assertThat(steps.get(2).getNextStepKey().getName(), equalTo(GENERATE_ROLLUP_STEP_NAME)); + assertTrue(steps.get(2) instanceof CleanupTargetIndexStep); + assertThat(steps.get(2).getKey().getName(), equalTo(CleanupTargetIndexStep.NAME)); + assertThat(steps.get(2).getNextStepKey().getName(), equalTo(ReadOnlyStep.NAME)); - assertTrue(steps.get(3) instanceof GenerateUniqueIndexNameStep); - assertThat(steps.get(3).getKey().getName(), equalTo(GENERATE_ROLLUP_STEP_NAME)); - assertThat(steps.get(3).getNextStepKey().getName(), equalTo(RollupStep.NAME)); + assertTrue(steps.get(3) instanceof ReadOnlyStep); + assertThat(steps.get(3).getKey().getName(), equalTo(ReadOnlyStep.NAME)); + assertThat(steps.get(3).getNextStepKey().getName(), equalTo(GENERATE_ROLLUP_STEP_NAME)); - assertTrue(steps.get(4) instanceof RollupStep); - assertThat(steps.get(4).getKey().getName(), equalTo(RollupStep.NAME)); - assertThat(steps.get(4).getNextStepKey().getName(), equalTo(CopyExecutionStateStep.NAME)); + assertTrue(steps.get(4) instanceof GenerateUniqueIndexNameStep); + assertThat(steps.get(4).getKey().getName(), equalTo(GENERATE_ROLLUP_STEP_NAME)); + assertThat(steps.get(4).getNextStepKey().getName(), equalTo(RollupStep.NAME)); - assertTrue(steps.get(5) instanceof CopyExecutionStateStep); - assertThat(steps.get(5).getKey().getName(), equalTo(CopyExecutionStateStep.NAME)); - assertThat(steps.get(5).getNextStepKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); + assertTrue(steps.get(5) instanceof RollupStep); + assertThat(steps.get(5).getKey().getName(), equalTo(RollupStep.NAME)); + assertThat(steps.get(5).getNextStepKey().getName(), equalTo(CopyExecutionStateStep.NAME)); - assertTrue(steps.get(6) instanceof BranchingStep); - assertThat(steps.get(6).getKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); - expectThrows(IllegalStateException.class, () -> steps.get(6).getNextStepKey()); - assertThat(((BranchingStep) steps.get(6)).getNextStepKeyOnFalse().getName(), equalTo(DeleteStep.NAME)); - assertThat(((BranchingStep) steps.get(6)).getNextStepKeyOnTrue().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); + assertTrue(steps.get(6) instanceof CopyExecutionStateStep); + assertThat(steps.get(6).getKey().getName(), equalTo(CopyExecutionStateStep.NAME)); + assertThat(steps.get(6).getNextStepKey().getName(), equalTo(CopySettingsStep.NAME)); - assertTrue(steps.get(7) instanceof ReplaceDataStreamBackingIndexStep); - assertThat(steps.get(7).getKey().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); - assertThat(steps.get(7).getNextStepKey().getName(), equalTo(DeleteStep.NAME)); + assertTrue(steps.get(7) instanceof CopySettingsStep); + assertThat(steps.get(7).getKey().getName(), equalTo(CopySettingsStep.NAME)); + assertThat(steps.get(7).getNextStepKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); - assertTrue(steps.get(8) instanceof DeleteStep); - assertThat(steps.get(8).getKey().getName(), equalTo(DeleteStep.NAME)); - assertThat(steps.get(8).getNextStepKey(), equalTo(nextStepKey)); + assertTrue(steps.get(8) instanceof BranchingStep); + assertThat(steps.get(8).getKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); + expectThrows(IllegalStateException.class, () -> steps.get(8).getNextStepKey()); + assertThat(((BranchingStep) steps.get(8)).getNextStepKeyOnFalse().getName(), equalTo(DeleteStep.NAME)); + assertThat(((BranchingStep) steps.get(8)).getNextStepKeyOnTrue().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); + + assertTrue(steps.get(9) instanceof ReplaceDataStreamBackingIndexStep); + assertThat(steps.get(9).getKey().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); + assertThat(steps.get(9).getNextStepKey().getName(), equalTo(DeleteStep.NAME)); + + assertTrue(steps.get(10) instanceof DeleteStep); + assertThat(steps.get(10).getKey().getName(), equalTo(DeleteStep.NAME)); + assertThat(steps.get(10).getNextStepKey(), equalTo(nextStepKey)); } public void testEqualsAndHashCode() { diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index f764de476bc54..80429e60521df 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.RollupILMAction; import org.elasticsearch.xpack.core.rollup.RollupActionConfigTests; @@ -43,9 +44,12 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getOnlyIndexSettings; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.index; import static org.elasticsearch.xpack.TimeSeriesRestDriver.rolloverMaxOneDocCondition; import static org.elasticsearch.xpack.TimeSeriesRestDriver.updatePolicy; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; public class RollupActionIT extends ESRestTestCase { @@ -136,6 +140,11 @@ public void testRollupIndex() throws Exception { String rollupIndex = getRollupIndexName(index); assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(index)), 30, TimeUnit.SECONDS); + assertBusy(() -> assertThat(getStepKeyForIndex(client(), rollupIndex), equalTo(PhaseCompleteStep.finalStep(phaseName).getKey()))); + assertBusy(() -> { + Map settings = getOnlyIndexSettings(client(), rollupIndex); + assertThat(settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey()), equalTo(index)); + }); } public void testRollupIndexInTheHotPhase() throws Exception { @@ -202,7 +211,11 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(originalIndex)), 30, TimeUnit.SECONDS); - // assertBusy(() -> assertThat(getStepKeyForIndex(client(), rollupIndex), equalTo(PhaseCompleteStep.finalStep("hot").getKey()))); + assertBusy(() -> assertThat(getStepKeyForIndex(client(), rollupIndex), equalTo(PhaseCompleteStep.finalStep("hot").getKey()))); + assertBusy(() -> { + Map settings = getOnlyIndexSettings(client(), rollupIndex); + assertThat(settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey()), equalTo(originalIndex)); + }); } public void testTsdbDataStreams() throws Exception { From 4d0a602ab39c2d8ba959cb86b4462e5b28155fd1 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Sat, 9 Jul 2022 00:01:07 +0300 Subject: [PATCH 15/28] Added tests for CleanupTargetIndexStep --- .../core/ilm/CleanupTargetIndexStep.java | 28 +++ .../core/ilm/CleanupTargetIndexStepTests.java | 195 ++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStep.java index 609fc65835d26..55f373d46a292 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStep.java @@ -18,6 +18,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; +import java.util.Objects; import java.util.function.Function; /** @@ -49,6 +50,14 @@ public boolean isRetryable() { return true; } + Function getSourceIndexNameSupplier() { + return sourceIndexNameSupplier; + } + + Function getTargetIndexNameSupplier() { + return targetIndexNameSupplier; + } + @Override void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentClusterState, ActionListener listener) { final String sourceIndexName = sourceIndexNameSupplier.apply(indexMetadata); @@ -98,4 +107,23 @@ public void onFailure(Exception e) { } }); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CleanupTargetIndexStep that = (CleanupTargetIndexStep) o; + return super.equals(o) + && Objects.equals(targetIndexNameSupplier, that.targetIndexNameSupplier) + && Objects.equals(sourceIndexNameSupplier, that.sourceIndexNameSupplier); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), targetIndexNameSupplier, sourceIndexNameSupplier); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java new file mode 100644 index 0000000000000..f822bb60f718c --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.xpack.core.ilm.Step.StepKey; + +import java.util.Map; +import java.util.function.Function; + +import static org.elasticsearch.xpack.core.ilm.GenerateUniqueIndexNameStep.generateValidIndexName; +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.is; + +public class CleanupTargetIndexStepTests extends AbstractStepTestCase { + + @Override + public CleanupTargetIndexStep createRandomInstance() { + StepKey stepKey = randomStepKey(); + StepKey nextStepKey = randomStepKey(); + + return new CleanupTargetIndexStep( + stepKey, + nextStepKey, + client, + (indexMetadata) -> randomAlphaOfLengthBetween(1, 10), + (indexMetadata) -> randomAlphaOfLengthBetween(1, 10) + ); + } + + @Override + protected CleanupTargetIndexStep copyInstance(CleanupTargetIndexStep instance) { + return new CleanupTargetIndexStep( + instance.getKey(), + instance.getNextStepKey(), + instance.getClient(), + instance.getSourceIndexNameSupplier(), + instance.getTargetIndexNameSupplier() + ); + } + + @Override + public CleanupTargetIndexStep mutateInstance(CleanupTargetIndexStep instance) { + StepKey key = instance.getKey(); + StepKey nextKey = instance.getNextStepKey(); + Function sourceIndexNameSupplier = instance.getSourceIndexNameSupplier(); + Function targetIndexNameSupplier = instance.getTargetIndexNameSupplier(); + + switch (between(0, 3)) { + case 0 -> key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + case 1 -> nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + case 2 -> sourceIndexNameSupplier = (indexMetadata) -> randomAlphaOfLengthBetween(11, 15) + indexMetadata.getIndex().getName(); + case 3 -> targetIndexNameSupplier = (indexMetadata) -> randomAlphaOfLengthBetween(11, 15) + indexMetadata.getIndex().getName(); + default -> throw new AssertionError("Illegal randomisation branch"); + } + return new CleanupTargetIndexStep(key, nextKey, instance.getClient(), sourceIndexNameSupplier, targetIndexNameSupplier); + } + + public void testPerformActionDoesntFailIfShrinkingIndexNameIsMissing() { + String indexName = randomAlphaOfLength(10); + String policyName = "test-ilm-policy"; + + IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)); + + IndexMetadata indexMetadata = indexMetadataBuilder.build(); + + ClusterState clusterState = ClusterState.builder(emptyClusterState()) + .metadata(Metadata.builder().put(indexMetadata, true).build()) + .build(); + + CleanupTargetIndexStep cleanupShrinkIndexStep = createRandomInstance(); + cleanupShrinkIndexStep.performAction(indexMetadata, clusterState, null, new ActionListener<>() { + @Override + public void onResponse(Void unused) {} + + @Override + public void onFailure(Exception e) { + fail( + "expecting the step to not report any failure if there isn't any shrink index name stored in the ILM execution " + + "state but got:" + + e.getMessage() + ); + } + }); + } + + public void testPerformAction() { + String indexName = randomAlphaOfLength(10); + String policyName = "test-ilm-policy"; + String shrinkIndexName = generateValidIndexName("shrink-", indexName); + Map ilmCustom = Map.of("shrink_index_name", shrinkIndexName); + + IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, ilmCustom) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)); + IndexMetadata indexMetadata = indexMetadataBuilder.build(); + + ClusterState clusterState = ClusterState.builder(emptyClusterState()) + .metadata(Metadata.builder().put(indexMetadata, true).build()) + .build(); + + try (NoOpClient client = getDeleteIndexRequestAssertingClient(shrinkIndexName)) { + CleanupTargetIndexStep step = new CleanupTargetIndexStep( + randomStepKey(), + randomStepKey(), + client, + (metadata) -> indexName, + (metadata) -> shrinkIndexName + ); + step.performAction(indexMetadata, clusterState, null, ActionListener.noop()); + } + } + + public void testDeleteSkippedIfManagedIndexIsShrunkAndSourceDoesntExist() { + String sourceIndex = randomAlphaOfLength(10); + String policyName = "test-ilm-policy"; + String shrinkIndexName = generateValidIndexName("shrink-", sourceIndex); + Map ilmCustom = Map.of("shrink_index_name", shrinkIndexName); + + IndexMetadata.Builder shrunkIndexMetadataBuilder = IndexMetadata.builder(shrinkIndexName) + .settings( + settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName) + .put(IndexMetadata.INDEX_RESIZE_SOURCE_NAME_KEY, sourceIndex) + ) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, ilmCustom) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)); + IndexMetadata shrunkIndexMetadata = shrunkIndexMetadataBuilder.build(); + + ClusterState clusterState = ClusterState.builder(emptyClusterState()) + .metadata(Metadata.builder().put(shrunkIndexMetadata, true).build()) + .build(); + + try (NoOpClient client = getFailingIfCalledClient()) { + CleanupTargetIndexStep step = new CleanupTargetIndexStep( + randomStepKey(), + randomStepKey(), + client, + (metadata) -> sourceIndex, + (metadata) -> shrinkIndexName + ); + step.performAction(shrunkIndexMetadata, clusterState, null, ActionListener.noop()); + } + } + + private NoOpClient getDeleteIndexRequestAssertingClient(String shrinkIndexName) { + return new NoOpClient(getTestName()) { + @Override + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + assertThat(action.name(), is(DeleteIndexAction.NAME)); + assertTrue(request instanceof DeleteIndexRequest); + assertThat(((DeleteIndexRequest) request).indices(), arrayContaining(shrinkIndexName)); + } + }; + } + + private NoOpClient getFailingIfCalledClient() { + return new NoOpClient(getTestName()) { + @Override + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + throw new IllegalStateException( + "not expecting client to be called, but received request [" + request + "] for action [" + action + "]" + ); + } + }; + } +} From 3dac1483ea7d87f4f1cd068d1289f52a4a8bf175 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Sat, 9 Jul 2022 00:27:23 +0300 Subject: [PATCH 16/28] Fix broken tests --- .../xpack/core/ilm/RollupStepTests.java | 49 +++++++++++++------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java index 2a6b625c7d53d..daa2b1f33b799 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java @@ -22,11 +22,10 @@ import org.elasticsearch.xpack.core.rollup.action.RollupAction; import org.mockito.Mockito; -import java.util.Collections; import java.util.List; -import java.util.Map; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; +import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -45,14 +44,16 @@ public RollupStep createRandomInstance() { public RollupStep mutateInstance(RollupStep instance) { StepKey key = instance.getKey(); StepKey nextKey = instance.getNextStepKey(); + RollupActionConfig config = instance.getConfig(); - switch (between(0, 1)) { + switch (between(0, 2)) { case 0 -> key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); case 1 -> nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + case 2 -> config = RollupActionConfigTests.randomConfig(); default -> throw new AssertionError("Illegal randomisation branch"); } - return new RollupStep(key, nextKey, instance.getClient(), instance.getConfig()); + return new RollupStep(key, nextKey, instance.getClient(), config); } @Override @@ -60,13 +61,19 @@ public RollupStep copyInstance(RollupStep instance) { return new RollupStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(), instance.getConfig()); } - private IndexMetadata getIndexMetadata(String index) { - Map ilmCustom = Collections.singletonMap("rollup_index_name", "rollup-index"); + private IndexMetadata getIndexMetadata(String index, String lifecycleName, RollupStep step) { + LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); + lifecycleState.setPhase(step.getKey().getPhase()); + lifecycleState.setAction(step.getKey().getAction()); + lifecycleState.setStep(step.getKey().getName()); + lifecycleState.setIndexCreationDate(randomNonNegativeLong()); + lifecycleState.setRollupIndexName("rollup-index"); + return IndexMetadata.builder(index) - .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, "test-ilm-policy")) + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)) .numberOfShards(randomIntBetween(1, 5)) .numberOfReplicas(randomIntBetween(0, 5)) - .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, ilmCustom) + .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()) .build(); } @@ -77,11 +84,11 @@ private static void assertRollupActionRequest(RollupAction.Request request, Stri } public void testPerformAction() throws Exception { - String index = randomAlphaOfLength(5); - IndexMetadata indexMetadata = getIndexMetadata(index); - + String lifecycleName = randomAlphaOfLength(5); RollupStep step = createRandomInstance(); + String index = randomAlphaOfLength(5); + IndexMetadata indexMetadata = getIndexMetadata(index, lifecycleName, step); mockClientRollupCall(index); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata, true)).build(); @@ -89,14 +96,24 @@ public void testPerformAction() throws Exception { } public void testPerformActionFailureInvalidExecutionState() { + String lifecycleName = randomAlphaOfLength(5); + RollupStep step = createRandomInstance(); + + LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); + lifecycleState.setPhase(step.getKey().getPhase()); + lifecycleState.setAction(step.getKey().getAction()); + lifecycleState.setStep(step.getKey().getName()); + lifecycleState.setIndexCreationDate(randomNonNegativeLong()); + IndexMetadata indexMetadata = IndexMetadata.builder(randomAlphaOfLength(10)) - .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, "test-ilm-policy")) + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)) .numberOfShards(randomIntBetween(1, 5)) .numberOfReplicas(randomIntBetween(0, 5)) + .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()) .build(); + String policyName = indexMetadata.getLifecyclePolicyName(); String indexName = indexMetadata.getIndex().getName(); - RollupStep step = createRandomInstance(); step.performAction(indexMetadata, emptyClusterState(), null, new ActionListener<>() { @Override public void onResponse(Void unused) { @@ -115,11 +132,11 @@ public void onFailure(Exception e) { } public void testPerformActionOnDataStream() throws Exception { + RollupStep step = createRandomInstance(); + String lifecycleName = randomAlphaOfLength(5); String dataStreamName = "test-datastream"; String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1); - IndexMetadata indexMetadata = getIndexMetadata(backingIndexName); - - RollupStep step = createRandomInstance(); + IndexMetadata indexMetadata = getIndexMetadata(backingIndexName, lifecycleName, step); mockClientRollupCall(backingIndexName); From 4f2770e286a0bbb92fbcd58097dd32a5cf122395 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Sat, 9 Jul 2022 00:50:34 +0300 Subject: [PATCH 17/28] checkstyle --- .../xpack/core/ilm/CleanupTargetIndexStepTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java index f822bb60f718c..f764ab2844c7c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java @@ -58,7 +58,7 @@ protected CleanupTargetIndexStep copyInstance(CleanupTargetIndexStep instance) { public CleanupTargetIndexStep mutateInstance(CleanupTargetIndexStep instance) { StepKey key = instance.getKey(); StepKey nextKey = instance.getNextStepKey(); - Function sourceIndexNameSupplier = instance.getSourceIndexNameSupplier(); + Function sourceIndexNameSupplier = instance.getSourceIndexNameSupplier(); Function targetIndexNameSupplier = instance.getTargetIndexNameSupplier(); switch (between(0, 3)) { From 9d762a3002355b61953180ea186d20d46113d777 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Mon, 11 Jul 2022 15:23:29 +0300 Subject: [PATCH 18/28] Added more tests --- .../xpack/core/ilm/RollupStep.java | 2 +- .../core/ilm/CleanupTargetIndexStepTests.java | 1 - .../xpack/core/ilm/RollupStepTests.java | 97 +++++++++++++++++++ .../xpack/ilm/actions/RollupActionIT.java | 14 ++- 4 files changed, 109 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index 6fd2c00d34999..cfe206c4b8150 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -70,7 +70,7 @@ public void performAction( IndexMetadata rollupIndexMetadata = currentState.metadata().index(rollupIndexName); if (rollupIndexMetadata != null) { - IndexMetadata.RollupTaskStatus rollupIndexStatus = IndexMetadata.INDEX_ROLLUP_STATUS.get(indexMetadata.getSettings()); + IndexMetadata.RollupTaskStatus rollupIndexStatus = IndexMetadata.INDEX_ROLLUP_STATUS.get(rollupIndexMetadata.getSettings()); if (IndexMetadata.RollupTaskStatus.SUCCESS.equals(rollupIndexStatus)) { logger.warn( "skipping [{}] step for index [{}] as part of policy [{}] as the rollup index [{}] already exists", diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java index f764ab2844c7c..efdaced753622 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupTargetIndexStepTests.java @@ -81,7 +81,6 @@ public void testPerformActionDoesntFailIfShrinkingIndexNameIsMissing() { .numberOfReplicas(randomIntBetween(0, 5)); IndexMetadata indexMetadata = indexMetadataBuilder.build(); - ClusterState clusterState = ClusterState.builder(emptyClusterState()) .metadata(Metadata.builder().put(indexMetadata, true).build()) .build(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java index daa2b1f33b799..db42ca0acea03 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java @@ -23,9 +23,11 @@ import org.mockito.Mockito; import java.util.List; +import java.util.Map; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; +import static org.elasticsearch.xpack.core.ilm.RollupILMAction.ROLLUP_INDEX_PREFIX; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -146,6 +148,101 @@ public void testPerformActionOnDataStream() throws Exception { PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); } + /** + * Test rollup step when a successfully completed rollup index already exists. + */ + public void testPerformActionCompletedRollupIndexExists() { + String sourceIndexName = randomAlphaOfLength(10); + String lifecycleName = randomAlphaOfLength(5); + RollupStep step = createRandomInstance(); + + LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); + lifecycleState.setPhase(step.getKey().getPhase()); + lifecycleState.setAction(step.getKey().getAction()); + lifecycleState.setStep(step.getKey().getName()); + lifecycleState.setIndexCreationDate(randomNonNegativeLong()); + + String rollupIndex = GenerateUniqueIndexNameStep.generateValidIndexName(ROLLUP_INDEX_PREFIX, sourceIndexName); + lifecycleState.setRollupIndexName(rollupIndex); + + IndexMetadata sourceIndexMetadata = IndexMetadata.builder(sourceIndexName) + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)) + .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)) + .build(); + + // Create a successfully completed rollup index (index.rollup.status: success) + IndexMetadata indexMetadata = IndexMetadata.builder(rollupIndex) + .settings(settings(Version.CURRENT).put(IndexMetadata.INDEX_ROLLUP_STATUS.getKey(), IndexMetadata.RollupTaskStatus.SUCCESS)) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + Map indices = Map.of(rollupIndex, indexMetadata); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(Metadata.builder().indices(indices)).build(); + + Mockito.doThrow(new IllegalStateException("Rollup action should not be invoked")) + .when(client) + .execute(Mockito.any(), Mockito.any(), Mockito.any()); + + step.performAction(sourceIndexMetadata, clusterState, null, new ActionListener<>() { + @Override + public void onResponse(Void unused) {} + + @Override + public void onFailure(Exception e) { + fail("onFailure should not be called in this test, called with exception: " + e.getMessage()); + } + }); + } + + /** + * Test rollup step when an in-progress rollup index already exists. + */ + public void testPerformActionRollupInProgressIndexExists() { + String sourceIndexName = randomAlphaOfLength(10); + String lifecycleName = randomAlphaOfLength(5); + RollupStep step = createRandomInstance(); + + LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); + lifecycleState.setPhase(step.getKey().getPhase()); + lifecycleState.setAction(step.getKey().getAction()); + lifecycleState.setStep(step.getKey().getName()); + lifecycleState.setIndexCreationDate(randomNonNegativeLong()); + + String rollupIndex = GenerateUniqueIndexNameStep.generateValidIndexName(ROLLUP_INDEX_PREFIX, sourceIndexName); + lifecycleState.setRollupIndexName(rollupIndex); + + IndexMetadata sourceIndexMetadata = IndexMetadata.builder(sourceIndexName) + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)) + .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)) + .build(); + + // Create an in-progress rollup index (index.rollup.status: started) + IndexMetadata indexMetadata = IndexMetadata.builder(rollupIndex) + .settings(settings(Version.CURRENT).put(IndexMetadata.INDEX_ROLLUP_STATUS.getKey(), IndexMetadata.RollupTaskStatus.STARTED)) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + Map indices = Map.of(rollupIndex, indexMetadata); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(Metadata.builder().indices(indices)).build(); + + step.performAction(sourceIndexMetadata, clusterState, null, new ActionListener<>() { + @Override + public void onResponse(Void unused) { + fail("onResponse should not be called in this test, because there's an in-progress rollup index"); + } + + @Override + public void onFailure(Exception e) { + assertTrue(e instanceof IllegalStateException); + assertTrue(e.getMessage().contains("already exists with rollup status [started]")); + } + }); + } + private void mockClientRollupCall(String sourceIndex) { Mockito.doAnswer(invocation -> { RollupAction.Request request = (RollupAction.Request) invocation.getArguments()[1]; diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index 80429e60521df..b5e8017e118ca 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -57,6 +57,7 @@ public class RollupActionIT extends ESRestTestCase { private String index; private String policy; private String alias; + private String dataStream; private static final String TEMPLATE = """ { @@ -94,7 +95,16 @@ public void refreshAbstractions() { index = "index-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); policy = "policy-" + randomAlphaOfLength(5); alias = "alias-" + randomAlphaOfLength(5); - logger.info("--> running [{}] with index [{}], alias [{}] and policy [{}]", getTestName(), index, alias, policy); + dataStream = "ds-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + + logger.info( + "--> running [{}] with index [{}], data stream [{}], alias [{}] and policy [{}]", + getTestName(), + index, + dataStream, + alias, + policy + ); } private void createIndex(String index, String alias) throws IOException { @@ -219,8 +229,6 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { } public void testTsdbDataStreams() throws Exception { - final String dataStream = "k8s-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); - // Create the ILM policy createNewSingletonPolicy(client(), policy, "warm", new RollupILMAction(RollupActionConfigTests.randomConfig())); From 5bbb446600b913283970de450e5773c827835f9a Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Mon, 11 Jul 2022 15:57:10 +0300 Subject: [PATCH 19/28] Added WaitForIndexColorStep to rollup ilm action --- .../xpack/core/ilm/RollupILMAction.java | 13 +++++-- .../xpack/core/ilm/WaitForIndexColorStep.java | 34 ++++++++++++++----- .../core/ilm/WaitForIndexColorStepTests.java | 14 ++++++-- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 3ae3635e3f838..653c8377f0893 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.ilm; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; @@ -99,7 +100,8 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyStep.NAME); StepKey cleanupRollupIndexKey = new StepKey(phase, NAME, CleanupTargetIndexStep.NAME); StepKey generateRollupIndexNameKey = new StepKey(phase, NAME, GENERATE_ROLLUP_STEP_NAME); - StepKey rollupKey = new StepKey(phase, NAME, NAME); + StepKey rollupKey = new StepKey(phase, NAME, RollupStep.NAME); + StepKey waitForGreenRestoredIndexKey = new StepKey(phase, NAME, WaitForIndexColorStep.NAME); StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME); StepKey copyLifecyclePolicySettingKey = new StepKey(phase, NAME, CopySettingsStep.NAME); StepKey dataStreamCheckBranchingKey = new StepKey(phase, NAME, CONDITIONAL_DATASTREAM_CHECK_KEY); @@ -134,7 +136,13 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { ); // Here is where the actual rollup action takes place - RollupStep rollupStep = new RollupStep(rollupKey, copyMetadataKey, client, config); + RollupStep rollupStep = new RollupStep(rollupKey, waitForGreenRestoredIndexKey, client, config); + WaitForIndexColorStep waitForGreenIndexHealthStep = new WaitForIndexColorStep( + waitForGreenRestoredIndexKey, + copyMetadataKey, + ClusterHealthStatus.GREEN, + (indexName, lifecycleState) -> lifecycleState.rollupIndexName() + ); CopyExecutionStateStep copyExecutionStateStep = new CopyExecutionStateStep( copyMetadataKey, copyLifecyclePolicySettingKey, @@ -179,6 +187,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { readOnlyStep, generateRollupIndexNameStep, rollupStep, + waitForGreenIndexHealthStep, copyExecutionStateStep, copySettingsStep, isDataStreamBranchingStep, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStep.java index 8c3ce8c7d3967..179fdf69e6b7b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStep.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.core.Nullable; @@ -23,6 +24,7 @@ import java.io.IOException; import java.util.Locale; import java.util.Objects; +import java.util.function.BiFunction; /** * Wait Step for index based on color. Optionally derives the index name using the provided prefix (if any). @@ -34,30 +36,41 @@ class WaitForIndexColorStep extends ClusterStateWaitStep { private static final Logger logger = LogManager.getLogger(WaitForIndexColorStep.class); private final ClusterHealthStatus color; - @Nullable - private final String indexNamePrefix; + + private final BiFunction indexNameSupplier; WaitForIndexColorStep(StepKey key, StepKey nextStepKey, ClusterHealthStatus color) { - this(key, nextStepKey, color, null); + this(key, nextStepKey, color, (index, lifecycleState) -> index); } WaitForIndexColorStep(StepKey key, StepKey nextStepKey, ClusterHealthStatus color, @Nullable String indexNamePrefix) { super(key, nextStepKey); this.color = color; - this.indexNamePrefix = indexNamePrefix; + this.indexNameSupplier = (index, lifecycleState) -> indexNamePrefix + index; + } + + WaitForIndexColorStep( + StepKey key, + StepKey nextStepKey, + ClusterHealthStatus color, + BiFunction indexNameSupplier + ) { + super(key, nextStepKey); + this.color = color; + this.indexNameSupplier = indexNameSupplier; } public ClusterHealthStatus getColor() { return this.color; } - public String getIndexNamePrefix() { - return indexNamePrefix; + BiFunction getIndexNameSupplier() { + return indexNameSupplier; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), this.color, this.indexNamePrefix); + return Objects.hash(super.hashCode(), this.color, this.indexNameSupplier); } @Override @@ -69,12 +82,15 @@ public boolean equals(Object obj) { return false; } WaitForIndexColorStep other = (WaitForIndexColorStep) obj; - return super.equals(obj) && Objects.equals(this.color, other.color) && Objects.equals(this.indexNamePrefix, other.indexNamePrefix); + return super.equals(obj) + && Objects.equals(this.color, other.color) + && Objects.equals(this.indexNameSupplier, other.indexNameSupplier); } @Override public Result isConditionMet(Index index, ClusterState clusterState) { - String indexName = indexNamePrefix != null ? indexNamePrefix + index.getName() : index.getName(); + LifecycleExecutionState lifecycleExecutionState = clusterState.metadata().index(index.getName()).getLifecycleExecutionState(); + String indexName = indexNameSupplier.apply(index.getName(), lifecycleExecutionState); IndexMetadata indexMetadata = clusterState.metadata().index(indexName); // check if the (potentially) derived index exists if (indexMetadata == null) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStepTests.java index 9e1b7a09dc6a2..9259c63f243e9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStepTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.RoutingTable; @@ -21,6 +22,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.xpack.core.ilm.Step.StepKey; +import java.util.function.BiFunction; + import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -48,7 +51,7 @@ protected WaitForIndexColorStep mutateInstance(WaitForIndexColorStep instance) { StepKey key = instance.getKey(); StepKey nextKey = instance.getNextStepKey(); ClusterHealthStatus color = instance.getColor(), newColor = randomColor(); - String indexPrefix = instance.getIndexNamePrefix(); + BiFunction indexNameSupplier = instance.getIndexNameSupplier(); while (color.equals(newColor)) { newColor = randomColor(); @@ -60,12 +63,17 @@ protected WaitForIndexColorStep mutateInstance(WaitForIndexColorStep instance) { case 2 -> color = newColor; } - return new WaitForIndexColorStep(key, nextKey, color, indexPrefix); + return new WaitForIndexColorStep(key, nextKey, color, indexNameSupplier); } @Override protected WaitForIndexColorStep copyInstance(WaitForIndexColorStep instance) { - return new WaitForIndexColorStep(instance.getKey(), instance.getNextStepKey(), instance.getColor(), instance.getIndexNamePrefix()); + return new WaitForIndexColorStep( + instance.getKey(), + instance.getNextStepKey(), + instance.getColor(), + instance.getIndexNameSupplier() + ); } public void testConditionMetForGreen() { From e6ea553ce8021555a2c788ea9796273704dcab9e Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Mon, 11 Jul 2022 18:07:57 +0300 Subject: [PATCH 20/28] Fix broken test --- .../xpack/core/ilm/RollupILMActionTests.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 5f74cf8698117..5cf813065632d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -58,7 +58,7 @@ public void testToSteps() { ); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(11, steps.size()); + assertEquals(12, steps.size()); assertTrue(steps.get(0) instanceof CheckNotDataStreamWriteIndexStep); assertThat(steps.get(0).getKey().getName(), equalTo(CheckNotDataStreamWriteIndexStep.NAME)); @@ -82,29 +82,33 @@ public void testToSteps() { assertTrue(steps.get(5) instanceof RollupStep); assertThat(steps.get(5).getKey().getName(), equalTo(RollupStep.NAME)); - assertThat(steps.get(5).getNextStepKey().getName(), equalTo(CopyExecutionStateStep.NAME)); + assertThat(steps.get(5).getNextStepKey().getName(), equalTo(WaitForIndexColorStep.NAME)); - assertTrue(steps.get(6) instanceof CopyExecutionStateStep); - assertThat(steps.get(6).getKey().getName(), equalTo(CopyExecutionStateStep.NAME)); - assertThat(steps.get(6).getNextStepKey().getName(), equalTo(CopySettingsStep.NAME)); + assertTrue(steps.get(6) instanceof WaitForIndexColorStep); + assertThat(steps.get(6).getKey().getName(), equalTo(WaitForIndexColorStep.NAME)); + assertThat(steps.get(6).getNextStepKey().getName(), equalTo(CopyExecutionStateStep.NAME)); - assertTrue(steps.get(7) instanceof CopySettingsStep); - assertThat(steps.get(7).getKey().getName(), equalTo(CopySettingsStep.NAME)); - assertThat(steps.get(7).getNextStepKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); + assertTrue(steps.get(7) instanceof CopyExecutionStateStep); + assertThat(steps.get(7).getKey().getName(), equalTo(CopyExecutionStateStep.NAME)); + assertThat(steps.get(7).getNextStepKey().getName(), equalTo(CopySettingsStep.NAME)); - assertTrue(steps.get(8) instanceof BranchingStep); - assertThat(steps.get(8).getKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); - expectThrows(IllegalStateException.class, () -> steps.get(8).getNextStepKey()); - assertThat(((BranchingStep) steps.get(8)).getNextStepKeyOnFalse().getName(), equalTo(DeleteStep.NAME)); - assertThat(((BranchingStep) steps.get(8)).getNextStepKeyOnTrue().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); + assertTrue(steps.get(8) instanceof CopySettingsStep); + assertThat(steps.get(8).getKey().getName(), equalTo(CopySettingsStep.NAME)); + assertThat(steps.get(8).getNextStepKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); - assertTrue(steps.get(9) instanceof ReplaceDataStreamBackingIndexStep); - assertThat(steps.get(9).getKey().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); - assertThat(steps.get(9).getNextStepKey().getName(), equalTo(DeleteStep.NAME)); + assertTrue(steps.get(9) instanceof BranchingStep); + assertThat(steps.get(9).getKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); + expectThrows(IllegalStateException.class, () -> steps.get(9).getNextStepKey()); + assertThat(((BranchingStep) steps.get(9)).getNextStepKeyOnFalse().getName(), equalTo(DeleteStep.NAME)); + assertThat(((BranchingStep) steps.get(9)).getNextStepKeyOnTrue().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); - assertTrue(steps.get(10) instanceof DeleteStep); - assertThat(steps.get(10).getKey().getName(), equalTo(DeleteStep.NAME)); - assertThat(steps.get(10).getNextStepKey(), equalTo(nextStepKey)); + assertTrue(steps.get(10) instanceof ReplaceDataStreamBackingIndexStep); + assertThat(steps.get(10).getKey().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); + assertThat(steps.get(10).getNextStepKey().getName(), equalTo(DeleteStep.NAME)); + + assertTrue(steps.get(11) instanceof DeleteStep); + assertThat(steps.get(11).getKey().getName(), equalTo(DeleteStep.NAME)); + assertThat(steps.get(11).getNextStepKey(), equalTo(nextStepKey)); } public void testEqualsAndHashCode() { From 6c90a7a9a5f3a7311ad76ccdecce1c9487eae441 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Wed, 13 Jul 2022 15:03:46 +0300 Subject: [PATCH 21/28] Added step to swap aliases and delete source index --- .../xpack/core/ilm/RollupILMAction.java | 21 +++++-- .../xpack/core/ilm/ShrinkSetAliasStep.java | 2 +- .../SwapAliasesAndDeleteSourceIndexStep.java | 58 +++++++++++++++---- .../xpack/core/ilm/WaitForIndexColorStep.java | 4 +- .../xpack/core/ilm/RollupILMActionTests.java | 8 ++- ...pAliasesAndDeleteSourceIndexStepTests.java | 15 +++-- .../xpack/ilm/actions/RollupActionIT.java | 2 + 7 files changed, 83 insertions(+), 27 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 653c8377f0893..07d5d79196a71 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -101,12 +101,13 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey cleanupRollupIndexKey = new StepKey(phase, NAME, CleanupTargetIndexStep.NAME); StepKey generateRollupIndexNameKey = new StepKey(phase, NAME, GENERATE_ROLLUP_STEP_NAME); StepKey rollupKey = new StepKey(phase, NAME, RollupStep.NAME); - StepKey waitForGreenRestoredIndexKey = new StepKey(phase, NAME, WaitForIndexColorStep.NAME); + StepKey waitForGreenRollupIndexKey = new StepKey(phase, NAME, WaitForIndexColorStep.NAME); StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME); StepKey copyLifecyclePolicySettingKey = new StepKey(phase, NAME, CopySettingsStep.NAME); StepKey dataStreamCheckBranchingKey = new StepKey(phase, NAME, CONDITIONAL_DATASTREAM_CHECK_KEY); StepKey replaceDataStreamIndexKey = new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME); StepKey deleteIndexKey = new StepKey(phase, NAME, DeleteStep.NAME); + StepKey swapAliasesKey = new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME); CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep( checkNotWriteIndex, @@ -136,9 +137,10 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { ); // Here is where the actual rollup action takes place - RollupStep rollupStep = new RollupStep(rollupKey, waitForGreenRestoredIndexKey, client, config); + RollupStep rollupStep = new RollupStep(rollupKey, waitForGreenRollupIndexKey, client, config); + // Wait until the Rollup index health is green WaitForIndexColorStep waitForGreenIndexHealthStep = new WaitForIndexColorStep( - waitForGreenRestoredIndexKey, + waitForGreenRollupIndexKey, copyMetadataKey, ClusterHealthStatus.GREEN, (indexName, lifecycleState) -> lifecycleState.rollupIndexName() @@ -164,7 +166,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { // then deleting the source index. BranchingStep isDataStreamBranchingStep = new BranchingStep( dataStreamCheckBranchingKey, - deleteIndexKey, + swapAliasesKey, replaceDataStreamIndexKey, (index, clusterState) -> { IndexAbstraction indexAbstraction = clusterState.metadata().getIndicesLookup().get(index.getName()); @@ -180,6 +182,14 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { ); DeleteStep deleteSourceIndexStep = new DeleteStep(deleteIndexKey, nextStepKey, client); + SwapAliasesAndDeleteSourceIndexStep swapAliasesAndDeleteSourceIndexStep = new SwapAliasesAndDeleteSourceIndexStep( + swapAliasesKey, + nextStepKey, + client, + (indexName, lifecycleState) -> lifecycleState.rollupIndexName(), + false + ); + return List.of( checkNotWriteIndexStep, waitForNoFollowersStep, @@ -192,7 +202,8 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { copySettingsStep, isDataStreamBranchingStep, replaceDataStreamBackingIndex, - deleteSourceIndexStep + deleteSourceIndexStep, + swapAliasesAndDeleteSourceIndexStep ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkSetAliasStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkSetAliasStep.java index 766149d7d2171..ecccd72e1f4d1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkSetAliasStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkSetAliasStep.java @@ -38,7 +38,7 @@ public void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState cu // get target shrink index LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState(); String targetIndexName = getShrinkIndexName(indexName, lifecycleState); - deleteSourceIndexAndTransferAliases(getClient(), indexMetadata, targetIndexName, listener); + deleteSourceIndexAndTransferAliases(getClient(), indexMetadata, targetIndexName, listener, true); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SwapAliasesAndDeleteSourceIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SwapAliasesAndDeleteSourceIndexStep.java index 39d2ae4ddea11..b5bbd65f0d2c7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SwapAliasesAndDeleteSourceIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SwapAliasesAndDeleteSourceIndexStep.java @@ -14,10 +14,12 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.core.TimeValue; import java.util.Locale; import java.util.Objects; +import java.util.function.BiFunction; /** * This step swaps all the aliases from the source index to the restored index and deletes the source index. This is useful in scenarios @@ -27,11 +29,32 @@ public class SwapAliasesAndDeleteSourceIndexStep extends AsyncActionStep { public static final String NAME = "swap-aliases"; private static final Logger logger = LogManager.getLogger(SwapAliasesAndDeleteSourceIndexStep.class); - private final String targetIndexPrefix; + /** + * Supplier function that returns the name of the target index where aliases will + * point to + */ + private final BiFunction targetIndexNameSupplier; + + /** + * if true, this method will create an alias named as the source index and will link it + * to the target index + */ + private final boolean createSourceIndexAlias; public SwapAliasesAndDeleteSourceIndexStep(StepKey key, StepKey nextStepKey, Client client, String targetIndexPrefix) { + this(key, nextStepKey, client, (index, lifecycleState) -> targetIndexPrefix + index, true); + } + + public SwapAliasesAndDeleteSourceIndexStep( + StepKey key, + StepKey nextStepKey, + Client client, + BiFunction targetIndexNameSupplier, + boolean createSourceIndexAlias + ) { super(key, nextStepKey, client); - this.targetIndexPrefix = targetIndexPrefix; + this.targetIndexNameSupplier = targetIndexNameSupplier; + this.createSourceIndexAlias = createSourceIndexAlias; } @Override @@ -39,8 +62,12 @@ public boolean isRetryable() { return true; } - public String getTargetIndexPrefix() { - return targetIndexPrefix; + BiFunction getTargetIndexNameSupplier() { + return targetIndexNameSupplier; + } + + boolean getCreateSourceIndexAlias() { + return createSourceIndexAlias; } @Override @@ -51,7 +78,7 @@ public void performAction( ActionListener listener ) { String originalIndex = indexMetadata.getIndex().getName(); - final String targetIndexName = targetIndexPrefix + originalIndex; + final String targetIndexName = targetIndexNameSupplier.apply(originalIndex, indexMetadata.getLifecycleExecutionState()); IndexMetadata targetIndexMetadata = currentClusterState.metadata().index(targetIndexName); if (targetIndexMetadata == null) { @@ -68,7 +95,7 @@ public void performAction( return; } - deleteSourceIndexAndTransferAliases(getClient(), indexMetadata, targetIndexName, listener); + deleteSourceIndexAndTransferAliases(getClient(), indexMetadata, targetIndexName, listener, createSourceIndexAlias); } /** @@ -76,17 +103,24 @@ public void performAction( * index. *

* The is_write_index will *not* be set on the target index as this operation is currently executed on read-only indices. + * @param createSourceIndexAlias if true, this method will create an alias named as the source index and will link it + * to the target index */ static void deleteSourceIndexAndTransferAliases( Client client, IndexMetadata sourceIndex, String targetIndex, - ActionListener listener + ActionListener listener, + boolean createSourceIndexAlias ) { String sourceIndexName = sourceIndex.getIndex().getName(); IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest().masterNodeTimeout(TimeValue.MAX_VALUE) - .addAliasAction(IndicesAliasesRequest.AliasActions.removeIndex().index(sourceIndexName)) - .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(targetIndex).alias(sourceIndexName)); + .addAliasAction(IndicesAliasesRequest.AliasActions.removeIndex().index(sourceIndexName)); + + if (createSourceIndexAlias) { + // create an alias with the same name as the source index and link it to the target index + aliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(targetIndex).alias(sourceIndexName)); + } // copy over other aliases from source index sourceIndex.getAliases().values().forEach(aliasMetaDataToAdd -> { // inherit all alias properties except `is_write_index` @@ -116,7 +150,7 @@ public boolean indexSurvives() { @Override public int hashCode() { - return Objects.hash(super.hashCode(), targetIndexPrefix); + return Objects.hash(super.hashCode(), targetIndexNameSupplier, createSourceIndexAlias); } @Override @@ -128,6 +162,8 @@ public boolean equals(Object obj) { return false; } SwapAliasesAndDeleteSourceIndexStep other = (SwapAliasesAndDeleteSourceIndexStep) obj; - return super.equals(obj) && Objects.equals(targetIndexPrefix, other.targetIndexPrefix); + return super.equals(obj) + && Objects.equals(targetIndexNameSupplier, other.targetIndexNameSupplier) + && createSourceIndexAlias == other.createSourceIndexAlias; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStep.java index 179fdf69e6b7b..59982b4d7931d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStep.java @@ -44,9 +44,7 @@ class WaitForIndexColorStep extends ClusterStateWaitStep { } WaitForIndexColorStep(StepKey key, StepKey nextStepKey, ClusterHealthStatus color, @Nullable String indexNamePrefix) { - super(key, nextStepKey); - this.color = color; - this.indexNameSupplier = (index, lifecycleState) -> indexNamePrefix + index; + this(key, nextStepKey, color, (index, lifecycleState) -> indexNamePrefix + index); } WaitForIndexColorStep( diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 5cf813065632d..0b4a7f086f0d5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -58,7 +58,7 @@ public void testToSteps() { ); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(12, steps.size()); + assertEquals(13, steps.size()); assertTrue(steps.get(0) instanceof CheckNotDataStreamWriteIndexStep); assertThat(steps.get(0).getKey().getName(), equalTo(CheckNotDataStreamWriteIndexStep.NAME)); @@ -99,7 +99,7 @@ public void testToSteps() { assertTrue(steps.get(9) instanceof BranchingStep); assertThat(steps.get(9).getKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); expectThrows(IllegalStateException.class, () -> steps.get(9).getNextStepKey()); - assertThat(((BranchingStep) steps.get(9)).getNextStepKeyOnFalse().getName(), equalTo(DeleteStep.NAME)); + assertThat(((BranchingStep) steps.get(9)).getNextStepKeyOnFalse().getName(), equalTo(SwapAliasesAndDeleteSourceIndexStep.NAME)); assertThat(((BranchingStep) steps.get(9)).getNextStepKeyOnTrue().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); assertTrue(steps.get(10) instanceof ReplaceDataStreamBackingIndexStep); @@ -109,6 +109,10 @@ public void testToSteps() { assertTrue(steps.get(11) instanceof DeleteStep); assertThat(steps.get(11).getKey().getName(), equalTo(DeleteStep.NAME)); assertThat(steps.get(11).getNextStepKey(), equalTo(nextStepKey)); + + assertTrue(steps.get(12) instanceof SwapAliasesAndDeleteSourceIndexStep); + assertThat(steps.get(12).getKey().getName(), equalTo(SwapAliasesAndDeleteSourceIndexStep.NAME)); + assertThat(steps.get(12).getNextStepKey(), equalTo(nextStepKey)); } public void testEqualsAndHashCode() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SwapAliasesAndDeleteSourceIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SwapAliasesAndDeleteSourceIndexStepTests.java index fa634813e3fc4..40c1f787f4a2b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SwapAliasesAndDeleteSourceIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SwapAliasesAndDeleteSourceIndexStepTests.java @@ -17,12 +17,14 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.xpack.core.ilm.Step.StepKey; import java.util.Arrays; import java.util.List; +import java.util.function.BiFunction; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -43,7 +45,8 @@ protected SwapAliasesAndDeleteSourceIndexStep copyInstance(SwapAliasesAndDeleteS instance.getKey(), instance.getNextStepKey(), instance.getClient(), - instance.getTargetIndexPrefix() + instance.getTargetIndexNameSupplier(), + instance.getCreateSourceIndexAlias() ); } @@ -51,14 +54,16 @@ protected SwapAliasesAndDeleteSourceIndexStep copyInstance(SwapAliasesAndDeleteS public SwapAliasesAndDeleteSourceIndexStep mutateInstance(SwapAliasesAndDeleteSourceIndexStep instance) { StepKey key = instance.getKey(); StepKey nextKey = instance.getNextStepKey(); - String restoredIndexPrefix = instance.getTargetIndexPrefix(); - switch (between(0, 2)) { + BiFunction indexNameSupplier = instance.getTargetIndexNameSupplier(); + boolean createSourceIndexAlias = instance.getCreateSourceIndexAlias(); + switch (between(0, 3)) { case 0 -> key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); case 1 -> nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); - case 2 -> restoredIndexPrefix += randomAlphaOfLength(5); + case 2 -> indexNameSupplier = (index, state) -> index + randomAlphaOfLength(5); + case 3 -> createSourceIndexAlias = createSourceIndexAlias == false; default -> throw new AssertionError("Illegal randomisation branch"); } - return new SwapAliasesAndDeleteSourceIndexStep(key, nextKey, instance.getClient(), restoredIndexPrefix); + return new SwapAliasesAndDeleteSourceIndexStep(key, nextKey, instance.getClient(), indexNameSupplier, createSourceIndexAlias); } public void testPerformAction() { diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index b5e8017e118ca..9aff53afe4b5b 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -155,6 +155,7 @@ public void testRollupIndex() throws Exception { Map settings = getOnlyIndexSettings(client(), rollupIndex); assertThat(settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey()), equalTo(index)); }); + assertBusy(() -> assertTrue(aliasExists(rollupIndex, alias))); } public void testRollupIndexInTheHotPhase() throws Exception { @@ -226,6 +227,7 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { Map settings = getOnlyIndexSettings(client(), rollupIndex); assertThat(settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey()), equalTo(originalIndex)); }); + assertBusy(() -> assertTrue(aliasExists(rollupIndex, alias))); } public void testTsdbDataStreams() throws Exception { From 6d91aaa63c0398878e2bd5c4c804f60038b169f9 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Wed, 13 Jul 2022 16:35:03 +0300 Subject: [PATCH 22/28] Changed parameters for RollupILMAction Removed the `config` object and added `fixed_interval` param right in the `rollup` action --- .../xpack/core/ilm/RollupILMAction.java | 49 +++++++++++-------- .../xpack/core/ilm/RollupStep.java | 18 ++++--- .../xpack/core/ilm/RollupILMActionTests.java | 13 ++--- .../xpack/core/ilm/RollupStepTests.java | 16 +++--- .../ilm/TimeseriesLifecycleTypeTests.java | 3 +- .../xpack/ilm/actions/RollupActionIT.java | 18 ++++--- 6 files changed, 63 insertions(+), 54 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 07d5d79196a71..d07ebc69bd2f4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -25,6 +26,8 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; + /** * A {@link LifecycleAction} which calls {@link org.elasticsearch.xpack.core.rollup.action.RollupAction} on an index */ @@ -33,48 +36,52 @@ public class RollupILMAction implements LifecycleAction { public static final String NAME = "rollup"; public static final String ROLLUP_INDEX_PREFIX = "rollup-"; public static final String CONDITIONAL_DATASTREAM_CHECK_KEY = BranchingStep.NAME + "-on-datastream-check"; + public static final String GENERATE_ROLLUP_STEP_NAME = "generate-rollup-name"; + private static final ParseField FIXED_INTERVAL_FIELD = new ParseField(RollupActionConfig.FIXED_INTERVAL); - private static final ParseField CONFIG_FIELD = new ParseField("config"); - - @SuppressWarnings("unchecked") +// @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, - a -> new RollupILMAction((RollupActionConfig) a[0]) + a -> new RollupILMAction((DateHistogramInterval) a[0]) ); - public static final String GENERATE_ROLLUP_STEP_NAME = "generate-rollup-name"; - - private final RollupActionConfig config; static { PARSER.declareField( - ConstructingObjectParser.constructorArg(), - (p, c) -> RollupActionConfig.fromXContent(p), - CONFIG_FIELD, - ObjectParser.ValueType.OBJECT + constructorArg(), + p -> new DateHistogramInterval(p.text()), + FIXED_INTERVAL_FIELD, + ObjectParser.ValueType.STRING ); } + private final DateHistogramInterval fixedInterval; + public static RollupILMAction parse(XContentParser parser) { return PARSER.apply(parser, null); } - public RollupILMAction(RollupActionConfig config) { - this.config = config; + public RollupILMAction(DateHistogramInterval fixedInterval) { + if (fixedInterval == null) { + throw new IllegalArgumentException("Parameter [" + FIXED_INTERVAL_FIELD.getPreferredName() + "] is required."); + } + this.fixedInterval = fixedInterval; } public RollupILMAction(StreamInput in) throws IOException { - this(new RollupActionConfig(in)); + this(new DateHistogramInterval(in)); } @Override public void writeTo(StreamOutput out) throws IOException { - config.writeTo(out); + fixedInterval.writeTo(out); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CONFIG_FIELD.getPreferredName(), config); + if (fixedInterval != null) { + builder.field(FIXED_INTERVAL_FIELD.getPreferredName(), fixedInterval.toString()); + } builder.endObject(); return builder; } @@ -84,8 +91,8 @@ public String getWriteableName() { return NAME; } - public RollupActionConfig config() { - return config; + public DateHistogramInterval fixedInterval() { + return fixedInterval; } @Override @@ -137,7 +144,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { ); // Here is where the actual rollup action takes place - RollupStep rollupStep = new RollupStep(rollupKey, waitForGreenRollupIndexKey, client, config); + RollupStep rollupStep = new RollupStep(rollupKey, waitForGreenRollupIndexKey, client, fixedInterval); // Wait until the Rollup index health is green WaitForIndexColorStep waitForGreenIndexHealthStep = new WaitForIndexColorStep( waitForGreenRollupIndexKey, @@ -213,12 +220,12 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; RollupILMAction that = (RollupILMAction) o; - return Objects.equals(this.config, that.config); + return Objects.equals(this.fixedInterval, that.fixedInterval); } @Override public int hashCode() { - return Objects.hash(config); + return Objects.hash(fixedInterval); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index cfe206c4b8150..8a64260d5eda7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -16,13 +16,14 @@ import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.common.Strings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.xpack.core.rollup.RollupActionConfig; import org.elasticsearch.xpack.core.rollup.action.RollupAction; import java.util.Objects; /** - * ILM step that invokes the rollup action for an index using a {@link RollupActionConfig}. The rollup + * ILM step that invokes the rollup action for an index using a {@link DateHistogramInterval}. The rollup * action produces a rollup index using a prefix prepended to the original index name for the name of the rollup * index. Also, the rollup action deletes the source index at the end, so no DeleteStep is required after this * step. @@ -32,11 +33,11 @@ public class RollupStep extends AsyncActionStep { private static final Logger logger = LogManager.getLogger(RollupStep.class); - private final RollupActionConfig config; + private final DateHistogramInterval fixedInterval; - public RollupStep(StepKey key, StepKey nextStepKey, Client client, RollupActionConfig config) { + public RollupStep(StepKey key, StepKey nextStepKey, Client client, DateHistogramInterval fixedInterval) { super(key, nextStepKey, client); - this.config = config; + this.fixedInterval = fixedInterval; } @Override @@ -100,6 +101,7 @@ public void performAction( return; } + RollupActionConfig config = new RollupActionConfig(fixedInterval); RollupAction.Request request = new RollupAction.Request(indexName, rollupIndexName, config).masterNodeTimeout(TimeValue.MAX_VALUE); // Currently, RollupAction always acknowledges action was complete when no exceptions are thrown. getClient().execute( @@ -109,13 +111,13 @@ public void performAction( ); } - public RollupActionConfig getConfig() { - return config; + public DateHistogramInterval getFixedInterval() { + return fixedInterval; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), config); + return Objects.hash(super.hashCode(), fixedInterval); } @Override @@ -127,6 +129,6 @@ public boolean equals(Object obj) { return false; } RollupStep other = (RollupStep) obj; - return super.equals(obj) && Objects.equals(config, other.config); + return super.equals(obj) && Objects.equals(fixedInterval, other.fixedInterval); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 0b4a7f086f0d5..9c467edd9aa37 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -12,8 +12,6 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.ilm.Step.StepKey; import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; -import org.elasticsearch.xpack.core.rollup.RollupActionConfig; -import org.elasticsearch.xpack.core.rollup.RollupActionConfigTests; import java.util.List; @@ -24,7 +22,7 @@ public class RollupILMActionTests extends AbstractActionTestCase { static RollupILMAction randomInstance() { - return new RollupILMAction(RollupActionConfigTests.randomConfig()); + return new RollupILMAction(ConfigTestHelpers.randomInterval()); } @Override @@ -49,7 +47,7 @@ public boolean isSafeAction() { @Override public void testToSteps() { - RollupILMAction action = new RollupILMAction(RollupActionConfigTests.randomConfig()); + RollupILMAction action = new RollupILMAction(ConfigTestHelpers.randomInterval()); String phase = randomAlphaOfLengthBetween(1, 10); StepKey nextStepKey = new StepKey( randomAlphaOfLengthBetween(1, 10), @@ -120,15 +118,14 @@ public void testEqualsAndHashCode() { } RollupILMAction copy(RollupILMAction rollupILMAction) { - return new RollupILMAction(rollupILMAction.config()); + return new RollupILMAction(rollupILMAction.fixedInterval()); } RollupILMAction notCopy(RollupILMAction rollupILMAction) { DateHistogramInterval fixedInterval = randomValueOtherThan( - rollupILMAction.config().getFixedInterval(), + rollupILMAction.fixedInterval(), ConfigTestHelpers::randomInterval ); - RollupActionConfig newConfig = new RollupActionConfig(fixedInterval); - return new RollupILMAction(newConfig); + return new RollupILMAction(fixedInterval); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java index db42ca0acea03..264c3e7e3966c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java @@ -16,9 +16,9 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.xpack.core.ilm.Step.StepKey; -import org.elasticsearch.xpack.core.rollup.RollupActionConfig; -import org.elasticsearch.xpack.core.rollup.RollupActionConfigTests; +import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.elasticsearch.xpack.core.rollup.action.RollupAction; import org.mockito.Mockito; @@ -38,29 +38,29 @@ public class RollupStepTests extends AbstractStepTestCase { public RollupStep createRandomInstance() { StepKey stepKey = randomStepKey(); StepKey nextStepKey = randomStepKey(); - RollupActionConfig config = RollupActionConfigTests.randomConfig(); - return new RollupStep(stepKey, nextStepKey, client, config); + DateHistogramInterval fixedInterval = ConfigTestHelpers.randomInterval(); + return new RollupStep(stepKey, nextStepKey, client, fixedInterval); } @Override public RollupStep mutateInstance(RollupStep instance) { StepKey key = instance.getKey(); StepKey nextKey = instance.getNextStepKey(); - RollupActionConfig config = instance.getConfig(); + DateHistogramInterval fixedInterval = instance.getFixedInterval(); switch (between(0, 2)) { case 0 -> key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); case 1 -> nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); - case 2 -> config = RollupActionConfigTests.randomConfig(); + case 2 -> fixedInterval = ConfigTestHelpers.randomInterval(); default -> throw new AssertionError("Illegal randomisation branch"); } - return new RollupStep(key, nextKey, instance.getClient(), config); + return new RollupStep(key, nextKey, instance.getClient(), fixedInterval); } @Override public RollupStep copyInstance(RollupStep instance) { - return new RollupStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(), instance.getConfig()); + return new RollupStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(), instance.getFixedInterval()); } private IndexMetadata getIndexMetadata(String index, String lifecycleName, RollupStep step) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index 066a1582c8b6a..3e577a63dd4d2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.rollup.RollupActionConfig; import java.util.ArrayList; import java.util.Arrays; @@ -72,7 +71,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase { // keeping the migrate action disabled as otherwise it could conflict with the allocate action if both are randomly selected for the // same phase private static final MigrateAction TEST_MIGRATE_ACTION = MigrateAction.DISABLED; - private static final RollupILMAction TEST_ROLLUP_ACTION = new RollupILMAction(new RollupActionConfig(DateHistogramInterval.DAY)); + private static final RollupILMAction TEST_ROLLUP_ACTION = new RollupILMAction(DateHistogramInterval.DAY); public void testValidatePhases() { boolean invalid = randomBoolean(); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index 9aff53afe4b5b..cd42fc32d3ae6 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -31,7 +31,7 @@ import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.RollupILMAction; -import org.elasticsearch.xpack.core.rollup.RollupActionConfigTests; +import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.junit.Before; import java.io.IOException; @@ -143,7 +143,7 @@ public void testRollupIndex() throws Exception { index(client(), index, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); String phaseName = randomFrom("warm", "cold"); - createNewSingletonPolicy(client(), policy, phaseName, new RollupILMAction(RollupActionConfigTests.randomConfig())); + createNewSingletonPolicy(client(), policy, phaseName, new RollupILMAction(ConfigTestHelpers.randomInterval())); updatePolicy(client(), index, policy); assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(index)), 30, TimeUnit.SECONDS); @@ -155,7 +155,9 @@ public void testRollupIndex() throws Exception { Map settings = getOnlyIndexSettings(client(), rollupIndex); assertThat(settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey()), equalTo(index)); }); - assertBusy(() -> assertTrue(aliasExists(rollupIndex, alias))); + assertBusy( + () -> assertTrue("Alias [" + alias + "] does not point to index [" + rollupIndex + "]", aliasExists(rollupIndex, alias)) + ); } public void testRollupIndexInTheHotPhase() throws Exception { @@ -164,7 +166,7 @@ public void testRollupIndexInTheHotPhase() throws Exception { ResponseException e = expectThrows( ResponseException.class, - () -> createNewSingletonPolicy(client(), policy, "hot", new RollupILMAction(RollupActionConfigTests.randomConfig())) + () -> createNewSingletonPolicy(client(), policy, "hot", new RollupILMAction(ConfigTestHelpers.randomInterval())) ); assertTrue( e.getMessage().contains("the [rollup] action(s) may not be used in the [hot] phase without an accompanying [rollover] action") @@ -179,7 +181,7 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { RolloverAction.NAME, new RolloverAction(null, null, null, 1L, null), RollupILMAction.NAME, - new RollupILMAction(RollupActionConfigTests.randomConfig()) + new RollupILMAction(ConfigTestHelpers.randomInterval()) ); Map phases = Map.of("hot", new Phase("hot", TimeValue.ZERO, hotActions)); LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); @@ -227,12 +229,14 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { Map settings = getOnlyIndexSettings(client(), rollupIndex); assertThat(settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey()), equalTo(originalIndex)); }); - assertBusy(() -> assertTrue(aliasExists(rollupIndex, alias))); + assertBusy( + () -> assertTrue("Alias [" + alias + "] does not point to index [" + rollupIndex + "]", aliasExists(rollupIndex, alias)) + ); } public void testTsdbDataStreams() throws Exception { // Create the ILM policy - createNewSingletonPolicy(client(), policy, "warm", new RollupILMAction(RollupActionConfigTests.randomConfig())); + createNewSingletonPolicy(client(), policy, "warm", new RollupILMAction(ConfigTestHelpers.randomInterval())); // Create a template Request createIndexTemplateRequest = new Request("POST", "/_index_template/" + dataStream); From 8bb250c9a20fd32d45028b4793fae223de493962 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Wed, 13 Jul 2022 16:49:00 +0300 Subject: [PATCH 23/28] checkstyle --- .../org/elasticsearch/xpack/core/ilm/RollupILMAction.java | 1 - .../elasticsearch/xpack/core/ilm/RollupILMActionTests.java | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index d07ebc69bd2f4..f56ccfeb5126e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -39,7 +39,6 @@ public class RollupILMAction implements LifecycleAction { public static final String GENERATE_ROLLUP_STEP_NAME = "generate-rollup-name"; private static final ParseField FIXED_INTERVAL_FIELD = new ParseField(RollupActionConfig.FIXED_INTERVAL); -// @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, a -> new RollupILMAction((DateHistogramInterval) a[0]) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 9c467edd9aa37..48416267466a1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -122,10 +122,7 @@ RollupILMAction copy(RollupILMAction rollupILMAction) { } RollupILMAction notCopy(RollupILMAction rollupILMAction) { - DateHistogramInterval fixedInterval = randomValueOtherThan( - rollupILMAction.fixedInterval(), - ConfigTestHelpers::randomInterval - ); + DateHistogramInterval fixedInterval = randomValueOtherThan(rollupILMAction.fixedInterval(), ConfigTestHelpers::randomInterval); return new RollupILMAction(fixedInterval); } } From 78baf977a545e428b79e08637467037ed28ba621 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Thu, 14 Jul 2022 16:55:47 +0300 Subject: [PATCH 24/28] Fix tests --- .../org/elasticsearch/xpack/ilm/actions/RollupActionIT.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index cd42fc32d3ae6..45ac161415aa0 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -115,7 +115,6 @@ private void createIndex(String index, String alias) throws IOException { .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("metricset")) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2006-01-08T23:40:53.384Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z") - .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) .put(LifecycleSettings.LIFECYCLE_NAME, policy); XContentBuilder builder = XContentFactory.jsonBuilder() @@ -229,9 +228,6 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { Map settings = getOnlyIndexSettings(client(), rollupIndex); assertThat(settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey()), equalTo(originalIndex)); }); - assertBusy( - () -> assertTrue("Alias [" + alias + "] does not point to index [" + rollupIndex + "]", aliasExists(rollupIndex, alias)) - ); } public void testTsdbDataStreams() throws Exception { From 5893ce04bd556bfe31c8175d68573dd24b20dd1b Mon Sep 17 00:00:00 2001 From: Christos Soulios <1561376+csoulios@users.noreply.github.com> Date: Tue, 19 Jul 2022 16:11:48 +0300 Subject: [PATCH 25/28] Update x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java Co-authored-by: Lee Hinman --- .../org/elasticsearch/xpack/core/ilm/RollupILMAction.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index f56ccfeb5126e..dddb7d79e24a6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -78,9 +78,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (fixedInterval != null) { - builder.field(FIXED_INTERVAL_FIELD.getPreferredName(), fixedInterval.toString()); - } + builder.field(FIXED_INTERVAL_FIELD.getPreferredName(), fixedInterval.toString()); builder.endObject(); return builder; } From 5bdea594fb3013619c5dbf94190a6c0679390961 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Tue, 19 Jul 2022 19:29:21 +0300 Subject: [PATCH 26/28] Addressing reviewer comments --- .../cluster/metadata/IndexMetadata.java | 3 +- .../xpack/core/ilm/RollupILMAction.java | 28 ++++++--- .../xpack/core/ilm/RollupStep.java | 57 +++++++++++++------ .../xpack/core/ilm/RollupILMActionTests.java | 2 +- 4 files changed, 62 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index ab0f3d91e1e33..032a5acc90c90 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -1138,6 +1138,7 @@ public Index getResizeSourceIndex() { ); public enum RollupTaskStatus { + UNKNOWN, STARTED, SUCCESS; @@ -1150,7 +1151,7 @@ public String toString() { public static final Setting INDEX_ROLLUP_STATUS = Setting.enumSetting( RollupTaskStatus.class, INDEX_ROLLUP_STATUS_KEY, - RollupTaskStatus.SUCCESS, + RollupTaskStatus.UNKNOWN, Property.IndexScope, Property.InternalIndex ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index dddb7d79e24a6..59e595c396637 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -105,7 +105,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey cleanupRollupIndexKey = new StepKey(phase, NAME, CleanupTargetIndexStep.NAME); StepKey generateRollupIndexNameKey = new StepKey(phase, NAME, GENERATE_ROLLUP_STEP_NAME); StepKey rollupKey = new StepKey(phase, NAME, RollupStep.NAME); - StepKey waitForGreenRollupIndexKey = new StepKey(phase, NAME, WaitForIndexColorStep.NAME); + StepKey waitForRollupIndexKey = new StepKey(phase, NAME, WaitForIndexColorStep.NAME); StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME); StepKey copyLifecyclePolicySettingKey = new StepKey(phase, NAME, CopySettingsStep.NAME); StepKey dataStreamCheckBranchingKey = new StepKey(phase, NAME, CONDITIONAL_DATASTREAM_CHECK_KEY); @@ -141,14 +141,22 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { ); // Here is where the actual rollup action takes place - RollupStep rollupStep = new RollupStep(rollupKey, waitForGreenRollupIndexKey, client, fixedInterval); - // Wait until the Rollup index health is green - WaitForIndexColorStep waitForGreenIndexHealthStep = new WaitForIndexColorStep( - waitForGreenRollupIndexKey, - copyMetadataKey, - ClusterHealthStatus.GREEN, - (indexName, lifecycleState) -> lifecycleState.rollupIndexName() + RollupStep rollupStep = new RollupStep(rollupKey, waitForRollupIndexKey, client, fixedInterval); + + // Wait until the rollup index is recovered. We again wait until the configured threshold is breached and + // if the rollup index has not successfully recovered until then, we rewind to the "cleanup-rollup-index" + // step to delete this unsuccessful rollup index and retry the operation by generating a new rollup index + // name and attempting to rollup again + ClusterStateWaitUntilThresholdStep rollupAllocatedStep = new ClusterStateWaitUntilThresholdStep( + new WaitForIndexColorStep( + waitForRollupIndexKey, + copyMetadataKey, + ClusterHealthStatus.YELLOW, + (indexName, lifecycleState) -> lifecycleState.rollupIndexName() + ), + cleanupRollupIndexKey ); + CopyExecutionStateStep copyExecutionStateStep = new CopyExecutionStateStep( copyMetadataKey, copyLifecyclePolicySettingKey, @@ -157,6 +165,8 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { ); // Copy the index.lifecycle.name setting to the rollup index settings + // TODO: This step is going to be removed when downsampling action copies all settings + // from source to rollup index (https://github.com/elastic/elasticsearch/pull/88565) CopySettingsStep copySettingsStep = new CopySettingsStep( copyLifecyclePolicySettingKey, dataStreamCheckBranchingKey, @@ -201,7 +211,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { readOnlyStep, generateRollupIndexNameStep, rollupStep, - waitForGreenIndexHealthStep, + rollupAllocatedStep, copyExecutionStateStep, copySettingsStep, isDataStreamBranchingStep, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index 8a64260d5eda7..200ab54292a84 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; @@ -24,9 +25,9 @@ /** * ILM step that invokes the rollup action for an index using a {@link DateHistogramInterval}. The rollup - * action produces a rollup index using a prefix prepended to the original index name for the name of the rollup - * index. Also, the rollup action deletes the source index at the end, so no DeleteStep is required after this - * step. + * index name is retrieved from the lifecycle state {@link LifecycleExecutionState#rollupIndexName()} + * index. If a rollup index with the same name has been already successfully created, this step + * will be skipped. */ public class RollupStep extends AsyncActionStep { public static final String NAME = "rollup"; @@ -72,6 +73,8 @@ public void performAction( IndexMetadata rollupIndexMetadata = currentState.metadata().index(rollupIndexName); if (rollupIndexMetadata != null) { IndexMetadata.RollupTaskStatus rollupIndexStatus = IndexMetadata.INDEX_ROLLUP_STATUS.get(rollupIndexMetadata.getSettings()); + // Rollup index has already been created with the generated name and its status is "success". + // So we skip index rollup creation. if (IndexMetadata.RollupTaskStatus.SUCCESS.equals(rollupIndexStatus)) { logger.warn( "skipping [{}] step for index [{}] as part of policy [{}] as the rollup index [{}] already exists", @@ -82,25 +85,45 @@ public void performAction( ); listener.onResponse(null); } else { - listener.onFailure( - new IllegalStateException( - "failing [" - + RollupStep.NAME - + "] step for index [" - + indexName - + "] as part of policy [" - + policyName - + "] because the rollup index [" - + rollupIndexName - + "] already exists with rollup status [" - + rollupIndexStatus - + "]" - ) + logger.warn( + "[{}] step for index [{}] as part of policy [{}] found the rollup index [{}] already exists. Deleting it.", + RollupStep.NAME, + indexName, + policyName, + rollupIndexName ); + // Rollup index has already been created with the generated name but its status is not "success". + // So we delete the index and proceed with executing the rollup step. + DeleteIndexRequest deleteRequest = new DeleteIndexRequest(rollupIndexName); + getClient().admin().indices().delete(deleteRequest, ActionListener.wrap(response -> { + if (response.isAcknowledged()) { + performRollupIndex(indexName, rollupIndexName, listener); + } else { + listener.onFailure( + new IllegalStateException( + "failing [" + + RollupStep.NAME + + "] step for index [" + + indexName + + "] as part of policy [" + + policyName + + "] because the rollup index [" + + rollupIndexName + + "] already exists with rollup status [" + + rollupIndexStatus + + "]" + ) + ); + } + }, listener::onFailure)); } return; } + performRollupIndex(indexName, rollupIndexName, listener); + } + + private void performRollupIndex(String indexName, String rollupIndexName, ActionListener listener) { RollupActionConfig config = new RollupActionConfig(fixedInterval); RollupAction.Request request = new RollupAction.Request(indexName, rollupIndexName, config).masterNodeTimeout(TimeValue.MAX_VALUE); // Currently, RollupAction always acknowledges action was complete when no exceptions are thrown. diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 48416267466a1..ae71a0a93fdea 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -82,7 +82,7 @@ public void testToSteps() { assertThat(steps.get(5).getKey().getName(), equalTo(RollupStep.NAME)); assertThat(steps.get(5).getNextStepKey().getName(), equalTo(WaitForIndexColorStep.NAME)); - assertTrue(steps.get(6) instanceof WaitForIndexColorStep); + assertTrue(steps.get(6) instanceof ClusterStateWaitUntilThresholdStep); assertThat(steps.get(6).getKey().getName(), equalTo(WaitForIndexColorStep.NAME)); assertThat(steps.get(6).getNextStepKey().getName(), equalTo(CopyExecutionStateStep.NAME)); From 704e9a9ef5356da1648f6fe41506a9082d987d72 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Mon, 25 Jul 2022 20:25:48 +0300 Subject: [PATCH 27/28] Removed the copy-settings step After merging PR #88565, that delegates this task to the actual rollup step --- .../xpack/core/ilm/RollupILMAction.java | 14 +------- .../xpack/core/ilm/RollupILMActionTests.java | 34 ++++++++----------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 59e595c396637..7440dc1118210 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -107,7 +107,6 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey rollupKey = new StepKey(phase, NAME, RollupStep.NAME); StepKey waitForRollupIndexKey = new StepKey(phase, NAME, WaitForIndexColorStep.NAME); StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME); - StepKey copyLifecyclePolicySettingKey = new StepKey(phase, NAME, CopySettingsStep.NAME); StepKey dataStreamCheckBranchingKey = new StepKey(phase, NAME, CONDITIONAL_DATASTREAM_CHECK_KEY); StepKey replaceDataStreamIndexKey = new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME); StepKey deleteIndexKey = new StepKey(phase, NAME, DeleteStep.NAME); @@ -159,19 +158,9 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { CopyExecutionStateStep copyExecutionStateStep = new CopyExecutionStateStep( copyMetadataKey, - copyLifecyclePolicySettingKey, - (indexName, lifecycleState) -> lifecycleState.rollupIndexName(), - nextStepKey - ); - - // Copy the index.lifecycle.name setting to the rollup index settings - // TODO: This step is going to be removed when downsampling action copies all settings - // from source to rollup index (https://github.com/elastic/elasticsearch/pull/88565) - CopySettingsStep copySettingsStep = new CopySettingsStep( - copyLifecyclePolicySettingKey, dataStreamCheckBranchingKey, (indexName, lifecycleState) -> lifecycleState.rollupIndexName(), - LifecycleSettings.LIFECYCLE_NAME + nextStepKey ); // By the time we get to this step we have 2 indices, the source and the rollup one. We now need to choose an index @@ -213,7 +202,6 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { rollupStep, rollupAllocatedStep, copyExecutionStateStep, - copySettingsStep, isDataStreamBranchingStep, replaceDataStreamBackingIndex, deleteSourceIndexStep, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index ae71a0a93fdea..c82ef50c9be71 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -56,7 +56,7 @@ public void testToSteps() { ); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(13, steps.size()); + assertEquals(12, steps.size()); assertTrue(steps.get(0) instanceof CheckNotDataStreamWriteIndexStep); assertThat(steps.get(0).getKey().getName(), equalTo(CheckNotDataStreamWriteIndexStep.NAME)); @@ -88,29 +88,25 @@ public void testToSteps() { assertTrue(steps.get(7) instanceof CopyExecutionStateStep); assertThat(steps.get(7).getKey().getName(), equalTo(CopyExecutionStateStep.NAME)); - assertThat(steps.get(7).getNextStepKey().getName(), equalTo(CopySettingsStep.NAME)); + assertThat(steps.get(7).getNextStepKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); - assertTrue(steps.get(8) instanceof CopySettingsStep); - assertThat(steps.get(8).getKey().getName(), equalTo(CopySettingsStep.NAME)); - assertThat(steps.get(8).getNextStepKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); + assertTrue(steps.get(8) instanceof BranchingStep); + assertThat(steps.get(8).getKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); + expectThrows(IllegalStateException.class, () -> steps.get(8).getNextStepKey()); + assertThat(((BranchingStep) steps.get(8)).getNextStepKeyOnFalse().getName(), equalTo(SwapAliasesAndDeleteSourceIndexStep.NAME)); + assertThat(((BranchingStep) steps.get(8)).getNextStepKeyOnTrue().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); - assertTrue(steps.get(9) instanceof BranchingStep); - assertThat(steps.get(9).getKey().getName(), equalTo(CONDITIONAL_DATASTREAM_CHECK_KEY)); - expectThrows(IllegalStateException.class, () -> steps.get(9).getNextStepKey()); - assertThat(((BranchingStep) steps.get(9)).getNextStepKeyOnFalse().getName(), equalTo(SwapAliasesAndDeleteSourceIndexStep.NAME)); - assertThat(((BranchingStep) steps.get(9)).getNextStepKeyOnTrue().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); + assertTrue(steps.get(9) instanceof ReplaceDataStreamBackingIndexStep); + assertThat(steps.get(9).getKey().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); + assertThat(steps.get(9).getNextStepKey().getName(), equalTo(DeleteStep.NAME)); - assertTrue(steps.get(10) instanceof ReplaceDataStreamBackingIndexStep); - assertThat(steps.get(10).getKey().getName(), equalTo(ReplaceDataStreamBackingIndexStep.NAME)); - assertThat(steps.get(10).getNextStepKey().getName(), equalTo(DeleteStep.NAME)); + assertTrue(steps.get(10) instanceof DeleteStep); + assertThat(steps.get(10).getKey().getName(), equalTo(DeleteStep.NAME)); + assertThat(steps.get(10).getNextStepKey(), equalTo(nextStepKey)); - assertTrue(steps.get(11) instanceof DeleteStep); - assertThat(steps.get(11).getKey().getName(), equalTo(DeleteStep.NAME)); + assertTrue(steps.get(11) instanceof SwapAliasesAndDeleteSourceIndexStep); + assertThat(steps.get(11).getKey().getName(), equalTo(SwapAliasesAndDeleteSourceIndexStep.NAME)); assertThat(steps.get(11).getNextStepKey(), equalTo(nextStepKey)); - - assertTrue(steps.get(12) instanceof SwapAliasesAndDeleteSourceIndexStep); - assertThat(steps.get(12).getKey().getName(), equalTo(SwapAliasesAndDeleteSourceIndexStep.NAME)); - assertThat(steps.get(12).getNextStepKey(), equalTo(nextStepKey)); } public void testEqualsAndHashCode() { From 7345a57ff42b461904e0722171298f935fad92f7 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Tue, 26 Jul 2022 14:31:49 +0300 Subject: [PATCH 28/28] Added assertion for rollup status --- .../xpack/ilm/actions/RollupActionIT.java | 63 ++++++++++++++----- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java index 45ac161415aa0..623c713e20674 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RollupActionIT.java @@ -10,8 +10,10 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexMetadata.RollupTaskStatus; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; @@ -145,14 +147,19 @@ public void testRollupIndex() throws Exception { createNewSingletonPolicy(client(), policy, phaseName, new RollupILMAction(ConfigTestHelpers.randomInterval())); updatePolicy(client(), index, policy); - assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(index)), 30, TimeUnit.SECONDS); - String rollupIndex = getRollupIndexName(index); + String rollupIndex = waitAndGetRollupIndexName(client(), index); + assertNotNull("Cannot retrieve rollup index name", rollupIndex); assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(index)), 30, TimeUnit.SECONDS); - assertBusy(() -> assertThat(getStepKeyForIndex(client(), rollupIndex), equalTo(PhaseCompleteStep.finalStep(phaseName).getKey()))); + assertBusy( + () -> assertThat(getStepKeyForIndex(client(), rollupIndex), equalTo(PhaseCompleteStep.finalStep(phaseName).getKey())), + 30, + TimeUnit.SECONDS + ); assertBusy(() -> { Map settings = getOnlyIndexSettings(client(), rollupIndex); - assertThat(settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey()), equalTo(index)); + assertEquals(index, settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey())); + assertEquals(RollupTaskStatus.SUCCESS.toString(), settings.get(IndexMetadata.INDEX_ROLLUP_STATUS.getKey())); }); assertBusy( () -> assertTrue("Alias [" + alias + "] does not point to index [" + rollupIndex + "]", aliasExists(rollupIndex, alias)) @@ -218,15 +225,19 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { randomAlphaOfLength(5) ); - assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(originalIndex)), 30, TimeUnit.SECONDS); - String rollupIndex = getRollupIndexName(originalIndex); - + String rollupIndex = waitAndGetRollupIndexName(client(), originalIndex); + assertNotNull("Cannot retrieve rollup index name", rollupIndex); assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(originalIndex)), 30, TimeUnit.SECONDS); - assertBusy(() -> assertThat(getStepKeyForIndex(client(), rollupIndex), equalTo(PhaseCompleteStep.finalStep("hot").getKey()))); + assertBusy( + () -> assertThat(getStepKeyForIndex(client(), rollupIndex), equalTo(PhaseCompleteStep.finalStep("hot").getKey())), + 30, + TimeUnit.SECONDS + ); assertBusy(() -> { Map settings = getOnlyIndexSettings(client(), rollupIndex); - assertThat(settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey()), equalTo(originalIndex)); + assertEquals(originalIndex, settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey())); + assertEquals(RollupTaskStatus.SUCCESS.toString(), settings.get(IndexMetadata.INDEX_ROLLUP_STATUS.getKey())); }); } @@ -255,23 +266,41 @@ public void testTsdbDataStreams() throws Exception { // Manual rollover the original index such that it's not the write index in the data stream anymore rolloverMaxOneDocCondition(client(), dataStream); - assertBusy(() -> assertNotNull("Cannot retrieve rollup index name", getRollupIndexName(backingIndexName)), 30, TimeUnit.SECONDS); - String rollupIndex = getRollupIndexName(backingIndexName); + String rollupIndex = waitAndGetRollupIndexName(client(), backingIndexName); + assertNotNull("Cannot retrieve rollup index name", rollupIndex); assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(backingIndexName)), 30, TimeUnit.SECONDS); + assertBusy(() -> { + Map settings = getOnlyIndexSettings(client(), rollupIndex); + assertEquals(backingIndexName, settings.get(IndexMetadata.INDEX_ROLLUP_SOURCE_NAME.getKey())); + assertEquals(RollupTaskStatus.SUCCESS.toString(), settings.get(IndexMetadata.INDEX_ROLLUP_STATUS.getKey())); + }); } /** - * gets the generated rollup index name for a given index by looking at newly created indices that match the rollup index name pattern + * Gets the generated rollup index name for a given index by looking at newly created indices that match the rollup index name pattern * - * @param index the name of the source index used to generate the rollup index name + * @param originalIndexName the name of the source index used to generate the rollup index name * @return the name of the rollup index for a given index, null if none exist - * @throws IOException if request fails */ - private String getRollupIndexName(String index) throws IOException { - Response response = client().performRequest( - new Request("GET", "/" + RollupILMAction.ROLLUP_INDEX_PREFIX + "*-" + index + "/?expand_wildcards=all") + public String waitAndGetRollupIndexName(RestClient client, String originalIndexName) throws InterruptedException { + final String[] rollupIndexName = new String[1]; + waitUntil(() -> { + try { + rollupIndexName[0] = getRollupIndexName(client, originalIndexName); + return rollupIndexName[0] != null; + } catch (IOException e) { + return false; + } + }, 60, TimeUnit.SECONDS); + logger.info("--> original index name is [{}], rollup index name is [{}]", originalIndexName, rollupIndexName[0]); + return rollupIndexName[0]; + } + + public static String getRollupIndexName(RestClient client, String originalIndexName) throws IOException { + Response response = client.performRequest( + new Request("GET", "/" + RollupILMAction.ROLLUP_INDEX_PREFIX + "*-" + originalIndexName + "/?expand_wildcards=all") ); Map asMap = responseAsMap(response); if (asMap.size() == 1) {