From 87e1997f51eecc34bc9b03104e92c823e42fae7f Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Wed, 12 Jul 2023 09:24:07 -0700 Subject: [PATCH] Revert "Remove Restriction of strictly using only docrep for system indices and hidden indices. (#8502)" (#8657) This reverts commit ef4881f0486a16519e5680624eabc524265e335b. Signed-off-by: Rishikesh1159 --- .../src/main/java/org/opensearch/Version.java | 59 +++----- .../SegmentReplicationClusterSettingIT.java | 66 +++++++++ .../remotestore/CreateRemoteIndexIT.java | 52 +++++++ .../metadata/MetadataCreateIndexService.java | 28 +++- .../MetadataCreateIndexServiceTests.java | 129 +++++++++++++++--- 5 files changed, 270 insertions(+), 64 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 7dbd30c567b95..f85da63bdbb1f 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -61,50 +61,28 @@ public class Version implements Comparable, ToXContentFragment { * All listed versions MUST be released versions, except the last major, the last minor and the last revison. ONLY those are required * as unreleased versions. * - * Example: assume the last release is 7.3.0 - * The unreleased last major is the next major release, e.g. _8_.0.0 - * The unreleased last minor is the current major with a upped minor: 7._4_.0 - * The unreleased revision is the very release with a upped revision 7.3._1_ + * Example: assume the last release is 2.4.0 + * The unreleased last major is the next major release, e.g. _3_.0.0 + * The unreleased last minor is the current major with a upped minor: 2._5_.0 + * The unreleased revision is the very release with a upped revision 2.4._1_ */ public static final int V_EMPTY_ID = 0; public static final Version V_EMPTY = new Version(V_EMPTY_ID, org.apache.lucene.util.Version.LATEST); - public static final Version V_1_0_0 = new Version(1000099, org.apache.lucene.util.Version.LUCENE_8_8_2); - public static final Version V_1_1_0 = new Version(1010099, org.apache.lucene.util.Version.LUCENE_8_9_0); - public static final Version V_1_2_0 = new Version(1020099, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_2_1 = new Version(1020199, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_2_2 = new Version(1020299, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_2_3 = new Version(1020399, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_2_4 = new Version(1020499, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_2_5 = new Version(1020599, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_0 = new Version(1030099, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_1 = new Version(1030199, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_2 = new Version(1030299, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_3 = new Version(1030399, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_4 = new Version(1030499, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_5 = new Version(1030599, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_6 = new Version(1030699, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_7 = new Version(1030799, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_8 = new Version(1030899, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_9 = new Version(1030999, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_10 = new Version(1031099, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_11 = new Version(1031199, org.apache.lucene.util.Version.LUCENE_8_10_1); - public static final Version V_1_3_12 = new Version(1031299, org.apache.lucene.util.Version.LUCENE_8_10_1); + // RELEASED public static final Version V_2_0_0 = new Version(2000099, org.apache.lucene.util.Version.LUCENE_9_1_0); public static final Version V_2_0_1 = new Version(2000199, org.apache.lucene.util.Version.LUCENE_9_1_0); - public static final Version V_2_0_2 = new Version(2000299, org.apache.lucene.util.Version.LUCENE_9_1_0); public static final Version V_2_1_0 = new Version(2010099, org.apache.lucene.util.Version.LUCENE_9_2_0); - public static final Version V_2_1_1 = new Version(2010199, org.apache.lucene.util.Version.LUCENE_9_2_0); public static final Version V_2_2_0 = new Version(2020099, org.apache.lucene.util.Version.LUCENE_9_3_0); public static final Version V_2_2_1 = new Version(2020199, org.apache.lucene.util.Version.LUCENE_9_3_0); - public static final Version V_2_2_2 = new Version(2020299, org.apache.lucene.util.Version.LUCENE_9_3_0); public static final Version V_2_3_0 = new Version(2030099, org.apache.lucene.util.Version.LUCENE_9_3_0); - public static final Version V_2_3_1 = new Version(2030199, org.apache.lucene.util.Version.LUCENE_9_3_0); public static final Version V_2_4_0 = new Version(2040099, org.apache.lucene.util.Version.LUCENE_9_4_1); public static final Version V_2_4_1 = new Version(2040199, org.apache.lucene.util.Version.LUCENE_9_4_2); - public static final Version V_2_4_2 = new Version(2040299, org.apache.lucene.util.Version.LUCENE_9_4_2); public static final Version V_2_5_0 = new Version(2050099, org.apache.lucene.util.Version.LUCENE_9_4_2); public static final Version V_2_5_1 = new Version(2050199, org.apache.lucene.util.Version.LUCENE_9_4_2); + + // UNRELEASED + public static final Version V_2_4_2 = new Version(2040299, org.apache.lucene.util.Version.LUCENE_9_4_2); public static final Version V_2_6_0 = new Version(2060099, org.apache.lucene.util.Version.LUCENE_9_5_0); public static final Version V_2_6_1 = new Version(2060199, org.apache.lucene.util.Version.LUCENE_9_5_0); public static final Version V_2_7_0 = new Version(2070099, org.apache.lucene.util.Version.LUCENE_9_5_0); @@ -112,7 +90,9 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version V_2_8_1 = new Version(2080199, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0); - public static final Version CURRENT = V_2_9_0; + public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_7_0); + public static final Version CURRENT = V_3_0_0; public static Version fromId(int id) { final Version known = LegacyESVersion.idToVersion.get(id); @@ -262,6 +242,8 @@ private static Version fromStringSlow(String version) { this.revision = (byte) ((id / 100) % 100); this.build = (byte) (id % 100); this.luceneVersion = Objects.requireNonNull(luceneVersion); + this.minCompatVersion = null; + this.minIndexCompatVersion = null; } public boolean after(Version version) { @@ -308,17 +290,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws * lazily once. */ static class DeclaredVersionsHolder { - // use LegacyESVersion.class since it inherits Version fields - protected static final List DECLARED_VERSIONS = Collections.unmodifiableList(getDeclaredVersions(LegacyESVersion.class)); + protected static final List DECLARED_VERSIONS = Collections.unmodifiableList(getDeclaredVersions(Version.class)); } // lazy initialized because we don't yet have the declared versions ready when instantiating the cached Version // instances - private Version minCompatVersion; + protected Version minCompatVersion = null; // lazy initialized because we don't yet have the declared versions ready when instantiating the cached Version // instances - private Version minIndexCompatVersion; + protected Version minIndexCompatVersion = null; /** * Returns the minimum compatible version based on the current @@ -341,7 +322,7 @@ protected Version computeMinCompatVersion() { // we don't have LegacyESVersion.V_6 constants, so set it to its last minor return LegacyESVersion.fromId(6080099); } else if (major == 2) { - return LegacyESVersion.V_7_10_0; + return LegacyESVersion.fromId(7100099); } else if (major == 6) { // force the minimum compatibility for version 6 to 5.6 since we don't reference version 5 anymore return LegacyESVersion.fromId(5060099); @@ -386,14 +367,14 @@ public Version minimumIndexCompatibilityVersion() { return res; } - private Version computeMinIndexCompatVersion() { + protected Version computeMinIndexCompatVersion() { final int bwcMajor; if (major == 5) { bwcMajor = 2; // we jumped from 2 to 5 } else if (major == 7 || major == 1) { return LegacyESVersion.fromId(6000026); } else if (major == 2) { - return LegacyESVersion.V_7_0_0; + return LegacyESVersion.fromId(7000099); } else { bwcMajor = major - 1; } @@ -412,7 +393,7 @@ public boolean isCompatible(Version version) { boolean compatible = onOrAfter(version.minimumCompatibilityVersion()) && version.onOrAfter(minimumCompatibilityVersion()); // OpenSearch version 1 is the functional equivalent of predecessor version 7 - // OpenSearch version 2 is the functional equivalent of predecessor unreleased version "8" + // OpenSearch version 2 is the functional equivalent of predecessor version 8 // todo refactor this logic after removing deprecated features int a = major; int b = version.major; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java index c1326c1b50c9e..57578cdbfa8e8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java @@ -15,8 +15,16 @@ import org.opensearch.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SystemIndexPlugin; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; + +import java.util.Collection; +import java.util.Collections; +import java.util.Arrays; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE; @@ -52,6 +60,40 @@ protected Settings nodeSettings(int nodeOrdinal) { .build(); } + public static class TestPlugin extends Plugin implements SystemIndexPlugin { + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList( + new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System index for [" + getTestClass().getName() + ']') + ); + } + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(SegmentReplicationClusterSettingIT.TestPlugin.class, MockTransportService.TestPlugin.class); + } + + public void testSystemIndexWithSegmentReplicationClusterSetting() throws Exception { + + // Starting two nodes with primary and replica shards respectively. + final String primaryNode = internalCluster().startNode(); + createIndex(SYSTEM_INDEX_NAME); + ensureYellowAndNoInitializingShards(SYSTEM_INDEX_NAME); + final String replicaNode = internalCluster().startNode(); + ensureGreen(SYSTEM_INDEX_NAME); + final GetSettingsResponse response = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(SYSTEM_INDEX_NAME).includeDefaults(true)) + .actionGet(); + assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); + + // Verify index setting isSegRepEnabled is false. + Index index = resolveIndex(SYSTEM_INDEX_NAME); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, primaryNode); + assertEquals(indicesService.indexService(index).getIndexSettings().isSegRepEnabled(), false); + } + public void testIndexReplicationSettingOverridesSegRepClusterSetting() throws Exception { Settings settings = Settings.builder().put(CLUSTER_SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build(); final String ANOTHER_INDEX = "test-index"; @@ -123,4 +165,28 @@ public void testIndexReplicationSettingOverridesDocRepClusterSetting() throws Ex assertEquals(indicesService.indexService(anotherIndex).getIndexSettings().isSegRepEnabled(), false); } + public void testHiddenIndicesWithReplicationStrategyClusterSetting() throws Exception { + final String primaryNode = internalCluster().startNode(); + final String replicaNode = internalCluster().startNode(); + prepareCreate( + INDEX_NAME, + Settings.builder() + // we want to set index as hidden + .put("index.hidden", true) + ).get(); + ensureGreen(INDEX_NAME); + + // Verify that document replication strategy is used for hidden indices. + final GetSettingsResponse response = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(INDEX_NAME).includeDefaults(true)) + .actionGet(); + assertEquals(response.getSetting(INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); + + // Verify index setting isSegRepEnabled. + Index index = resolveIndex(INDEX_NAME); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, primaryNode); + assertEquals(indicesService.indexService(index).getIndexSettings().isSegRepEnabled(), false); + } + } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index fda344acad166..46966e289e75e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -12,15 +12,24 @@ import org.junit.Before; import org.opensearch.action.admin.indices.get.GetIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexResponse; +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.index.IndexSettings; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SystemIndexPlugin; import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.OpenSearchIntegTestCase; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + import static org.hamcrest.Matchers.containsString; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY; @@ -59,6 +68,20 @@ protected Settings nodeSettings(int nodeOriginal) { return builder.build(); } + public static class TestPlugin extends Plugin implements SystemIndexPlugin { + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList( + new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System index for [" + getTestClass().getName() + ']') + ); + } + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(CreateRemoteIndexIT.TestPlugin.class); + } + @Override protected Settings featureFlagSettings() { return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE, "true").build(); @@ -107,6 +130,35 @@ public void testDefaultRemoteStoreNoUserOverride() throws Exception { ); } + private static final String SYSTEM_INDEX_NAME = ".test-system-index"; + + public void testSystemIndexWithRemoteStoreClusterSetting() throws Exception { + createIndex(SYSTEM_INDEX_NAME); + ensureGreen(SYSTEM_INDEX_NAME); + final GetSettingsResponse response = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(SYSTEM_INDEX_NAME).includeDefaults(true)) + .actionGet(); + // Verify that Document replication strategy is used + assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); + assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REMOTE_STORE_ENABLED), "false"); + } + + public void testSystemIndexWithRemoteStoreIndexSettings() throws Exception { + prepareCreate( + SYSTEM_INDEX_NAME, + Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).put(SETTING_REMOTE_STORE_ENABLED, true) + ).get(); + ensureGreen(SYSTEM_INDEX_NAME); + final GetSettingsResponse response = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(SYSTEM_INDEX_NAME).includeDefaults(true)) + .actionGet(); + // Verify that Document replication strategy is used + assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); + assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REMOTE_STORE_ENABLED), "false"); + } + public void testRemoteStoreDisabledByUser() throws Exception { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index ceb716fef4cc2..ae09bfe363d99 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -73,6 +73,7 @@ import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -137,6 +138,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING; +import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; @@ -584,7 +586,8 @@ private ClusterState applyCreateIndexRequestWithV1Templates( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders + indexSettingProviders, + systemIndices.validateSystemIndex(request.index()) ); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -648,7 +651,8 @@ private ClusterState applyCreateIndexRequestWithV2Template( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders + indexSettingProviders, + systemIndices.validateSystemIndex(request.index()) ); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -728,7 +732,8 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders + indexSettingProviders, + sourceMetadata.isSystem() ); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -811,7 +816,8 @@ static Settings aggregateIndexSettings( Settings settings, IndexScopedSettings indexScopedSettings, ShardLimitValidator shardLimitValidator, - Set indexSettingProviders + Set indexSettingProviders, + boolean isSystemIndex ) { // Create builders for the template and request settings. We transform these into builders // because we may want settings to be "removed" from these prior to being set on the new @@ -901,8 +907,18 @@ static Settings aggregateIndexSettings( indexSettingsBuilder.put(IndexMetadata.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName()); indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); - updateReplicationStrategy(indexSettingsBuilder, request.settings(), settings); - updateRemoteStoreSettings(indexSettingsBuilder, request.settings(), settings); + if (isSystemIndex || IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings())) { + logger.warn( + "Setting replication.type: DOCUMENT will be used for Index until Segment Replication supports System and Hidden indices" + ); + indexSettingsBuilder.put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT); + if (FeatureFlags.isEnabled(REMOTE_STORE)) { + indexSettingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, false); + } + } else { + updateReplicationStrategy(indexSettingsBuilder, request.settings(), settings); + updateRemoteStoreSettings(indexSettingsBuilder, request.settings(), settings); + } if (sourceMetadata != null) { assert request.resizeType() != null; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 5291607cb6945..f20f6ae440e0a 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -826,7 +826,8 @@ public void testAggregateSettingsAppliesSettingsFromTemplatesAndRequest() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1")); @@ -888,7 +889,8 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertThat(resolvedAliases.get(0).getSearchRouting(), equalTo("fromRequest")); @@ -910,7 +912,8 @@ public void testDefaultSettings() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("1")); @@ -925,7 +928,8 @@ public void testSettingsFromClusterState() { Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 15).build(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("15")); @@ -962,7 +966,8 @@ public void testTemplateOrder() throws Exception { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); List resolvedAliases = resolveAndValidateAliases( request.index(), @@ -1001,7 +1006,8 @@ public void testAggregateIndexSettingsIgnoresTemplatesOnCreateFromSourceIndex() Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertThat(aggregatedIndexSettings.get("templateSetting"), is(nullValue())); @@ -1223,7 +1229,8 @@ public void testRemoteStoreNoUserOverrideConflictingReplicationTypeIndexSettings settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ) ); assertThat( @@ -1254,7 +1261,8 @@ public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettin settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1286,7 +1294,8 @@ public void testRemoteStoreNoUserOverrideIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1320,7 +1329,8 @@ public void testRemoteStoreDisabledByUserIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1354,7 +1364,8 @@ public void testRemoteStoreTranslogDisabledByUserIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1391,7 +1402,8 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1425,7 +1437,8 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1459,7 +1472,8 @@ public void testRemoteStoreOverrideReplicationTypeIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1541,7 +1555,8 @@ public void testSoftDeletesDisabledIsRejected() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); }); assertThat( @@ -1570,7 +1585,8 @@ public void testValidateTranslogRetentionSettings() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertWarnings( "Translog retention settings [index.translog.retention.age] " @@ -1617,7 +1633,8 @@ public void testDeprecatedSimpleFSStoreSettings() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertWarnings( "[simplefs] is deprecated and will be removed in 2.0. Use [niofs], which offers equal " @@ -1636,7 +1653,8 @@ public void testClusterReplicationSetting() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); } @@ -1656,12 +1674,85 @@ public void testIndexSettingOverridesClusterReplicationSetting() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); // Verify if index setting overrides cluster replication setting assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); } + public void testHiddenIndexUsesDocumentReplication() { + Settings settings = Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build(); + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + final Settings.Builder requestSettings = Settings.builder(); + // Set index setting replication type as DOCUMENT + requestSettings.put("index.hidden", true); + request.settings(requestSettings.build()); + Settings indexSettings = aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet(), + false + ); + // Verify replication type is Document Replication + assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); + } + + public void testSystemIndexUsesDocumentReplication() { + Settings settings = Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build(); + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + final Settings.Builder requestSettings = Settings.builder(); + request.settings(requestSettings.build()); + // set isSystemIndex parameter as true + Settings indexSettings = aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet(), + true + ); + // Verify replication type is Document Replication + assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); + } + + public void testRemoteStoreDisabledForSystemIndices() { + Settings settings = Settings.builder() + .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) + .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) + .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") + .put(CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey(), true) + .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") + .build(); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + final Settings.Builder requestSettings = Settings.builder(); + request.settings(requestSettings.build()); + // set isSystemIndex parameter as true + Settings indexSettings = aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet(), + true + ); + // Verify that remote store is disabled. + assertEquals(indexSettings.get(SETTING_REMOTE_STORE_ENABLED), "false"); + assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); + } + private IndexTemplateMetadata addMatchingTemplate(Consumer configurator) { IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*"); configurator.accept(builder);