[Remote Store] Add support to create index with remote store by defau…

…lt (opensearch-project#6932) Signed-off-by: Varun Bansal <bansvaru@amazon.com>
austintlee · Apr 28, 2023 · b0951b0 · b0951b0
1 parent 21c7fee
commit b0951b0
Show file tree

Hide file tree

Showing 11 changed files with 1,021 additions and 10 deletions.
diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml
@@ -87,6 +87,19 @@ ${path.logs}
 #
 #action.destructive_requires_name: true
 #
+# ---------------------------------- Remote Store -----------------------------------
+# Controls whether cluster imposes index creation only with remote store enabled
+# cluster.remote_store.enabled: true
+#
+# Repository to use for segment upload while enforcing remote store for an index
+# cluster.remote_store.repository: my-repo-1
+#
+# Controls whether cluster imposes index creation only with translog remote store enabled
+# cluster.remote_store.translog.enabled: true
+#
+# Repository to use for translog upload while enforcing remote store for an index
+# cluster.remote_store.translog.repository: my-repo-1
+#
 # ---------------------------------- Experimental Features -----------------------------------
 #
 # Gates the visibility of the index setting that allows changing of replication type.

diff --git a/...nalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java b/...nalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java
@@ -0,0 +1,93 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotestore;
+
+import org.opensearch.action.admin.indices.get.GetIndexRequest;
+import org.opensearch.action.admin.indices.get.GetIndexResponse;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED;
+import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST)
+public class CreateRemoteIndexClusterDefaultDocRep extends CreateRemoteIndexIT {
+
+    @Override
+    protected Settings nodeSettings(int nodeOriginal) {
+        Settings settings = super.nodeSettings(nodeOriginal);
+        Settings.Builder builder = Settings.builder()
+            .put(settings)
+            .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT);
+        return builder.build();
+    }
+
+    @Override
+    public void testRemoteStoreTranslogDisabledByUser() throws Exception {
+        final int numReplicas = internalCluster().numDataNodes();
+        Settings settings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)
+            .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
+            .put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false)
+            .build();
+        assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get());
+        GetIndexResponse getIndexResponse = client().admin()
+            .indices()
+            .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true))
+            .get();
+        Settings indexSettings = getIndexResponse.settings().get("test-idx-1");
+        verifyRemoteStoreIndexSettings(indexSettings, "true", "my-segment-repo-1", "false", null, ReplicationType.SEGMENT.toString(), null);
+    }
+
+    @Override
+    public void testDefaultRemoteStoreNoUserOverride() throws Exception {
+        final int numReplicas = internalCluster().numDataNodes();
+        Settings settings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)
+            .build();
+        IllegalArgumentException exc = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()
+        );
+        assertThat(
+            exc.getMessage(),
+            containsString("Cannot enable [index.remote_store.enabled] when [cluster.indices.replication.strategy] is DOCUMENT")
+        );
+    }
+
+    public void testDefaultRemoteStoreNoUserOverrideExceptReplicationTypeSegment() throws Exception {
+        final int numReplicas = internalCluster().numDataNodes();
+        Settings settings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)
+            .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
+            .build();
+        assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get());
+        GetIndexResponse getIndexResponse = client().admin()
+            .indices()
+            .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true))
+            .get();
+        Settings indexSettings = getIndexResponse.settings().get("test-idx-1");
+        verifyRemoteStoreIndexSettings(
+            indexSettings,
+            "true",
+            "my-segment-repo-1",
+            "true",
+            "my-translog-repo-1",
+            ReplicationType.SEGMENT.toString(),
+            null
+        );
+    }
+}