[Backport 2.x] [Remote Store] Backport remote segment store changes

CHANGELOG.md

@@ -8,8 +8,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added @dreamer-89 as an Opensearch maintainer ([#4342](https://github.com/opensearch-project/OpenSearch/pull/4342))
 
 ### Changed
- - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
- - Update to Netty 4.1.80.Final ([#4359](https://github.com/opensearch-project/OpenSearch/pull/4359))
+- Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
+- Update to Netty 4.1.80.Final ([#4359](https://github.com/opensearch-project/OpenSearch/pull/4359))
+- Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240))
 
 ### Deprecated
 

client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java

@@ -885,7 +885,8 @@ public void testApiNamingConventions() throws Exception {
             "nodes.hot_threads",
             "nodes.usage",
             "nodes.reload_secure_settings",
-            "search_shards", };
+            "search_shards",
+            "remote_store.restore", };
         List<String> booleanReturnMethods = Arrays.asList("security.enable_user", "security.disable_user", "security.change_password");
         Set<String> deprecatedMethods = new HashSet<>();
         deprecatedMethods.add("indices.force_merge");

rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.restore.json

@@ -0,0 +1,34 @@
+{
+  "remote_store.restore":{
+    "documentation":{
+      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/remote-store#restore",
+      "description":"Restores from remote store."
+    },
+    "stability":"experimental",
+    "url":{
+      "paths":[
+        {
+          "path":"/_remotestore/_restore",
+          "methods":[
+            "POST"
+          ]
+        }
+      ]
+    },
+    "params":{
+      "cluster_manager_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to cluster-manager node"
+      },
+      "wait_for_completion":{
+        "type":"boolean",
+        "description":"Should this request wait until the operation has completed before returning",
+        "default":false
+      }
+    },
+    "body":{
+      "description":"A comma separated list of index IDs",
+      "required":true
+    }
+  }
+}

server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java

@@ -675,7 +675,8 @@ public static final IndexShard newIndexShard(
             () -> {},
             RetentionLeaseSyncer.EMPTY,
             cbs,
-            SegmentReplicationCheckpointPublisher.EMPTY
+            SegmentReplicationCheckpointPublisher.EMPTY,
+            null
         );
     }
 

server/src/main/java/org/opensearch/action/ActionModule.java

@@ -61,6 +61,8 @@
 import org.opensearch.action.admin.cluster.node.usage.TransportNodesUsageAction;
 import org.opensearch.action.admin.cluster.remote.RemoteInfoAction;
 import org.opensearch.action.admin.cluster.remote.TransportRemoteInfoAction;
+import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreAction;
+import org.opensearch.action.admin.cluster.remotestore.restore.TransportRestoreRemoteStoreAction;
 import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryAction;
 import org.opensearch.action.admin.cluster.repositories.cleanup.TransportCleanupRepositoryAction;
 import org.opensearch.action.admin.cluster.repositories.delete.DeleteRepositoryAction;
@@ -261,6 +263,7 @@
 import org.opensearch.common.settings.IndexScopedSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsFilter;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.seqno.RetentionLeaseActions;
 import org.opensearch.indices.SystemIndices;
 import org.opensearch.indices.breaker.CircuitBreakerService;
@@ -308,6 +311,7 @@
 import org.opensearch.rest.action.admin.cluster.RestPutStoredScriptAction;
 import org.opensearch.rest.action.admin.cluster.RestReloadSecureSettingsAction;
 import org.opensearch.rest.action.admin.cluster.RestRemoteClusterInfoAction;
+import org.opensearch.rest.action.admin.cluster.RestRestoreRemoteStoreAction;
 import org.opensearch.rest.action.admin.cluster.RestRestoreSnapshotAction;
 import org.opensearch.rest.action.admin.cluster.RestSnapshotsStatusAction;
 import org.opensearch.rest.action.admin.cluster.RestVerifyRepositoryAction;
@@ -657,6 +661,9 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(DeleteDanglingIndexAction.INSTANCE, TransportDeleteDanglingIndexAction.class);
         actions.register(FindDanglingIndexAction.INSTANCE, TransportFindDanglingIndexAction.class);
 
+        // Remote Store
+        actions.register(RestoreRemoteStoreAction.INSTANCE, TransportRestoreRemoteStoreAction.class);
+
         return unmodifiableMap(actions.getRegistry());
     }
 
@@ -842,6 +849,11 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
             }
         }
         registerHandler.accept(new RestCatAction(catActions));
+
+        // Remote Store APIs
+        if (FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE)) {
+            registerHandler.accept(new RestRestoreRemoteStoreAction());
+        }
     }
 
     @Override

server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreAction.java

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.admin.cluster.remotestore.restore;
+
+import org.opensearch.action.ActionType;
+
+/**
+ * Restore remote store action
+ *
+ * @opensearch.internal
+ */
+public final class RestoreRemoteStoreAction extends ActionType<RestoreRemoteStoreResponse> {
+
+    public static final RestoreRemoteStoreAction INSTANCE = new RestoreRemoteStoreAction();
+    public static final String NAME = "cluster:admin/remotestore/restore";
+
+    private RestoreRemoteStoreAction() {
+        super(NAME, RestoreRemoteStoreResponse::new);
+    }
+}

server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java

@@ -0,0 +1,183 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.admin.cluster.remotestore.restore;
+
+import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.action.support.IndicesOptions;
+import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
+import org.opensearch.common.Strings;
+import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.common.xcontent.ToXContentObject;
+import org.opensearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.opensearch.action.ValidateActions.addValidationError;
+
+/**
+ * Restore remote store request
+ *
+ * @opensearch.internal
+ */
+public class RestoreRemoteStoreRequest extends ClusterManagerNodeRequest<RestoreRemoteStoreRequest> implements ToXContentObject {
+
+    private String[] indices = Strings.EMPTY_ARRAY;
+    private Boolean waitForCompletion;
+
+    public RestoreRemoteStoreRequest() {}
+
+    public RestoreRemoteStoreRequest(StreamInput in) throws IOException {
+        super(in);
+        indices = in.readStringArray();
+        waitForCompletion = in.readOptionalBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringArray(indices);
+        out.writeOptionalBoolean(waitForCompletion);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (indices == null || indices.length == 0) {
+            validationException = addValidationError("indices are missing", validationException);
+        }
+        return validationException;
+    }
+
+    /**
+     * Sets the list of indices that should be restored from the remote store
+     * <p>
+     * The list of indices supports multi-index syntax. For example: "+test*" ,"-test42" will index all indices with
+     * prefix "test" except index "test42". Aliases are not supported. An empty list or {"_all"} will restore all open
+     * indices in the cluster.
+     *
+     * @param indices list of indices
+     * @return this request
+     */
+    public RestoreRemoteStoreRequest indices(String... indices) {
+        this.indices = indices;
+        return this;
+    }
+
+    /**
+     * Sets the list of indices that should be restored from the remote store
+     * <p>
+     * The list of indices supports multi-index syntax. For example: "+test*" ,"-test42" will index all indices with
+     * prefix "test" except index "test42". Aliases are not supported. An empty list or {"_all"} will restore all open
+     * indices in the cluster.
+     *
+     * @param indices list of indices
+     * @return this request
+     */
+    public RestoreRemoteStoreRequest indices(List<String> indices) {
+        this.indices = indices.toArray(new String[indices.size()]);
+        return this;
+    }
+
+    /**
+     * Returns list of indices that should be restored from the remote store
+     */
+    public String[] indices() {
+        return indices;
+    }
+
+    /**
+     * If this parameter is set to true the operation will wait for completion of restore process before returning.
+     *
+     * @param waitForCompletion if true the operation will wait for completion
+     * @return this request
+     */
+    public RestoreRemoteStoreRequest waitForCompletion(boolean waitForCompletion) {
+        this.waitForCompletion = waitForCompletion;
+        return this;
+    }
+
+    /**
+     * Returns wait for completion setting
+     *
+     * @return true if the operation will wait for completion
+     */
+    public boolean waitForCompletion() {
+        return waitForCompletion;
+    }
+
+    /**
+     * Parses restore definition
+     *
+     * @param source restore definition
+     * @return this request
+     */
+    @SuppressWarnings("unchecked")
+    public RestoreRemoteStoreRequest source(Map<String, Object> source) {
+        for (Map.Entry<String, Object> entry : source.entrySet()) {
+            String name = entry.getKey();
+            if (name.equals("indices")) {
+                if (entry.getValue() instanceof String) {
+                    indices(Strings.splitStringByCommaToArray((String) entry.getValue()));
+                } else if (entry.getValue() instanceof ArrayList) {
+                    indices((ArrayList<String>) entry.getValue());
+                } else {
+                    throw new IllegalArgumentException("malformed indices section, should be an array of strings");
+                }
+            } else {
+                if (IndicesOptions.isIndicesOptions(name) == false) {
+                    throw new IllegalArgumentException("Unknown parameter " + name);
+                }
+            }
+        }
+        return this;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.startArray("indices");
+        for (String index : indices) {
+            builder.value(index);
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getDescription() {
+        return "remote_store";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RestoreRemoteStoreRequest that = (RestoreRemoteStoreRequest) o;
+        return waitForCompletion == that.waitForCompletion && Arrays.equals(indices, that.indices);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Objects.hash(waitForCompletion);
+        result = 31 * result + Arrays.hashCode(indices);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}