Skip to content

Commit

Permalink
Restore ClusterState version during remote state restore
Browse files Browse the repository at this point in the history 
Signed-off-by: bansvaru <bansvaru@amazon.com>
  • Loading branch information
@linuxpi
linuxpi committed Oct 23, 2023
1 parent ef4b327 commit dd2a41a
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 21 deletions.
19 changes: 18 additions & 1 deletion ...c/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@

import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;

import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
Expand Down Expand Up @@ -86,9 +88,24 @@ public void testFullClusterRestoreStaleDelete() throws Exception {

assertEquals(10, repository.blobStore().blobContainer(baseMetadataPath.add("manifest")).listBlobsByPrefix("manifest").size());

Optional<ClusterMetadataManifest> clusterMetadataManifest = remoteClusterStateService.getLatestClusterMetadataManifest(
clusterService().state().getClusterName().value(),
getClusterState().metadata().clusterUUID()
);
if (clusterMetadataManifest.isEmpty()) {
throw new IllegalStateException(
String.format(
Locale.ROOT,
"Latest cluster metadata manifest is not present for the provided clusterUUID: %s",
getClusterState().metadata().clusterUUID()
)
);
}

Map<String, IndexMetadata> indexMetadataMap = remoteClusterStateService.getLatestMetadata(
cluster().getClusterName(),
getClusterState().metadata().clusterUUID()
getClusterState().metadata().clusterUUID(),
clusterMetadataManifest.get()
).getIndices();
assertEquals(0, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas());
assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards());
Expand Down
15 changes: 5 additions & 10 deletions server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -718,21 +718,16 @@ private IndexMetadata getIndexMetadata(String clusterName, String clusterUUID, U
* @param clusterName name of the cluster
* @return {@link IndexMetadata}
*/
public Metadata getLatestMetadata(String clusterName, String clusterUUID) {
public Metadata getLatestMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) {
start();
Optional<ClusterMetadataManifest> clusterMetadataManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID);
if (!clusterMetadataManifest.isPresent()) {
throw new IllegalStateException(
String.format(Locale.ROOT, "Latest cluster metadata manifest is not present for the provided clusterUUID: %s", clusterUUID)
);
}

// Fetch Global Metadata
Metadata globalMetadata = getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest.get());
Metadata globalMetadata = getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest);

// Fetch Index Metadata
Map<String, IndexMetadata> indices = getIndexMetadataMap(clusterName, clusterUUID, clusterMetadataManifest.get());
Map<String, IndexMetadata> indices = getIndexMetadataMap(clusterName, clusterUUID, clusterMetadataManifest);

return Metadata.builder(globalMetadata).indices(indices).build();
return Metadata.builder(globalMetadata).version(clusterMetadataManifest.getStateVersion()).indices(indices).build();
}

private Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) {
Expand Down
21 changes: 20 additions & 1 deletion server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.gateway.remote.ClusterMetadataManifest;
import org.opensearch.gateway.remote.RemoteClusterStateService;
import org.opensearch.indices.ShardLimitValidator;
import org.opensearch.repositories.IndexId;
Expand Down Expand Up @@ -149,7 +150,24 @@ public RemoteRestoreResult restore(
if (currentState.metadata().clusterUUID().equals(restoreClusterUUID)) {
throw new IllegalArgumentException("clusterUUID to restore from should be different from current cluster UUID");
}
remoteMetadata = remoteClusterStateService.getLatestMetadata(currentState.getClusterName().value(), restoreClusterUUID);
Optional<ClusterMetadataManifest> clusterMetadataManifest = remoteClusterStateService.getLatestClusterMetadataManifest(
currentState.getClusterName().value(),
restoreClusterUUID
);
if (clusterMetadataManifest.isEmpty()) {
throw new IllegalStateException(
String.format(
Locale.ROOT,
"Latest cluster metadata manifest is not present for the provided clusterUUID: %s",
restoreClusterUUID
)
);
}
remoteMetadata = remoteClusterStateService.getLatestMetadata(
currentState.getClusterName().value(),
restoreClusterUUID,
clusterMetadataManifest.get()
);
remoteMetadata.getIndices().values().forEach(indexMetadata -> {
indexMetadataMap.put(indexMetadata.getIndex().getName(), new Tuple<>(true, indexMetadata));
});
Expand Down Expand Up @@ -255,6 +273,7 @@ private RemoteRestoreResult executeRestore(
}

private void restoreGlobalMetadata(Metadata.Builder mdBuilder, Metadata remoteMetadata) {
mdBuilder.version(remoteMetadata.version());
if (remoteMetadata.persistentSettings() != null) {
Settings settings = remoteMetadata.persistentSettings();
clusterService.getClusterSettings().validateUpdate(settings);
Expand Down
29 changes: 20 additions & 9 deletions server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -655,9 +655,11 @@ public void testReadLatestMetadataManifestSuccessButNoIndexMetadata() throws IOE

remoteClusterStateService.start();
assertEquals(
remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
.getIndices()
.size(),
remoteClusterStateService.getLatestMetadata(
clusterState.getClusterName().value(),
clusterState.metadata().clusterUUID(),
expectedManifest
).getIndices().size(),
0
);
}
Expand All @@ -684,8 +686,11 @@ public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOExceptio
remoteClusterStateService.start();
Exception e = assertThrows(
IllegalStateException.class,
() -> remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
.getIndices()
() -> remoteClusterStateService.getLatestMetadata(
clusterState.getClusterName().value(),
clusterState.metadata().clusterUUID(),
expectedManifest
).getIndices()
);
assertEquals(e.getMessage(), "Error while downloading IndexMetadata - " + uploadedIndexMetadata.getUploadedFilename());
}
Expand Down Expand Up @@ -748,7 +753,8 @@ public void testReadGlobalMetadata() throws IOException {

Metadata metadata = remoteClusterStateService.getLatestMetadata(
clusterState.getClusterName().value(),
clusterState.metadata().clusterUUID()
clusterState.metadata().clusterUUID(),
expectedManifest
);

assertTrue(Metadata.isGlobalStateEquals(metadata, expactedMetadata));
Expand Down Expand Up @@ -783,7 +789,11 @@ public void testReadGlobalMetadataIOException() throws IOException {
remoteClusterStateService.start();
Exception e = assertThrows(
IllegalStateException.class,
() -> remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
() -> remoteClusterStateService.getLatestMetadata(
clusterState.getClusterName().value(),
clusterState.metadata().clusterUUID(),
expectedManifest
)
);
assertEquals(e.getMessage(), "Error while downloading Global Metadata - " + globalIndexMetadataName);
}
Expand Down Expand Up @@ -814,15 +824,16 @@ public void testReadLatestIndexMetadataSuccess() throws IOException {
.nodeId("nodeA")
.opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
.previousClusterUUID("prev-cluster-uuid")
.globalMetadataFileName("global-metadata-file")
// .globalMetadataFileName("global-metadata-file")
.codecVersion(ClusterMetadataManifest.CODEC_V0)
.build();

mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata));

Map<String, IndexMetadata> indexMetadataMap = remoteClusterStateService.getLatestMetadata(
clusterState.getClusterName().value(),
clusterState.metadata().clusterUUID()
clusterState.metadata().clusterUUID(),
expectedManifest
).getIndices();

assertEquals(indexMetadataMap.size(), 1);
Expand Down

0 comments on commit dd2a41a

Please sign in to comment.