From 379fe9048a09d5b847f3e892ef8dda1aa48e819c Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 13 Aug 2024 12:43:39 +0530 Subject: [PATCH 01/29] Add cluster state checksum in manifest Signed-off-by: Himshikha Gupta --- .../common/settings/ClusterSettings.java | 1 + .../remote/ClusterMetadataManifest.java | 74 +++- .../gateway/remote/ClusterStateChecksum.java | 419 ++++++++++++++++++ .../remote/RemoteClusterStateService.java | 44 +- .../gateway/remote/RemoteManifestManager.java | 5 +- .../model/RemoteClusterMetadataManifest.java | 7 +- 6 files changed, 541 insertions(+), 9 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index a73e5d44b7e02..2cae5fe071f77 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -735,6 +735,7 @@ public void apply(Settings value, Settings current, Settings previous) { IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING, RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, + RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING, // Admission Control Settings AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE, diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 71815b6ee324c..4da7fb8a824d4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -45,6 +45,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { // also we introduce index routing-metadata, diff and other attributes as part of manifest // required for state publication public static final int CODEC_V3 = 3; // In Codec V3, we have introduced new diff field in diff-manifest's routing_table_diff + public static final int CODEC_V4 = 4; // In Codec V4, we are adding checksum of cluster state. private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); @@ -73,6 +74,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { ); private static final ParseField UPLOADED_CLUSTER_STATE_CUSTOM_METADATA = new ParseField("uploaded_cluster_state_custom_metadata"); private static final ParseField DIFF_MANIFEST = new ParseField("diff_manifest"); + private static final ParseField CHECKSUM = new ParseField("checksum"); private static ClusterMetadataManifest.Builder manifestV0Builder(Object[] fields) { return ClusterMetadataManifest.builder() @@ -114,6 +116,10 @@ private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields return manifestV2Builder(fields); } + private static ClusterMetadataManifest.Builder manifestV4Builder(Object[] fields) { + return manifestV3Builder(fields).checksum(checksum(fields)); + } + private static long term(Object[] fields) { return (long) fields[0]; } @@ -216,6 +222,10 @@ private static ClusterStateDiffManifest diffManifest(Object[] fields) { return (ClusterStateDiffManifest) fields[23]; } + private static ClusterStateChecksum checksum(Object[] fields) { + return (ClusterStateChecksum) fields[24]; + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> manifestV0Builder(fields).build() @@ -236,13 +246,20 @@ private static ClusterStateDiffManifest diffManifest(Object[] fields) { fields -> manifestV3Builder(fields).build() ); - private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V3; + private static final ConstructingObjectParser PARSER_V4 = new ConstructingObjectParser<>( + "cluster_metadata_manifest", + fields -> manifestV4Builder(fields).build() + ); + + private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V4; static { declareParser(PARSER_V0, CODEC_V0); declareParser(PARSER_V1, CODEC_V1); declareParser(PARSER_V2, CODEC_V2); declareParser(PARSER_V3, CODEC_V3); + declareParser(PARSER_V4, CODEC_V4); + } private static void declareParser(ConstructingObjectParser parser, long codec_version) { @@ -324,6 +341,13 @@ private static void declareParser(ConstructingObjectParser= CODEC_V4) { + parser.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> ClusterStateChecksum.fromXContent(p), + CHECKSUM + ); + } } private final int codecVersion; @@ -351,6 +375,7 @@ private static void declareParser(ConstructingObjectParser uploadedClusterStateCustomMap; private final ClusterStateDiffManifest diffManifest; + private ClusterStateChecksum clusterStateChecksum; public List getIndices() { return indices; @@ -459,6 +484,10 @@ public List getIndicesRouting() { return indicesRouting; } + public ClusterStateChecksum getClusterStateChecksum() { + return clusterStateChecksum; + } + public ClusterMetadataManifest( long clusterTerm, long version, @@ -484,7 +513,8 @@ public ClusterMetadataManifest( UploadedMetadataAttribute uploadedTransientSettingsMetadata, UploadedMetadataAttribute uploadedHashesOfConsistentSettings, Map uploadedClusterStateCustomMap, - ClusterStateDiffManifest diffManifest + ClusterStateDiffManifest diffManifest, + ClusterStateChecksum clusterStateChecksum ) { this.clusterTerm = clusterTerm; this.stateVersion = version; @@ -515,6 +545,7 @@ public ClusterMetadataManifest( this.uploadedClusterStateCustomMap = Collections.unmodifiableMap( uploadedClusterStateCustomMap != null ? uploadedClusterStateCustomMap : new HashMap<>() ); + this.clusterStateChecksum = clusterStateChecksum; } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -528,6 +559,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.previousClusterUUID = in.readString(); this.clusterUUIDCommitted = in.readBoolean(); + clusterStateChecksum = null; if (in.getVersion().onOrAfter(Version.V_2_15_0)) { this.codecVersion = in.readInt(); this.uploadedCoordinationMetadata = new UploadedMetadataAttribute(in); @@ -590,6 +622,9 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedHashesOfConsistentSettings = null; this.uploadedClusterStateCustomMap = null; } + if (in.getVersion().onOrAfter(Version.V_2_17_0) && in.readBoolean()) { + clusterStateChecksum = new ClusterStateChecksum(in); + } } public static Builder builder() { @@ -687,6 +722,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } + if (onOrAfterCodecVersion(CODEC_V4)) { + if (getClusterStateChecksum() != null) { + builder.startObject(CHECKSUM.getPreferredName()); + getClusterStateChecksum().toXContent(builder, params); + builder.endObject(); + } + } return builder; } @@ -746,6 +788,14 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); } + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { + if (clusterStateChecksum != null) { + out.writeBoolean(true); + clusterStateChecksum.writeTo(out); + } else { + out.writeBoolean(false); + } + } } @Override @@ -781,7 +831,8 @@ public boolean equals(Object o) { && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata) && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings) && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap) - && Objects.equals(diffManifest, that.diffManifest); + && Objects.equals(diffManifest, that.diffManifest) + && Objects.equals(clusterStateChecksum, that.clusterStateChecksum); } @Override @@ -811,7 +862,8 @@ public int hashCode() { uploadedTransientSettingsMetadata, uploadedHashesOfConsistentSettings, uploadedClusterStateCustomMap, - diffManifest + diffManifest, + clusterStateChecksum ); } @@ -836,6 +888,10 @@ public static ClusterMetadataManifest fromXContentV2(XContentParser parser) thro return PARSER_V2.parse(parser, null); } + public static ClusterMetadataManifest fromXContentV3(XContentParser parser) throws IOException { + return PARSER_V3.parse(parser, null); + } + public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException { return CURRENT_PARSER.parse(parser, null); } @@ -872,6 +928,7 @@ public static class Builder { private UploadedMetadataAttribute hashesOfConsistentSettings; private Map clusterStateCustomMetadataMap; private ClusterStateDiffManifest diffManifest; + private ClusterStateChecksum checksum; public Builder indices(List indices) { this.indices = indices; @@ -1011,6 +1068,11 @@ public Builder diffManifest(ClusterStateDiffManifest diffManifest) { return this; } + public Builder checksum(ClusterStateChecksum checksum) { + this.checksum = checksum; + return this; + } + public Builder() { indices = new ArrayList<>(); customMetadataMap = new HashMap<>(); @@ -1043,6 +1105,7 @@ public Builder(ClusterMetadataManifest manifest) { this.diffManifest = manifest.diffManifest; this.hashesOfConsistentSettings = manifest.uploadedHashesOfConsistentSettings; this.clusterStateCustomMetadataMap = manifest.uploadedClusterStateCustomMap; + this.checksum = manifest.clusterStateChecksum; } public ClusterMetadataManifest build() { @@ -1071,7 +1134,8 @@ public ClusterMetadataManifest build() { transientSettingsMetadata, hashesOfConsistentSettings, clusterStateCustomMetadataMap, - diffManifest + diffManifest, + checksum ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java new file mode 100644 index 0000000000000..d2dd8c84820ed --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -0,0 +1,419 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParseException; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +import com.jcraft.jzlib.JZlib; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * Stores checksum for all components in cluster state. This will be used to ensure cluster state is same across all nodes in the cluster. + */ +public class ClusterStateChecksum implements ToXContentFragment, Writeable { + + private static final String ROUTING_TABLE_CS = "routing_table"; + private static final String NODES_CS = "discovery_nodes"; + private static final String BLOCKS_CS = "blocks"; + private static final String CUSTOMS_CS = "customs"; + private static final String COORDINATION_MD_CS = "coordination_md"; + private static final String SETTINGS_MD_CS = "settings_md"; + private static final String TRANSIENT_SETTINGS_MD_CS = "transient_settings_md"; + private static final String TEMPLATES_MD_CS = "templated_md"; + private static final String CUSTOM_MD_CS = "customs_md"; + private static final String HASHES_MD_CS = "hashes_md"; + private static final String INDICES_CS = "indices_md"; + private static final String CLUSTER_STATE_CS = "cluster_state"; + private static final Logger logger = LogManager.getLogger(ClusterStateChecksum.class); + + long routingTableChecksum; + long nodesChecksum; + long blocksChecksum; + long clusterStateCustomsChecksum; + long coordinationMetadataChecksum; + long settingMetadataChecksum; + long transientSettingsMetadataChecksum; + long templatesMetadataChecksum; + long customMetadataMapChecksum; + long hashesOfConsistentSettingsChecksum; + long indicesChecksum; + long clusterStateChecksum; + + public ClusterStateChecksum(ClusterState clusterState) { + try ( + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out) + ) { + clusterState.routingTable().writeTo(checksumOut); + routingTableChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.nodes().writeTo(checksumOut); + nodesChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.coordinationMetadata().writeTo(checksumOut); + coordinationMetadataChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + Settings.writeSettingsToStream(clusterState.metadata().persistentSettings(), checksumOut); + settingMetadataChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + Settings.writeSettingsToStream(clusterState.metadata().transientSettings(), checksumOut); + transientSettingsMetadataChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.metadata().templatesMetadata().writeTo(checksumOut); + templatesMetadataChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.metadata().customs().forEach((key, custom) -> { + try { + custom.writeTo(checksumOut); + } catch (IOException e) { + logger.error("Failed to create checksum for custom metadata.", e); + throw new RemoteStateTransferException("Failed to create checksum for custom metadata.", e); + } + }); + customMetadataMapChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeTo(checksumOut); + hashesOfConsistentSettingsChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.metadata().indices().forEach(((s, indexMetadata) -> { + try { + indexMetadata.writeTo(checksumOut); + } catch (IOException e) { + logger.error("Failed to create checksum for index metadata.", e); + throw new RemoteStateTransferException("Failed to create checksum for index metadata.", e); + } + })); + indicesChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.blocks().writeTo(checksumOut); + blocksChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.customs().forEach((key, value) -> { + try { + checksumOut.writeNamedWriteable(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + clusterStateCustomsChecksum = checksumOut.getChecksum(); + } catch (IOException e) { + logger.error("Failed to create checksum for cluster state.", e); + throw new RemoteStateTransferException("Failed to create checksum for cluster state.", e); + } + clusterStateChecksum = JZlib.crc32_combine(routingTableChecksum, nodesChecksum, 8); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, blocksChecksum, 8); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, clusterStateCustomsChecksum, 8); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, coordinationMetadataChecksum, 8); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, settingMetadataChecksum, 8); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, transientSettingsMetadataChecksum, 8); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, templatesMetadataChecksum, 8); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, customMetadataMapChecksum, 8); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, hashesOfConsistentSettingsChecksum, 8); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, indicesChecksum, 8); + } + + public static ClusterStateChecksum.Builder builder() { + return new ClusterStateChecksum.Builder(); + } + + public ClusterStateChecksum( + long routingTableChecksum, + long nodesChecksum, + long blocksChecksum, + long clusterStateCustomsChecksum, + long coordinationMetadataChecksum, + long settingMetadataChecksum, + long transientSettingsMetadataChecksum, + long templatesMetadataChecksum, + long customMetadataMapChecksum, + long hashesOfConsistentSettingsChecksum, + long indicesChecksum, + long clusterStateChecksum + ) { + this.routingTableChecksum = routingTableChecksum; + this.nodesChecksum = nodesChecksum; + this.blocksChecksum = blocksChecksum; + this.clusterStateCustomsChecksum = clusterStateCustomsChecksum; + this.coordinationMetadataChecksum = coordinationMetadataChecksum; + this.settingMetadataChecksum = settingMetadataChecksum; + this.transientSettingsMetadataChecksum = transientSettingsMetadataChecksum; + this.templatesMetadataChecksum = templatesMetadataChecksum; + this.customMetadataMapChecksum = customMetadataMapChecksum; + this.hashesOfConsistentSettingsChecksum = hashesOfConsistentSettingsChecksum; + this.indicesChecksum = indicesChecksum; + this.clusterStateChecksum = clusterStateChecksum; + } + + public ClusterStateChecksum(StreamInput in) throws IOException { + routingTableChecksum = in.readLong(); + nodesChecksum = in.readLong(); + blocksChecksum = in.readLong(); + clusterStateCustomsChecksum = in.readLong(); + coordinationMetadataChecksum = in.readLong(); + settingMetadataChecksum = in.readLong(); + transientSettingsMetadataChecksum = in.readLong(); + templatesMetadataChecksum = in.readLong(); + customMetadataMapChecksum = in.readLong(); + hashesOfConsistentSettingsChecksum = in.readLong(); + indicesChecksum = in.readLong(); + clusterStateChecksum = in.readLong(); + } + + public static ClusterStateChecksum fromXContent(XContentParser parser) throws IOException { + ClusterStateChecksum.Builder builder = new ClusterStateChecksum.Builder(); + if (parser.currentToken() == null) { // fresh parser? move to next token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + XContentParser.Token token; + String currentFieldName = parser.currentName(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + switch (currentFieldName) { + case ROUTING_TABLE_CS: + builder.routingTableChecksum(parser.longValue()); + break; + case NODES_CS: + builder.nodesChecksum(parser.longValue()); + break; + case BLOCKS_CS: + builder.blocksChecksum(parser.longValue()); + break; + case CUSTOMS_CS: + builder.clusterStateCustomsChecksum(parser.longValue()); + break; + case COORDINATION_MD_CS: + builder.coordinationMetadataChecksum(parser.longValue()); + break; + case SETTINGS_MD_CS: + builder.settingMetadataChecksum(parser.longValue()); + break; + case TRANSIENT_SETTINGS_MD_CS: + builder.transientSettingsMetadataChecksum(parser.longValue()); + break; + case TEMPLATES_MD_CS: + builder.templatesMetadataChecksum(parser.longValue()); + break; + case CUSTOM_MD_CS: + builder.customMetadataMapChecksum(parser.longValue()); + break; + case HASHES_MD_CS: + builder.hashesOfConsistentSettingsChecksum(parser.longValue()); + break; + case INDICES_CS: + builder.indicesChecksum(parser.longValue()); + break; + case CLUSTER_STATE_CS: + builder.clusterStateChecksum(parser.longValue()); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else { + throw new XContentParseException("Unexpected token [" + token + "]"); + } + } + return builder.build(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(routingTableChecksum); + out.writeLong(nodesChecksum); + out.writeLong(blocksChecksum); + out.writeLong(clusterStateCustomsChecksum); + out.writeLong(coordinationMetadataChecksum); + out.writeLong(settingMetadataChecksum); + out.writeLong(transientSettingsMetadataChecksum); + out.writeLong(templatesMetadataChecksum); + out.writeLong(customMetadataMapChecksum); + out.writeLong(hashesOfConsistentSettingsChecksum); + out.writeLong(indicesChecksum); + out.writeLong(clusterStateChecksum); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(ROUTING_TABLE_CS, routingTableChecksum); + builder.field(NODES_CS, nodesChecksum); + builder.field(BLOCKS_CS, blocksChecksum); + builder.field(CUSTOMS_CS, clusterStateCustomsChecksum); + builder.field(COORDINATION_MD_CS, coordinationMetadataChecksum); + builder.field(SETTINGS_MD_CS, settingMetadataChecksum); + builder.field(TRANSIENT_SETTINGS_MD_CS, transientSettingsMetadataChecksum); + builder.field(TEMPLATES_MD_CS, templatesMetadataChecksum); + builder.field(CUSTOM_MD_CS, customMetadataMapChecksum); + builder.field(HASHES_MD_CS, hashesOfConsistentSettingsChecksum); + builder.field(INDICES_CS, indicesChecksum); + builder.field(CLUSTER_STATE_CS, clusterStateChecksum); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterStateChecksum that = (ClusterStateChecksum) o; + return routingTableChecksum == that.routingTableChecksum + && nodesChecksum == that.nodesChecksum + && blocksChecksum == that.blocksChecksum + && clusterStateCustomsChecksum == that.clusterStateCustomsChecksum + && coordinationMetadataChecksum == that.coordinationMetadataChecksum + && settingMetadataChecksum == that.settingMetadataChecksum + && transientSettingsMetadataChecksum == that.transientSettingsMetadataChecksum + && templatesMetadataChecksum == that.templatesMetadataChecksum + && customMetadataMapChecksum == that.customMetadataMapChecksum + && hashesOfConsistentSettingsChecksum == that.hashesOfConsistentSettingsChecksum + && indicesChecksum == that.indicesChecksum + && clusterStateChecksum == that.clusterStateChecksum; + } + + @Override + public int hashCode() { + return Objects.hash( + routingTableChecksum, + nodesChecksum, + blocksChecksum, + clusterStateCustomsChecksum, + coordinationMetadataChecksum, + settingMetadataChecksum, + transientSettingsMetadataChecksum, + templatesMetadataChecksum, + customMetadataMapChecksum, + hashesOfConsistentSettingsChecksum, + indicesChecksum, + clusterStateChecksum + ); + } + + public static class Builder { + long routingTableChecksum; + long nodesChecksum; + long blocksChecksum; + long clusterStateCustomsChecksum; + long coordinationMetadataChecksum; + long settingMetadataChecksum; + long transientSettingsMetadataChecksum; + long templatesMetadataChecksum; + long customMetadataMapChecksum; + long hashesOfConsistentSettingsChecksum; + long indicesChecksum; + long clusterStateChecksum; + + public Builder routingTableChecksum(long routingTableChecksum) { + this.routingTableChecksum = routingTableChecksum; + return this; + } + + public Builder nodesChecksum(long nodesChecksum) { + this.nodesChecksum = nodesChecksum; + return this; + } + + public Builder blocksChecksum(long blocksChecksum) { + this.blocksChecksum = blocksChecksum; + return this; + } + + public Builder clusterStateCustomsChecksum(long clusterStateCustomsChecksum) { + this.clusterStateCustomsChecksum = clusterStateCustomsChecksum; + return this; + } + + public Builder coordinationMetadataChecksum(long coordinationMetadataChecksum) { + this.coordinationMetadataChecksum = coordinationMetadataChecksum; + return this; + } + + public Builder settingMetadataChecksum(long settingMetadataChecksum) { + this.settingMetadataChecksum = settingMetadataChecksum; + return this; + } + + public Builder transientSettingsMetadataChecksum(long transientSettingsMetadataChecksum) { + this.transientSettingsMetadataChecksum = transientSettingsMetadataChecksum; + return this; + } + + public Builder templatesMetadataChecksum(long templatesMetadataChecksum) { + this.templatesMetadataChecksum = templatesMetadataChecksum; + return this; + } + + public Builder customMetadataMapChecksum(long customMetadataMapChecksum) { + this.customMetadataMapChecksum = customMetadataMapChecksum; + return this; + } + + public Builder hashesOfConsistentSettingsChecksum(long hashesOfConsistentSettingsChecksum) { + this.hashesOfConsistentSettingsChecksum = hashesOfConsistentSettingsChecksum; + return this; + } + + public Builder indicesChecksum(long indicesChecksum) { + this.indicesChecksum = indicesChecksum; + return this; + } + + public Builder clusterStateChecksum(long clusterStateChecksum) { + this.clusterStateChecksum = clusterStateChecksum; + return this; + } + + public ClusterStateChecksum build() { + return new ClusterStateChecksum( + routingTableChecksum, + nodesChecksum, + blocksChecksum, + clusterStateCustomsChecksum, + coordinationMetadataChecksum, + settingMetadataChecksum, + transientSettingsMetadataChecksum, + templatesMetadataChecksum, + customMetadataMapChecksum, + hashesOfConsistentSettingsChecksum, + indicesChecksum, + clusterStateChecksum + ); + } + } + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 674279f2251bd..9a1314ded9180 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -137,6 +137,13 @@ public class RemoteClusterStateService implements Closeable { Setting.Property.NodeScope ); + public static final Setting REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING = Setting.boolSetting( + "cluster.remote_store.state.checksum_validation.enabled", + false, + Property.Dynamic, + Property.NodeScope + ); + private TimeValue remoteStateReadTimeout; private final String nodeId; private final Supplier repositoriesService; @@ -148,6 +155,7 @@ public class RemoteClusterStateService implements Closeable { private BlobStoreTransferService blobStoreTransferService; private RemoteRoutingTableService remoteRoutingTableService; private volatile TimeValue slowWriteLoggingThreshold; + private boolean checksumValidationEnabled; private final RemotePersistenceStats remoteStateStats; private RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; @@ -194,6 +202,9 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); this.remoteStateReadTimeout = clusterSettings.get(REMOTE_STATE_READ_TIMEOUT_SETTING); clusterSettings.addSettingsUpdateConsumer(REMOTE_STATE_READ_TIMEOUT_SETTING, this::setRemoteStateReadTimeout); + this.checksumValidationEnabled = clusterSettings.get(REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING); + clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING, this::setChecksumValidationEnabled); + this.remoteStateStats = new RemotePersistenceStats(); this.namedWriteableRegistry = namedWriteableRegistry; this.indexMetadataUploadListeners = indexMetadataUploadListeners; @@ -252,6 +263,7 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat uploadedMetadataResults, previousClusterUUID, clusterStateDiffManifest, + checksumValidationEnabled ? new ClusterStateChecksum(clusterState) : null, false ); @@ -443,6 +455,7 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), clusterStateDiffManifest, + checksumValidationEnabled ? new ClusterStateChecksum(clusterState) : null, false ); @@ -878,6 +891,7 @@ public RemoteClusterStateManifestInfo markLastStateAsCommitted(ClusterState clus uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), previousManifest.getDiffManifest(), + checksumValidationEnabled ? previousManifest.getClusterStateChecksum(): null, true ); if (!previousManifest.isClusterUUIDCommitted() && committedManifestDetails.getClusterMetadataManifest().isClusterUUIDCommitted()) { @@ -962,6 +976,10 @@ private void setSlowWriteLoggingThreshold(TimeValue slowWriteLoggingThreshold) { this.slowWriteLoggingThreshold = slowWriteLoggingThreshold; } + private void setChecksumValidationEnabled(Boolean checksumValidationEnabled) { + this.checksumValidationEnabled = checksumValidationEnabled; + } + // Package private for unit test RemoteRoutingTableService getRemoteRoutingTableService() { return this.remoteRoutingTableService; @@ -1312,7 +1330,7 @@ public ClusterState getClusterStateForManifest( boolean includeEphemeral ) throws IOException { if (manifest.onOrAfterCodecVersion(CODEC_V2)) { - return readClusterStateInParallel( + ClusterState clusterState = readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), manifest, manifest.getClusterUUID(), @@ -1331,6 +1349,11 @@ public ClusterState getClusterStateForManifest( false, includeEphemeral ); + + if (checksumValidationEnabled && manifest.getClusterStateChecksum()!=null) { + validateClusterStateFromChecksum(manifest.getClusterStateChecksum(), clusterState); + } + return clusterState; } else { ClusterState clusterState = readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), @@ -1437,11 +1460,28 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C indexRoutingTables.remove(indexName); } - return clusterStateBuilder.stateUUID(manifest.getStateUUID()) + ClusterState clusterState = clusterStateBuilder.stateUUID(manifest.getStateUUID()) .version(manifest.getStateVersion()) .metadata(metadataBuilder) .routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indexRoutingTables)) .build(); + + if (checksumValidationEnabled && manifest.getClusterStateChecksum()!=null) { + validateClusterStateFromChecksum(manifest.getClusterStateChecksum(), clusterState); + } + return clusterState; + } + + private void validateClusterStateFromChecksum(ClusterStateChecksum clusterStateChecksum, ClusterState clusterState) { + ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState); + if (!newClusterStateChecksum.equals(clusterStateChecksum)) { + logger.error( + "Cluster state checksums do not match. Checksum from manifest {}, checksum from created cluster state {}", + clusterStateChecksum, + newClusterStateChecksum + ); + throw new IllegalStateException("Cluster state checksums do not match."); + } } /** diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index cb09de1a6ec44..c72ba8dea6464 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -97,6 +97,7 @@ RemoteClusterStateManifestInfo uploadManifest( RemoteClusterStateUtils.UploadedMetadataResults uploadedMetadataResult, String previousClusterUUID, ClusterStateDiffManifest clusterDiffManifest, + ClusterStateChecksum clusterStateChecksum, boolean committed ) { synchronized (this) { @@ -124,8 +125,10 @@ RemoteClusterStateManifestInfo uploadManifest( .metadataVersion(clusterState.metadata().version()) .transientSettingsMetadata(uploadedMetadataResult.uploadedTransientSettingsMetadata) .clusterStateCustomMetadataMap(uploadedMetadataResult.uploadedClusterStateCustomMetadataMap) - .hashesOfConsistentSettings(uploadedMetadataResult.uploadedHashesOfConsistentSettings); + .hashesOfConsistentSettings(uploadedMetadataResult.uploadedHashesOfConsistentSettings) + .checksum(clusterStateChecksum); final ClusterMetadataManifest manifest = manifestBuilder.build(); + logger.trace("uploading manifest [{}]", manifest); String manifestFileName = writeMetadataManifest(clusterState.metadata().clusterUUID(), manifest); return new RemoteClusterStateManifestInfo(manifest, manifestFileName); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index acaae3173315a..f51c0db135b54 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -35,7 +35,7 @@ public class RemoteClusterMetadataManifest extends AbstractRemoteWritableBlobEnt public static final int SPLITTED_MANIFEST_FILE_LENGTH = 6; public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V4; public static final String COMMITTED = "C"; public static final String PUBLISHED = "P"; @@ -53,6 +53,9 @@ public class RemoteClusterMetadataManifest extends AbstractRemoteWritableBlobEnt public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V2 = new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV2); + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V3 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV3); + /** * Manifest format compatible with codec v2, where we introduced codec versions/global metadata. */ @@ -152,6 +155,8 @@ private ChecksumBlobStoreFormat getClusterMetadataManif long codecVersion = getManifestCodecVersion(); if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { return CLUSTER_METADATA_MANIFEST_FORMAT; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V3) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V3; } else if (codecVersion == ClusterMetadataManifest.CODEC_V2) { return CLUSTER_METADATA_MANIFEST_FORMAT_V2; } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { From 64af94a38651af7809378e4878541809e3894737 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Fri, 16 Aug 2024 15:51:47 +0530 Subject: [PATCH 02/29] Add UTs Signed-off-by: Himshikha Gupta --- .../gateway/remote/ClusterStateChecksum.java | 91 +++++-- .../remote/RemoteClusterStateService.java | 27 +- .../gateway/remote/RemoteManifestManager.java | 2 + .../remote/ClusterMetadataManifestTests.java | 132 +++++++++- .../remote/ClusterStateChecksumTests.java | 162 ++++++++++++ .../RemoteClusterStateServiceTests.java | 239 ++++++++++++++++++ 6 files changed, 621 insertions(+), 32 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index d2dd8c84820ed..8751ec2292045 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -24,6 +24,8 @@ import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import com.jcraft.jzlib.JZlib; @@ -35,18 +37,19 @@ */ public class ClusterStateChecksum implements ToXContentFragment, Writeable { - private static final String ROUTING_TABLE_CS = "routing_table"; - private static final String NODES_CS = "discovery_nodes"; - private static final String BLOCKS_CS = "blocks"; - private static final String CUSTOMS_CS = "customs"; - private static final String COORDINATION_MD_CS = "coordination_md"; - private static final String SETTINGS_MD_CS = "settings_md"; - private static final String TRANSIENT_SETTINGS_MD_CS = "transient_settings_md"; - private static final String TEMPLATES_MD_CS = "templated_md"; - private static final String CUSTOM_MD_CS = "customs_md"; - private static final String HASHES_MD_CS = "hashes_md"; - private static final String INDICES_CS = "indices_md"; + static final String ROUTING_TABLE_CS = "routing_table"; + static final String NODES_CS = "discovery_nodes"; + static final String BLOCKS_CS = "blocks"; + static final String CUSTOMS_CS = "customs"; + static final String COORDINATION_MD_CS = "coordination_md"; + static final String SETTINGS_MD_CS = "settings_md"; + static final String TRANSIENT_SETTINGS_MD_CS = "transient_settings_md"; + static final String TEMPLATES_MD_CS = "templated_md"; + static final String CUSTOM_MD_CS = "customs_md"; + static final String HASHES_MD_CS = "hashes_md"; + static final String INDICES_CS = "indices_md"; private static final String CLUSTER_STATE_CS = "cluster_state"; + private static final int CHECKSUM_SIZE = 8; private static final Logger logger = LogManager.getLogger(ClusterStateChecksum.class); long routingTableChecksum; @@ -133,16 +136,20 @@ public ClusterStateChecksum(ClusterState clusterState) { logger.error("Failed to create checksum for cluster state.", e); throw new RemoteStateTransferException("Failed to create checksum for cluster state.", e); } - clusterStateChecksum = JZlib.crc32_combine(routingTableChecksum, nodesChecksum, 8); - clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, blocksChecksum, 8); - clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, clusterStateCustomsChecksum, 8); - clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, coordinationMetadataChecksum, 8); - clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, settingMetadataChecksum, 8); - clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, transientSettingsMetadataChecksum, 8); - clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, templatesMetadataChecksum, 8); - clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, customMetadataMapChecksum, 8); - clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, hashesOfConsistentSettingsChecksum, 8); - clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, indicesChecksum, 8); + createClusterStateChecksum(); + } + + private void createClusterStateChecksum() { + clusterStateChecksum = JZlib.crc32_combine(routingTableChecksum, nodesChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, blocksChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, clusterStateCustomsChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, coordinationMetadataChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, settingMetadataChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, transientSettingsMetadataChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, templatesMetadataChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, customMetadataMapChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, hashesOfConsistentSettingsChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, indicesChecksum, CHECKSUM_SIZE); } public static ClusterStateChecksum.Builder builder() { @@ -324,6 +331,48 @@ public int hashCode() { ); } + public List getMismatchEntities(ClusterStateChecksum otherClusterStateChecksum) { + if (this.clusterStateChecksum == otherClusterStateChecksum.clusterStateChecksum) { + logger.info("No mismatch in checksums."); + return List.of(); + } + List mismatches = new ArrayList<>(); + addIfMismatch(this.routingTableChecksum, otherClusterStateChecksum.routingTableChecksum, ROUTING_TABLE_CS, mismatches); + addIfMismatch(this.nodesChecksum, otherClusterStateChecksum.nodesChecksum, NODES_CS, mismatches); + addIfMismatch(this.blocksChecksum, otherClusterStateChecksum.blocksChecksum, BLOCKS_CS, mismatches); + addIfMismatch(this.clusterStateCustomsChecksum, otherClusterStateChecksum.clusterStateCustomsChecksum, CUSTOMS_CS, mismatches); + addIfMismatch( + this.coordinationMetadataChecksum, + otherClusterStateChecksum.coordinationMetadataChecksum, + COORDINATION_MD_CS, + mismatches + ); + addIfMismatch(this.settingMetadataChecksum, otherClusterStateChecksum.settingMetadataChecksum, SETTINGS_MD_CS, mismatches); + addIfMismatch( + this.transientSettingsMetadataChecksum, + otherClusterStateChecksum.transientSettingsMetadataChecksum, + TRANSIENT_SETTINGS_MD_CS, + mismatches + ); + addIfMismatch(this.templatesMetadataChecksum, otherClusterStateChecksum.templatesMetadataChecksum, TEMPLATES_MD_CS, mismatches); + addIfMismatch(this.customMetadataMapChecksum, otherClusterStateChecksum.customMetadataMapChecksum, CUSTOM_MD_CS, mismatches); + addIfMismatch( + this.hashesOfConsistentSettingsChecksum, + otherClusterStateChecksum.hashesOfConsistentSettingsChecksum, + HASHES_MD_CS, + mismatches + ); + addIfMismatch(this.indicesChecksum, otherClusterStateChecksum.indicesChecksum, INDICES_CS, mismatches); + + return mismatches; + } + + private void addIfMismatch(long checksum, long otherChecksum, String entityName, List mismatches) { + if (checksum != otherChecksum) { + mismatches.add(entityName); + } + } + public static class Builder { long routingTableChecksum; long nodesChecksum; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 9a1314ded9180..99aad901cbd38 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -203,7 +203,10 @@ public RemoteClusterStateService( this.remoteStateReadTimeout = clusterSettings.get(REMOTE_STATE_READ_TIMEOUT_SETTING); clusterSettings.addSettingsUpdateConsumer(REMOTE_STATE_READ_TIMEOUT_SETTING, this::setRemoteStateReadTimeout); this.checksumValidationEnabled = clusterSettings.get(REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING); - clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING, this::setChecksumValidationEnabled); + clusterSettings.addSettingsUpdateConsumer( + REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING, + this::setChecksumValidationEnabled + ); this.remoteStateStats = new RemotePersistenceStats(); this.namedWriteableRegistry = namedWriteableRegistry; @@ -891,7 +894,7 @@ public RemoteClusterStateManifestInfo markLastStateAsCommitted(ClusterState clus uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), previousManifest.getDiffManifest(), - checksumValidationEnabled ? previousManifest.getClusterStateChecksum(): null, + checksumValidationEnabled ? previousManifest.getClusterStateChecksum() : null, true ); if (!previousManifest.isClusterUUIDCommitted() && committedManifestDetails.getClusterMetadataManifest().isClusterUUIDCommitted()) { @@ -1330,7 +1333,7 @@ public ClusterState getClusterStateForManifest( boolean includeEphemeral ) throws IOException { if (manifest.onOrAfterCodecVersion(CODEC_V2)) { - ClusterState clusterState = readClusterStateInParallel( + ClusterState clusterState = readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), manifest, manifest.getClusterUUID(), @@ -1350,7 +1353,7 @@ public ClusterState getClusterStateForManifest( includeEphemeral ); - if (checksumValidationEnabled && manifest.getClusterStateChecksum()!=null) { + if (checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { validateClusterStateFromChecksum(manifest.getClusterStateChecksum(), clusterState); } return clusterState; @@ -1466,21 +1469,25 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C .routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indexRoutingTables)) .build(); - if (checksumValidationEnabled && manifest.getClusterStateChecksum()!=null) { + if (checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { validateClusterStateFromChecksum(manifest.getClusterStateChecksum(), clusterState); } return clusterState; } - private void validateClusterStateFromChecksum(ClusterStateChecksum clusterStateChecksum, ClusterState clusterState) { + void validateClusterStateFromChecksum(ClusterStateChecksum clusterStateChecksum, ClusterState clusterState) { ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState); if (!newClusterStateChecksum.equals(clusterStateChecksum)) { + List failedValidation = newClusterStateChecksum.getMismatchEntities(clusterStateChecksum); logger.error( - "Cluster state checksums do not match. Checksum from manifest {}, checksum from created cluster state {}", - clusterStateChecksum, - newClusterStateChecksum + () -> new ParameterizedMessage( + "Cluster state checksums do not match. Checksum from manifest {}, checksum from created cluster state {}. Entities failing validation {}", + clusterStateChecksum, + newClusterStateChecksum, + failedValidation + ) ); - throw new IllegalStateException("Cluster state checksums do not match."); + throw new IllegalStateException("Cluster state checksums do not match. Validation failed for " + failedValidation); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index c72ba8dea6464..6972c1f4949c4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; @@ -129,6 +130,7 @@ RemoteClusterStateManifestInfo uploadManifest( .checksum(clusterStateChecksum); final ClusterMetadataManifest manifest = manifestBuilder.build(); logger.trace("uploading manifest [{}]", manifest); + logger.trace(() -> new ParameterizedMessage("[{}] uploading manifest", manifest)); String manifestFileName = writeMetadataManifest(clusterState.metadata().clusterUUID(), manifest); return new RemoteClusterStateManifestInfo(manifest, manifestFileName); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 8a6dd6bc96e72..a3c3ce2d679bb 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -9,15 +9,24 @@ package org.opensearch.gateway.remote; import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.DiffableUtils; +import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexGraveyard; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -162,7 +171,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { .opensearchVersion(Version.CURRENT) .nodeId("B10RX1f5RJenMQvYccCgSQ") .committed(true) - .codecVersion(ClusterMetadataManifest.CODEC_V3) + .codecVersion(ClusterMetadataManifest.CODEC_V4) .indices(randomUploadedIndexMetadataList()) .previousClusterUUID("yfObdx8KSMKKrXf8UyHhM") .clusterUUIDCommitted(true) @@ -201,6 +210,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "indicesRoutingDiffPath" ) ) + .checksum(new ClusterStateChecksum(createClusterState())) .build(); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( @@ -482,6 +492,22 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { } ); } + { + // Mutate checksum + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.checksum(null); + return builder.build(); + } + ); + } } public void testClusterMetadataManifestXContentV2() throws IOException { @@ -613,6 +639,74 @@ public void testClusterMetadataManifestXContentV3() throws IOException { } } + public void testClusterMetadataManifestXContentV4() throws IOException { + UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); + UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); + final DiffableUtils.MapDiff> routingTableIncrementalDiff = Mockito.mock( + DiffableUtils.MapDiff.class + ); + ClusterStateChecksum checksum = new ClusterStateChecksum(createClusterState()); + ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(ClusterMetadataManifest.CODEC_V4) + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .coordinationMetadata(uploadedMetadataAttribute) + .settingMetadata(uploadedMetadataAttribute) + .templatesMetadata(uploadedMetadataAttribute) + .customMetadataMap( + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + CUSTOM_METADATA + CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + CUSTOM_METADATA + CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + CUSTOM_METADATA + CUSTOM_DELIMITER + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) + ) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ) + .routingTableVersion(1L) + .indicesRouting(Collections.singletonList(uploadedIndexMetadata)) + .discoveryNodesMetadata(uploadedMetadataAttribute) + .clusterBlocksMetadata(uploadedMetadataAttribute) + .transientSettingsMetadata(uploadedMetadataAttribute) + .hashesOfConsistentSettings(uploadedMetadataAttribute) + .clusterStateCustomMetadataMap(Collections.emptyMap()) + .diffManifest( + new ClusterStateDiffManifest( + RemoteClusterStateServiceTests.generateClusterStateWithOneIndex().build(), + ClusterState.EMPTY_STATE, + routingTableIncrementalDiff, + uploadedMetadataAttribute.getUploadedFilename() + ) + ) + .checksum(checksum) + .build(); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); + assertEquals(originalManifest, fromXContentManifest); + } + } + public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); @@ -736,4 +830,40 @@ private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedInde return uploadedIndexMetadata; } + static ClusterState createClusterState() { + final Index index = new Index("test-index", "index-uuid"); + final Settings idxSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) + .build(); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); + final Settings settings = Settings.builder().put("mock-settings", true).build(); + final TemplatesMetadata templatesMetadata = TemplatesMetadata.builder() + .put(IndexTemplateMetadata.builder("template1").settings(idxSettings).patterns(List.of("test*")).build()) + .build(); + final RemoteClusterStateTestUtils.CustomMetadata1 customMetadata = new RemoteClusterStateTestUtils.CustomMetadata1( + "custom-metadata-1" + ); + return ClusterState.builder(ClusterName.DEFAULT) + .version(1L) + .stateUUID("state-uuid") + .metadata( + Metadata.builder() + .version(randomNonNegativeLong()) + .put(indexMetadata, true) + .clusterUUID("cluster-uuid") + .coordinationMetadata(coordinationMetadata) + .persistentSettings(settings) + .templates(templatesMetadata) + .hashesOfConsistentSettings(Map.of("key1", "value1", "key2", "value2")) + .putCustom(customMetadata.getWriteableName(), customMetadata) + .build() + ) + .routingTable(RoutingTable.builder().addAsNew(indexMetadata).version(1L).build()) + .build(); + } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java new file mode 100644 index 0000000000000..670d71fa05c8b --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java @@ -0,0 +1,162 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class ClusterStateChecksumTests extends OpenSearchTestCase { + + public void testClusterStateChecksumEmptyClusterState() { + ClusterStateChecksum checksum = new ClusterStateChecksum(ClusterState.EMPTY_STATE); + assertNotNull(checksum); + } + + public void testClusterStateChecksum() { + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + assertNotNull(checksum); + assertTrue(checksum.routingTableChecksum != 0); + assertTrue(checksum.nodesChecksum != 0); + assertTrue(checksum.blocksChecksum != 0); + assertTrue(checksum.clusterStateCustomsChecksum != 0); + assertTrue(checksum.coordinationMetadataChecksum != 0); + assertTrue(checksum.settingMetadataChecksum != 0); + assertTrue(checksum.transientSettingsMetadataChecksum != 0); + assertTrue(checksum.templatesMetadataChecksum != 0); + assertTrue(checksum.customMetadataMapChecksum != 0); + assertTrue(checksum.hashesOfConsistentSettingsChecksum != 0); + assertTrue(checksum.indicesChecksum != 0); + assertTrue(checksum.clusterStateChecksum != 0); + } + + public void testXContentConversion() throws IOException { + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + checksum.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterStateChecksum parsedChecksum = ClusterStateChecksum.fromXContent(parser); + assertEquals(checksum, parsedChecksum); + } + } + + public void testSerialization() throws IOException { + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + BytesStreamOutput output = new BytesStreamOutput(); + checksum.writeTo(output); + + try (StreamInput in = output.bytes().streamInput()) { + ClusterStateChecksum deserializedChecksum = new ClusterStateChecksum(in); + assertEquals(checksum, deserializedChecksum); + } + } + + public void testGetMismatchEntities() { + ClusterState clsState1 = generateClusterState(); + ClusterStateChecksum checksum = new ClusterStateChecksum(clsState1); + assertTrue(checksum.getMismatchEntities(checksum).isEmpty()); + + ClusterStateChecksum checksum2 = new ClusterStateChecksum(clsState1); + assertTrue(checksum.getMismatchEntities(checksum2).isEmpty()); + + ClusterState clsState2 = ClusterState.builder(ClusterName.DEFAULT) + .routingTable(RoutingTable.builder().build()) + .nodes(DiscoveryNodes.builder().build()) + .blocks(ClusterBlocks.builder().build()) + .customs(Map.of()) + .metadata(Metadata.EMPTY_METADATA) + .build(); + ClusterStateChecksum checksum3 = new ClusterStateChecksum(clsState2); + List mismatches = checksum.getMismatchEntities(checksum3); + assertFalse(mismatches.isEmpty()); + assertEquals(11, mismatches.size()); + assertEquals(ClusterStateChecksum.ROUTING_TABLE_CS, mismatches.get(0)); + assertEquals(ClusterStateChecksum.NODES_CS, mismatches.get(1)); + assertEquals(ClusterStateChecksum.BLOCKS_CS, mismatches.get(2)); + assertEquals(ClusterStateChecksum.CUSTOMS_CS, mismatches.get(3)); + assertEquals(ClusterStateChecksum.COORDINATION_MD_CS, mismatches.get(4)); + assertEquals(ClusterStateChecksum.SETTINGS_MD_CS, mismatches.get(5)); + assertEquals(ClusterStateChecksum.TRANSIENT_SETTINGS_MD_CS, mismatches.get(6)); + assertEquals(ClusterStateChecksum.TEMPLATES_MD_CS, mismatches.get(7)); + assertEquals(ClusterStateChecksum.CUSTOM_MD_CS, mismatches.get(8)); + assertEquals(ClusterStateChecksum.HASHES_MD_CS, mismatches.get(9)); + assertEquals(ClusterStateChecksum.INDICES_CS, mismatches.get(10)); + } + + private ClusterState generateClusterState() { + final Index index = new Index("test-index", "index-uuid"); + final Settings idxSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) + .put(IndexMetadata.INDEX_READ_ONLY_SETTING.getKey(), true) + .build(); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); + final Settings settings = Settings.builder().put("mock-settings", true).build(); + final TemplatesMetadata templatesMetadata = TemplatesMetadata.builder() + .put(IndexTemplateMetadata.builder("template1").settings(idxSettings).patterns(List.of("test*")).build()) + .build(); + final RemoteClusterStateTestUtils.CustomMetadata1 customMetadata1 = new RemoteClusterStateTestUtils.CustomMetadata1( + "custom-metadata-1" + ); + RemoteClusterStateTestUtils.TestClusterStateCustom1 clusterStateCustom1 = new RemoteClusterStateTestUtils.TestClusterStateCustom1( + "custom-1" + ); + return ClusterState.builder(ClusterName.DEFAULT) + .version(1L) + .stateUUID("state-uuid") + .metadata( + Metadata.builder() + .version(1L) + .put(indexMetadata, true) + .clusterUUID("cluster-uuid") + .coordinationMetadata(coordinationMetadata) + .persistentSettings(settings) + .transientSettings(settings) + .templates(templatesMetadata) + .hashesOfConsistentSettings(Map.of("key1", "value1", "key2", "value2")) + .putCustom(customMetadata1.getWriteableName(), customMetadata1) + .indices(Map.of(indexMetadata.getIndex().getName(), indexMetadata)) + .build() + ) + .nodes(DiscoveryNodes.builder().clusterManagerNodeId("test-node").build()) + .blocks(ClusterBlocks.builder().addBlocks(indexMetadata).build()) + .customs(Map.of(clusterStateCustom1.getWriteableName(), clusterStateCustom1)) + .routingTable(RoutingTable.builder().addAsNew(indexMetadata).version(1L).build()) + .build(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 59ca62dff2aa7..b3f6330ba3778 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -165,6 +165,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -2882,6 +2883,244 @@ private void initializeRoutingTable() { ); } + private void initializeWithChecksumEnabled() { + Settings newSettings = Settings.builder() + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true) + .build(); + clusterSettings.applySettings(newSettings); + + Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + remoteClusterStateService = new RemoteClusterStateService( + "test-node-id", + repositoriesServiceSupplier, + newSettings, + clusterService, + () -> 0L, + threadPool, + List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)), + writableRegistry() + ); + } + + public void testWriteFullMetadataSuccessWithChecksumValidationEnabled() throws IOException { + initializeWithChecksumEnabled(); + mockBlobStoreObjects(); + when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); + + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "index-uuid", + "routing-filename", + INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of(uploadedIndexMetadata)) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1L) + .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) + .checksum(new ClusterStateChecksum(clusterState)) + .build(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID())); + assertThat(manifest.getRoutingTableVersion(), is(expectedManifest.getRoutingTableVersion())); + assertThat(manifest.getIndicesRouting().get(0).getIndexName(), is(uploadedIndiceRoutingMetadata.getIndexName())); + assertThat(manifest.getIndicesRouting().get(0).getIndexUUID(), is(uploadedIndiceRoutingMetadata.getIndexUUID())); + assertThat(manifest.getIndicesRouting().get(0).getUploadedFilename(), notNullValue()); + assertThat(manifest.getClusterStateChecksum(), is(expectedManifest.getClusterStateChecksum())); + } + + public void testWriteIncrementalMetadataSuccessWithChecksumValidationEnabled() throws IOException { + initializeWithChecksumEnabled(); + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + mockBlobStoreObjects(); + final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); + final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT) + .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata)) + .build(); + + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() + .indices(Collections.emptyList()) + .checksum(new ClusterStateChecksum(clusterState)) + .build(); + when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); + + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata( + previousClusterState, + clusterState, + previousManifest + ).getClusterMetadataManifest(); + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "index-uuid", + "routing-filename", + INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of(uploadedIndexMetadata)) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) + .checksum(new ClusterStateChecksum(clusterState)) + .build(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getRoutingTableVersion(), is(expectedManifest.getRoutingTableVersion())); + assertThat(manifest.getIndicesRouting().get(0).getIndexName(), is(uploadedIndiceRoutingMetadata.getIndexName())); + assertThat(manifest.getIndicesRouting().get(0).getIndexUUID(), is(uploadedIndiceRoutingMetadata.getIndexUUID())); + assertThat(manifest.getIndicesRouting().get(0).getUploadedFilename(), notNullValue()); + assertThat(manifest.getClusterStateChecksum(), is(expectedManifest.getClusterStateChecksum())); + } + + public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullChecksum() throws IOException { + initializeWithChecksumEnabled(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().build(); + mockBlobStoreObjects(); + remoteClusterStateService.start(); + RemoteReadResult mockedResult = mock(RemoteReadResult.class); + RemoteIndexMetadataManager mockedIndexManager = mock(RemoteIndexMetadataManager.class); + RemoteGlobalMetadataManager mockedGlobalMetadataManager = mock(RemoteGlobalMetadataManager.class); + RemoteClusterStateAttributesManager mockedClusterStateAttributeManager = mock(RemoteClusterStateAttributesManager.class); + remoteClusterStateService.setRemoteIndexMetadataManager(mockedIndexManager); + remoteClusterStateService.setRemoteGlobalMetadataManager(mockedGlobalMetadataManager); + remoteClusterStateService.setRemoteClusterStateAttributesManager(mockedClusterStateAttributeManager); + ArgumentCaptor> listenerArgumentCaptor = ArgumentCaptor.forClass( + LatchedActionListener.class + ); + doAnswer(invocation -> { + listenerArgumentCaptor.getValue().onResponse(mockedResult); + return null; + }).when(mockedIndexManager).readAsync(any(), any(), listenerArgumentCaptor.capture()); + doAnswer(invocation -> { + listenerArgumentCaptor.getValue().onResponse(mockedResult); + return null; + }).when(mockedGlobalMetadataManager).readAsync(any(), any(), listenerArgumentCaptor.capture()); + doAnswer(invocation -> { + listenerArgumentCaptor.getValue().onResponse(mockedResult); + return null; + }).when(mockedClusterStateAttributeManager).readAsync(anyString(), any(), listenerArgumentCaptor.capture()); + when(mockedResult.getComponent()).thenReturn(COORDINATION_METADATA); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + ClusterState clusterState = mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); + verify(mockService, times(1)).readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + verify(mockService, times(0)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + } + + public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws IOException { + initializeWithChecksumEnabled(); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).build(); + mockBlobStoreObjects(); + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + doReturn(clusterState).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); + verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + } + + public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMismatch() throws IOException { + initializeWithChecksumEnabled(); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).build(); + mockBlobStoreObjects(); + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + ClusterState clusterStateWrong = ClusterState.builder(clusterState).routingTable(RoutingTable.EMPTY_ROUTING_TABLE).build(); + doReturn(clusterStateWrong).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + expectThrows( + IllegalStateException.class, + () -> mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true) + ); + verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + } + private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, Collections.emptyMap()); } From 859da0731a847fc5b74425c44746ce104b479395 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Fri, 16 Aug 2024 15:54:46 +0530 Subject: [PATCH 03/29] comments address Signed-off-by: Himshikha Gupta --- .../org/opensearch/gateway/remote/RemoteManifestManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 6972c1f4949c4..c47a0ebef91de 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -129,7 +129,6 @@ RemoteClusterStateManifestInfo uploadManifest( .hashesOfConsistentSettings(uploadedMetadataResult.uploadedHashesOfConsistentSettings) .checksum(clusterStateChecksum); final ClusterMetadataManifest manifest = manifestBuilder.build(); - logger.trace("uploading manifest [{}]", manifest); logger.trace(() -> new ParameterizedMessage("[{}] uploading manifest", manifest)); String manifestFileName = writeMetadataManifest(clusterState.metadata().clusterUUID(), manifest); return new RemoteClusterStateManifestInfo(manifest, manifestFileName); From 57d4709521c1d900c530ee489bebf9a151e91834 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 21 Aug 2024 13:28:29 +0530 Subject: [PATCH 04/29] gradle fix Signed-off-by: Himshikha Gupta --- .../org/opensearch/gateway/remote/ClusterStateChecksum.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index 8751ec2292045..7f07b5864375c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -373,6 +373,9 @@ private void addIfMismatch(long checksum, long otherChecksum, String entityName, } } + /** + * Builder for ClusterStateChecksum + */ public static class Builder { long routingTableChecksum; long nodesChecksum; From 9853636a3aa7b6e35400f4f9384d001e22598882 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 21 Aug 2024 15:46:54 +0530 Subject: [PATCH 05/29] enable checksum validation in IT Signed-off-by: Himshikha Gupta --- .../opensearch/gateway/remote/RemoteRoutingTableServiceIT.java | 1 + .../org/opensearch/gateway/remote/RemoteStatePublicationIT.java | 1 + 2 files changed, 2 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java index b0d046cbdf3db..d47659ff10581 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java @@ -66,6 +66,7 @@ protected Settings nodeSettings(int nodeOrdinal) { ) .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, REMOTE_ROUTING_TABLE_REPO) .put(REMOTE_PUBLICATION_EXPERIMENTAL, true) + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index 07d6e1379ced8..1b8584050ee39 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -80,6 +80,7 @@ protected Settings nodeSettings(int nodeOrdinal) { .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName) .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE) .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true) .build(); } From 0f54c72d0fbf070f429c3fb8b863c682c6e5125d Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 21 Aug 2024 16:24:02 +0530 Subject: [PATCH 06/29] adding changelog entry Signed-off-by: Himshikha Gupta --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6943e538fdcc..493e797f18507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support filtering on a large list encoded by bitmap ([#14774](https://github.com/opensearch-project/OpenSearch/pull/14774)) - Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153)) - Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895)) +- [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) From 0dd6d0110112383ce47931f02673e1628fa1f0e5 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 22 Aug 2024 15:14:11 +0530 Subject: [PATCH 07/29] sorting checksum components Signed-off-by: Himshikha Gupta --- .../opensearch/cluster/AbstractDiffable.java | 7 ++ .../org/opensearch/cluster/ClusterState.java | 18 ++- .../org/opensearch/cluster/DiffableUtils.java | 9 ++ .../coordination/CoordinationMetadata.java | 15 +++ .../cluster/metadata/ComponentTemplate.java | 11 ++ .../metadata/ComponentTemplateMetadata.java | 15 +++ .../cluster/metadata/DiffableStringMap.java | 4 + .../cluster/metadata/IndexMetadata.java | 61 ++++++++++ .../opensearch/cluster/metadata/Metadata.java | 2 +- .../cluster/metadata/TemplatesMetadata.java | 12 ++ .../cluster/node/DiscoveryNode.java | 28 +++++ .../cluster/node/DiscoveryNodes.java | 22 ++++ .../cluster/routing/IndexRoutingTable.java | 15 +++ .../routing/IndexShardRoutingTable.java | 12 ++ .../cluster/routing/RoutingTable.java | 21 ++++ .../gateway/remote/ClusterStateChecksum.java | 52 ++++++-- .../remote/RemoteClusterStateService.java | 111 +++++++++++++++--- .../remote/ClusterStateChecksumTests.java | 38 +++++- .../RemoteClusterStateServiceTests.java | 6 +- 19 files changed, 424 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java b/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java index 74af3472433ba..da22dd59289b9 100644 --- a/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java +++ b/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java @@ -83,6 +83,13 @@ private static class CompleteDiff> implements Diff { this.part = part; } + @Override + public String toString() { + return "CompleteDiff{" + + "part=" + part + + '}'; + } + /** * Creates simple diff without changes */ diff --git a/server/src/main/java/org/opensearch/cluster/ClusterState.java b/server/src/main/java/org/opensearch/cluster/ClusterState.java index 9e63f961d241d..3d9ba1e016496 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterState.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterState.java @@ -156,7 +156,7 @@ default boolean isPrivate() { } - private static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); + public static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); public static final String UNKNOWN_UUID = "_na_"; @@ -839,6 +839,22 @@ private static class ClusterStateDiff implements Diff { minimumClusterManagerNodesOnPublishingClusterManager = after.minimumClusterManagerNodesOnPublishingClusterManager; } + @Override + public String toString() { + return "ClusterStateDiff{" + + "toVersion=" + toVersion + + ", fromUuid='" + fromUuid + '\'' + + ", toUuid='" + toUuid + '\'' + + ", clusterName=" + clusterName + + ", routingTable=" + routingTable + + ", nodes=" + nodes + + ", metadata=" + metadata + + ", blocks=" + blocks + + ", customs=" + customs + + ", minimumClusterManagerNodesOnPublishingClusterManager=" + minimumClusterManagerNodesOnPublishingClusterManager + + '}'; + } + ClusterStateDiff(StreamInput in, DiscoveryNode localNode) throws IOException { clusterName = new ClusterName(in); fromUuid = in.readString(); diff --git a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java index d21cd354bf659..9ee2cf3ddd083 100644 --- a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java +++ b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java @@ -271,6 +271,15 @@ public Map getUpserts() { return upserts; } + @Override + public String toString() { + return "MapDiff{" + + "deletes=" + deletes + + ", diffs=" + diffs + + ", upserts=" + upserts + + '}'; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeCollection(deletes, (o, v) -> keySerializer.writeKey(v, o)); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java index 53398d6f3f98f..3d02d58e26f15 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java @@ -47,6 +47,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -149,6 +150,14 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(votingConfigExclusions); } + public void writeToSorted(StreamOutput out) throws IOException { + out.writeLong(term); + lastCommittedConfiguration.writeToSorted(out); + lastAcceptedConfiguration.writeToSorted(out); + List sortedList = votingConfigExclusions.stream().sorted(Comparator.comparing(VotingConfigExclusion::getNodeId)).collect(Collectors.toList()); + out.writeCollection(sortedList); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.field(TERM_PARSE_FIELD.getPreferredName(), term) @@ -391,6 +400,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringArray(nodeIds.toArray(new String[0])); } + public void writeToSorted(StreamOutput out) throws IOException { + String[] nodeIds = this.nodeIds.toArray(new String[0]); + Arrays.sort(nodeIds); + out.writeStringArray(nodeIds); + } + public boolean hasQuorum(Collection votes) { final HashSet intersection = new HashSet<>(nodeIds); intersection.retainAll(votes); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java index abc3712ee07e3..54658d8304d5b 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java @@ -133,6 +133,17 @@ public void writeTo(StreamOutput out) throws IOException { } } + public void writeToSorted(StreamOutput out) throws IOException { + this.template.writeTo(out); + out.writeOptionalVLong(this.version); + if (this.metadata == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeMap(this.metadata); + } + } + @Override public int hashCode() { return Objects.hash(template, version, metadata); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java index d19743e643d12..c75e2fff0e47c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java @@ -46,6 +46,7 @@ import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; +import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.Map; @@ -122,6 +123,20 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(this.componentTemplates, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); } + public void writeToSorted(StreamOutput out) throws IOException { + out.writeVInt(this.componentTemplates.size()); + this.componentTemplates.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEachOrdered(entry-> + { + try { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + ); + } + public static ComponentTemplateMetadata fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java index 5865891c8a7f9..da81c2275109e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java @@ -81,6 +81,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap((Map) (Map) innerMap); } + public void writeToSorted(StreamOutput out) throws IOException { + out.writeMapWithConsistentOrder((Map) (Map) innerMap); + } + @Override public Diff diff(DiffableStringMap previousState) { return new DiffableStringMapDiff(previousState, this); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index df0d2609ad83d..547c093a48a8f 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -87,6 +87,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeSet; import java.util.function.Function; import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_PARAM; @@ -1212,6 +1213,66 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isSystem); } + public void writeToSorted(StreamOutput out) throws IOException { + out.writeString(index.getName()); // uuid will come as part of settings + out.writeLong(version); + out.writeVLong(mappingVersion); + out.writeVLong(settingsVersion); + out.writeVLong(aliasesVersion); + out.writeInt(routingNumShards); + out.writeByte(state.id()); + writeSettingsToStream(settings, out); + out.writeVLongArray(primaryTerms); + out.writeVInt(mappings.size()); + mappings.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { + try { + entry.getValue().writeTo(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + out.writeVInt(aliases.size()); + aliases.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { + try { + entry.getValue().writeTo(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + out.writeVInt(customData.size()); + customData.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { + try { + out.writeString(entry.getKey()); + entry.getValue().writeToSorted(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + out.writeVInt(inSyncAllocationIds.size()); + inSyncAllocationIds.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { + try { + out.writeVInt(entry.getKey()); + DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(entry.getValue()), out); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + out.writeVInt(rolloverInfos.size()); + rolloverInfos.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { + try { + entry.getValue().writeTo(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + out.writeBoolean(isSystem); + } + public boolean isSystem() { return isSystem; } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 4da6c68b40733..7423c0116939d 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -253,7 +253,7 @@ static Custom fromXContent(XContentParser parser, String name) throws IOExceptio public static final String GLOBAL_STATE_FILE_PREFIX = "global-"; - private static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); + public static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); private final String clusterUUID; private final boolean clusterUUIDCommitted; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index 6ecc471c5e0ae..b2f45f0ea4cce 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -65,6 +66,17 @@ public void writeTo(StreamOutput out) throws IOException { } } + public void writeToSorted(StreamOutput out) throws IOException { + out.writeVInt(templates.size()); + templates.values().stream().sorted(Comparator.comparing(IndexTemplateMetadata::getName)).forEach(template -> { + try { + template.writeTo(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 653f81830ed17..166719acade02 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -47,6 +47,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Locale; @@ -379,6 +380,33 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVersion(version); } + public void writeToSorted(StreamOutput out) throws IOException { + out.writeString(nodeName); + out.writeString(nodeId); + out.writeString(ephemeralId); + out.writeString(hostName); + out.writeString(hostAddress); + address.writeTo(out); + out.writeVInt(attributes.size()); + attributes.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { + try { + out.writeString(entry.getKey()); + out.writeString(entry.getValue()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + }); + out.writeVInt(roles.size()); + for (final DiscoveryNodeRole role : roles) { + final DiscoveryNodeRole compatibleRole = role.getCompatibilityRole(out.getVersion()); + out.writeString(compatibleRole.roleName()); + out.writeString(compatibleRole.roleNameAbbreviation()); + out.writeBoolean(compatibleRole.canContainData()); + } + out.writeVersion(version); + } + /** * The address that the node can be communicated with. */ diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index 2ebcd8096893d..fc2a65929fbfd 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -47,7 +47,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -55,6 +57,8 @@ import java.util.Map; import java.util.Objects; import java.util.Spliterators; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -700,6 +704,24 @@ public void writeTo(StreamOutput out) throws IOException { } } + public void writeToSorted(StreamOutput out) throws IOException { + if (clusterManagerNodeId == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeString(clusterManagerNodeId); + } + out.writeVInt(nodes.size()); + + TreeSet sortedNodes = new TreeSet<>(Comparator.comparing(DiscoveryNode::getId)); + sortedNodes.addAll(nodes.values()); + + for (DiscoveryNode node : sortedNodes) { + node.writeToSorted(out); + } + + } + public static DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws IOException { Builder builder = new Builder(); if (in.readBoolean()) { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index 7c179f6d4d8fd..13b755d9d20d0 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -376,6 +377,20 @@ public void writeTo(StreamOutput out) throws IOException { } } + public void writeToSorted(StreamOutput out) throws IOException { + index.writeTo(out); + out.writeVInt(shards.size()); + shards.values().stream().sorted(Comparator.comparing(IndexShardRoutingTable::getShardId)).forEach(indexShard -> + { + try { + IndexShardRoutingTable.Builder.writeToSorted(indexShard, out); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + ); + } + public static Builder builder(Index index) { return new Builder(index); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index 479143fa9a2f0..4af6635a24a58 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -1135,6 +1135,18 @@ public static void writeToThin(IndexShardRoutingTable indexShard, StreamOutput o } } + public static void writeToSorted(IndexShardRoutingTable indexShard, StreamOutput out) throws IOException { + out.writeVInt(indexShard.shardId.id()); + out.writeVInt(indexShard.shards.size()); + //Order allocated shards by allocationId + indexShard.shards.stream().filter(shardRouting -> shardRouting.allocationId()!=null).sorted(Comparator.comparing(o -> o.allocationId().getId())).forEach(shardRouting -> { + try { + shardRouting.writeToThin(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } } @Override diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 6c7b94f316da2..430e9fffad015 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -52,6 +52,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -403,6 +404,18 @@ public void writeTo(StreamOutput out) throws IOException { } } + public void writeToSorted(StreamOutput out) throws IOException { + out.writeLong(version); + out.writeVInt(indicesRouting.size()); + indicesRouting.values().stream().sorted(Comparator.comparing(index -> index.getIndex().getName())).forEach(index -> { + try { + index.writeToSorted(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + private static class RoutingTableDiff implements Diff { private final long version; @@ -422,6 +435,14 @@ private static class RoutingTableDiff implements Diff { indicesRouting = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), DIFF_VALUE_READER); } + @Override + public String toString() { + return "RoutingTableDiff{" + + "version=" + version + + ", indicesRouting=" + indicesRouting + + '}'; + } + @Override public RoutingTable apply(RoutingTable part) { return new RoutingTable(version, indicesRouting.apply(part.indicesRouting)); diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index 7f07b5864375c..f2acded5ef89e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -10,8 +10,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; @@ -25,8 +27,13 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.TreeSet; import com.jcraft.jzlib.JZlib; @@ -44,7 +51,7 @@ public class ClusterStateChecksum implements ToXContentFragment, Writeable { static final String COORDINATION_MD_CS = "coordination_md"; static final String SETTINGS_MD_CS = "settings_md"; static final String TRANSIENT_SETTINGS_MD_CS = "transient_settings_md"; - static final String TEMPLATES_MD_CS = "templated_md"; + static final String TEMPLATES_MD_CS = "templates_md"; static final String CUSTOM_MD_CS = "customs_md"; static final String HASHES_MD_CS = "hashes_md"; static final String INDICES_CS = "indices_md"; @@ -70,17 +77,18 @@ public ClusterStateChecksum(ClusterState clusterState) { BytesStreamOutput out = new BytesStreamOutput(); BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out) ) { - clusterState.routingTable().writeTo(checksumOut); + clusterState.routingTable().writeToSorted(checksumOut); routingTableChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.nodes().writeTo(checksumOut); + clusterState.nodes().writeToSorted(checksumOut); nodesChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.coordinationMetadata().writeTo(checksumOut); + clusterState.coordinationMetadata().writeToSorted(checksumOut); coordinationMetadataChecksum = checksumOut.getChecksum(); + //Settings create sortedMap by default, so no explicit sorting required here. checksumOut.reset(); Settings.writeSettingsToStream(clusterState.metadata().persistentSettings(), checksumOut); settingMetadataChecksum = checksumOut.getChecksum(); @@ -90,13 +98,13 @@ public ClusterStateChecksum(ClusterState clusterState) { transientSettingsMetadataChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.metadata().templatesMetadata().writeTo(checksumOut); + clusterState.metadata().templatesMetadata().writeToSorted(checksumOut); templatesMetadataChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.metadata().customs().forEach((key, custom) -> { + clusterState.metadata().customs().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { try { - custom.writeTo(checksumOut); + entry.getValue().writeTo(checksumOut); } catch (IOException e) { logger.error("Failed to create checksum for custom metadata.", e); throw new RemoteStateTransferException("Failed to create checksum for custom metadata.", e); @@ -105,13 +113,13 @@ public ClusterStateChecksum(ClusterState clusterState) { customMetadataMapChecksum = checksumOut.getChecksum(); checksumOut.reset(); - ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeTo(checksumOut); + ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeToSorted(checksumOut); hashesOfConsistentSettingsChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.metadata().indices().forEach(((s, indexMetadata) -> { + clusterState.metadata().indices().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach((entry -> { try { - indexMetadata.writeTo(checksumOut); + entry.getValue().writeToSorted(checksumOut); } catch (IOException e) { logger.error("Failed to create checksum for index metadata.", e); throw new RemoteStateTransferException("Failed to create checksum for index metadata.", e); @@ -124,9 +132,9 @@ public ClusterStateChecksum(ClusterState clusterState) { blocksChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.customs().forEach((key, value) -> { + clusterState.customs().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { try { - checksumOut.writeNamedWriteable(value); + checksumOut.writeNamedWriteable(entry.getValue()); } catch (IOException e) { throw new RuntimeException(e); } @@ -331,9 +339,27 @@ public int hashCode() { ); } + @Override + public String toString() { + return "ClusterStateChecksum{" + + "routingTableChecksum=" + routingTableChecksum + + ", nodesChecksum=" + nodesChecksum + + ", blocksChecksum=" + blocksChecksum + + ", clusterStateCustomsChecksum=" + clusterStateCustomsChecksum + + ", coordinationMetadataChecksum=" + coordinationMetadataChecksum + + ", settingMetadataChecksum=" + settingMetadataChecksum + + ", transientSettingsMetadataChecksum=" + transientSettingsMetadataChecksum + + ", templatesMetadataChecksum=" + templatesMetadataChecksum + + ", customMetadataMapChecksum=" + customMetadataMapChecksum + + ", hashesOfConsistentSettingsChecksum=" + hashesOfConsistentSettingsChecksum + + ", indicesChecksum=" + indicesChecksum + + ", clusterStateChecksum=" + clusterStateChecksum + + '}'; + } + public List getMismatchEntities(ClusterStateChecksum otherClusterStateChecksum) { if (this.clusterStateChecksum == otherClusterStateChecksum.clusterStateChecksum) { - logger.info("No mismatch in checksums."); + logger.debug("No mismatch in checksums."); return List.of(); } List mismatches = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 04a6fa53e4fef..d3624e76b27e1 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -20,6 +20,7 @@ import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Manifest; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.Metadata.XContentContext; import org.opensearch.cluster.metadata.TemplatesMetadata; @@ -89,6 +90,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static org.opensearch.cluster.ClusterState.CUSTOM_VALUE_SERIALIZER; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2; @@ -1353,8 +1355,8 @@ public ClusterState getClusterStateForManifest( includeEphemeral ); - if (checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { - validateClusterStateFromChecksum(manifest.getClusterStateChecksum(), clusterState); + if (includeEphemeral && checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { + validateClusterStateFromChecksum(manifest, clusterState, clusterName, localNodeId, true); } return clusterState; } else { @@ -1470,25 +1472,102 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C .build(); if (checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { - validateClusterStateFromChecksum(manifest.getClusterStateChecksum(), clusterState); + validateClusterStateFromChecksum(manifest, clusterState, previousState.getClusterName().value(), localNodeId,false); } return clusterState; } - void validateClusterStateFromChecksum(ClusterStateChecksum clusterStateChecksum, ClusterState clusterState) { + void validateClusterStateFromChecksum(ClusterMetadataManifest manifest, ClusterState clusterState,String clusterName, String localNodeId, boolean isFullStateDownload) throws IOException { ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState); - if (!newClusterStateChecksum.equals(clusterStateChecksum)) { - List failedValidation = newClusterStateChecksum.getMismatchEntities(clusterStateChecksum); - logger.error( - () -> new ParameterizedMessage( - "Cluster state checksums do not match. Checksum from manifest {}, checksum from created cluster state {}. Entities failing validation {}", - clusterStateChecksum, - newClusterStateChecksum, - failedValidation - ) - ); - throw new IllegalStateException("Cluster state checksums do not match. Validation failed for " + failedValidation); - } + List failedValidation = newClusterStateChecksum.getMismatchEntities(manifest.getClusterStateChecksum()); + if(!failedValidation.isEmpty()) { + logger.error( + () -> new ParameterizedMessage( + "Cluster state checksums do not match. Checksum from manifest {}, checksum from created cluster state {}. Entities failing validation {}", + manifest.getClusterStateChecksum(), + newClusterStateChecksum, + failedValidation + ) + ); + if(isFullStateDownload) { + throw new IllegalStateException("Cluster state checksums do not match during full state read. Validation failed for " + failedValidation); + } + //download full cluster state and match against state created for the failing entities + ClusterState fullClusterState = readClusterStateInParallel( + ClusterState.builder(new ClusterName(clusterName)).build(), + manifest, + manifest.getClusterUUID(), + localNodeId, + manifest.getIndices(), + manifest.getCustomMetadataMap(), + manifest.getCoordinationMetadata() != null, + manifest.getSettingsMetadata() != null, + manifest.getTransientSettingsMetadata() != null, + manifest.getTemplatesMetadata() != null, + manifest.getDiscoveryNodesMetadata() != null, + manifest.getClusterBlocksMetadata() != null, + manifest.getIndicesRouting(), + manifest.getHashesOfConsistentSettings() != null, + manifest.getClusterStateCustomMap() , + false, + true + ); + for(String failedEntity : failedValidation) { + switch (failedEntity) { + case ClusterStateChecksum.ROUTING_TABLE_CS: + Diff routingTableDiff = fullClusterState.routingTable().diff(clusterState.routingTable()); + logger.error(() -> new ParameterizedMessage("Failing Diff in routing table {}", routingTableDiff)); + break; + case ClusterStateChecksum.NODES_CS: + logger.error(() -> new ParameterizedMessage("Failing Diff in discovery nodes {}", + fullClusterState.nodes().diff(clusterState.nodes()))); + break; + case ClusterStateChecksum.BLOCKS_CS: + logger.error(() -> new ParameterizedMessage("Failing Diff in cluster blocks {}", + fullClusterState.blocks().diff(clusterState.blocks()))); + break; + case ClusterStateChecksum.CUSTOMS_CS: + logger.error(() -> new ParameterizedMessage("Failing Diff in cluster state customs {}", + DiffableUtils.diff(clusterState.customs(), fullClusterState.customs(), DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER))); + break; + case ClusterStateChecksum.COORDINATION_MD_CS : + logger.error(() -> new ParameterizedMessage("Failing Diff in coordination md. current md {}, full state md {}", + clusterState.metadata().coordinationMetadata(), fullClusterState.metadata().coordinationMetadata())); + break; + case ClusterStateChecksum.TRANSIENT_SETTINGS_MD_CS: + logger.error(() -> new ParameterizedMessage("Failing Diff in transient settings md. current md {}, full state md {}", + clusterState.metadata().transientSettings(), fullClusterState.metadata().transientSettings())); + + break; + case ClusterStateChecksum.SETTINGS_MD_CS: + logger.error(() -> new ParameterizedMessage("Failing Diff in settings md. current md {}, full state md {}", + clusterState.metadata().settings(), fullClusterState.metadata().settings())); + + break; + case ClusterStateChecksum.HASHES_MD_CS: + logger.error(() -> new ParameterizedMessage("Failing Diff in hashes md {}", + ((DiffableStringMap)fullClusterState.metadata().hashesOfConsistentSettings()).diff((DiffableStringMap)clusterState.metadata().hashesOfConsistentSettings()))); + break; + case ClusterStateChecksum.TEMPLATES_MD_CS: + logger.error(() -> new ParameterizedMessage("Failing Diff in templates md{}", + fullClusterState.metadata().templatesMetadata().diff(clusterState.metadata().templatesMetadata()))); + break; + case ClusterStateChecksum.CUSTOM_MD_CS: + logger.error(() -> new ParameterizedMessage("Failing Diff in customs md {}", + DiffableUtils.diff(clusterState.metadata().customs(), fullClusterState.metadata().customs(), DiffableUtils.getStringKeySerializer(), Metadata.CUSTOM_VALUE_SERIALIZER))); + break; + case ClusterStateChecksum.INDICES_CS: + logger.error(() -> new ParameterizedMessage("Failing Diff in index md {}", + DiffableUtils.diff(clusterState.metadata().indices(), fullClusterState.metadata().indices(), DiffableUtils.getStringKeySerializer()))); + break; + default: + logger.error(() -> new ParameterizedMessage("Unknown failed entity {}", + failedEntity)); + break; + } + } + throw new IllegalStateException("Cluster state checksums do not match during diff read. Validation failed for " + failedValidation); + } } /** diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java index 670d71fa05c8b..12dc1cf18c055 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.metadata.IndexTemplateMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -24,6 +25,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -114,6 +116,30 @@ public void testGetMismatchEntities() { assertEquals(ClusterStateChecksum.INDICES_CS, mismatches.get(10)); } + public void testGetMismatchEntitiesUnorderedInput() { + ClusterState state1 = generateClusterState(); + DiscoveryNode node1 = DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9200), "node1"); + DiscoveryNode node2 = DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9201), "node2"); + DiscoveryNode node3 = DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9202), "node3"); + + DiscoveryNodes nodes1 = DiscoveryNodes.builder().clusterManagerNodeId("test-node") + .add(node1) + .add(node2) + .add(node3) + .build(); + DiscoveryNodes nodes2 = DiscoveryNodes.builder().clusterManagerNodeId("test-node") + .add(node2) + .add(node3) + .build(); + nodes2 = nodes2.newNode(node1); + ClusterState state2 = ClusterState.builder(state1).nodes(nodes1).build(); + ClusterState state3 = ClusterState.builder(state1).nodes(nodes2).build(); + + ClusterStateChecksum checksum1 = new ClusterStateChecksum(state2); + ClusterStateChecksum checksum2 = new ClusterStateChecksum(state3); + assertEquals(checksum2, checksum1); + } + private ClusterState generateClusterState() { final Index index = new Index("test-index", "index-uuid"); final Settings idxSettings = Settings.builder() @@ -125,6 +151,16 @@ private ClusterState generateClusterState() { .numberOfShards(1) .numberOfReplicas(0) .build(); + final Index index2 = new Index("test-index2", "index-uuid2"); + final Settings idxSettings2 = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index2.getUUID()) + .put(IndexMetadata.INDEX_READ_ONLY_SETTING.getKey(), true) + .build(); + final IndexMetadata indexMetadata2 = new IndexMetadata.Builder(index2.getName()).settings(idxSettings2) + .numberOfShards(1) + .numberOfReplicas(2) + .build(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); final Settings settings = Settings.builder().put("mock-settings", true).build(); final TemplatesMetadata templatesMetadata = TemplatesMetadata.builder() @@ -156,7 +192,7 @@ private ClusterState generateClusterState() { .nodes(DiscoveryNodes.builder().clusterManagerNodeId("test-node").build()) .blocks(ClusterBlocks.builder().addBlocks(indexMetadata).build()) .customs(Map.of(clusterStateCustom1.getWriteableName(), clusterStateCustom1)) - .routingTable(RoutingTable.builder().addAsNew(indexMetadata).version(1L).build()) + .routingTable(RoutingTable.builder().addAsNew(indexMetadata).addAsNew(indexMetadata2).version(1L).build()) .build(); } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index db4f4737531f7..eeda9d84ca4c2 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -3048,7 +3048,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullC eq(false), eq(true) ); - verify(mockService, times(0)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + //verify(mockService, times(0)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); } public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws IOException { @@ -3081,7 +3081,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws eq(true) ); mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); - verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + //verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); } public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMismatch() throws IOException { @@ -3118,7 +3118,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma IllegalStateException.class, () -> mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true) ); - verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + //verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); } private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { From 815d659635b63f7ae8c1412d920ef79b3d5a9d7f Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 27 Aug 2024 12:17:22 +0530 Subject: [PATCH 08/29] refactoring Signed-off-by: Himshikha Gupta --- .../core/common/io/stream/StreamOutput.java | 28 ++ .../opensearch/cluster/AbstractDiffable.java | 4 +- .../org/opensearch/cluster/ClusterState.java | 36 ++- .../org/opensearch/cluster/DiffableUtils.java | 6 +- .../coordination/CoordinationMetadata.java | 4 +- .../metadata/ComponentTemplateMetadata.java | 16 +- .../cluster/metadata/IndexMetadata.java | 62 ++--- .../cluster/node/DiscoveryNode.java | 12 +- .../cluster/node/DiscoveryNodes.java | 14 +- .../cluster/routing/IndexRoutingTable.java | 14 +- .../routing/IndexShardRoutingTable.java | 19 +- .../cluster/routing/RoutingTable.java | 5 +- .../gateway/remote/ClusterStateChecksum.java | 48 ++-- .../remote/RemoteClusterStateService.java | 245 +++++++++++------- .../remote/ClusterStateChecksumTests.java | 13 +- .../RemoteClusterStateServiceTests.java | 6 +- 16 files changed, 287 insertions(+), 245 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java index b7599265aece3..b69c8fcae88c8 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java @@ -75,6 +75,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; @@ -641,6 +642,33 @@ public final void writeMap(final Map map, final Writer keyWriter } } + public final void writeMapOrderedByKey(final Map map, Comparator> entryComparator, final Writer keyWriter, final Writer valueWriter) throws IOException { + writeVInt(map.size()); + map.entrySet().stream().sorted(entryComparator).forEachOrdered(entry-> + { + try { + keyWriter.write(this, entry.getKey()); + valueWriter.write(this, entry.getValue()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + ); + } + + public final void writeMapValuesOrderedByKey(final Map map, Comparator> entryComparator, final Writer valueWriter) throws IOException { + writeVInt(map.size()); + map.entrySet().stream().sorted(entryComparator).forEachOrdered(entry-> + { + try { + valueWriter.write(this, entry.getValue()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + ); + } + /** * Writes an {@link Instant} to the stream with nanosecond resolution */ diff --git a/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java b/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java index da22dd59289b9..770a0c171e084 100644 --- a/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java +++ b/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java @@ -85,9 +85,7 @@ private static class CompleteDiff> implements Diff { @Override public String toString() { - return "CompleteDiff{" + - "part=" + part + - '}'; + return "CompleteDiff{" + "part=" + part + '}'; } /** diff --git a/server/src/main/java/org/opensearch/cluster/ClusterState.java b/server/src/main/java/org/opensearch/cluster/ClusterState.java index 3d9ba1e016496..1350c52997791 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterState.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterState.java @@ -841,18 +841,30 @@ private static class ClusterStateDiff implements Diff { @Override public String toString() { - return "ClusterStateDiff{" + - "toVersion=" + toVersion + - ", fromUuid='" + fromUuid + '\'' + - ", toUuid='" + toUuid + '\'' + - ", clusterName=" + clusterName + - ", routingTable=" + routingTable + - ", nodes=" + nodes + - ", metadata=" + metadata + - ", blocks=" + blocks + - ", customs=" + customs + - ", minimumClusterManagerNodesOnPublishingClusterManager=" + minimumClusterManagerNodesOnPublishingClusterManager + - '}'; + return "ClusterStateDiff{" + + "toVersion=" + + toVersion + + ", fromUuid='" + + fromUuid + + '\'' + + ", toUuid='" + + toUuid + + '\'' + + ", clusterName=" + + clusterName + + ", routingTable=" + + routingTable + + ", nodes=" + + nodes + + ", metadata=" + + metadata + + ", blocks=" + + blocks + + ", customs=" + + customs + + ", minimumClusterManagerNodesOnPublishingClusterManager=" + + minimumClusterManagerNodesOnPublishingClusterManager + + '}'; } ClusterStateDiff(StreamInput in, DiscoveryNode localNode) throws IOException { diff --git a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java index 9ee2cf3ddd083..dbd454720c69b 100644 --- a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java +++ b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java @@ -273,11 +273,7 @@ public Map getUpserts() { @Override public String toString() { - return "MapDiff{" + - "deletes=" + deletes + - ", diffs=" + diffs + - ", upserts=" + upserts + - '}'; + return "MapDiff{" + "deletes=" + deletes + ", diffs=" + diffs + ", upserts=" + upserts + '}'; } @Override diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java index 3d02d58e26f15..9e7315cb405b2 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java @@ -154,7 +154,9 @@ public void writeToSorted(StreamOutput out) throws IOException { out.writeLong(term); lastCommittedConfiguration.writeToSorted(out); lastAcceptedConfiguration.writeToSorted(out); - List sortedList = votingConfigExclusions.stream().sorted(Comparator.comparing(VotingConfigExclusion::getNodeId)).collect(Collectors.toList()); + List sortedList = votingConfigExclusions.stream() + .sorted(Comparator.comparing(VotingConfigExclusion::getNodeId)) + .collect(Collectors.toList()); out.writeCollection(sortedList); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java index c75e2fff0e47c..c189f621b1960 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java @@ -46,7 +46,6 @@ import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; -import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.Map; @@ -124,16 +123,11 @@ public void writeTo(StreamOutput out) throws IOException { } public void writeToSorted(StreamOutput out) throws IOException { - out.writeVInt(this.componentTemplates.size()); - this.componentTemplates.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEachOrdered(entry-> - { - try { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + out.writeMapOrderedByKey( + this.componentTemplates, + Map.Entry.comparingByKey(), + StreamOutput::writeString, + (stream, val) -> val.writeTo(stream) ); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 547c093a48a8f..d763453bf3aab 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1223,53 +1223,21 @@ public void writeToSorted(StreamOutput out) throws IOException { out.writeByte(state.id()); writeSettingsToStream(settings, out); out.writeVLongArray(primaryTerms); - out.writeVInt(mappings.size()); - mappings.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { - try { - entry.getValue().writeTo(out); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - out.writeVInt(aliases.size()); - aliases.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { - try { - entry.getValue().writeTo(out); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - out.writeVInt(customData.size()); - customData.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { - try { - out.writeString(entry.getKey()); - entry.getValue().writeToSorted(out); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - out.writeVInt(inSyncAllocationIds.size()); - inSyncAllocationIds.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { - try { - out.writeVInt(entry.getKey()); - DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(entry.getValue()), out); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - out.writeVInt(rolloverInfos.size()); - rolloverInfos.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { - try { - entry.getValue().writeTo(out); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - + out.writeMapValuesOrderedByKey(mappings, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); + out.writeMapValuesOrderedByKey(aliases, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); + out.writeMapOrderedByKey( + customData, + Map.Entry.comparingByKey(), + StreamOutput::writeString, + (stream, val) -> val.writeToSorted(stream) + ); + out.writeMapOrderedByKey( + inSyncAllocationIds, + Map.Entry.comparingByKey(), + StreamOutput::writeVInt, + (stream, val) -> DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(val), stream) + ); + out.writeMapValuesOrderedByKey(rolloverInfos, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); out.writeBoolean(isSystem); } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 166719acade02..6ae13378560a4 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -47,7 +47,6 @@ import java.io.IOException; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Locale; @@ -387,16 +386,9 @@ public void writeToSorted(StreamOutput out) throws IOException { out.writeString(hostName); out.writeString(hostAddress); address.writeTo(out); - out.writeVInt(attributes.size()); - attributes.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { - try { - out.writeString(entry.getKey()); - out.writeString(entry.getValue()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + out.writeMapOrderedByKey(attributes, Map.Entry.comparingByKey(), StreamOutput::writeString, StreamOutput::writeString); + out.writeVInt(roles.size()); for (final DiscoveryNodeRole role : roles) { final DiscoveryNodeRole compatibleRole = role.getCompatibilityRole(out.getVersion()); diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index fc2a65929fbfd..dab2ee1d4d33f 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -47,9 +47,7 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -57,8 +55,6 @@ import java.util.Map; import java.util.Objects; import java.util.Spliterators; -import java.util.TreeMap; -import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -711,15 +707,7 @@ public void writeToSorted(StreamOutput out) throws IOException { out.writeBoolean(true); out.writeString(clusterManagerNodeId); } - out.writeVInt(nodes.size()); - - TreeSet sortedNodes = new TreeSet<>(Comparator.comparing(DiscoveryNode::getId)); - sortedNodes.addAll(nodes.values()); - - for (DiscoveryNode node : sortedNodes) { - node.writeToSorted(out); - } - + out.writeMapValuesOrderedByKey(nodes, Map.Entry.comparingByKey(), (stream, val) -> val.writeToSorted(stream)); } public static DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws IOException { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index 13b755d9d20d0..15e6ec06ee861 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -380,15 +380,13 @@ public void writeTo(StreamOutput out) throws IOException { public void writeToSorted(StreamOutput out) throws IOException { index.writeTo(out); out.writeVInt(shards.size()); - shards.values().stream().sorted(Comparator.comparing(IndexShardRoutingTable::getShardId)).forEach(indexShard -> - { - try { - IndexShardRoutingTable.Builder.writeToSorted(indexShard, out); - } catch (IOException e) { - throw new RuntimeException(e); - } + shards.values().stream().sorted(Comparator.comparing(IndexShardRoutingTable::getShardId)).forEach(indexShard -> { + try { + IndexShardRoutingTable.Builder.writeToSorted(indexShard, out); + } catch (IOException e) { + throw new RuntimeException(e); } - ); + }); } public static Builder builder(Index index) { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index 4af6635a24a58..1e078bc40ebc4 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -1138,14 +1138,17 @@ public static void writeToThin(IndexShardRoutingTable indexShard, StreamOutput o public static void writeToSorted(IndexShardRoutingTable indexShard, StreamOutput out) throws IOException { out.writeVInt(indexShard.shardId.id()); out.writeVInt(indexShard.shards.size()); - //Order allocated shards by allocationId - indexShard.shards.stream().filter(shardRouting -> shardRouting.allocationId()!=null).sorted(Comparator.comparing(o -> o.allocationId().getId())).forEach(shardRouting -> { - try { - shardRouting.writeToThin(out); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + // Order allocated shards by allocationId + indexShard.shards.stream() + .filter(shardRouting -> shardRouting.allocationId() != null) + .sorted(Comparator.comparing(o -> o.allocationId().getId())) + .forEach(shardRouting -> { + try { + shardRouting.writeToThin(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 430e9fffad015..53936f78122cf 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -437,10 +437,7 @@ private static class RoutingTableDiff implements Diff { @Override public String toString() { - return "RoutingTableDiff{" + - "version=" + version + - ", indicesRouting=" + indicesRouting + - '}'; + return "RoutingTableDiff{" + "version=" + version + ", indicesRouting=" + indicesRouting + '}'; } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index f2acded5ef89e..d7c4eaaa094eb 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -10,10 +10,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.DiffableStringMap; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; @@ -27,13 +25,9 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.TreeSet; import com.jcraft.jzlib.JZlib; @@ -88,7 +82,7 @@ public ClusterStateChecksum(ClusterState clusterState) { clusterState.coordinationMetadata().writeToSorted(checksumOut); coordinationMetadataChecksum = checksumOut.getChecksum(); - //Settings create sortedMap by default, so no explicit sorting required here. + // Settings create sortedMap by default, so no explicit sorting required here. checksumOut.reset(); Settings.writeSettingsToStream(clusterState.metadata().persistentSettings(), checksumOut); settingMetadataChecksum = checksumOut.getChecksum(); @@ -341,20 +335,32 @@ public int hashCode() { @Override public String toString() { - return "ClusterStateChecksum{" + - "routingTableChecksum=" + routingTableChecksum + - ", nodesChecksum=" + nodesChecksum + - ", blocksChecksum=" + blocksChecksum + - ", clusterStateCustomsChecksum=" + clusterStateCustomsChecksum + - ", coordinationMetadataChecksum=" + coordinationMetadataChecksum + - ", settingMetadataChecksum=" + settingMetadataChecksum + - ", transientSettingsMetadataChecksum=" + transientSettingsMetadataChecksum + - ", templatesMetadataChecksum=" + templatesMetadataChecksum + - ", customMetadataMapChecksum=" + customMetadataMapChecksum + - ", hashesOfConsistentSettingsChecksum=" + hashesOfConsistentSettingsChecksum + - ", indicesChecksum=" + indicesChecksum + - ", clusterStateChecksum=" + clusterStateChecksum + - '}'; + return "ClusterStateChecksum{" + + "routingTableChecksum=" + + routingTableChecksum + + ", nodesChecksum=" + + nodesChecksum + + ", blocksChecksum=" + + blocksChecksum + + ", clusterStateCustomsChecksum=" + + clusterStateCustomsChecksum + + ", coordinationMetadataChecksum=" + + coordinationMetadataChecksum + + ", settingMetadataChecksum=" + + settingMetadataChecksum + + ", transientSettingsMetadataChecksum=" + + transientSettingsMetadataChecksum + + ", templatesMetadataChecksum=" + + templatesMetadataChecksum + + ", customMetadataMapChecksum=" + + customMetadataMapChecksum + + ", hashesOfConsistentSettingsChecksum=" + + hashesOfConsistentSettingsChecksum + + ", indicesChecksum=" + + indicesChecksum + + ", clusterStateChecksum=" + + clusterStateChecksum + + '}'; } public List getMismatchEntities(ClusterStateChecksum otherClusterStateChecksum) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index d3624e76b27e1..2e0d92f8a6eb4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -20,7 +20,6 @@ import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.metadata.Manifest; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.Metadata.XContentContext; import org.opensearch.cluster.metadata.TemplatesMetadata; @@ -1472,102 +1471,170 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C .build(); if (checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { - validateClusterStateFromChecksum(manifest, clusterState, previousState.getClusterName().value(), localNodeId,false); + validateClusterStateFromChecksum(manifest, clusterState, previousState.getClusterName().value(), localNodeId, false); } return clusterState; } - void validateClusterStateFromChecksum(ClusterMetadataManifest manifest, ClusterState clusterState,String clusterName, String localNodeId, boolean isFullStateDownload) throws IOException { + void validateClusterStateFromChecksum( + ClusterMetadataManifest manifest, + ClusterState clusterState, + String clusterName, + String localNodeId, + boolean isFullStateDownload + ) throws IOException { ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState); - List failedValidation = newClusterStateChecksum.getMismatchEntities(manifest.getClusterStateChecksum()); - if(!failedValidation.isEmpty()) { - logger.error( - () -> new ParameterizedMessage( - "Cluster state checksums do not match. Checksum from manifest {}, checksum from created cluster state {}. Entities failing validation {}", - manifest.getClusterStateChecksum(), - newClusterStateChecksum, - failedValidation - ) - ); - if(isFullStateDownload) { - throw new IllegalStateException("Cluster state checksums do not match during full state read. Validation failed for " + failedValidation); - } - //download full cluster state and match against state created for the failing entities - ClusterState fullClusterState = readClusterStateInParallel( - ClusterState.builder(new ClusterName(clusterName)).build(), - manifest, - manifest.getClusterUUID(), - localNodeId, - manifest.getIndices(), - manifest.getCustomMetadataMap(), - manifest.getCoordinationMetadata() != null, - manifest.getSettingsMetadata() != null, - manifest.getTransientSettingsMetadata() != null, - manifest.getTemplatesMetadata() != null, - manifest.getDiscoveryNodesMetadata() != null, - manifest.getClusterBlocksMetadata() != null, - manifest.getIndicesRouting(), - manifest.getHashesOfConsistentSettings() != null, - manifest.getClusterStateCustomMap() , - false, - true + List failedValidation = newClusterStateChecksum.getMismatchEntities(manifest.getClusterStateChecksum()); + if (!failedValidation.isEmpty()) { + logger.error( + () -> new ParameterizedMessage( + "Cluster state checksums do not match. Checksum from manifest {}, checksum from created cluster state {}. Entities failing validation {}", + manifest.getClusterStateChecksum(), + newClusterStateChecksum, + failedValidation + ) + ); + if (isFullStateDownload) { + throw new IllegalStateException( + "Cluster state checksums do not match during full state read. Validation failed for " + failedValidation ); - for(String failedEntity : failedValidation) { - switch (failedEntity) { - case ClusterStateChecksum.ROUTING_TABLE_CS: - Diff routingTableDiff = fullClusterState.routingTable().diff(clusterState.routingTable()); - logger.error(() -> new ParameterizedMessage("Failing Diff in routing table {}", routingTableDiff)); - break; - case ClusterStateChecksum.NODES_CS: - logger.error(() -> new ParameterizedMessage("Failing Diff in discovery nodes {}", - fullClusterState.nodes().diff(clusterState.nodes()))); - break; - case ClusterStateChecksum.BLOCKS_CS: - logger.error(() -> new ParameterizedMessage("Failing Diff in cluster blocks {}", - fullClusterState.blocks().diff(clusterState.blocks()))); - break; - case ClusterStateChecksum.CUSTOMS_CS: - logger.error(() -> new ParameterizedMessage("Failing Diff in cluster state customs {}", - DiffableUtils.diff(clusterState.customs(), fullClusterState.customs(), DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER))); - break; - case ClusterStateChecksum.COORDINATION_MD_CS : - logger.error(() -> new ParameterizedMessage("Failing Diff in coordination md. current md {}, full state md {}", - clusterState.metadata().coordinationMetadata(), fullClusterState.metadata().coordinationMetadata())); - break; - case ClusterStateChecksum.TRANSIENT_SETTINGS_MD_CS: - logger.error(() -> new ParameterizedMessage("Failing Diff in transient settings md. current md {}, full state md {}", - clusterState.metadata().transientSettings(), fullClusterState.metadata().transientSettings())); - - break; - case ClusterStateChecksum.SETTINGS_MD_CS: - logger.error(() -> new ParameterizedMessage("Failing Diff in settings md. current md {}, full state md {}", - clusterState.metadata().settings(), fullClusterState.metadata().settings())); - - break; - case ClusterStateChecksum.HASHES_MD_CS: - logger.error(() -> new ParameterizedMessage("Failing Diff in hashes md {}", - ((DiffableStringMap)fullClusterState.metadata().hashesOfConsistentSettings()).diff((DiffableStringMap)clusterState.metadata().hashesOfConsistentSettings()))); - break; - case ClusterStateChecksum.TEMPLATES_MD_CS: - logger.error(() -> new ParameterizedMessage("Failing Diff in templates md{}", - fullClusterState.metadata().templatesMetadata().diff(clusterState.metadata().templatesMetadata()))); - break; - case ClusterStateChecksum.CUSTOM_MD_CS: - logger.error(() -> new ParameterizedMessage("Failing Diff in customs md {}", - DiffableUtils.diff(clusterState.metadata().customs(), fullClusterState.metadata().customs(), DiffableUtils.getStringKeySerializer(), Metadata.CUSTOM_VALUE_SERIALIZER))); - break; - case ClusterStateChecksum.INDICES_CS: - logger.error(() -> new ParameterizedMessage("Failing Diff in index md {}", - DiffableUtils.diff(clusterState.metadata().indices(), fullClusterState.metadata().indices(), DiffableUtils.getStringKeySerializer()))); - break; - default: - logger.error(() -> new ParameterizedMessage("Unknown failed entity {}", - failedEntity)); - break; - } + } + // download full cluster state and match against state created for the failing entities + ClusterState fullClusterState = readClusterStateInParallel( + ClusterState.builder(new ClusterName(clusterName)).build(), + manifest, + manifest.getClusterUUID(), + localNodeId, + manifest.getIndices(), + manifest.getCustomMetadataMap(), + manifest.getCoordinationMetadata() != null, + manifest.getSettingsMetadata() != null, + manifest.getTransientSettingsMetadata() != null, + manifest.getTemplatesMetadata() != null, + manifest.getDiscoveryNodesMetadata() != null, + manifest.getClusterBlocksMetadata() != null, + manifest.getIndicesRouting(), + manifest.getHashesOfConsistentSettings() != null, + manifest.getClusterStateCustomMap(), + false, + true + ); + for (String failedEntity : failedValidation) { + switch (failedEntity) { + case ClusterStateChecksum.ROUTING_TABLE_CS: + Diff routingTableDiff = fullClusterState.routingTable().diff(clusterState.routingTable()); + logger.error(() -> new ParameterizedMessage("Failing Diff in routing table {}", routingTableDiff)); + break; + case ClusterStateChecksum.NODES_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in discovery nodes {}", + fullClusterState.nodes().diff(clusterState.nodes()) + ) + ); + break; + case ClusterStateChecksum.BLOCKS_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in cluster blocks {}", + fullClusterState.blocks().diff(clusterState.blocks()) + ) + ); + break; + case ClusterStateChecksum.CUSTOMS_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in cluster state customs {}", + DiffableUtils.diff( + clusterState.customs(), + fullClusterState.customs(), + DiffableUtils.getStringKeySerializer(), + CUSTOM_VALUE_SERIALIZER + ) + ) + ); + break; + case ClusterStateChecksum.COORDINATION_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in coordination md. current md {}, full state md {}", + clusterState.metadata().coordinationMetadata(), + fullClusterState.metadata().coordinationMetadata() + ) + ); + break; + case ClusterStateChecksum.TRANSIENT_SETTINGS_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in transient settings md. current md {}, full state md {}", + clusterState.metadata().transientSettings(), + fullClusterState.metadata().transientSettings() + ) + ); + + break; + case ClusterStateChecksum.SETTINGS_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in settings md. current md {}, full state md {}", + clusterState.metadata().settings(), + fullClusterState.metadata().settings() + ) + ); + + break; + case ClusterStateChecksum.HASHES_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in hashes md {}", + ((DiffableStringMap) fullClusterState.metadata().hashesOfConsistentSettings()).diff( + (DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings() + ) + ) + ); + break; + case ClusterStateChecksum.TEMPLATES_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in templates md{}", + fullClusterState.metadata().templatesMetadata().diff(clusterState.metadata().templatesMetadata()) + ) + ); + break; + case ClusterStateChecksum.CUSTOM_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in customs md {}", + DiffableUtils.diff( + clusterState.metadata().customs(), + fullClusterState.metadata().customs(), + DiffableUtils.getStringKeySerializer(), + Metadata.CUSTOM_VALUE_SERIALIZER + ) + ) + ); + break; + case ClusterStateChecksum.INDICES_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in index md {}", + DiffableUtils.diff( + clusterState.metadata().indices(), + fullClusterState.metadata().indices(), + DiffableUtils.getStringKeySerializer() + ) + ) + ); + break; + default: + logger.error(() -> new ParameterizedMessage("Unknown failed entity {}", failedEntity)); + break; } - throw new IllegalStateException("Cluster state checksums do not match during diff read. Validation failed for " + failedValidation); } + throw new IllegalStateException( + "Cluster state checksums do not match during diff read. Validation failed for " + failedValidation + ); + } } /** diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java index 12dc1cf18c055..c5955f5d06070 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java @@ -122,15 +122,8 @@ public void testGetMismatchEntitiesUnorderedInput() { DiscoveryNode node2 = DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9201), "node2"); DiscoveryNode node3 = DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9202), "node3"); - DiscoveryNodes nodes1 = DiscoveryNodes.builder().clusterManagerNodeId("test-node") - .add(node1) - .add(node2) - .add(node3) - .build(); - DiscoveryNodes nodes2 = DiscoveryNodes.builder().clusterManagerNodeId("test-node") - .add(node2) - .add(node3) - .build(); + DiscoveryNodes nodes1 = DiscoveryNodes.builder().clusterManagerNodeId("test-node").add(node1).add(node2).add(node3).build(); + DiscoveryNodes nodes2 = DiscoveryNodes.builder().clusterManagerNodeId("test-node").add(node2).add(node3).build(); nodes2 = nodes2.newNode(node1); ClusterState state2 = ClusterState.builder(state1).nodes(nodes1).build(); ClusterState state3 = ClusterState.builder(state1).nodes(nodes2).build(); @@ -139,7 +132,7 @@ public void testGetMismatchEntitiesUnorderedInput() { ClusterStateChecksum checksum2 = new ClusterStateChecksum(state3); assertEquals(checksum2, checksum1); } - + private ClusterState generateClusterState() { final Index index = new Index("test-index", "index-uuid"); final Settings idxSettings = Settings.builder() diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index eeda9d84ca4c2..2e168553c090e 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -3048,7 +3048,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullC eq(false), eq(true) ); - //verify(mockService, times(0)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + // verify(mockService, times(0)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); } public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws IOException { @@ -3081,7 +3081,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws eq(true) ); mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); - //verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + // verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); } public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMismatch() throws IOException { @@ -3118,7 +3118,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma IllegalStateException.class, () -> mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true) ); - //verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + // verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); } private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { From 48bc77dcd8f6213b74e8e66b43966ce63267dd64 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 27 Aug 2024 15:07:34 +0530 Subject: [PATCH 09/29] templates md ordering Signed-off-by: Himshikha Gupta --- .../cluster/metadata/IndexTemplateMetadata.java | 11 +++++++++++ .../cluster/metadata/TemplatesMetadata.java | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java index 3d532208bcfe2..08d93f38c9d87 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java @@ -257,6 +257,17 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalVInt(version); } + public void writeToSorted(StreamOutput out) throws IOException { + out.writeString(name); + out.writeInt(order); + Collections.sort(patterns); + out.writeStringCollection(patterns); + Settings.writeSettingsToStream(settings, out); + out.writeMapOrderedByKey(mappings, Map.Entry.comparingByKey(), StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + out.writeMapValuesOrderedByKey(aliases, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); + out.writeOptionalVInt(version); + } + @Override public String toString() { try { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index b2f45f0ea4cce..a7b91a701d948 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -70,7 +70,7 @@ public void writeToSorted(StreamOutput out) throws IOException { out.writeVInt(templates.size()); templates.values().stream().sorted(Comparator.comparing(IndexTemplateMetadata::getName)).forEach(template -> { try { - template.writeTo(out); + template.writeToSorted(out); } catch (IOException e) { throw new RuntimeException(e); } From 72fda7619852a9a679a1047e36236317fb53c4aa Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 28 Aug 2024 12:37:25 +0530 Subject: [PATCH 10/29] refactoring sorting methods Signed-off-by: Himshikha Gupta --- .../core/common/io/stream/StreamOutput.java | 20 ++++++++++++++---- .../coordination/CoordinationMetadata.java | 9 ++------ .../metadata/ComponentTemplateMetadata.java | 2 +- .../cluster/metadata/IndexMetadata.java | 13 ++++++------ .../metadata/IndexTemplateMetadata.java | 8 ++++--- .../opensearch/cluster/metadata/Template.java | 21 +++++++++++++++++++ .../cluster/metadata/TemplatesMetadata.java | 9 +------- .../cluster/node/DiscoveryNode.java | 2 +- .../cluster/node/DiscoveryNodes.java | 2 +- .../cluster/routing/IndexRoutingTable.java | 10 +-------- .../routing/IndexShardRoutingTable.java | 4 +++- .../cluster/routing/RoutingTable.java | 10 +-------- .../opensearch/common/settings/Settings.java | 4 ++++ 13 files changed, 64 insertions(+), 50 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java index b69c8fcae88c8..446422db5b63f 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java @@ -87,6 +87,7 @@ import java.util.Map; import java.util.Set; import java.util.function.IntFunction; +import java.util.stream.Collectors; /** * A stream from another node to this node. Technically, it can also be streamed from a byte array but that is mostly for testing. @@ -549,6 +550,11 @@ public void writeStringArray(String[] array) throws IOException { } } + public void writeStringArrayOrdered(String[] array) throws IOException { + Arrays.sort(array); + writeStringArray(array); + } + /** * Writes a string array, for nullable string, writes it as 0 (empty string). */ @@ -642,7 +648,7 @@ public final void writeMap(final Map map, final Writer keyWriter } } - public final void writeMapOrderedByKey(final Map map, Comparator> entryComparator, final Writer keyWriter, final Writer valueWriter) throws IOException { + public final void writeMapOrdered(final Map map, Comparator> entryComparator, final Writer keyWriter, final Writer valueWriter) throws IOException { writeVInt(map.size()); map.entrySet().stream().sorted(entryComparator).forEachOrdered(entry-> { @@ -650,20 +656,20 @@ public final void writeMapOrderedByKey(final Map map, Comparator void writeMapValuesOrderedByKey(final Map map, Comparator> entryComparator, final Writer valueWriter) throws IOException { + public final void writeMapValuesOrdered(final Map map, Comparator> entryComparator, final Writer valueWriter) throws IOException { writeVInt(map.size()); map.entrySet().stream().sorted(entryComparator).forEachOrdered(entry-> { try { valueWriter.write(this, entry.getValue()); } catch (IOException e) { - throw new RuntimeException(e); + throw new RuntimeException("Failed to write map values.", e); } } ); @@ -1205,6 +1211,12 @@ public void writeCollection(final Collection collection) th writeCollection(collection, (o, v) -> v.writeTo(o)); } + public void writeCollectionOrdered(final Collection collection, Comparator comparator, final Writer writer) throws IOException { + List sortedList = collection.stream().sorted(comparator).collect(Collectors.toList()); + writeCollection(sortedList, writer); + } + + /** * Writes a list of {@link Writeable} objects */ diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java index 9e7315cb405b2..90e6d6f577184 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java @@ -154,10 +154,7 @@ public void writeToSorted(StreamOutput out) throws IOException { out.writeLong(term); lastCommittedConfiguration.writeToSorted(out); lastAcceptedConfiguration.writeToSorted(out); - List sortedList = votingConfigExclusions.stream() - .sorted(Comparator.comparing(VotingConfigExclusion::getNodeId)) - .collect(Collectors.toList()); - out.writeCollection(sortedList); + out.writeCollectionOrdered(votingConfigExclusions, Comparator.comparing(VotingConfigExclusion::getNodeId), (o, v) -> v.writeTo(o)); } @Override @@ -403,9 +400,7 @@ public void writeTo(StreamOutput out) throws IOException { } public void writeToSorted(StreamOutput out) throws IOException { - String[] nodeIds = this.nodeIds.toArray(new String[0]); - Arrays.sort(nodeIds); - out.writeStringArray(nodeIds); + out.writeStringArrayOrdered(nodeIds.toArray(new String[0])); } public boolean hasQuorum(Collection votes) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java index c189f621b1960..a762d4ebd47e1 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java @@ -123,7 +123,7 @@ public void writeTo(StreamOutput out) throws IOException { } public void writeToSorted(StreamOutput out) throws IOException { - out.writeMapOrderedByKey( + out.writeMapOrdered( this.componentTemplates, Map.Entry.comparingByKey(), StreamOutput::writeString, diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index d763453bf3aab..ac39c37412a10 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -96,6 +96,7 @@ import static org.opensearch.cluster.node.DiscoveryNodeFilters.OpType.OR; import static org.opensearch.common.settings.Settings.readSettingsFromStream; import static org.opensearch.common.settings.Settings.writeSettingsToStream; +import static org.opensearch.common.settings.Settings.writeSettingsToStreamSorted; /** * Index metadata information @@ -1221,23 +1222,23 @@ public void writeToSorted(StreamOutput out) throws IOException { out.writeVLong(aliasesVersion); out.writeInt(routingNumShards); out.writeByte(state.id()); - writeSettingsToStream(settings, out); + writeSettingsToStreamSorted(settings, out); out.writeVLongArray(primaryTerms); - out.writeMapValuesOrderedByKey(mappings, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); - out.writeMapValuesOrderedByKey(aliases, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); - out.writeMapOrderedByKey( + out.writeMapValuesOrdered(mappings, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); + out.writeMapValuesOrdered(aliases, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); + out.writeMapOrdered( customData, Map.Entry.comparingByKey(), StreamOutput::writeString, (stream, val) -> val.writeToSorted(stream) ); - out.writeMapOrderedByKey( + out.writeMapOrdered( inSyncAllocationIds, Map.Entry.comparingByKey(), StreamOutput::writeVInt, (stream, val) -> DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(val), stream) ); - out.writeMapValuesOrderedByKey(rolloverInfos, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); + out.writeMapValuesOrdered(rolloverInfos, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); out.writeBoolean(isSystem); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java index 08d93f38c9d87..ed1fd99f3f7f1 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java @@ -260,11 +260,13 @@ public void writeTo(StreamOutput out) throws IOException { public void writeToSorted(StreamOutput out) throws IOException { out.writeString(name); out.writeInt(order); + //patterns is an immutable list so we need to copy before sorting. + List patterns = new ArrayList<>(this.patterns); Collections.sort(patterns); out.writeStringCollection(patterns); - Settings.writeSettingsToStream(settings, out); - out.writeMapOrderedByKey(mappings, Map.Entry.comparingByKey(), StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); - out.writeMapValuesOrderedByKey(aliases, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); + Settings.writeSettingsToStreamSorted(settings, out); + out.writeMapOrdered(mappings, Map.Entry.comparingByKey(), StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + out.writeMapValuesOrdered(aliases, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); out.writeOptionalVInt(version); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Template.java b/server/src/main/java/org/opensearch/cluster/metadata/Template.java index bd110c6af8975..d356466d5a85f 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Template.java @@ -156,6 +156,27 @@ public void writeTo(StreamOutput out) throws IOException { } } + public void writeToSorted(StreamOutput out) throws IOException { + if (this.settings == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + Settings.writeSettingsToStreamSorted(this.settings, out); + } + if (this.mappings == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + this.mappings.writeTo(out); + } + if (this.aliases == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeMapOrdered(this.aliases, Map.Entry.comparingByKey(), StreamOutput::writeString, (stream, aliasMetadata) -> aliasMetadata.writeTo(stream)); + } + } + @Override public int hashCode() { return Objects.hash(settings, mappings, aliases); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index a7b91a701d948..76149f54dfade 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -17,7 +17,6 @@ import java.io.IOException; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -68,13 +67,7 @@ public void writeTo(StreamOutput out) throws IOException { public void writeToSorted(StreamOutput out) throws IOException { out.writeVInt(templates.size()); - templates.values().stream().sorted(Comparator.comparing(IndexTemplateMetadata::getName)).forEach(template -> { - try { - template.writeToSorted(out); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + out.writeMapValuesOrdered(templates, Map.Entry.comparingByKey(), (stream, value) -> value.writeToSorted(stream)); } @Override diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 6ae13378560a4..bc5cec7817700 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -387,7 +387,7 @@ public void writeToSorted(StreamOutput out) throws IOException { out.writeString(hostAddress); address.writeTo(out); - out.writeMapOrderedByKey(attributes, Map.Entry.comparingByKey(), StreamOutput::writeString, StreamOutput::writeString); + out.writeMapOrdered(attributes, Map.Entry.comparingByKey(), StreamOutput::writeString, StreamOutput::writeString); out.writeVInt(roles.size()); for (final DiscoveryNodeRole role : roles) { diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index dab2ee1d4d33f..6b54d0a87f62e 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -707,7 +707,7 @@ public void writeToSorted(StreamOutput out) throws IOException { out.writeBoolean(true); out.writeString(clusterManagerNodeId); } - out.writeMapValuesOrderedByKey(nodes, Map.Entry.comparingByKey(), (stream, val) -> val.writeToSorted(stream)); + out.writeMapValuesOrdered(nodes, Map.Entry.comparingByKey(), (stream, val) -> val.writeToSorted(stream)); } public static DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws IOException { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index 15e6ec06ee861..380c707b69b84 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -53,7 +53,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -379,14 +378,7 @@ public void writeTo(StreamOutput out) throws IOException { public void writeToSorted(StreamOutput out) throws IOException { index.writeTo(out); - out.writeVInt(shards.size()); - shards.values().stream().sorted(Comparator.comparing(IndexShardRoutingTable::getShardId)).forEach(indexShard -> { - try { - IndexShardRoutingTable.Builder.writeToSorted(indexShard, out); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + out.writeMapValuesOrdered(shards, Map.Entry.comparingByKey(), (stream, value) -> IndexShardRoutingTable.Builder.writeToSorted(value, stream)); } public static Builder builder(Index index) { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index 1e078bc40ebc4..37d7e69f7855b 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.Nullable; @@ -1146,7 +1147,8 @@ public static void writeToSorted(IndexShardRoutingTable indexShard, StreamOutput try { shardRouting.writeToThin(out); } catch (IOException e) { - throw new RuntimeException(e); + logger.error(() -> new ParameterizedMessage("Failed to write shard {}. Exception {}", indexShard, e)); + throw new RuntimeException("Failed to write IndexShardRoutingTable", e); } }); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 53936f78122cf..517ad279d54d8 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -52,7 +52,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -406,14 +405,7 @@ public void writeTo(StreamOutput out) throws IOException { public void writeToSorted(StreamOutput out) throws IOException { out.writeLong(version); - out.writeVInt(indicesRouting.size()); - indicesRouting.values().stream().sorted(Comparator.comparing(index -> index.getIndex().getName())).forEach(index -> { - try { - index.writeToSorted(out); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + out.writeMapValuesOrdered(indicesRouting, Map.Entry.comparingByKey(), (stream, value) -> value.writeToSorted(stream)); } private static class RoutingTableDiff implements Diff { diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index 9da47ff3aa700..36f44d745704c 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -589,6 +589,10 @@ public static void writeSettingsToStream(Settings settings, StreamOutput out) th } } + public static void writeSettingsToStreamSorted(Settings settings, StreamOutput out) throws IOException { + out.writeMapOrdered(settings.settings, Map.Entry.comparingByKey(), StreamOutput::writeString, StreamOutput::writeGenericValue); + } + /** * Returns a builder to be used in order to build settings. */ From d9bceb9824f8038e97bfaca915aea48e7833c7fa Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 28 Aug 2024 17:30:14 +0530 Subject: [PATCH 11/29] refactoring Signed-off-by: Himshikha Gupta --- .../stream/BufferedChecksumStreamOutput.java | 93 +++++++++++++++++++ .../core/common/io/stream/StreamOutput.java | 41 +------- .../common/io/stream/VerifiableWriteable.java | 15 +++ .../coordination/CoordinationMetadata.java | 15 +-- .../cluster/metadata/ComponentTemplate.java | 11 --- .../metadata/ComponentTemplateMetadata.java | 9 -- .../cluster/metadata/DiffableStringMap.java | 4 - .../cluster/metadata/IndexMetadata.java | 25 ++--- .../metadata/IndexTemplateMetadata.java | 15 ++- .../opensearch/cluster/metadata/Template.java | 21 ----- .../cluster/metadata/TemplatesMetadata.java | 9 +- .../cluster/node/DiscoveryNode.java | 33 +++---- .../cluster/node/DiscoveryNodes.java | 19 ++-- .../cluster/routing/IndexRoutingTable.java | 7 +- .../routing/IndexShardRoutingTable.java | 2 +- .../cluster/routing/RoutingTable.java | 8 +- .../opensearch/common/settings/Settings.java | 4 - .../gateway/remote/ClusterStateChecksum.java | 36 ++----- 18 files changed, 184 insertions(+), 183 deletions(-) create mode 100644 libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index 422f956c0cd47..f9bdb361f6608 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -33,9 +33,20 @@ package org.opensearch.core.common.io.stream; import org.apache.lucene.store.BufferedChecksum; +import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; import java.util.zip.CRC32; import java.util.zip.Checksum; @@ -90,4 +101,86 @@ public void reset() throws IOException { public void resetDigest() { digest.reset(); } + + @Override + public void writeMap(@Nullable Map map) throws IOException { + Map newMap = new TreeMap<>(map); + writeGenericValue(newMap); + } + + @Override + public void writeMap(final Map map, final Writeable.Writer keyWriter, final Writeable.Writer valueWriter) throws IOException { + writeVInt(map.size()); + map.entrySet().stream().sorted(Comparator.comparing(o -> ( o.getKey().hashCode()))).forEachOrdered(entry-> + { + try { + keyWriter.write(this, entry.getKey()); + valueWriter.write(this, entry.getValue()); + } catch (IOException e) { + throw new RuntimeException("Failed to write map.", e); + } + } + ); + } + + /** + * + * @param map + * @param valueWriter + * @param + * @param + * @throws IOException + */ + @Override + public void writeMapValues(final Map map, final Writeable.Writer valueWriter) throws IOException { + writeVInt(map.size()); + map.entrySet().stream().sorted(Comparator.comparing(o -> ( o.getKey().hashCode()))).forEachOrdered(entry-> + { + try { + valueWriter.write(this, entry.getValue()); + } catch (IOException e) { + throw new RuntimeException("Failed to write map values.", e); + } + } + ); + } + + @Override + public void writeStringArray(String[] array) throws IOException { + Arrays.sort(array); + super.writeStringArray(array); + } + + @Override + public void writeCollection(final Collection collection) throws IOException { + List sortedList = collection.stream().sorted(Comparator.comparing(Object::hashCode)).collect(Collectors.toList()); + writeCollection(sortedList, (o, v) -> v.writeTo(o)); + } + + @Override + public void writeStringCollection(final Collection collection) throws IOException { + List listCollection = new ArrayList<>(collection); + Collections.sort(listCollection); + writeCollection(listCollection, StreamOutput::writeString); + } + + @Override + public void writeList(List list) throws IOException { + //copy to new to handle unmodifiable lists + List newList = new ArrayList<>(list); + newList.sort(Comparator.comparing(Object::hashCode)); + writeCollection(newList); + } + + @Override + public void writeOptionalStringCollection(final Collection collection) throws IOException { + if (collection != null) { + List listCollection = new ArrayList<>(collection); + Collections.sort(listCollection); + writeBoolean(true); + writeCollection(listCollection, StreamOutput::writeString); + } else { + writeBoolean(false); + } + } } diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java index 446422db5b63f..3358b39f8a782 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java @@ -550,11 +550,6 @@ public void writeStringArray(String[] array) throws IOException { } } - public void writeStringArrayOrdered(String[] array) throws IOException { - Arrays.sort(array); - writeStringArray(array); - } - /** * Writes a string array, for nullable string, writes it as 0 (empty string). */ @@ -640,7 +635,7 @@ public final void writeMapOfLists(final Map> map, final Writer * @param keyWriter The key writer * @param valueWriter The value writer */ - public final void writeMap(final Map map, final Writer keyWriter, final Writer valueWriter) throws IOException { + public void writeMap(final Map map, final Writer keyWriter, final Writer valueWriter) throws IOException { writeVInt(map.size()); for (final Map.Entry entry : map.entrySet()) { keyWriter.write(this, entry.getKey()); @@ -648,31 +643,11 @@ public final void writeMap(final Map map, final Writer keyWriter } } - public final void writeMapOrdered(final Map map, Comparator> entryComparator, final Writer keyWriter, final Writer valueWriter) throws IOException { - writeVInt(map.size()); - map.entrySet().stream().sorted(entryComparator).forEachOrdered(entry-> - { - try { - keyWriter.write(this, entry.getKey()); - valueWriter.write(this, entry.getValue()); - } catch (IOException e) { - throw new RuntimeException("Failed to write map.", e); - } - } - ); - } - - public final void writeMapValuesOrdered(final Map map, Comparator> entryComparator, final Writer valueWriter) throws IOException { + public void writeMapValues(final Map map, final Writer valueWriter) throws IOException { writeVInt(map.size()); - map.entrySet().stream().sorted(entryComparator).forEachOrdered(entry-> - { - try { - valueWriter.write(this, entry.getValue()); - } catch (IOException e) { - throw new RuntimeException("Failed to write map values.", e); - } - } - ); + for (final Map.Entry entry : map.entrySet()) { + valueWriter.write(this, entry.getValue()); + } } /** @@ -1211,12 +1186,6 @@ public void writeCollection(final Collection collection) th writeCollection(collection, (o, v) -> v.writeTo(o)); } - public void writeCollectionOrdered(final Collection collection, Comparator comparator, final Writer writer) throws IOException { - List sortedList = collection.stream().sorted(comparator).collect(Collectors.toList()); - writeCollection(sortedList, writer); - } - - /** * Writes a list of {@link Writeable} objects */ diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java new file mode 100644 index 0000000000000..cda69980231cc --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.core.common.io.stream; + +import java.io.IOException; + +public interface VerifiableWriteable extends Writeable { + void writeVerifiableTo(StreamOutput out) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java index 90e6d6f577184..2b1f2b220e754 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java @@ -37,6 +37,7 @@ import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; @@ -60,7 +61,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class CoordinationMetadata implements Writeable, ToXContentFragment { +public class CoordinationMetadata implements VerifiableWriteable, ToXContentFragment { public static final CoordinationMetadata EMPTY_METADATA = builder().build(); @@ -150,11 +151,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(votingConfigExclusions); } - public void writeToSorted(StreamOutput out) throws IOException { - out.writeLong(term); - lastCommittedConfiguration.writeToSorted(out); - lastAcceptedConfiguration.writeToSorted(out); - out.writeCollectionOrdered(votingConfigExclusions, Comparator.comparing(VotingConfigExclusion::getNodeId), (o, v) -> v.writeTo(o)); + @Override + public void writeVerifiableTo(StreamOutput out) throws IOException { + writeTo(out); } @Override @@ -399,10 +398,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringArray(nodeIds.toArray(new String[0])); } - public void writeToSorted(StreamOutput out) throws IOException { - out.writeStringArrayOrdered(nodeIds.toArray(new String[0])); - } - public boolean hasQuorum(Collection votes) { final HashSet intersection = new HashSet<>(nodeIds); intersection.retainAll(votes); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java index 54658d8304d5b..abc3712ee07e3 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java @@ -133,17 +133,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - public void writeToSorted(StreamOutput out) throws IOException { - this.template.writeTo(out); - out.writeOptionalVLong(this.version); - if (this.metadata == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - out.writeMap(this.metadata); - } - } - @Override public int hashCode() { return Objects.hash(template, version, metadata); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java index a762d4ebd47e1..d19743e643d12 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java @@ -122,15 +122,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(this.componentTemplates, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); } - public void writeToSorted(StreamOutput out) throws IOException { - out.writeMapOrdered( - this.componentTemplates, - Map.Entry.comparingByKey(), - StreamOutput::writeString, - (stream, val) -> val.writeTo(stream) - ); - } - public static ComponentTemplateMetadata fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java index da81c2275109e..5865891c8a7f9 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java @@ -81,10 +81,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap((Map) (Map) innerMap); } - public void writeToSorted(StreamOutput out) throws IOException { - out.writeMapWithConsistentOrder((Map) (Map) innerMap); - } - @Override public Diff diff(DiffableStringMap previousState) { return new DiffableStringMapDiff(previousState, this); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index ac39c37412a10..78a2165eb5e69 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -56,6 +56,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; @@ -96,7 +97,6 @@ import static org.opensearch.cluster.node.DiscoveryNodeFilters.OpType.OR; import static org.opensearch.common.settings.Settings.readSettingsFromStream; import static org.opensearch.common.settings.Settings.writeSettingsToStream; -import static org.opensearch.common.settings.Settings.writeSettingsToStreamSorted; /** * Index metadata information @@ -104,7 +104,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class IndexMetadata implements Diffable, ToXContentFragment { +public class IndexMetadata implements Diffable, ToXContentFragment, VerifiableWriteable { public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock( 5, @@ -1214,7 +1214,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isSystem); } - public void writeToSorted(StreamOutput out) throws IOException { + @Override + public void writeVerifiableTo(StreamOutput out) throws IOException { out.writeString(index.getName()); // uuid will come as part of settings out.writeLong(version); out.writeVLong(mappingVersion); @@ -1222,23 +1223,17 @@ public void writeToSorted(StreamOutput out) throws IOException { out.writeVLong(aliasesVersion); out.writeInt(routingNumShards); out.writeByte(state.id()); - writeSettingsToStreamSorted(settings, out); + writeSettingsToStream(settings, out); out.writeVLongArray(primaryTerms); - out.writeMapValuesOrdered(mappings, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); - out.writeMapValuesOrdered(aliases, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); - out.writeMapOrdered( - customData, - Map.Entry.comparingByKey(), - StreamOutput::writeString, - (stream, val) -> val.writeToSorted(stream) - ); - out.writeMapOrdered( + out.writeMapValues(mappings,(stream, val) -> val.writeTo(stream)); + out.writeMapValues(aliases,(stream, val) -> val.writeTo(stream)); + out.writeMap(customData, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + out.writeMap( inSyncAllocationIds, - Map.Entry.comparingByKey(), StreamOutput::writeVInt, (stream, val) -> DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(val), stream) ); - out.writeMapValuesOrdered(rolloverInfos, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); + out.writeMapValues(rolloverInfos, (stream, val) -> val.writeTo(stream)); out.writeBoolean(isSystem); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java index ed1fd99f3f7f1..dd59776f77427 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java @@ -46,6 +46,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -67,7 +68,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class IndexTemplateMetadata extends AbstractDiffable { +public class IndexTemplateMetadata extends AbstractDiffable implements VerifiableWriteable { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(IndexTemplateMetadata.class); @@ -257,16 +258,14 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalVInt(version); } - public void writeToSorted(StreamOutput out) throws IOException { + @Override + public void writeVerifiableTo(StreamOutput out) throws IOException { out.writeString(name); out.writeInt(order); - //patterns is an immutable list so we need to copy before sorting. - List patterns = new ArrayList<>(this.patterns); - Collections.sort(patterns); out.writeStringCollection(patterns); - Settings.writeSettingsToStreamSorted(settings, out); - out.writeMapOrdered(mappings, Map.Entry.comparingByKey(), StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); - out.writeMapValuesOrdered(aliases, Map.Entry.comparingByKey(), (stream, val) -> val.writeTo(stream)); + Settings.writeSettingsToStream(settings, out); + out.writeMap(mappings, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); out.writeOptionalVInt(version); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Template.java b/server/src/main/java/org/opensearch/cluster/metadata/Template.java index d356466d5a85f..bd110c6af8975 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Template.java @@ -156,27 +156,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - public void writeToSorted(StreamOutput out) throws IOException { - if (this.settings == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - Settings.writeSettingsToStreamSorted(this.settings, out); - } - if (this.mappings == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - this.mappings.writeTo(out); - } - if (this.aliases == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - out.writeMapOrdered(this.aliases, Map.Entry.comparingByKey(), StreamOutput::writeString, (stream, aliasMetadata) -> aliasMetadata.writeTo(stream)); - } - } - @Override public int hashCode() { return Objects.hash(settings, mappings, aliases); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index 76149f54dfade..d99bacb1765c4 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -11,6 +11,7 @@ import org.opensearch.cluster.AbstractDiffable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -27,7 +28,7 @@ * @opensearch.api */ @PublicApi(since = "2.15.0") -public class TemplatesMetadata extends AbstractDiffable implements ToXContentFragment { +public class TemplatesMetadata extends AbstractDiffable implements ToXContentFragment, VerifiableWriteable { public static TemplatesMetadata EMPTY_METADATA = builder().build(); private final Map templates; @@ -65,9 +66,9 @@ public void writeTo(StreamOutput out) throws IOException { } } - public void writeToSorted(StreamOutput out) throws IOException { - out.writeVInt(templates.size()); - out.writeMapValuesOrdered(templates, Map.Entry.comparingByKey(), (stream, value) -> value.writeToSorted(stream)); + @Override + public void writeVerifiableTo(StreamOutput out) throws IOException { + out.writeMapValues(templates, (stream, value) -> value.writeVerifiableTo(stream)); } @Override diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index bc5cec7817700..2390665d5a627 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -39,6 +39,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.xcontent.ToXContentFragment; @@ -71,7 +72,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class DiscoveryNode implements Writeable, ToXContentFragment { +public class DiscoveryNode implements VerifiableWriteable, ToXContentFragment { static final String COORDINATING_ONLY = "coordinating_only"; @@ -358,37 +359,33 @@ public DiscoveryNode(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(nodeName); - out.writeString(nodeId); - out.writeString(ephemeralId); - out.writeString(hostName); - out.writeString(hostAddress); - address.writeTo(out); + writeNodeDetails(out); + out.writeVInt(attributes.size()); for (Map.Entry entry : attributes.entrySet()) { out.writeString(entry.getKey()); out.writeString(entry.getValue()); } - out.writeVInt(roles.size()); - for (final DiscoveryNodeRole role : roles) { - final DiscoveryNodeRole compatibleRole = role.getCompatibilityRole(out.getVersion()); - out.writeString(compatibleRole.roleName()); - out.writeString(compatibleRole.roleNameAbbreviation()); - out.writeBoolean(compatibleRole.canContainData()); - } - out.writeVersion(version); + writeRolesAndVersion(out); } - public void writeToSorted(StreamOutput out) throws IOException { + @Override + public void writeVerifiableTo(StreamOutput out) throws IOException { + writeNodeDetails(out); + out.writeMap(attributes, StreamOutput::writeString, StreamOutput::writeString); + writeRolesAndVersion(out); + } + + private void writeNodeDetails(StreamOutput out) throws IOException { out.writeString(nodeName); out.writeString(nodeId); out.writeString(ephemeralId); out.writeString(hostName); out.writeString(hostAddress); address.writeTo(out); + } - out.writeMapOrdered(attributes, Map.Entry.comparingByKey(), StreamOutput::writeString, StreamOutput::writeString); - + private void writeRolesAndVersion(StreamOutput out) throws IOException { out.writeVInt(roles.size()); for (final DiscoveryNodeRole role : roles) { final DiscoveryNodeRole compatibleRole = role.getCompatibilityRole(out.getVersion()); diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index 6b54d0a87f62e..e7b1ad30fc63a 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -43,6 +43,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.common.transport.TransportAddress; import java.io.IOException; @@ -66,7 +67,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class DiscoveryNodes extends AbstractDiffable implements Iterable { +public class DiscoveryNodes extends AbstractDiffable implements Iterable, VerifiableWriteable { public static final DiscoveryNodes EMPTY_NODES = builder().build(); @@ -688,26 +689,26 @@ public String shortSummary() { @Override public void writeTo(StreamOutput out) throws IOException { - if (clusterManagerNodeId == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - out.writeString(clusterManagerNodeId); - } + writeClusterManager(out); out.writeVInt(nodes.size()); for (DiscoveryNode node : this) { node.writeTo(out); } } - public void writeToSorted(StreamOutput out) throws IOException { + @Override + public void writeVerifiableTo(StreamOutput out) throws IOException { + writeClusterManager(out); + out.writeMapValues(nodes, (stream, val) -> val.writeVerifiableTo(stream)); + } + + private void writeClusterManager(StreamOutput out) throws IOException { if (clusterManagerNodeId == null) { out.writeBoolean(false); } else { out.writeBoolean(true); out.writeString(clusterManagerNodeId); } - out.writeMapValuesOrdered(nodes, Map.Entry.comparingByKey(), (stream, val) -> val.writeToSorted(stream)); } public static DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws IOException { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index 380c707b69b84..c86ce15cb9bcf 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -376,9 +376,12 @@ public void writeTo(StreamOutput out) throws IOException { } } - public void writeToSorted(StreamOutput out) throws IOException { + public void writeVerifiableTo(StreamOutput out) throws IOException { index.writeTo(out); - out.writeMapValuesOrdered(shards, Map.Entry.comparingByKey(), (stream, value) -> IndexShardRoutingTable.Builder.writeToSorted(value, stream)); + out.writeMapValues( + shards, + (stream, value) -> IndexShardRoutingTable.Builder.writeVerifiableTo(value, stream) + ); } public static Builder builder(Index index) { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index 37d7e69f7855b..564c922f78d48 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -1136,7 +1136,7 @@ public static void writeToThin(IndexShardRoutingTable indexShard, StreamOutput o } } - public static void writeToSorted(IndexShardRoutingTable indexShard, StreamOutput out) throws IOException { + public static void writeVerifiableTo(IndexShardRoutingTable indexShard, StreamOutput out) throws IOException { out.writeVInt(indexShard.shardId.id()); out.writeVInt(indexShard.shards.size()); // Order allocated shards by allocationId diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 517ad279d54d8..788b903072aa0 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -44,6 +44,7 @@ import org.opensearch.common.util.iterable.Iterables; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexNotFoundException; @@ -70,7 +71,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class RoutingTable implements Iterable, Diffable { +public class RoutingTable implements Iterable, Diffable, VerifiableWriteable { public static final RoutingTable EMPTY_ROUTING_TABLE = builder().build(); @@ -403,9 +404,10 @@ public void writeTo(StreamOutput out) throws IOException { } } - public void writeToSorted(StreamOutput out) throws IOException { + @Override + public void writeVerifiableTo(StreamOutput out) throws IOException { out.writeLong(version); - out.writeMapValuesOrdered(indicesRouting, Map.Entry.comparingByKey(), (stream, value) -> value.writeToSorted(stream)); + out.writeMapValues(indicesRouting, (stream, value) -> value.writeVerifiableTo(stream)); } private static class RoutingTableDiff implements Diff { diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index 36f44d745704c..9da47ff3aa700 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -589,10 +589,6 @@ public static void writeSettingsToStream(Settings settings, StreamOutput out) th } } - public static void writeSettingsToStreamSorted(Settings settings, StreamOutput out) throws IOException { - out.writeMapOrdered(settings.settings, Map.Entry.comparingByKey(), StreamOutput::writeString, StreamOutput::writeGenericValue); - } - /** * Returns a builder to be used in order to build settings. */ diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index d7c4eaaa094eb..cbf8935c5867a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -71,15 +71,15 @@ public ClusterStateChecksum(ClusterState clusterState) { BytesStreamOutput out = new BytesStreamOutput(); BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out) ) { - clusterState.routingTable().writeToSorted(checksumOut); + clusterState.routingTable().writeVerifiableTo(checksumOut); routingTableChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.nodes().writeToSorted(checksumOut); + clusterState.nodes().writeVerifiableTo(checksumOut); nodesChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.coordinationMetadata().writeToSorted(checksumOut); + clusterState.coordinationMetadata().writeVerifiableTo(checksumOut); coordinationMetadataChecksum = checksumOut.getChecksum(); // Settings create sortedMap by default, so no explicit sorting required here. @@ -92,33 +92,19 @@ public ClusterStateChecksum(ClusterState clusterState) { transientSettingsMetadataChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.metadata().templatesMetadata().writeToSorted(checksumOut); + clusterState.metadata().templatesMetadata().writeVerifiableTo(checksumOut); templatesMetadataChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.metadata().customs().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { - try { - entry.getValue().writeTo(checksumOut); - } catch (IOException e) { - logger.error("Failed to create checksum for custom metadata.", e); - throw new RemoteStateTransferException("Failed to create checksum for custom metadata.", e); - } - }); + checksumOut.writeMapValues(clusterState.metadata().customs(), (stream, value) -> value.writeTo(stream)); customMetadataMapChecksum = checksumOut.getChecksum(); checksumOut.reset(); - ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeToSorted(checksumOut); + ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeTo(checksumOut); hashesOfConsistentSettingsChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.metadata().indices().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach((entry -> { - try { - entry.getValue().writeToSorted(checksumOut); - } catch (IOException e) { - logger.error("Failed to create checksum for index metadata.", e); - throw new RemoteStateTransferException("Failed to create checksum for index metadata.", e); - } - })); + checksumOut.writeMapValues(clusterState.metadata().indices(), (stream, value) -> value.writeVerifiableTo(stream)); indicesChecksum = checksumOut.getChecksum(); checksumOut.reset(); @@ -126,13 +112,7 @@ public ClusterStateChecksum(ClusterState clusterState) { blocksChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.customs().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { - try { - checksumOut.writeNamedWriteable(entry.getValue()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + checksumOut.writeMapValues(clusterState.customs(), (stream, value) -> checksumOut.writeNamedWriteable(value)); clusterStateCustomsChecksum = checksumOut.getChecksum(); } catch (IOException e) { logger.error("Failed to create checksum for cluster state.", e); From 2c00559d333e0a6c42183d4bf11113b56858c79e Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 29 Aug 2024 12:50:39 +0530 Subject: [PATCH 12/29] fixing interface Signed-off-by: Himshikha Gupta --- .../stream/BufferedChecksumStreamOutput.java | 40 +++++++++---------- .../core/common/io/stream/StreamOutput.java | 2 - .../common/io/stream/VerifiableWriteable.java | 7 +++- .../coordination/CoordinationMetadata.java | 4 +- .../cluster/metadata/IndexMetadata.java | 11 ++--- .../metadata/IndexTemplateMetadata.java | 5 ++- .../cluster/metadata/TemplatesMetadata.java | 5 ++- .../cluster/node/DiscoveryNode.java | 4 +- .../cluster/node/DiscoveryNodes.java | 5 ++- .../cluster/routing/IndexRoutingTable.java | 16 +++++--- .../cluster/routing/RoutingTable.java | 5 ++- .../gateway/remote/ClusterStateChecksum.java | 3 +- .../remote/ClusterStateChecksumTests.java | 2 +- 13 files changed, 59 insertions(+), 50 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index f9bdb361f6608..f28a19a9489f2 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -42,7 +42,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -109,18 +108,17 @@ public void writeMap(@Nullable Map map) throws IOException { } @Override - public void writeMap(final Map map, final Writeable.Writer keyWriter, final Writeable.Writer valueWriter) throws IOException { + public void writeMap(final Map map, final Writeable.Writer keyWriter, final Writeable.Writer valueWriter) + throws IOException { writeVInt(map.size()); - map.entrySet().stream().sorted(Comparator.comparing(o -> ( o.getKey().hashCode()))).forEachOrdered(entry-> - { - try { - keyWriter.write(this, entry.getKey()); - valueWriter.write(this, entry.getValue()); - } catch (IOException e) { - throw new RuntimeException("Failed to write map.", e); - } + map.entrySet().stream().sorted(Comparator.comparing(o -> (o.getKey().hashCode()))).forEachOrdered(entry -> { + try { + keyWriter.write(this, entry.getKey()); + valueWriter.write(this, entry.getValue()); + } catch (IOException e) { + throw new RuntimeException("Failed to write map.", e); } - ); + }); } /** @@ -134,15 +132,13 @@ public void writeMap(final Map map, final Writeable.Writer keyWr @Override public void writeMapValues(final Map map, final Writeable.Writer valueWriter) throws IOException { writeVInt(map.size()); - map.entrySet().stream().sorted(Comparator.comparing(o -> ( o.getKey().hashCode()))).forEachOrdered(entry-> - { - try { - valueWriter.write(this, entry.getValue()); - } catch (IOException e) { - throw new RuntimeException("Failed to write map values.", e); - } + map.entrySet().stream().sorted(Comparator.comparing(o -> (o.getKey().hashCode()))).forEachOrdered(entry -> { + try { + valueWriter.write(this, entry.getValue()); + } catch (IOException e) { + throw new RuntimeException("Failed to write map values.", e); } - ); + }); } @Override @@ -153,7 +149,9 @@ public void writeStringArray(String[] array) throws IOException { @Override public void writeCollection(final Collection collection) throws IOException { - List sortedList = collection.stream().sorted(Comparator.comparing(Object::hashCode)).collect(Collectors.toList()); + List sortedList = collection.stream() + .sorted(Comparator.comparing(Object::hashCode)) + .collect(Collectors.toList()); writeCollection(sortedList, (o, v) -> v.writeTo(o)); } @@ -166,7 +164,7 @@ public void writeStringCollection(final Collection collection) throws IO @Override public void writeList(List list) throws IOException { - //copy to new to handle unmodifiable lists + // copy to new to handle unmodifiable lists List newList = new ArrayList<>(list); newList.sort(Comparator.comparing(Object::hashCode)); writeCollection(newList); diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java index 3358b39f8a782..26f83ac5413b2 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java @@ -75,7 +75,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; @@ -87,7 +86,6 @@ import java.util.Map; import java.util.Set; import java.util.function.IntFunction; -import java.util.stream.Collectors; /** * A stream from another node to this node. Technically, it can also be streamed from a byte array but that is mostly for testing. diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java index cda69980231cc..6683606e27195 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java @@ -10,6 +10,11 @@ import java.io.IOException; +/** + * Provides a method for serialization which will give ordered stream, creating same byte array on every invocation. + * This should be invoked with a stream that provides ordered serialization. + */ public interface VerifiableWriteable extends Writeable { - void writeVerifiableTo(StreamOutput out) throws IOException; + + void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException; } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java index 2b1f2b220e754..6ba7e1e19c79d 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java @@ -35,6 +35,7 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.util.set.Sets; import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.VerifiableWriteable; @@ -48,7 +49,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -152,7 +152,7 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public void writeVerifiableTo(StreamOutput out) throws IOException { + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { writeTo(out); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 78a2165eb5e69..d8af1277ccb0e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -54,6 +54,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.Assertions; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.VerifiableWriteable; @@ -1215,7 +1216,7 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public void writeVerifiableTo(StreamOutput out) throws IOException { + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { out.writeString(index.getName()); // uuid will come as part of settings out.writeLong(version); out.writeVLong(mappingVersion); @@ -1225,15 +1226,15 @@ public void writeVerifiableTo(StreamOutput out) throws IOException { out.writeByte(state.id()); writeSettingsToStream(settings, out); out.writeVLongArray(primaryTerms); - out.writeMapValues(mappings,(stream, val) -> val.writeTo(stream)); - out.writeMapValues(aliases,(stream, val) -> val.writeTo(stream)); - out.writeMap(customData, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(mappings, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); + out.writeMap(customData, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); out.writeMap( inSyncAllocationIds, StreamOutput::writeVInt, (stream, val) -> DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(val), stream) ); - out.writeMapValues(rolloverInfos, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(rolloverInfos, (stream, val) -> val.writeTo(stream)); out.writeBoolean(isSystem); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java index dd59776f77427..b09acc54653c4 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java @@ -44,6 +44,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.VerifiableWriteable; @@ -259,13 +260,13 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public void writeVerifiableTo(StreamOutput out) throws IOException { + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { out.writeString(name); out.writeInt(order); out.writeStringCollection(patterns); Settings.writeSettingsToStream(settings, out); out.writeMap(mappings, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); - out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); out.writeOptionalVInt(version); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index d99bacb1765c4..6f2145a3ef57f 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -10,6 +10,7 @@ import org.opensearch.cluster.AbstractDiffable; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.xcontent.ToXContentFragment; @@ -67,8 +68,8 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public void writeVerifiableTo(StreamOutput out) throws IOException { - out.writeMapValues(templates, (stream, value) -> value.writeVerifiableTo(stream)); + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + out.writeMapValues(templates, (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); } @Override diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 2390665d5a627..50086d458fab2 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -37,10 +37,10 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.VerifiableWriteable; -import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -370,7 +370,7 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public void writeVerifiableTo(StreamOutput out) throws IOException { + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { writeNodeDetails(out); out.writeMap(attributes, StreamOutput::writeString, StreamOutput::writeString); writeRolesAndVersion(out); diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index e7b1ad30fc63a..c8080a92ddfaa 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -41,6 +41,7 @@ import org.opensearch.common.regex.Regex; import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.VerifiableWriteable; @@ -697,9 +698,9 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public void writeVerifiableTo(StreamOutput out) throws IOException { + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { writeClusterManager(out); - out.writeMapValues(nodes, (stream, val) -> val.writeVerifiableTo(stream)); + out.writeMapValues(nodes, (stream, val) -> val.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); } private void writeClusterManager(StreamOutput out) throws IOException { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index c86ce15cb9bcf..7d6c52ecec4d4 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -45,8 +45,10 @@ import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource; import org.opensearch.common.Randomness; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; @@ -79,7 +81,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class IndexRoutingTable extends AbstractDiffable implements Iterable { +public class IndexRoutingTable extends AbstractDiffable implements Iterable, VerifiableWriteable { private final Index index; private final ShardShuffler shuffler; @@ -351,6 +353,11 @@ public int hashCode() { return result; } + @Override + public String toString() { + return "IndexRoutingTable{" + "shards=" + shards + ", index=" + index + '}'; + } + public static IndexRoutingTable readFrom(StreamInput in) throws IOException { Index index = new Index(in); Builder builder = new Builder(index); @@ -376,12 +383,9 @@ public void writeTo(StreamOutput out) throws IOException { } } - public void writeVerifiableTo(StreamOutput out) throws IOException { + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { index.writeTo(out); - out.writeMapValues( - shards, - (stream, value) -> IndexShardRoutingTable.Builder.writeVerifiableTo(value, stream) - ); + out.writeMapValues(shards, (stream, value) -> IndexShardRoutingTable.Builder.writeVerifiableTo(value, stream)); } public static Builder builder(Index index) { diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 788b903072aa0..ae2c3f6e4794f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -42,6 +42,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.util.iterable.Iterables; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.VerifiableWriteable; @@ -405,9 +406,9 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public void writeVerifiableTo(StreamOutput out) throws IOException { + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { out.writeLong(version); - out.writeMapValues(indicesRouting, (stream, value) -> value.writeVerifiableTo(stream)); + out.writeMapValues(indicesRouting, (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); } private static class RoutingTableDiff implements Diff { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index cbf8935c5867a..a50865ad213c6 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Objects; import com.jcraft.jzlib.JZlib; @@ -104,7 +103,7 @@ public ClusterStateChecksum(ClusterState clusterState) { hashesOfConsistentSettingsChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeMapValues(clusterState.metadata().indices(), (stream, value) -> value.writeVerifiableTo(stream)); + checksumOut.writeMapValues(clusterState.metadata().indices(), (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); indicesChecksum = checksumOut.getChecksum(); checksumOut.reset(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java index c5955f5d06070..4895d198c9749 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java @@ -179,7 +179,7 @@ private ClusterState generateClusterState() { .templates(templatesMetadata) .hashesOfConsistentSettings(Map.of("key1", "value1", "key2", "value2")) .putCustom(customMetadata1.getWriteableName(), customMetadata1) - .indices(Map.of(indexMetadata.getIndex().getName(), indexMetadata)) + .indices(Map.of(indexMetadata.getIndex().getName(), indexMetadata, indexMetadata2.getIndex().getName(), indexMetadata2)) .build() ) .nodes(DiscoveryNodes.builder().clusterManagerNodeId("test-node").build()) From d7bd138143c5a8cfd973dda781d63414faf3f4bc Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 29 Aug 2024 12:54:53 +0530 Subject: [PATCH 13/29] sorting vlongarray Signed-off-by: Himshikha Gupta --- .../core/common/io/stream/BufferedChecksumStreamOutput.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index f28a19a9489f2..f1bd024e36b48 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -147,6 +147,12 @@ public void writeStringArray(String[] array) throws IOException { super.writeStringArray(array); } + @Override + public void writeVLongArray(long[] values) throws IOException { + Arrays.sort(values); + super.writeVLongArray(values); + } + @Override public void writeCollection(final Collection collection) throws IOException { List sortedList = collection.stream() From 1e1dacce421250cab88d151d44b0386a397f2a0a Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 29 Aug 2024 17:21:30 +0530 Subject: [PATCH 14/29] map based comparison Signed-off-by: Himshikha Gupta --- .../stream/BufferedChecksumStreamOutput.java | 49 +++++++------------ .../cluster/block/ClusterBlocks.java | 20 +++++++- .../coordination/CoordinationMetadata.java | 6 ++- .../cluster/metadata/IndexMetadata.java | 13 ++--- .../metadata/IndexTemplateMetadata.java | 4 +- .../cluster/metadata/TemplatesMetadata.java | 6 ++- .../cluster/node/DiscoveryNode.java | 2 +- .../cluster/node/DiscoveryNodes.java | 6 ++- .../cluster/routing/IndexRoutingTable.java | 11 ++++- .../cluster/routing/RoutingTable.java | 6 ++- .../gateway/remote/ClusterStateChecksum.java | 21 ++++++-- 11 files changed, 94 insertions(+), 50 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index f1bd024e36b48..980ff577477e9 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -107,32 +107,30 @@ public void writeMap(@Nullable Map map) throws IOException { writeGenericValue(newMap); } - @Override - public void writeMap(final Map map, final Writeable.Writer keyWriter, final Writeable.Writer valueWriter) - throws IOException { + public void writeMapOrdered( + Map map, + final Writeable.Writer keyWriter, + final Writeable.Writer valueWriter, + Comparator> entryComparator + ) throws IOException { writeVInt(map.size()); - map.entrySet().stream().sorted(Comparator.comparing(o -> (o.getKey().hashCode()))).forEachOrdered(entry -> { + map.entrySet().stream().sorted(entryComparator).forEachOrdered(entry -> { try { keyWriter.write(this, entry.getKey()); valueWriter.write(this, entry.getValue()); } catch (IOException e) { - throw new RuntimeException("Failed to write map.", e); + throw new RuntimeException("Failed to write map values.", e); } }); } - /** - * - * @param map - * @param valueWriter - * @param - * @param - * @throws IOException - */ - @Override - public void writeMapValues(final Map map, final Writeable.Writer valueWriter) throws IOException { + public void writeMapValuesOrdered( + Map map, + final Writeable.Writer valueWriter, + Comparator> entryComparator + ) throws IOException { writeVInt(map.size()); - map.entrySet().stream().sorted(Comparator.comparing(o -> (o.getKey().hashCode()))).forEachOrdered(entry -> { + map.entrySet().stream().sorted(entryComparator).forEachOrdered(entry -> { try { valueWriter.write(this, entry.getValue()); } catch (IOException e) { @@ -153,12 +151,10 @@ public void writeVLongArray(long[] values) throws IOException { super.writeVLongArray(values); } - @Override - public void writeCollection(final Collection collection) throws IOException { - List sortedList = collection.stream() - .sorted(Comparator.comparing(Object::hashCode)) - .collect(Collectors.toList()); - writeCollection(sortedList, (o, v) -> v.writeTo(o)); + public void writeCollectionOrdered(Collection collection, final Writeable.Writer valueWriter, Comparator comparing) + throws IOException { + List sortedList = collection.stream().sorted(comparing).collect(Collectors.toList()); + writeCollection(sortedList, valueWriter); } @Override @@ -168,14 +164,6 @@ public void writeStringCollection(final Collection collection) throws IO writeCollection(listCollection, StreamOutput::writeString); } - @Override - public void writeList(List list) throws IOException { - // copy to new to handle unmodifiable lists - List newList = new ArrayList<>(list); - newList.sort(Comparator.comparing(Object::hashCode)); - writeCollection(newList); - } - @Override public void writeOptionalStringCollection(final Collection collection) throws IOException { if (collection != null) { @@ -187,4 +175,5 @@ public void writeOptionalStringCollection(final Collection collection) t writeBoolean(false); } } + } diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java index 02a20b7681ba7..9a2d1aab9529d 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java @@ -39,12 +39,15 @@ import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.util.set.Sets; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Collections; +import java.util.Comparator; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; @@ -62,7 +65,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class ClusterBlocks extends AbstractDiffable { +public class ClusterBlocks extends AbstractDiffable implements VerifiableWriteable { public static final ClusterBlocks EMPTY_CLUSTER_BLOCK = new ClusterBlocks(emptySet(), Map.of()); private final Set global; @@ -303,10 +306,25 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(indicesBlocks, StreamOutput::writeString, (o, s) -> writeBlockSet(s, o)); } + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + writeVerifiableBlockSet(global, out); + out.writeMapOrdered( + indicesBlocks, + StreamOutput::writeString, + (o, s) -> writeVerifiableBlockSet(s, (BufferedChecksumStreamOutput) o), + Map.Entry.comparingByKey() + ); + } + private static void writeBlockSet(Set blocks, StreamOutput out) throws IOException { out.writeCollection(blocks); } + private static void writeVerifiableBlockSet(Set blocks, BufferedChecksumStreamOutput out) throws IOException { + out.writeCollectionOrdered(blocks, (o, v) -> v.writeTo(o), Comparator.comparing(ClusterBlock::id)); + } + public static ClusterBlocks readFrom(StreamInput in) throws IOException { final Set global = readBlockSet(in); final Map> indicesBlocks = in.readMap(i -> i.readString().intern(), ClusterBlocks::readBlockSet); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java index 6ba7e1e19c79d..99be03a078e25 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -153,7 +154,10 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { - writeTo(out); + out.writeLong(term); + lastCommittedConfiguration.writeTo(out); + lastAcceptedConfiguration.writeTo(out); + out.writeCollectionOrdered(votingConfigExclusions, (o, v) -> v.writeTo(o), Comparator.comparing(VotingConfigExclusion::getNodeId)); } @Override diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index d8af1277ccb0e..8bf043ccf4210 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1226,15 +1226,16 @@ public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOExcepti out.writeByte(state.id()); writeSettingsToStream(settings, out); out.writeVLongArray(primaryTerms); - out.writeMapValues(mappings, (stream, val) -> val.writeTo(stream)); - out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); - out.writeMap(customData, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); - out.writeMap( + out.writeMapValuesOrdered(mappings, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); + out.writeMapValuesOrdered(aliases, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); + out.writeMapOrdered(customData, StreamOutput::writeString, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); + out.writeMapOrdered( inSyncAllocationIds, StreamOutput::writeVInt, - (stream, val) -> DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(val), stream) + (stream, val) -> DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(val), stream), + Map.Entry.comparingByKey() ); - out.writeMapValues(rolloverInfos, (stream, val) -> val.writeTo(stream)); + out.writeMapValuesOrdered(rolloverInfos, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); out.writeBoolean(isSystem); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java index b09acc54653c4..2c523901f5455 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java @@ -265,8 +265,8 @@ public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOExcepti out.writeInt(order); out.writeStringCollection(patterns); Settings.writeSettingsToStream(settings, out); - out.writeMap(mappings, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); - out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); + out.writeMapOrdered(mappings, StreamOutput::writeString, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); + out.writeMapValuesOrdered(aliases, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); out.writeOptionalVInt(version); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index 6f2145a3ef57f..abfaf8667fbeb 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -69,7 +69,11 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { - out.writeMapValues(templates, (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); + out.writeMapValuesOrdered( + templates, + (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream), + Map.Entry.comparingByKey() + ); } @Override diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 50086d458fab2..b1e7b7a5abca6 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -372,7 +372,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { writeNodeDetails(out); - out.writeMap(attributes, StreamOutput::writeString, StreamOutput::writeString); + out.writeMapOrdered(attributes, StreamOutput::writeString, StreamOutput::writeString, Map.Entry.comparingByKey()); writeRolesAndVersion(out); } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index c8080a92ddfaa..e95c51d5c58bb 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -700,7 +700,11 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { writeClusterManager(out); - out.writeMapValues(nodes, (stream, val) -> val.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); + out.writeMapValuesOrdered( + nodes, + (stream, val) -> val.writeVerifiableTo((BufferedChecksumStreamOutput) stream), + Map.Entry.comparingByKey() + ); } private void writeClusterManager(StreamOutput out) throws IOException { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index 7d6c52ecec4d4..137478b95cf17 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -81,7 +81,10 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class IndexRoutingTable extends AbstractDiffable implements Iterable, VerifiableWriteable { +public class IndexRoutingTable extends AbstractDiffable + implements + Iterable, + VerifiableWriteable { private final Index index; private final ShardShuffler shuffler; @@ -385,7 +388,11 @@ public void writeTo(StreamOutput out) throws IOException { public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { index.writeTo(out); - out.writeMapValues(shards, (stream, value) -> IndexShardRoutingTable.Builder.writeVerifiableTo(value, stream)); + out.writeMapValuesOrdered( + shards, + (stream, value) -> IndexShardRoutingTable.Builder.writeVerifiableTo(value, stream), + Map.Entry.comparingByKey() + ); } public static Builder builder(Index index) { diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index ae2c3f6e4794f..297e9b4493a60 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -408,7 +408,11 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { out.writeLong(version); - out.writeMapValues(indicesRouting, (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); + out.writeMapValuesOrdered( + indicesRouting, + (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream), + Map.Entry.comparingByKey() + ); } private static class RoutingTableDiff implements Diff { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index a50865ad213c6..dddbc2acc74ec 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; import com.jcraft.jzlib.JZlib; @@ -95,7 +96,11 @@ public ClusterStateChecksum(ClusterState clusterState) { templatesMetadataChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeMapValues(clusterState.metadata().customs(), (stream, value) -> value.writeTo(stream)); + checksumOut.writeMapValuesOrdered( + clusterState.metadata().customs(), + (stream, value) -> value.writeTo(stream), + Map.Entry.comparingByKey() + ); customMetadataMapChecksum = checksumOut.getChecksum(); checksumOut.reset(); @@ -103,15 +108,23 @@ public ClusterStateChecksum(ClusterState clusterState) { hashesOfConsistentSettingsChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeMapValues(clusterState.metadata().indices(), (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); + checksumOut.writeMapValuesOrdered( + clusterState.metadata().indices(), + (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream), + Map.Entry.comparingByKey() + ); indicesChecksum = checksumOut.getChecksum(); checksumOut.reset(); - clusterState.blocks().writeTo(checksumOut); + clusterState.blocks().writeVerifiableTo(checksumOut); blocksChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeMapValues(clusterState.customs(), (stream, value) -> checksumOut.writeNamedWriteable(value)); + checksumOut.writeMapValuesOrdered( + clusterState.customs(), + (stream, value) -> checksumOut.writeNamedWriteable(value), + Map.Entry.comparingByKey() + ); clusterStateCustomsChecksum = checksumOut.getChecksum(); } catch (IOException e) { logger.error("Failed to create checksum for cluster state.", e); From 0dfbae0f13955470708f688a4d21eeea8284e574 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 29 Aug 2024 22:39:35 +0530 Subject: [PATCH 15/29] removing redundant code post refactoring Signed-off-by: Himshikha Gupta --- .../opensearch/core/common/io/stream/StreamOutput.java | 9 +-------- .../opensearch/cluster/metadata/TemplatesMetadata.java | 5 +++++ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java index 26f83ac5413b2..b7599265aece3 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java @@ -633,7 +633,7 @@ public final void writeMapOfLists(final Map> map, final Writer * @param keyWriter The key writer * @param valueWriter The value writer */ - public void writeMap(final Map map, final Writer keyWriter, final Writer valueWriter) throws IOException { + public final void writeMap(final Map map, final Writer keyWriter, final Writer valueWriter) throws IOException { writeVInt(map.size()); for (final Map.Entry entry : map.entrySet()) { keyWriter.write(this, entry.getKey()); @@ -641,13 +641,6 @@ public void writeMap(final Map map, final Writer keyWriter, fina } } - public void writeMapValues(final Map map, final Writer valueWriter) throws IOException { - writeVInt(map.size()); - for (final Map.Entry entry : map.entrySet()) { - valueWriter.write(this, entry.getValue()); - } - } - /** * Writes an {@link Instant} to the stream with nanosecond resolution */ diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index abfaf8667fbeb..d5d82bcb22dc6 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -91,6 +91,11 @@ public int hashCode() { return templates != null ? templates.hashCode() : 0; } + @Override + public String toString() { + return "TemplatesMetadata{" + "templates=" + templates + '}'; + } + /** * Builder for the templates metadata * From aaca759877ebe192a2de68bad3e5ab3b12d59674 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sat, 31 Aug 2024 23:17:50 +0530 Subject: [PATCH 16/29] More tests Signed-off-by: Bukhtawar Khan --- .../remote/ClusterStateChecksumTests.java | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java index 4895d198c9749..65e0dc3e44497 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java @@ -11,6 +11,8 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlock; +import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; @@ -27,12 +29,14 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -60,6 +64,25 @@ public void testClusterStateChecksum() { assertTrue(checksum.clusterStateChecksum != 0); } + public void testClusterStateMatchChecksum() { + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + ClusterStateChecksum newChecksum = new ClusterStateChecksum(generateClusterState()); + assertNotNull(checksum); + assertNotNull(newChecksum); + assertEquals(checksum.routingTableChecksum, newChecksum.routingTableChecksum); + assertEquals(checksum.nodesChecksum, newChecksum.nodesChecksum); + assertEquals(checksum.blocksChecksum, newChecksum.blocksChecksum); + assertEquals(checksum.clusterStateCustomsChecksum, newChecksum.clusterStateCustomsChecksum); + assertEquals(checksum.coordinationMetadataChecksum, newChecksum.coordinationMetadataChecksum); + assertEquals(checksum.settingMetadataChecksum, newChecksum.settingMetadataChecksum); + assertEquals(checksum.transientSettingsMetadataChecksum, newChecksum.transientSettingsMetadataChecksum); + assertEquals(checksum.templatesMetadataChecksum, newChecksum.templatesMetadataChecksum); + assertEquals(checksum.customMetadataMapChecksum, newChecksum.customMetadataMapChecksum); + assertEquals(checksum.hashesOfConsistentSettingsChecksum, newChecksum.hashesOfConsistentSettingsChecksum); + assertEquals(checksum.indicesChecksum, newChecksum.indicesChecksum); + assertEquals(checksum.clusterStateChecksum, newChecksum.clusterStateChecksum); + } + public void testXContentConversion() throws IOException { ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); final XContentBuilder builder = JsonXContent.contentBuilder(); @@ -183,7 +206,11 @@ private ClusterState generateClusterState() { .build() ) .nodes(DiscoveryNodes.builder().clusterManagerNodeId("test-node").build()) - .blocks(ClusterBlocks.builder().addBlocks(indexMetadata).build()) + .blocks(ClusterBlocks.builder() + .addBlocks(indexMetadata) + .addGlobalBlock(new ClusterBlock(1, "block", true, true, true, RestStatus.ACCEPTED, EnumSet.of(ClusterBlockLevel.READ))) + .addGlobalBlock(new ClusterBlock(2, "block-name", false, true, true, RestStatus.OK, EnumSet.of(ClusterBlockLevel.WRITE))) + .build()) .customs(Map.of(clusterStateCustom1.getWriteableName(), clusterStateCustom1)) .routingTable(RoutingTable.builder().addAsNew(indexMetadata).addAsNew(indexMetadata2).version(1L).build()) .build(); From 94d5fcc707c20ee16a5717f3dca44996b2aaa10a Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 2 Sep 2024 13:18:11 +0530 Subject: [PATCH 17/29] simplifying comparator Signed-off-by: Himshikha Gupta --- .../stream/BufferedChecksumStreamOutput.java | 33 ++++++----- .../core/common/io/stream/StreamOutput.java | 2 +- .../org/opensearch/cluster/ClusterState.java | 48 ++++++++-------- .../org/opensearch/cluster/DiffableUtils.java | 9 ++- .../cluster/block/ClusterBlock.java | 8 ++- .../cluster/block/ClusterBlocks.java | 13 +---- .../coordination/CoordinationMetadata.java | 12 ++-- .../cluster/metadata/IndexMetadata.java | 13 ++--- .../metadata/IndexTemplateMetadata.java | 4 +- .../cluster/metadata/TemplatesMetadata.java | 6 +- .../cluster/node/DiscoveryNode.java | 2 +- .../cluster/node/DiscoveryNodes.java | 6 +- .../cluster/routing/IndexRoutingTable.java | 6 +- .../cluster/routing/RoutingTable.java | 6 +- .../gateway/remote/ClusterStateChecksum.java | 18 ++---- .../cluster/block/ClusterBlocksTests.java | 55 +++++++++++++++++++ .../CoordinationMetadataTests.java | 31 +++++++++++ .../metadata/IndexTemplateMetadataTests.java | 34 ++++++++++++ .../remote/ClusterStateChecksumTests.java | 2 +- 19 files changed, 203 insertions(+), 105 deletions(-) create mode 100644 server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index 980ff577477e9..8ad629fc29e48 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -107,32 +107,31 @@ public void writeMap(@Nullable Map map) throws IOException { writeGenericValue(newMap); } - public void writeMapOrdered( + @Override + public void writeMap( Map map, final Writeable.Writer keyWriter, - final Writeable.Writer valueWriter, - Comparator> entryComparator + final Writeable.Writer valueWriter ) throws IOException { writeVInt(map.size()); - map.entrySet().stream().sorted(entryComparator).forEachOrdered(entry -> { + map.keySet().stream().sorted().forEachOrdered(key -> { try { - keyWriter.write(this, entry.getKey()); - valueWriter.write(this, entry.getValue()); + keyWriter.write(this, key); + valueWriter.write(this, map.get(key)); } catch (IOException e) { throw new RuntimeException("Failed to write map values.", e); } }); } - public void writeMapValuesOrdered( + public void writeMapValues( Map map, - final Writeable.Writer valueWriter, - Comparator> entryComparator + final Writeable.Writer valueWriter ) throws IOException { writeVInt(map.size()); - map.entrySet().stream().sorted(entryComparator).forEachOrdered(entry -> { + map.keySet().stream().sorted().forEachOrdered(key -> { try { - valueWriter.write(this, entry.getValue()); + valueWriter.write(this, map.get(key)); } catch (IOException e) { throw new RuntimeException("Failed to write map values.", e); } @@ -151,10 +150,14 @@ public void writeVLongArray(long[] values) throws IOException { super.writeVLongArray(values); } - public void writeCollectionOrdered(Collection collection, final Writeable.Writer valueWriter, Comparator comparing) - throws IOException { - List sortedList = collection.stream().sorted(comparing).collect(Collectors.toList()); - writeCollection(sortedList, valueWriter); + @Override + public void writeCollection(final Collection collection) throws IOException { + if(!collection.isEmpty() && collection.iterator().next() instanceof Comparable) { + List sortedList = collection.stream().sorted().collect(Collectors.toList()); + super.writeCollection(sortedList, (o, v) -> v.writeTo(o)); + } else { + super.writeCollection(collection, (o, v) -> v.writeTo(o)); + } } @Override diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java index b7599265aece3..867fc99cc6a56 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java @@ -633,7 +633,7 @@ public final void writeMapOfLists(final Map> map, final Writer * @param keyWriter The key writer * @param valueWriter The value writer */ - public final void writeMap(final Map map, final Writer keyWriter, final Writer valueWriter) throws IOException { + public void writeMap(final Map map, final Writer keyWriter, final Writer valueWriter) throws IOException { writeVInt(map.size()); for (final Map.Entry entry : map.entrySet()) { keyWriter.write(this, entry.getKey()); diff --git a/server/src/main/java/org/opensearch/cluster/ClusterState.java b/server/src/main/java/org/opensearch/cluster/ClusterState.java index 1350c52997791..054233305726d 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterState.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterState.java @@ -841,30 +841,30 @@ private static class ClusterStateDiff implements Diff { @Override public String toString() { - return "ClusterStateDiff{" - + "toVersion=" - + toVersion - + ", fromUuid='" - + fromUuid - + '\'' - + ", toUuid='" - + toUuid - + '\'' - + ", clusterName=" - + clusterName - + ", routingTable=" - + routingTable - + ", nodes=" - + nodes - + ", metadata=" - + metadata - + ", blocks=" - + blocks - + ", customs=" - + customs - + ", minimumClusterManagerNodesOnPublishingClusterManager=" - + minimumClusterManagerNodesOnPublishingClusterManager - + '}'; + return new StringBuilder().append("ClusterStateDiff{toVersion=") + .append(toVersion) + .append(", fromUuid='") + .append(fromUuid) + .append('\'') + .append(", toUuid='") + .append(toUuid) + .append('\'') + .append(", clusterName=") + .append(clusterName) + .append(", routingTable=") + .append(routingTable) + .append(", nodes=") + .append(nodes) + .append(", metadata=") + .append(metadata) + .append(", blocks=") + .append(blocks) + .append(", customs=") + .append(customs) + .append(", minimumClusterManagerNodesOnPublishingClusterManager=") + .append(minimumClusterManagerNodesOnPublishingClusterManager) + .append("}") + .toString(); } ClusterStateDiff(StreamInput in, DiscoveryNode localNode) throws IOException { diff --git a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java index dbd454720c69b..4b5dcaa52cc50 100644 --- a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java +++ b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java @@ -273,7 +273,14 @@ public Map getUpserts() { @Override public String toString() { - return "MapDiff{" + "deletes=" + deletes + ", diffs=" + diffs + ", upserts=" + upserts + '}'; + return new StringBuilder().append("MapDiff{deletes=") + .append(deletes) + .append(", diffs=") + .append(diffs) + .append(", upserts=") + .append(upserts) + .append("}") + .toString(); } @Override diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java index 5fa897c0b1185..7c0a7a2a6b837 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java @@ -52,7 +52,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class ClusterBlock implements Writeable, ToXContentFragment { +public class ClusterBlock implements Writeable, ToXContentFragment, Comparable { private final int id; @Nullable @@ -217,7 +217,13 @@ public int hashCode() { return Objects.hash(id, uuid); } + @Override + public int compareTo(ClusterBlock block) { + return Integer.compare(block.id(), this.id()); + } + public boolean isAllowReleaseResources() { return allowReleaseResources; } + } diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java index 9a2d1aab9529d..615ea18315cd1 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java @@ -47,7 +47,6 @@ import java.io.IOException; import java.util.Collections; -import java.util.Comparator; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; @@ -308,23 +307,13 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { - writeVerifiableBlockSet(global, out); - out.writeMapOrdered( - indicesBlocks, - StreamOutput::writeString, - (o, s) -> writeVerifiableBlockSet(s, (BufferedChecksumStreamOutput) o), - Map.Entry.comparingByKey() - ); + writeTo(out); } private static void writeBlockSet(Set blocks, StreamOutput out) throws IOException { out.writeCollection(blocks); } - private static void writeVerifiableBlockSet(Set blocks, BufferedChecksumStreamOutput out) throws IOException { - out.writeCollectionOrdered(blocks, (o, v) -> v.writeTo(o), Comparator.comparing(ClusterBlock::id)); - } - public static ClusterBlocks readFrom(StreamInput in) throws IOException { final Set global = readBlockSet(in); final Map> indicesBlocks = in.readMap(i -> i.readString().intern(), ClusterBlocks::readBlockSet); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java index 99be03a078e25..d7291a3689192 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java @@ -49,7 +49,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -154,10 +153,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { - out.writeLong(term); - lastCommittedConfiguration.writeTo(out); - lastAcceptedConfiguration.writeTo(out); - out.writeCollectionOrdered(votingConfigExclusions, (o, v) -> v.writeTo(o), Comparator.comparing(VotingConfigExclusion::getNodeId)); + writeTo(out); } @Override @@ -283,7 +279,7 @@ public CoordinationMetadata build() { * @opensearch.api */ @PublicApi(since = "1.0.0") - public static class VotingConfigExclusion implements Writeable, ToXContentFragment { + public static class VotingConfigExclusion implements Writeable, ToXContentFragment, Comparable { public static final String MISSING_VALUE_MARKER = "_absent_"; private final String nodeId; private final String nodeName; @@ -372,6 +368,10 @@ public String toString() { return sb.toString(); } + @Override + public int compareTo(VotingConfigExclusion votingConfigExclusion) { + return votingConfigExclusion.getNodeId().compareTo(this.getNodeId()); + } } /** diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 8bf043ccf4210..d8af1277ccb0e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1226,16 +1226,15 @@ public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOExcepti out.writeByte(state.id()); writeSettingsToStream(settings, out); out.writeVLongArray(primaryTerms); - out.writeMapValuesOrdered(mappings, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); - out.writeMapValuesOrdered(aliases, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); - out.writeMapOrdered(customData, StreamOutput::writeString, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); - out.writeMapOrdered( + out.writeMapValues(mappings, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); + out.writeMap(customData, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + out.writeMap( inSyncAllocationIds, StreamOutput::writeVInt, - (stream, val) -> DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(val), stream), - Map.Entry.comparingByKey() + (stream, val) -> DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(val), stream) ); - out.writeMapValuesOrdered(rolloverInfos, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); + out.writeMapValues(rolloverInfos, (stream, val) -> val.writeTo(stream)); out.writeBoolean(isSystem); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java index 2c523901f5455..b09acc54653c4 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java @@ -265,8 +265,8 @@ public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOExcepti out.writeInt(order); out.writeStringCollection(patterns); Settings.writeSettingsToStream(settings, out); - out.writeMapOrdered(mappings, StreamOutput::writeString, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); - out.writeMapValuesOrdered(aliases, (stream, val) -> val.writeTo(stream), Map.Entry.comparingByKey()); + out.writeMap(mappings, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); out.writeOptionalVInt(version); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index d5d82bcb22dc6..8a378deccdab4 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -69,11 +69,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { - out.writeMapValuesOrdered( - templates, - (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream), - Map.Entry.comparingByKey() - ); + out.writeMapValues(templates, (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); } @Override diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index b1e7b7a5abca6..50086d458fab2 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -372,7 +372,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { writeNodeDetails(out); - out.writeMapOrdered(attributes, StreamOutput::writeString, StreamOutput::writeString, Map.Entry.comparingByKey()); + out.writeMap(attributes, StreamOutput::writeString, StreamOutput::writeString); writeRolesAndVersion(out); } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index e95c51d5c58bb..c8080a92ddfaa 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -700,11 +700,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { writeClusterManager(out); - out.writeMapValuesOrdered( - nodes, - (stream, val) -> val.writeVerifiableTo((BufferedChecksumStreamOutput) stream), - Map.Entry.comparingByKey() - ); + out.writeMapValues(nodes, (stream, val) -> val.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); } private void writeClusterManager(StreamOutput out) throws IOException { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index 137478b95cf17..2f2088c687024 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -388,11 +388,7 @@ public void writeTo(StreamOutput out) throws IOException { public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { index.writeTo(out); - out.writeMapValuesOrdered( - shards, - (stream, value) -> IndexShardRoutingTable.Builder.writeVerifiableTo(value, stream), - Map.Entry.comparingByKey() - ); + out.writeMapValues(shards, (stream, value) -> IndexShardRoutingTable.Builder.writeVerifiableTo(value, stream)); } public static Builder builder(Index index) { diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 297e9b4493a60..ae2c3f6e4794f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -408,11 +408,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { out.writeLong(version); - out.writeMapValuesOrdered( - indicesRouting, - (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream), - Map.Entry.comparingByKey() - ); + out.writeMapValues(indicesRouting, (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); } private static class RoutingTableDiff implements Diff { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index dddbc2acc74ec..11087570e0d21 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Objects; import com.jcraft.jzlib.JZlib; @@ -96,11 +95,7 @@ public ClusterStateChecksum(ClusterState clusterState) { templatesMetadataChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeMapValuesOrdered( - clusterState.metadata().customs(), - (stream, value) -> value.writeTo(stream), - Map.Entry.comparingByKey() - ); + checksumOut.writeMapValues(clusterState.metadata().customs(), (stream, value) -> value.writeTo(stream)); customMetadataMapChecksum = checksumOut.getChecksum(); checksumOut.reset(); @@ -108,10 +103,9 @@ public ClusterStateChecksum(ClusterState clusterState) { hashesOfConsistentSettingsChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeMapValuesOrdered( + checksumOut.writeMapValues( clusterState.metadata().indices(), - (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream), - Map.Entry.comparingByKey() + (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream) ); indicesChecksum = checksumOut.getChecksum(); @@ -120,11 +114,7 @@ public ClusterStateChecksum(ClusterState clusterState) { blocksChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeMapValuesOrdered( - clusterState.customs(), - (stream, value) -> checksumOut.writeNamedWriteable(value), - Map.Entry.comparingByKey() - ); + checksumOut.writeMapValues(clusterState.customs(), (stream, value) -> checksumOut.writeNamedWriteable(value)); clusterStateCustomsChecksum = checksumOut.getChecksum(); } catch (IOException e) { logger.error("Failed to create checksum for cluster state.", e); diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java new file mode 100644 index 0000000000000..839e831d38b1b --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.block; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; + +import static org.opensearch.cluster.block.ClusterBlockTests.randomClusterBlock; + +public class ClusterBlocksTests extends OpenSearchTestCase { + + public void testWriteVerifiableTo() throws Exception { + ClusterBlock clusterBlock1 = randomClusterBlock(); + ClusterBlock clusterBlock2 = randomClusterBlock(); + ClusterBlock clusterBlock3 = randomClusterBlock(); + + ClusterBlocks clusterBlocks = ClusterBlocks.builder() + .addGlobalBlock(clusterBlock1) + .addGlobalBlock(clusterBlock2) + .addGlobalBlock(clusterBlock3) + .addIndexBlock("index-1", clusterBlock1) + .addIndexBlock("index-2", clusterBlock2) + .build(); + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + clusterBlocks.writeVerifiableTo(checksumOut); + StreamInput in = out.bytes().streamInput(); + ClusterBlocks result = ClusterBlocks.readFrom(in); + + assertEquals(clusterBlocks.global().size(), result.global().size()); + assertEquals(clusterBlocks.global(), result.global()); + assertEquals(clusterBlocks.indices().size(), result.indices().size()); + assertEquals(clusterBlocks.indices(), result.indices()); + + ClusterBlocks clusterBlocks2 = ClusterBlocks.builder() + .addGlobalBlock(clusterBlock3) + .addGlobalBlock(clusterBlock1) + .addGlobalBlock(clusterBlock2) + .addIndexBlock("index-2", clusterBlock2) + .addIndexBlock("index-1", clusterBlock1) + .build(); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + clusterBlocks2.writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } +} diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java index 290479941aaa9..74c25246c2aa4 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java @@ -33,10 +33,13 @@ import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion; import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfiguration; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -46,8 +49,11 @@ import java.io.IOException; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; @@ -231,4 +237,29 @@ public void testXContent() throws IOException { assertThat(originalMeta, equalTo(fromXContentMeta)); } } + + public void testWriteVerifiableTo() throws IOException { + VotingConfiguration votingConfiguration = randomVotingConfig(); + Set votingTombstones = randomVotingTombstones(); + CoordinationMetadata meta1 = new CoordinationMetadata(1, votingConfiguration, votingConfiguration, votingTombstones); + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + meta1.writeVerifiableTo(checksumOut); + StreamInput in = out.bytes().streamInput(); + CoordinationMetadata result = new CoordinationMetadata(in); + + assertEquals(meta1, result); + + VotingConfiguration votingConfiguration2 = new VotingConfiguration( + (Set) votingConfiguration.getNodeIds().stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)) + ); + Set votingTombstones2 = votingTombstones.stream() + .sorted(Comparator.comparing(VotingConfigExclusion::getNodeId)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + CoordinationMetadata meta2 = new CoordinationMetadata(1, votingConfiguration2, votingConfiguration2, votingTombstones2); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + meta2.writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java index 0ea2834cc3024..38d66da632dc5 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java @@ -31,11 +31,14 @@ package org.opensearch.cluster.metadata; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -232,6 +235,37 @@ public void testFromToXContent() throws Exception { } } + public void testWriteVerifiableTo() throws Exception { + String templateName = randomUnicodeOfCodepointLengthBetween(1, 10); + IndexTemplateMetadata.Builder templateBuilder = IndexTemplateMetadata.builder(templateName); + templateBuilder.patterns(Arrays.asList("pattern-1")); + int numAlias = 3; + for (int i = 0; i < numAlias; i++) { + AliasMetadata.Builder alias = AliasMetadata.builder(randomRealisticUnicodeOfLengthBetween(1, 100)); + alias.indexRouting(randomRealisticUnicodeOfLengthBetween(1, 100)); + + alias.searchRouting(randomRealisticUnicodeOfLengthBetween(1, 100)); + + templateBuilder.putAlias(alias); + } + templateBuilder.settings(Settings.builder().put("index.setting-1", randomLong())); + templateBuilder.settings(Settings.builder().put("index.setting-2", randomTimeValue())); + + templateBuilder.order(randomInt()); + + templateBuilder.version(between(0, 100)); + + templateBuilder.putMapping("doc", "{\"doc\":{\"properties\":{\"type\":\"text\"}}}"); + + IndexTemplateMetadata template = templateBuilder.build(); + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + template.writeVerifiableTo(checksumOut); + StreamInput in = out.bytes().streamInput(); + IndexTemplateMetadata result = IndexTemplateMetadata.readFrom(in); + assertThat(result, equalTo(template)); + } + public void testDeprecationWarningsOnMultipleMappings() throws IOException { IndexTemplateMetadata.Builder builder = IndexTemplateMetadata.builder("my-template"); builder.patterns(Arrays.asList("a", "b")); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java index 65e0dc3e44497..b466585acbcfb 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java @@ -174,7 +174,7 @@ private ClusterState generateClusterState() { .put(IndexMetadata.INDEX_READ_ONLY_SETTING.getKey(), true) .build(); final IndexMetadata indexMetadata2 = new IndexMetadata.Builder(index2.getName()).settings(idxSettings2) - .numberOfShards(1) + .numberOfShards(3) .numberOfReplicas(2) .build(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); From 4b9f937e22faa3c713faab30ed30b6e43e9e06ad Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 2 Sep 2024 14:51:54 +0530 Subject: [PATCH 18/29] spotless libs Signed-off-by: Himshikha Gupta --- .../stream/BufferedChecksumStreamOutput.java | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index 8ad629fc29e48..32efc35525797 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -41,7 +41,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -108,11 +107,8 @@ public void writeMap(@Nullable Map map) throws IOException { } @Override - public void writeMap( - Map map, - final Writeable.Writer keyWriter, - final Writeable.Writer valueWriter - ) throws IOException { + public void writeMap(Map map, final Writeable.Writer keyWriter, final Writeable.Writer valueWriter) + throws IOException { writeVInt(map.size()); map.keySet().stream().sorted().forEachOrdered(key -> { try { @@ -124,10 +120,7 @@ public void writeMap( }); } - public void writeMapValues( - Map map, - final Writeable.Writer valueWriter - ) throws IOException { + public void writeMapValues(Map map, final Writeable.Writer valueWriter) throws IOException { writeVInt(map.size()); map.keySet().stream().sorted().forEachOrdered(key -> { try { @@ -152,12 +145,8 @@ public void writeVLongArray(long[] values) throws IOException { @Override public void writeCollection(final Collection collection) throws IOException { - if(!collection.isEmpty() && collection.iterator().next() instanceof Comparable) { - List sortedList = collection.stream().sorted().collect(Collectors.toList()); - super.writeCollection(sortedList, (o, v) -> v.writeTo(o)); - } else { - super.writeCollection(collection, (o, v) -> v.writeTo(o)); - } + List sortedList = collection.stream().sorted().collect(Collectors.toList()); + super.writeCollection(sortedList, (o, v) -> v.writeTo(o)); } @Override From ed05014cc0d04ff92a68b97c91c9a0a957219576 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 2 Sep 2024 18:35:22 +0530 Subject: [PATCH 19/29] adding UTs Signed-off-by: Himshikha Gupta --- .../cluster/node/DiscoveryNodes.java | 33 +++++ .../cluster/metadata/IndexMetadataTests.java | 86 ++++++++++++ .../metadata/IndexTemplateMetadataTests.java | 29 ++-- .../cluster/node/DiscoveryNodeTests.java | 39 ++++++ .../cluster/node/DiscoveryNodesTests.java | 26 ++++ .../routing/IndexShardRoutingTableTests.java | 22 +++ .../cluster/routing/RoutingTableTests.java | 1 + .../remote/ClusterStateChecksumTests.java | 14 +- .../RemoteClusterStateServiceTests.java | 129 +++++++++++++++++- 9 files changed, 359 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index c8080a92ddfaa..ba81012e386d7 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -712,6 +712,39 @@ private void writeClusterManager(StreamOutput out) throws IOException { } } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DiscoveryNodes that = (DiscoveryNodes) o; + return Objects.equals(nodes, that.nodes) + && Objects.equals(dataNodes, that.dataNodes) + && Objects.equals(clusterManagerNodes, that.clusterManagerNodes) + && Objects.equals(ingestNodes, that.ingestNodes) + && Objects.equals(clusterManagerNodeId, that.clusterManagerNodeId) + && Objects.equals(localNodeId, that.localNodeId) + && Objects.equals(minNonClientNodeVersion, that.minNonClientNodeVersion) + && Objects.equals(maxNonClientNodeVersion, that.maxNonClientNodeVersion) + && Objects.equals(maxNodeVersion, that.maxNodeVersion) + && Objects.equals(minNodeVersion, that.minNodeVersion); + } + + @Override + public int hashCode() { + return Objects.hash( + nodes, + dataNodes, + clusterManagerNodes, + ingestNodes, + clusterManagerNodeId, + localNodeId, + minNonClientNodeVersion, + maxNonClientNodeVersion, + maxNodeVersion, + minNodeVersion + ); + } + public static DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws IOException { Builder builder = new Builder(); if (in.readBoolean()) { diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java index 393a652952771..fce3360b2d81f 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java @@ -43,6 +43,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; @@ -170,6 +171,91 @@ public void testIndexMetadataSerialization() throws IOException { } } + public void testWriteVerifiableTo() throws IOException { + int numberOfReplicas = randomIntBetween(0, 10); + final boolean system = randomBoolean(); + Map customMap = new HashMap<>(); + customMap.put(randomAlphaOfLength(5), randomAlphaOfLength(10)); + customMap.put(randomAlphaOfLength(10), randomAlphaOfLength(15)); + + RolloverInfo info1 = new RolloverInfo( + randomAlphaOfLength(5), + Arrays.asList( + new MaxAgeCondition(TimeValue.timeValueMillis(randomNonNegativeLong())), + new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())), + new MaxDocsCondition(randomNonNegativeLong()) + ), + randomNonNegativeLong() + ); + RolloverInfo info2 = new RolloverInfo( + randomAlphaOfLength(5), + Arrays.asList( + new MaxAgeCondition(TimeValue.timeValueMillis(randomNonNegativeLong())), + new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())), + new MaxDocsCondition(randomNonNegativeLong()) + ), + randomNonNegativeLong() + ); + + IndexMetadata metadata1 = IndexMetadata.builder("foo") + .settings( + Settings.builder() + .put("index.version.created", 1) + .put("index.number_of_shards", 4) + .put("index.number_of_replicas", numberOfReplicas) + .build() + ) + .creationDate(randomLong()) + .primaryTerm(0, 2) + .primaryTerm(1, 3) + .setRoutingNumShards(32) + .system(system) + .putCustom("my_custom", customMap) + .putCustom("my_custom2", customMap) + .putAlias(AliasMetadata.builder("alias-1").routing("routing-1").build()) + .putAlias(AliasMetadata.builder("alias-2").routing("routing-2").build()) + .putRolloverInfo(info1) + .putRolloverInfo(info2) + .putInSyncAllocationIds(0, Set.of("1", "2", "3")) + .build(); + + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + metadata1.writeVerifiableTo(checksumOut); + try (StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), writableRegistry())) { + IndexMetadata deserialized = IndexMetadata.readFrom(in); + assertEquals(metadata1, deserialized); + + } + + IndexMetadata metadata2 = IndexMetadata.builder(metadata1.getIndex().getName()) + .settings( + Settings.builder() + .put("index.number_of_replicas", numberOfReplicas) + .put("index.number_of_shards", 4) + .put("index.version.created", 1) + .build() + ) + .creationDate(metadata1.getCreationDate()) + .primaryTerm(1, 3) + .primaryTerm(0, 2) + .setRoutingNumShards(32) + .system(system) + .putCustom("my_custom2", customMap) + .putCustom("my_custom", customMap) + .putAlias(AliasMetadata.builder("alias-2").routing("routing-2").build()) + .putAlias(AliasMetadata.builder("alias-1").routing("routing-1").build()) + .putRolloverInfo(info2) + .putRolloverInfo(info1) + .putInSyncAllocationIds(0, Set.of("3", "1", "2")) + .build(); + + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + metadata2.writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } + public void testGetRoutingFactor() { Integer numShard = randomFrom(1, 2, 4, 8, 16); int routingFactor = IndexMetadata.getRoutingFactor(32, numShard); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java index 38d66da632dc5..6b0d5bcd980d8 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java @@ -238,23 +238,18 @@ public void testFromToXContent() throws Exception { public void testWriteVerifiableTo() throws Exception { String templateName = randomUnicodeOfCodepointLengthBetween(1, 10); IndexTemplateMetadata.Builder templateBuilder = IndexTemplateMetadata.builder(templateName); - templateBuilder.patterns(Arrays.asList("pattern-1")); - int numAlias = 3; + templateBuilder.patterns(Arrays.asList("pattern-1", "pattern-2")); + int numAlias = between(2, 5); for (int i = 0; i < numAlias; i++) { AliasMetadata.Builder alias = AliasMetadata.builder(randomRealisticUnicodeOfLengthBetween(1, 100)); alias.indexRouting(randomRealisticUnicodeOfLengthBetween(1, 100)); - alias.searchRouting(randomRealisticUnicodeOfLengthBetween(1, 100)); - templateBuilder.putAlias(alias); } templateBuilder.settings(Settings.builder().put("index.setting-1", randomLong())); templateBuilder.settings(Settings.builder().put("index.setting-2", randomTimeValue())); - templateBuilder.order(randomInt()); - templateBuilder.version(between(0, 100)); - templateBuilder.putMapping("doc", "{\"doc\":{\"properties\":{\"type\":\"text\"}}}"); IndexTemplateMetadata template = templateBuilder.build(); @@ -263,7 +258,25 @@ public void testWriteVerifiableTo() throws Exception { template.writeVerifiableTo(checksumOut); StreamInput in = out.bytes().streamInput(); IndexTemplateMetadata result = IndexTemplateMetadata.readFrom(in); - assertThat(result, equalTo(template)); + assertEquals(result, template); + + IndexTemplateMetadata.Builder templateBuilder2 = IndexTemplateMetadata.builder(templateName); + templateBuilder2.patterns(Arrays.asList("pattern-2", "pattern-1")); + template.getAliases() + .entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .forEachOrdered(entry -> templateBuilder2.putAlias(entry.getValue())); + templateBuilder2.settings(template.settings()); + templateBuilder2.order(template.order()); + templateBuilder2.version(template.version()); + templateBuilder2.putMapping("doc", template.mappings()); + + IndexTemplateMetadata template2 = templateBuilder.build(); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + template2.writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); } public void testDeprecationWarningsOnMultipleMappings() throws IOException { diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java index c8a6fc76ce820..525a53f3e6158 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java @@ -36,6 +36,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; @@ -46,6 +47,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -128,6 +130,43 @@ public void testDiscoveryNodeSerializationKeepsHost() throws Exception { assertEquals(transportAddress.getPort(), serialized.getAddress().getPort()); } + public void testWriteVerifiableTo() throws Exception { + InetAddress inetAddress = InetAddress.getByAddress("name1", new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1 }); + TransportAddress transportAddress = new TransportAddress(inetAddress, randomIntBetween(0, 65535)); + final Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)); + Map attributes = new HashMap<>(); + attributes.put("att-1", "test-repo"); + attributes.put("att-2", "test-repo"); + DiscoveryNode node = new DiscoveryNode("name1", "id1", transportAddress, attributes, roles, Version.CURRENT); + + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + node.writeVerifiableTo(checksumOut); + StreamInput in = out.bytes().streamInput(); + DiscoveryNode result = new DiscoveryNode(in); + assertEquals(result, node); + + Map attributes2 = new HashMap<>(); + attributes2.put("att-2", "test-repo"); + attributes2.put("att-1", "test-repo"); + + DiscoveryNode node2 = new DiscoveryNode( + node.getName(), + node.getId(), + node.getEphemeralId(), + node.getHostName(), + node.getHostAddress(), + transportAddress, + attributes2, + roles.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)), + Version.CURRENT + ); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + node2.writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } + public void testDiscoveryNodeRoleWithOldVersion() throws Exception { InetAddress inetAddress = InetAddress.getByAddress("name1", new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1 }); TransportAddress transportAddress = new TransportAddress(inetAddress, randomIntBetween(0, 65535)); diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java index d2450859dfcd4..61b86856c9ebc 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java @@ -36,10 +36,14 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -328,6 +332,28 @@ public void testDeprecatedMasterNodeFilter() { assertThat(discoveryNodes.resolveNodes("master:false", "_all"), arrayContainingInAnyOrder(allNodes)); } + public void testWriteVerifiableTo() throws IOException { + final DiscoveryNodes discoveryNodes = buildDiscoveryNodes(); + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + discoveryNodes.writeVerifiableTo(checksumOut); + StreamInput in = out.bytes().streamInput(); + DiscoveryNodes result = DiscoveryNodes.readFrom(in, discoveryNodes.getLocalNode()); + assertEquals(result, discoveryNodes); + + final DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder() + .clusterManagerNodeId(discoveryNodes.getClusterManagerNodeId()); + discoveryNodes.getNodes() + .entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .forEachOrdered(entry -> discoveryNodesBuilder.add(entry.getValue())); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + discoveryNodesBuilder.build().writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } + private static AtomicInteger idGenerator = new AtomicInteger(); private static List randomNodes(final int numNodes) { diff --git a/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java index 6bfe60980adf3..5f8dabdcd4e45 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java @@ -32,10 +32,13 @@ package org.opensearch.cluster.routing; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -77,6 +80,25 @@ public void testEquals() { assertEquals(table4, table5); } + public void testWriteVerifiableTo() throws IOException { + Index index = new Index("a", "b"); + ShardId shardId = new ShardId(index, 1); + ShardRouting shard1 = TestShardRouting.newShardRouting(shardId, "node-1", true, ShardRoutingState.STARTED); + ShardRouting shard2 = TestShardRouting.newShardRouting(shardId, "node-2", false, ShardRoutingState.STARTED); + ShardRouting shard3 = TestShardRouting.newShardRouting(shardId, null, false, ShardRoutingState.UNASSIGNED); + + IndexShardRoutingTable table1 = new IndexShardRoutingTable(shardId, Arrays.asList(shard1, shard2, shard3)); + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + IndexShardRoutingTable.Builder.writeVerifiableTo(table1, checksumOut); + + IndexShardRoutingTable table2 = new IndexShardRoutingTable(shardId, Arrays.asList(shard3, shard1, shard2)); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + IndexShardRoutingTable.Builder.writeVerifiableTo(table2, checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } + public void testShardsMatchingPredicate() { ShardId shardId = new ShardId(new Index("a", UUID.randomUUID().toString()), 0); ShardRouting primary = TestShardRouting.newShardRouting(shardId, "node-1", true, ShardRoutingState.STARTED); diff --git a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java index 97283f561d6d4..d8a67c09e442c 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java @@ -714,4 +714,5 @@ public static IndexMetadata updateActiveAllocations(IndexRoutingTable indexRouti } return imdBuilder.build(); } + } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java index b466585acbcfb..0203e56dd2d5c 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java @@ -206,11 +206,15 @@ private ClusterState generateClusterState() { .build() ) .nodes(DiscoveryNodes.builder().clusterManagerNodeId("test-node").build()) - .blocks(ClusterBlocks.builder() - .addBlocks(indexMetadata) - .addGlobalBlock(new ClusterBlock(1, "block", true, true, true, RestStatus.ACCEPTED, EnumSet.of(ClusterBlockLevel.READ))) - .addGlobalBlock(new ClusterBlock(2, "block-name", false, true, true, RestStatus.OK, EnumSet.of(ClusterBlockLevel.WRITE))) - .build()) + .blocks( + ClusterBlocks.builder() + .addBlocks(indexMetadata) + .addGlobalBlock(new ClusterBlock(1, "block", true, true, true, RestStatus.ACCEPTED, EnumSet.of(ClusterBlockLevel.READ))) + .addGlobalBlock( + new ClusterBlock(2, "block-name", false, true, true, RestStatus.OK, EnumSet.of(ClusterBlockLevel.WRITE)) + ) + .build() + ) .customs(Map.of(clusterStateCustom1.getWriteableName(), clusterStateCustom1)) .routingTable(RoutingTable.builder().addAsNew(indexMetadata).addAsNew(indexMetadata2).version(1L).build()) .build(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 2e168553c090e..ec44d693daaad 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -3028,7 +3028,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullC }).when(mockedClusterStateAttributeManager).readAsync(anyString(), any(), listenerArgumentCaptor.capture()); when(mockedResult.getComponent()).thenReturn(COORDINATION_METADATA); RemoteClusterStateService mockService = spy(remoteClusterStateService); - ClusterState clusterState = mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); + mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); verify(mockService, times(1)).readClusterStateInParallel( any(), eq(manifest), @@ -3048,7 +3048,13 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullC eq(false), eq(true) ); - // verify(mockService, times(0)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + verify(mockService, times(0)).validateClusterStateFromChecksum( + any(ClusterMetadataManifest.class), + any(ClusterState.class), + anyString(), + anyString(), + anyBoolean() + ); } public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws IOException { @@ -3057,7 +3063,6 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( new ClusterStateChecksum(clusterState) ).build(); - mockBlobStoreObjects(); remoteClusterStateService.start(); RemoteClusterStateService mockService = spy(remoteClusterStateService); doReturn(clusterState).when(mockService) @@ -3081,7 +3086,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws eq(true) ); mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); - // verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + verify(mockService, times(1)).validateClusterStateFromChecksum(manifest, clusterState, ClusterName.DEFAULT.value(), NODE_ID, true); } public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMismatch() throws IOException { @@ -3090,7 +3095,6 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( new ClusterStateChecksum(clusterState) ).build(); - mockBlobStoreObjects(); remoteClusterStateService.start(); RemoteClusterStateService mockService = spy(remoteClusterStateService); ClusterState clusterStateWrong = ClusterState.builder(clusterState).routingTable(RoutingTable.EMPTY_ROUTING_TABLE).build(); @@ -3118,7 +3122,115 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma IllegalStateException.class, () -> mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true) ); - // verify(mockService, times(1)).validateClusterStateFromChecksum(any(ClusterStateChecksum.class), any(ClusterState.class)); + verify(mockService, times(1)).validateClusterStateFromChecksum( + manifest, + clusterStateWrong, + ClusterName.DEFAULT.value(), + NODE_ID, + true + ); + } + + public void testGetClusterStateUsingDiffWithChecksum() throws IOException { + initializeWithChecksumEnabled(); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); + + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + + doReturn(clusterState).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(emptyList()), + eq(emptyMap()), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + eq(emptyList()), + anyBoolean(), + eq(emptyMap()), + anyBoolean(), + anyBoolean() + ); + mockService.getClusterStateUsingDiff(manifest, clusterState, NODE_ID); + + verify(mockService, times(1)).validateClusterStateFromChecksum( + eq(manifest), + any(ClusterState.class), + eq(ClusterName.DEFAULT.value()), + eq(NODE_ID), + eq(false) + ); + } + + public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOException { + initializeWithChecksumEnabled(); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); + + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + ClusterState clusterStateWrong = ClusterState.builder(clusterState).routingTable(RoutingTable.EMPTY_ROUTING_TABLE).build(); + doReturn(clusterStateWrong).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(emptyList()), + eq(emptyMap()), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + eq(emptyList()), + anyBoolean(), + eq(emptyMap()), + anyBoolean(), + anyBoolean() + ); + doReturn(clusterState).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + + expectThrows(IllegalStateException.class, () -> mockService.getClusterStateUsingDiff(manifest, clusterState, NODE_ID)); + verify(mockService, times(1)).validateClusterStateFromChecksum( + eq(manifest), + any(ClusterState.class), + eq(ClusterName.DEFAULT.value()), + eq(NODE_ID), + eq(false) + ); } private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { @@ -3670,6 +3782,8 @@ static ClusterMetadataManifest.Builder generateClusterMetadataManifestWithAllAtt return ClusterMetadataManifest.builder() .codecVersion(CODEC_V2) .clusterUUID("cluster-uuid") + .stateVersion(1L) + .stateUUID("state-uuid") .indices(List.of(new UploadedIndexMetadata("test-index", "test-index-uuid", "test-index-file__2"))) .customMetadataMap( Map.of( @@ -3695,7 +3809,8 @@ static ClusterMetadataManifest.Builder generateClusterMetadataManifestWithAllAtt "custom_2", new UploadedMetadataAttribute("custom_2", "test-cluster-state-custom2-file__1") ) - ); + ) + .routingTableVersion(1L); } public static DiscoveryNodes nodesWithLocalNodeClusterManager() { From 7f4947c3fc027548c66cdce0a7b86d6f29d33f75 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 2 Sep 2024 19:07:59 +0530 Subject: [PATCH 20/29] merge conflict fix Signed-off-by: Himshikha Gupta --- .../opensearch/gateway/remote/ClusterMetadataManifest.java | 3 ++- .../gateway/remote/RemoteClusterStateService.java | 1 - .../gateway/remote/RemoteClusterStateServiceTests.java | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index c7fb892d27756..2aa5904ab5b3e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -255,7 +255,7 @@ private static ClusterStateChecksum checksum(Object[] fields) { private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V4; - public static final int MANIFEST_CURRENT_CODEC_VERSION = CODEC_V3; + public static final int MANIFEST_CURRENT_CODEC_VERSION = CODEC_V4; private static final Map VERSION_TO_CODEC_MAPPING; @@ -279,6 +279,7 @@ private static ClusterStateChecksum checksum(Object[] fields) { versionToCodecMapping.put(version, ClusterMetadataManifest.CODEC_V3); } else if (version.onOrAfter(Version.V_2_17_0)) { versionToCodecMapping.put(version, ClusterMetadataManifest.CODEC_V4); + } } VERSION_TO_CODEC_MAPPING = Collections.unmodifiableMap(versionToCodecMapping); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 252949f0ccabd..3a234d7123482 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -1490,7 +1490,6 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C remoteStateStats.stateDiffDownloadTook(durationMillis); return clusterState; - return clusterState; } void validateClusterStateFromChecksum( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 267b8eec1f5c2..cffae395d8f1f 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -2948,8 +2948,11 @@ public void testWriteFullMetadataSuccessWithChecksumValidationEnabled() throws I final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") - .getClusterMetadataManifest(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( + clusterState, + "prev-cluster-uuid", + MANIFEST_CURRENT_CODEC_VERSION + ).getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( "test-index", From d6bcabb3e094245cbd0e9897cbad47e06b3d6f20 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 2 Sep 2024 20:52:35 +0530 Subject: [PATCH 21/29] gradle fix Signed-off-by: Himshikha Gupta --- .../java/org/opensearch/cluster/metadata/IndexMetadata.java | 3 +++ .../org/opensearch/gateway/remote/ClusterMetadataManifest.java | 2 +- .../opensearch/gateway/remote/RemoteClusterStateService.java | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index ab13cb346835c..cc6ade8f9d1d4 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1288,6 +1288,9 @@ public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOExcepti ); out.writeMapValues(rolloverInfos, (stream, val) -> val.writeTo(stream)); out.writeBoolean(isSystem); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalWriteable(context); + } } public boolean isSystem() { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 2aa5904ab5b3e..361b09b200068 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -275,7 +275,7 @@ private static ClusterStateChecksum checksum(Object[] fields) { versionToCodecMapping.put(version, ClusterMetadataManifest.CODEC_V1); } else if (version.onOrAfter(Version.V_2_15_0) && version.before(Version.V_2_16_0)) { versionToCodecMapping.put(version, ClusterMetadataManifest.CODEC_V2); - } else if (version.onOrAfter(Version.V_2_16_0)) { + } else if (version.onOrAfter(Version.V_2_16_0) && version.before(Version.V_2_17_0)) { versionToCodecMapping.put(version, ClusterMetadataManifest.CODEC_V3); } else if (version.onOrAfter(Version.V_2_17_0)) { versionToCodecMapping.put(version, ClusterMetadataManifest.CODEC_V4); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 3a234d7123482..4b9e387a98416 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -1364,7 +1364,6 @@ public ClusterState getClusterStateForManifest( if (includeEphemeral && checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { validateClusterStateFromChecksum(manifest, clusterState, clusterName, localNodeId, true); } - return clusterState; } else { ClusterState state = readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), From d7bfb15a7531549a927a19db262bfca72330e517 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 2 Sep 2024 21:57:41 +0530 Subject: [PATCH 22/29] resolve conflict fix Signed-off-by: Himshikha Gupta --- .../opensearch/gateway/remote/RemoteClusterStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 95c0dba198f42..88dda3953a5f9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -1514,7 +1514,7 @@ void validateClusterStateFromChecksum( String clusterName, String localNodeId, boolean isFullStateDownload - ) throws IOException { + ) { ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState); List failedValidation = newClusterStateChecksum.getMismatchEntities(manifest.getClusterStateChecksum()); if (!failedValidation.isEmpty()) { From fd0ed78ba07f8dd0fc234e5caf4153c8863be474 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 2 Sep 2024 22:08:42 +0530 Subject: [PATCH 23/29] test fix Signed-off-by: Himshikha Gupta --- .../remote/ClusterMetadataManifestTests.java | 71 +------------------ 1 file changed, 3 insertions(+), 68 deletions(-) diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 5202cc2331694..2f8531e122eeb 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -41,6 +41,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -646,9 +647,7 @@ public void testClusterMetadataManifestXContentV3() throws IOException { public void testClusterMetadataManifestXContentV4() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); - final DiffableUtils.MapDiff> routingTableIncrementalDiff = Mockito.mock( - DiffableUtils.MapDiff.class - ); + final StringKeyDiffProvider routingTableIncrementalDiff = Mockito.mock(StringKeyDiffProvider.class); ClusterStateChecksum checksum = new ClusterStateChecksum(createClusterState()); ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() .clusterTerm(1L) @@ -694,6 +693,7 @@ public void testClusterMetadataManifestXContentV4() throws IOException { new ClusterStateDiffManifest( RemoteClusterStateServiceTests.generateClusterStateWithOneIndex().build(), ClusterState.EMPTY_STATE, + CODEC_V4, routingTableIncrementalDiff, uploadedMetadataAttribute.getUploadedFilename() ) @@ -766,71 +766,6 @@ public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOExc } } - public void testClusterMetadataManifestXContentV4() throws IOException { - UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); - UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); - final StringKeyDiffProvider routingTableIncrementalDiff = Mockito.mock(StringKeyDiffProvider.class); - ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() - .clusterTerm(1L) - .stateVersion(1L) - .clusterUUID("test-cluster-uuid") - .stateUUID("test-state-uuid") - .opensearchVersion(Version.CURRENT) - .nodeId("test-node-id") - .committed(false) - .codecVersion(ClusterMetadataManifest.CODEC_V4) - .indices(Collections.singletonList(uploadedIndexMetadata)) - .previousClusterUUID("prev-cluster-uuid") - .clusterUUIDCommitted(true) - .coordinationMetadata(uploadedMetadataAttribute) - .settingMetadata(uploadedMetadataAttribute) - .templatesMetadata(uploadedMetadataAttribute) - .customMetadataMap( - Collections.unmodifiableList( - Arrays.asList( - new UploadedMetadataAttribute( - CUSTOM_METADATA + CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, - "custom--repositories-file" - ), - new UploadedMetadataAttribute( - CUSTOM_METADATA + CUSTOM_DELIMITER + IndexGraveyard.TYPE, - "custom--index_graveyard-file" - ), - new UploadedMetadataAttribute( - CUSTOM_METADATA + CUSTOM_DELIMITER + WeightedRoutingMetadata.TYPE, - "custom--weighted_routing_netadata-file" - ) - ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) - ) - .routingTableVersion(1L) - .indicesRouting(Collections.singletonList(uploadedIndexMetadata)) - .discoveryNodesMetadata(uploadedMetadataAttribute) - .clusterBlocksMetadata(uploadedMetadataAttribute) - .transientSettingsMetadata(uploadedMetadataAttribute) - .hashesOfConsistentSettings(uploadedMetadataAttribute) - .clusterStateCustomMetadataMap(Collections.emptyMap()) - .diffManifest( - new ClusterStateDiffManifest( - RemoteClusterStateServiceTests.generateClusterStateWithOneIndex().build(), - ClusterState.EMPTY_STATE, - CODEC_V4, - routingTableIncrementalDiff, - uploadedMetadataAttribute.getUploadedFilename() - ) - ) - .build(); - final XContentBuilder builder = JsonXContent.contentBuilder(); - builder.startObject(); - originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - - try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); - assertEquals(originalManifest, fromXContentManifest); - } - } - public static List randomUploadedIndexMetadataList() { final int size = randomIntBetween(1, 10); final List uploadedIndexMetadataList = new ArrayList<>(size); From 543c492c82f36fbc4c006eed1dbfbabd5872e258 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 2 Sep 2024 22:39:17 +0530 Subject: [PATCH 24/29] spotless Signed-off-by: Himshikha Gupta --- .../org/opensearch/gateway/remote/ClusterMetadataManifest.java | 3 ++- .../gateway/remote/ClusterMetadataManifestTests.java | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 4b9100b7a5a48..405e5cd784196 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -45,7 +45,8 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { // also we introduce index routing-metadata, diff and other attributes as part of manifest // required for state publication public static final int CODEC_V3 = 3; // In Codec V3, we have introduced new diff field in diff-manifest's routing_table_diff - public static final int CODEC_V4 = 4; // In Codec V4, we have removed upserts and delete field for routing table in diff manifest and added checksum of cluster state. + public static final int CODEC_V4 = 4; // In Codec V4, we have removed upserts and delete field for routing table in diff manifest and + // added checksum of cluster state. public static final int[] CODEC_VERSIONS = { CODEC_V0, CODEC_V1, CODEC_V2, CODEC_V3, CODEC_V4 }; private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 2f8531e122eeb..3f9aa1245cab3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -11,7 +11,6 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexGraveyard; import org.opensearch.cluster.metadata.IndexMetadata; @@ -22,8 +21,8 @@ import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; -import org.opensearch.common.settings.Settings; import org.opensearch.cluster.routing.StringKeyDiffProvider; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; From f8953235b8013afbb08d520b567e83940f017728 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 2 Sep 2024 23:34:49 +0530 Subject: [PATCH 25/29] custom key check Signed-off-by: Himshikha Gupta --- .../org/opensearch/gateway/remote/ClusterStateChecksum.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index 11087570e0d21..ac087e18fac54 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -95,7 +95,7 @@ public ClusterStateChecksum(ClusterState clusterState) { templatesMetadataChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeMapValues(clusterState.metadata().customs(), (stream, value) -> value.writeTo(stream)); + checksumOut.writeCollection(clusterState.metadata().customs().keySet(), StreamOutput::writeString); customMetadataMapChecksum = checksumOut.getChecksum(); checksumOut.reset(); @@ -114,7 +114,7 @@ public ClusterStateChecksum(ClusterState clusterState) { blocksChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeMapValues(clusterState.customs(), (stream, value) -> checksumOut.writeNamedWriteable(value)); + checksumOut.writeCollection(clusterState.customs().keySet(), StreamOutput::writeString); clusterStateCustomsChecksum = checksumOut.getChecksum(); } catch (IOException e) { logger.error("Failed to create checksum for cluster state.", e); From cf942090b9877b187a0b23b9dd5da58ae2446623 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 3 Sep 2024 00:11:09 +0530 Subject: [PATCH 26/29] pr comments Signed-off-by: Himshikha Gupta --- .../opensearch/cluster/routing/IndexShardRoutingTable.java | 6 ++++++ .../org/opensearch/gateway/remote/ClusterStateChecksum.java | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index 263aba4312c7e..4cc3300676986 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -63,6 +63,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -1174,17 +1175,22 @@ public static void writeVerifiableTo(IndexShardRoutingTable indexShard, StreamOu out.writeVInt(indexShard.shardId.id()); out.writeVInt(indexShard.shards.size()); // Order allocated shards by allocationId + AtomicInteger assignedShardCount = new AtomicInteger(); indexShard.shards.stream() .filter(shardRouting -> shardRouting.allocationId() != null) .sorted(Comparator.comparing(o -> o.allocationId().getId())) .forEach(shardRouting -> { try { + assignedShardCount.getAndIncrement(); shardRouting.writeToThin(out); } catch (IOException e) { logger.error(() -> new ParameterizedMessage("Failed to write shard {}. Exception {}", indexShard, e)); throw new RuntimeException("Failed to write IndexShardRoutingTable", e); } }); + // is primary assigned + out.writeBoolean(indexShard.primaryShard().allocationId() != null); + out.writeVInt(indexShard.shards.size() - assignedShardCount.get()); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index ac087e18fac54..d6739c4572d1a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -95,7 +95,7 @@ public ClusterStateChecksum(ClusterState clusterState) { templatesMetadataChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeCollection(clusterState.metadata().customs().keySet(), StreamOutput::writeString); + checksumOut.writeStringCollection(clusterState.metadata().customs().keySet()); customMetadataMapChecksum = checksumOut.getChecksum(); checksumOut.reset(); @@ -114,7 +114,7 @@ public ClusterStateChecksum(ClusterState clusterState) { blocksChecksum = checksumOut.getChecksum(); checksumOut.reset(); - checksumOut.writeCollection(clusterState.customs().keySet(), StreamOutput::writeString); + checksumOut.writeStringCollection(clusterState.customs().keySet()); clusterStateCustomsChecksum = checksumOut.getChecksum(); } catch (IOException e) { logger.error("Failed to create checksum for cluster state.", e); From 45cb5d4202a2bf28cf06ff470c2460f59ef8505e Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 3 Sep 2024 14:51:05 +0530 Subject: [PATCH 27/29] fixing IT Signed-off-by: Himshikha Gupta --- .../common/io/stream/BufferedChecksumStreamOutput.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index 32efc35525797..18bd53dc5d77c 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -133,14 +133,16 @@ public void writeMapValues(Map map, final Writeable.Writer value @Override public void writeStringArray(String[] array) throws IOException { - Arrays.sort(array); - super.writeStringArray(array); + String[] copyArray = Arrays.copyOf(array, array.length); + Arrays.sort(copyArray); + super.writeStringArray(copyArray); } @Override public void writeVLongArray(long[] values) throws IOException { - Arrays.sort(values); - super.writeVLongArray(values); + long[] copyValues = Arrays.copyOf(values, values.length); + Arrays.sort(copyValues); + super.writeVLongArray(copyValues); } @Override From ac2e2a2283ff2e417fb8e3208c4b93b2fd1ad939 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 3 Sep 2024 15:57:33 +0530 Subject: [PATCH 28/29] merge fix for indexmetadata Signed-off-by: Himshikha Gupta --- .../java/org/opensearch/cluster/metadata/IndexMetadata.java | 2 +- .../org/opensearch/cluster/metadata/IndexMetadataTests.java | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 2f01cde440fd4..bb470ea9e4ab8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1288,7 +1288,7 @@ public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOExcepti ); out.writeMapValues(rolloverInfos, (stream, val) -> val.writeTo(stream)); out.writeBoolean(isSystem); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeOptionalWriteable(context); } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java index 538890469abd6..a92a5b41e8580 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java @@ -225,11 +225,6 @@ public void testWriteVerifiableTo() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); metadata1.writeVerifiableTo(checksumOut); - try (StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), writableRegistry())) { - IndexMetadata deserialized = IndexMetadata.readFrom(in); - assertEquals(metadata1, deserialized); - - } IndexMetadata metadata2 = IndexMetadata.builder(metadata1.getIndex().getName()) .settings( From 18b8cd0ec3890bf9deb75306a38f4a51919b3063 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 3 Sep 2024 16:15:36 +0530 Subject: [PATCH 29/29] test fix after merge Signed-off-by: Himshikha Gupta --- .../gateway/remote/RemoteClusterStateServiceTests.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index eaae8c84f2209..8dd35e59c43c9 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -2972,7 +2972,15 @@ private void initializeWithChecksumEnabled() { clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)), + List.of( + new RemoteIndexPathUploader( + threadPool, + newSettings, + repositoriesServiceSupplier, + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE + ) + ), writableRegistry() ); }