Skip to content

Commit

Permalink
Split the cluster state remote global metadata file to metadata attri…
Browse files Browse the repository at this point in the history
…bute files

Signed-off-by: Shivansh Arora <hishiv@amazon.com>
  • Loading branch information
shiv0408 committed Feb 6, 2024
1 parent 904c9a9 commit 6bf7bc9
Show file tree
Hide file tree
Showing 5 changed files with 838 additions and 40 deletions.
62 changes: 41 additions & 21 deletions server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,27 @@ public enum XContentContext {
public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, ClusterState.FeatureAware {

EnumSet<XContentContext> context();

static Custom fromXContent(XContentParser parser, String name) throws IOException {
try {
return parser.namedObject(Custom.class, name, null);
} catch (NamedObjectNotFoundException e) {
logger.warn("Unknown custom object with type {}", name);
parser.skipChildren();
throw e;
}
}

static Custom fromXContent(XContentParser parser) throws IOException {
String currentFieldName = parser.currentName();
try {
return parser.namedObject(Custom.class, currentFieldName, null);
} catch (NamedObjectNotFoundException e) {
logger.warn("Unknown custom object with type {}", currentFieldName);
parser.skipChildren();
throw e;
}
}
}

public static final Setting<Integer> DEFAULT_REPLICA_COUNT_SETTING = Setting.intSetting(
Expand Down Expand Up @@ -260,7 +281,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
private final Settings settings;
private final DiffableStringMap hashesOfConsistentSettings;
private final Map<String, IndexMetadata> indices;
private final Map<String, IndexTemplateMetadata> templates;
private final TemplatesMetadata templates;
private final Map<String, Custom> customs;

private final transient int totalNumberOfShards; // Transient ? not serializable anyway?
Expand Down Expand Up @@ -304,7 +325,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
this.hashesOfConsistentSettings = hashesOfConsistentSettings;
this.indices = Collections.unmodifiableMap(indices);
this.customs = Collections.unmodifiableMap(customs);
this.templates = Collections.unmodifiableMap(templates);
this.templates = new TemplatesMetadata(templates);
int totalNumberOfShards = 0;
int totalOpenIndexShards = 0;
for (IndexMetadata cursor : indices.values()) {
Expand Down Expand Up @@ -806,13 +827,17 @@ public Map<String, IndexMetadata> getIndices() {
}

public Map<String, IndexTemplateMetadata> templates() {
return this.templates;
return this.templates.getTemplates();
}

public Map<String, IndexTemplateMetadata> getTemplates() {
return templates();
}

public TemplatesMetadata templatesMetadata() {
return this.templates;
}

public Map<String, ComponentTemplate> componentTemplates() {
return Optional.ofNullable((ComponentTemplateMetadata) this.custom(ComponentTemplateMetadata.TYPE))
.map(ComponentTemplateMetadata::componentTemplates)
Expand Down Expand Up @@ -945,7 +970,7 @@ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metada
if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) {
return false;
}
if (!metadata1.templates.equals(metadata2.templates())) {
if (!metadata1.templates.equals(metadata2.templates)) {
return false;
}
// Check if any persistent metadata needs to be saved
Expand Down Expand Up @@ -1012,7 +1037,7 @@ private static class MetadataDiff implements Diff<Metadata> {
persistentSettings = after.persistentSettings;
hashesOfConsistentSettings = after.hashesOfConsistentSettings.diff(before.hashesOfConsistentSettings);
indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer());
templates = DiffableUtils.diff(before.templates.getTemplates(), after.templates.getTemplates(), DiffableUtils.getStringKeySerializer());
customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
}

Expand Down Expand Up @@ -1059,7 +1084,7 @@ public Metadata apply(Metadata part) {
builder.persistentSettings(persistentSettings);
builder.hashesOfConsistentSettings(hashesOfConsistentSettings.apply(part.hashesOfConsistentSettings));
builder.indices(indices.apply(part.indices));
builder.templates(templates.apply(part.templates));
builder.templates(templates.apply(part.templates.getTemplates()));
builder.customs(customs.apply(part.customs));
return builder.build();
}
Expand Down Expand Up @@ -1103,10 +1128,7 @@ public void writeTo(StreamOutput out) throws IOException {
for (IndexMetadata indexMetadata : this) {
indexMetadata.writeTo(out);
}
out.writeVInt(templates.size());
for (final IndexTemplateMetadata cursor : templates.values()) {
cursor.writeTo(out);
}
templates.writeTo(out);
// filter out custom states not supported by the other node
int numberOfCustoms = 0;
for (final Custom cursor : customs.values()) {
Expand Down Expand Up @@ -1170,7 +1192,7 @@ public Builder(Metadata metadata) {
this.hashesOfConsistentSettings = metadata.hashesOfConsistentSettings;
this.version = metadata.version;
this.indices = new HashMap<>(metadata.indices);
this.templates = new HashMap<>(metadata.templates);
this.templates = new HashMap<>(metadata.templates.getTemplates());
this.customs = new HashMap<>(metadata.customs);
this.previousMetadata = metadata;
}
Expand Down Expand Up @@ -1249,6 +1271,11 @@ public Builder templates(Map<String, IndexTemplateMetadata> templates) {
return this;
}

public Builder templates(TemplatesMetadata templatesMetadata) {
this.templates.putAll(templatesMetadata.getTemplates());
return this;
}

public Builder put(String name, ComponentTemplate componentTemplate) {
Objects.requireNonNull(componentTemplate, "it is invalid to add a null component template: " + name);
Map<String, ComponentTemplate> existingTemplates = Optional.ofNullable(
Expand Down Expand Up @@ -1738,11 +1765,7 @@ public static void toXContent(Metadata metadata, XContentBuilder builder, ToXCon
builder.endObject();
}

builder.startObject("templates");
for (final IndexTemplateMetadata cursor : metadata.templates().values()) {
IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
}
builder.endObject();
metadata.templatesMetadata().toXContent(builder, params);

if (context == XContentContext.API) {
builder.startObject("indices");
Expand Down Expand Up @@ -1804,16 +1827,13 @@ public static Metadata fromXContent(XContentParser parser) throws IOException {
} else if ("hashes_of_consistent_settings".equals(currentFieldName)) {
builder.hashesOfConsistentSettings(parser.mapStrings());
} else if ("templates".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName()));
}
builder.templates(TemplatesMetadata.fromXContent(parser));
} else {
try {
Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
Custom custom = Custom.fromXContent(parser, currentFieldName);
builder.putCustom(custom.getWriteableName(), custom);
} catch (NamedObjectNotFoundException ex) {
logger.warn("Skipping unknown custom object with type {}", currentFieldName);
parser.skipChildren();
}
}
} else if (token.isValue()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,10 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO
XContentParser.Token token;
List<RepositoryMetadata> repository = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.START_OBJECT) {
// move to next token if parsing the whole object
token = parser.nextToken();
}
if (token == XContentParser.Token.FIELD_NAME) {
String name = parser.currentName();
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.cluster.metadata;

import org.opensearch.cluster.AbstractDiffable;
import org.opensearch.cluster.coordination.CoordinationMetadata;
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

public class TemplatesMetadata extends AbstractDiffable<TemplatesMetadata> implements ToXContentFragment {
public static TemplatesMetadata EMPTY_METADATA = builder().build();
private final Map<String, IndexTemplateMetadata> templates;

public TemplatesMetadata() {
this(Collections.emptyMap());
}

public TemplatesMetadata(Map<String, IndexTemplateMetadata> templates) {
this.templates = Collections.unmodifiableMap(templates);
}

public static Builder builder() {
return new Builder();
}

public Map<String, IndexTemplateMetadata> getTemplates() {
return this.templates;
}

public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException {
return Builder.fromXContent(parser);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
Builder.toXContent(this, builder, params);
return builder;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(templates.size());
for (final IndexTemplateMetadata cursor : templates.values()) {
cursor.writeTo(out);
}
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;

TemplatesMetadata that = (TemplatesMetadata) o;

return Objects.equals(templates, that.templates);
}

@Override
public int hashCode() {
return templates != null ? templates.hashCode() : 0;
}

public static class Builder {
private final Map<String, IndexTemplateMetadata> templates;

public Builder() {
this.templates = new HashMap<String, IndexTemplateMetadata>();
}

public Builder(Map<String, IndexTemplateMetadata> templates) {
this.templates = templates;
}

public Builder put(IndexTemplateMetadata.Builder template) {
return put(template.build());
}

public Builder put(IndexTemplateMetadata template) {
templates.put(template.name(), template);
return this;
}

public Builder removeTemplate(String templateName) {
templates.remove(templateName);
return this;
}

public Builder templates(Map<String, IndexTemplateMetadata> templates) {
this.templates.putAll(templates);
return this;
}

public TemplatesMetadata build() {
return new TemplatesMetadata(templates);
}

public static void toXContent(TemplatesMetadata templates, XContentBuilder builder, Params params) throws IOException {
// builder.startObject("templates-metadata");
for(IndexTemplateMetadata cursor : templates.getTemplates().values()) {
IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
}
// builder.endObject();
}

public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException {
Builder builder = new Builder();

// we might get here after the templates-metadata element, or on a fresh parser
XContentParser.Token token = parser.currentToken();
String currentFieldName = parser.currentName();
if (currentFieldName == null) {
token = parser.nextToken();
if (token == XContentParser.Token.START_OBJECT) {
// move to the field name
token = parser.nextToken();
if (token == XContentParser.Token.FIELD_NAME) {
// move to the next object
token = parser.nextToken();
}
}
currentFieldName = parser.currentName();
}
if (currentFieldName != null && token == XContentParser.Token.START_OBJECT) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName()));
}
}
return builder.build();
}
}
}
Loading

0 comments on commit 6bf7bc9

Please sign in to comment.