Skip to content

Commit

Permalink
Changed the setting name cluster.ignore_hidden_indexes to cluster.ign…
Browse files Browse the repository at this point in the history 
…ore_dot_indexes

Signed-off-by: Nishchay Malhotra <nishcha@amazon.com>
  • Loading branch information
@nishchay21
nishchay21 committed Oct 19, 2022
1 parent 4f6e176 commit 8e70553
Show file tree
Hide file tree
Showing 4 changed files with 66 additions and 66 deletions.
52 changes: 26 additions & 26 deletions server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,16 +76,16 @@
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST)
public class ClusterShardLimitIT extends OpenSearchIntegTestCase {
private static final String shardsPerNodeKey = ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey();
private static final String ignoreHiddenIndexKey = ShardLimitValidator.SETTING_CLUSTER_IGNORE_HIDDEN_INDEXES.getKey();
private static final String ignoreDotIndexKey = ShardLimitValidator.SETTING_CLUSTER_IGNORE_DOT_INDEXES.getKey();

public void testSettingClusterMaxShards() {
int shardsPerNode = between(1, 500_000);
setShardsPerNode(shardsPerNode);
}

public void testSettingIgnoreHiddenIndexes() {
boolean ignoreHiddenIndexes = randomBoolean();
setIgnoreHiddenIndex(ignoreHiddenIndexes);
public void testSettingIgnoreDotIndexes() {
boolean ignoreDotIndexes = randomBoolean();
setIgnoreDotIndex(ignoreDotIndexes);
}

public void testMinimumPerNode() {
Expand Down Expand Up @@ -147,16 +147,16 @@ public void testIndexCreationOverLimit() {

/**
* The test checks if the index starting with the dot can be created if the node has
* number of shards equivalent to the cluster.max_shards_per_node and the cluster.ignore_hidden_indexes
* setting is set to true. If the cluster.ignore_hidden_indexes is set to true index creation of
* number of shards equivalent to the cluster.max_shards_per_node and the cluster.ignore_Dot_indexes
* setting is set to true. If the cluster.ignore_Dot_indexes is set to true index creation of
* indexes starting with dot would succeed.
*/
public void testIndexCreationOverLimitForHiddenIndexesSucceeds() {
public void testIndexCreationOverLimitForDotIndexesSucceeds() {
int dataNodes = client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size();

// Setting the cluster.max_shards_per_node setting according to the data node count.
setShardsPerNode(dataNodes);
setIgnoreHiddenIndex(true);
setIgnoreDotIndex(true);

/*
Create an index that will bring us up to the limit. It would create index with primary equal to the
Expand Down Expand Up @@ -194,11 +194,11 @@ public void testIndexCreationOverLimitForHiddenIndexesSucceeds() {

/**
* The test checks if the index starting with the dot should not be created if the node has
* number of shards equivalent to the cluster.max_shards_per_node and the cluster.ignore_hidden_indexes
* setting is set to false. If the cluster.ignore_hidden_indexes is set to false index creation of
* number of shards equivalent to the cluster.max_shards_per_node and the cluster.ignore_Dot_indexes
* setting is set to false. If the cluster.ignore_Dot_indexes is set to false index creation of
* indexes starting with dot would fail as well.
*/
public void testIndexCreationOverLimitForHiddenIndexesFail() {
public void testIndexCreationOverLimitForDotIndexesFail() {
int dataNodes = client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size();
int maxAllowedShards = dataNodes * dataNodes;

Expand Down Expand Up @@ -249,8 +249,8 @@ public void testIndexCreationOverLimitForHiddenIndexesFail() {

/**
* The test checks if the index starting with the .ds- can be created if the node has
* number of shards equivalent to the cluster.max_shards_per_node and the cluster.ignore_hidden_indexes
* setting is set to true. If the cluster.ignore_hidden_indexes is set to true index creation of
* number of shards equivalent to the cluster.max_shards_per_node and the cluster.ignore_Dot_indexes
* setting is set to true. If the cluster.ignore_Dot_indexes is set to true index creation of
* indexes starting with dot would only succeed and dataStream indexes would still have validation applied.
*/
public void testIndexCreationOverLimitForDataStreamIndexes() {
Expand All @@ -259,7 +259,7 @@ public void testIndexCreationOverLimitForDataStreamIndexes() {

// Setting the cluster.max_shards_per_node setting according to the data node count.
setShardsPerNode(dataNodes);
setIgnoreHiddenIndex(true);
setIgnoreDotIndex(true);

/*
Create an index that will bring us up to the limit. It would create index with primary equal to the
Expand Down Expand Up @@ -590,7 +590,7 @@ public void testOpenIndexOverLimit() {
assertFalse(clusterState.getMetadata().hasIndex("snapshot-index"));
}

public void testIgnoreHiddenSettingOnMultipleNodes() throws IOException, InterruptedException {
public void testIgnoreDotSettingOnMultipleNodes() throws IOException, InterruptedException {
int maxAllowedShardsPerNode = 10, indexPrimaryShards = 11, indexReplicaShards = 1;

InternalTestCluster cluster = new InternalTestCluster(
Expand Down Expand Up @@ -630,13 +630,13 @@ public Path nodeConfigPath(int nodeOrdinal) {
cluster.beforeTest(random());

// Starting 3 ClusterManagerOnlyNode nodes
cluster.startClusterManagerOnlyNode(Settings.builder().put("cluster.ignore_hidden_indexes", true).build());
cluster.startClusterManagerOnlyNode(Settings.builder().put("cluster.ignore_hidden_indexes", false).build());
cluster.startClusterManagerOnlyNode(Settings.builder().put("cluster.ignore_hidden_indexes", false).build());
cluster.startClusterManagerOnlyNode(Settings.builder().put("cluster.ignore_dot_indexes", true).build());
cluster.startClusterManagerOnlyNode(Settings.builder().put("cluster.ignore_dot_indexes", false).build());
cluster.startClusterManagerOnlyNode(Settings.builder().put("cluster.ignore_dot_indexes", false).build());

// Starting 2 data nodes
cluster.startDataOnlyNode(Settings.builder().put("cluster.ignore_hidden_indexes", false).build());
cluster.startDataOnlyNode(Settings.builder().put("cluster.ignore_hidden_indexes", false).build());
cluster.startDataOnlyNode(Settings.builder().put("cluster.ignore_dot_indexes", false).build());
cluster.startDataOnlyNode(Settings.builder().put("cluster.ignore_dot_indexes", false).build());

// Setting max shards per node to be 10
cluster.client()
Expand All @@ -657,7 +657,7 @@ public Path nodeConfigPath(int nodeOrdinal) {
.get();

// As active ClusterManagerNode setting takes precedence killing the active one.
// This would be the first one where cluster.ignore_hidden_indexes is true because the above calls are blocking.
// This would be the first one where cluster.ignore_dot_indexes is true because the above calls are blocking.
cluster.stopCurrentClusterManagerNode();

// Waiting for all shards to get assigned
Expand Down Expand Up @@ -727,23 +727,23 @@ private void setShardsPerNode(int shardsPerNode) {
}
}

private void setIgnoreHiddenIndex(boolean ignoreHiddenIndex) {
private void setIgnoreDotIndex(boolean ignoreDotIndex) {
try {
ClusterUpdateSettingsResponse response;
if (frequently()) {
response = client().admin()
.cluster()
.prepareUpdateSettings()
.setPersistentSettings(Settings.builder().put(ignoreHiddenIndexKey, ignoreHiddenIndex).build())
.setPersistentSettings(Settings.builder().put(ignoreDotIndexKey, ignoreDotIndex).build())
.get();
assertEquals(ignoreHiddenIndex, response.getPersistentSettings().getAsBoolean(ignoreHiddenIndexKey, true));
assertEquals(ignoreDotIndex, response.getPersistentSettings().getAsBoolean(ignoreDotIndexKey, true));
} else {
response = client().admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(Settings.builder().put(ignoreHiddenIndexKey, ignoreHiddenIndex).build())
.setTransientSettings(Settings.builder().put(ignoreDotIndexKey, ignoreDotIndex).build())
.get();
assertEquals(ignoreHiddenIndex, response.getTransientSettings().getAsBoolean(ignoreHiddenIndexKey, true));
assertEquals(ignoreDotIndex, response.getTransientSettings().getAsBoolean(ignoreDotIndexKey, true));
}
} catch (IllegalArgumentException ex) {
fail(ex.getMessage());
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ public void apply(Settings value, Settings current, Settings previous) {
Metadata.SETTING_READ_ONLY_SETTING,
Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING,
ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE,
ShardLimitValidator.SETTING_CLUSTER_IGNORE_HIDDEN_INDEXES,
ShardLimitValidator.SETTING_CLUSTER_IGNORE_DOT_INDEXES,
RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING,
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING,
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING,
Expand Down
32 changes: 16 additions & 16 deletions server/src/main/java/org/opensearch/indices/ShardLimitValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,22 +66,22 @@ public class ShardLimitValidator {
Setting.Property.NodeScope
);

public static final Setting<Boolean> SETTING_CLUSTER_IGNORE_HIDDEN_INDEXES = Setting.boolSetting(
"cluster.ignore_hidden_indexes",
public static final Setting<Boolean> SETTING_CLUSTER_IGNORE_DOT_INDEXES = Setting.boolSetting(
"cluster.ignore_dot_indexes",
false,
Setting.Property.Dynamic,
Setting.Property.NodeScope
);

protected final AtomicInteger shardLimitPerNode = new AtomicInteger();
private final SystemIndices systemIndices;
private volatile boolean ignoreHiddenIndexes;
private volatile boolean ignoreDotIndexes;

public ShardLimitValidator(final Settings settings, ClusterService clusterService, SystemIndices systemIndices) {
this.shardLimitPerNode.set(SETTING_CLUSTER_MAX_SHARDS_PER_NODE.get(settings));
this.ignoreHiddenIndexes = SETTING_CLUSTER_IGNORE_HIDDEN_INDEXES.get(settings);
this.ignoreDotIndexes = SETTING_CLUSTER_IGNORE_DOT_INDEXES.get(settings);
clusterService.getClusterSettings().addSettingsUpdateConsumer(SETTING_CLUSTER_MAX_SHARDS_PER_NODE, this::setShardLimitPerNode);
clusterService.getClusterSettings().addSettingsUpdateConsumer(SETTING_CLUSTER_IGNORE_HIDDEN_INDEXES, this::setIgnoreHiddenIndexes);
clusterService.getClusterSettings().addSettingsUpdateConsumer(SETTING_CLUSTER_IGNORE_DOT_INDEXES, this::setIgnoreDotIndexes);
this.systemIndices = systemIndices;
}

Expand All @@ -97,14 +97,14 @@ public int getShardLimitPerNode() {
return shardLimitPerNode.get();
}

private void setIgnoreHiddenIndexes(boolean newValue) {
this.ignoreHiddenIndexes = newValue;
private void setIgnoreDotIndexes(boolean newValue) {
this.ignoreDotIndexes = newValue;
}

/**
* Checks whether an index can be created without going over the cluster shard limit.
* Validate shard limit only for non system indices as it is not hard limit anyways.
* Further also validates if the cluster.ignore_hidden_indexes is set to true.
* Further also validates if the cluster.ignore_dot_indexes is set to true.
* If so then it does not validate any index which starts with '.' except data-stream index.
*
* @param indexName the name of the index being created
Expand All @@ -115,10 +115,10 @@ private void setIgnoreHiddenIndexes(boolean newValue) {
public void validateShardLimit(final String indexName, final Settings settings, final ClusterState state) {
/*
Validate shard limit only for non system indices as it is not hard limit anyways.
Further also validates if the cluster.ignore_hidden_indexes is set to true.
Further also validates if the cluster.ignore_dot_indexes is set to true.
If so then it does not validate any index which starts with '.'.
*/
if ((systemIndices.validateSystemIndex(indexName) || validateHiddenIndex(indexName)) && !isDataStreamIndex(indexName)) {
if ((systemIndices.validateSystemIndex(indexName) || validateDotIndex(indexName)) && !isDataStreamIndex(indexName)) {
return;
}

Expand All @@ -137,7 +137,7 @@ public void validateShardLimit(final String indexName, final Settings settings,
/**
* Validates whether a list of indices can be opened without going over the cluster shard limit. Only counts indices which are
* currently closed and will be opened, ignores indices which are already open. Adding to this it validates the
* shard limit only for non system indices and if the cluster.ignore_hidden_indexes property is set to true
* shard limit only for non system indices and if the cluster.ignore_dot_indexes property is set to true
* then the indexes starting with '.' are ignored except the data-stream indexes.
*
* @param currentState The current cluster state.
Expand All @@ -148,12 +148,12 @@ public void validateShardLimit(ClusterState currentState, Index[] indicesToOpen)
int shardsToOpen = Arrays.stream(indicesToOpen)
/*
Validate shard limit only for non system indices as it is not hard limit anyways.
Further also validates if the cluster.ignore_hidden_indexes is set to true.
Further also validates if the cluster.ignore_dot_indexes is set to true.
If so then it does not validate any index which starts with '.'
however data-stream indexes are still validated.
*/
.filter(index -> !systemIndices.validateSystemIndex(index.getName()))
.filter(index -> !(validateHiddenIndex(index.getName()) && !isDataStreamIndex(index.getName())))
.filter(index -> !(validateDotIndex(index.getName()) && !isDataStreamIndex(index.getName())))
.filter(index -> currentState.metadata().index(index).getState().equals(IndexMetadata.State.CLOSE))
.mapToInt(index -> getTotalShardCount(currentState, index))
.sum();
Expand All @@ -172,13 +172,13 @@ private static int getTotalShardCount(ClusterState state, Index index) {
}

/**
* Returns true if the cluster.ignore_hidden_indexes property is set to true and the index name
* Returns true if the cluster.ignore_dot_indexes property is set to true and the index name
* starts with '.' else false.
*
* @param indexName The index which needs to be validated.
*/
private boolean validateHiddenIndex(String indexName) {
return this.ignoreHiddenIndexes && indexName.charAt(0) == '.';
private boolean validateDotIndex(String indexName) {
return this.ignoreDotIndexes && indexName.charAt(0) == '.';
}

/**
Expand Down
Loading

0 comments on commit 8e70553

Please sign in to comment.