Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CCR] Don't auto follow follow indices in the same cluster. #33944

Merged
merged 4 commits into from
Sep 26, 2018
Merged

[CCR] Don't auto follow follow indices in the same cluster. #33944

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
006f583
[CCR] Don't auto follow follow indices in the same cluster
martijnvg Sep 21, 2018
df20939
check for the use of _local_ cluster alias
martijnvg Sep 23, 2018
e0bd9be
Merge remote-tracking branch 'es/master' into ccr_do_not_auto_follow_…
martijnvg Sep 25, 2018
928aaea
fixed compile error
martijnvg Sep 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions ...ck/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.elasticsearch.index.Index;
import org.elasticsearch.license.LicenseUtils;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.ccr.Ccr;
import org.elasticsearch.xpack.ccr.CcrLicenseChecker;
import org.elasticsearch.xpack.ccr.CcrSettings;
import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
Expand Down Expand Up @@ -259,8 +260,8 @@ void autoFollowIndices() {
if (leaderClusterState != null) {
assert e == null;
final List<String> followedIndices = autoFollowMetadata.getFollowedLeaderIndexUUIDs().get(clusterAlias);
final List<Index> leaderIndicesToFollow =
getLeaderIndicesToFollow(autoFollowPattern, leaderClusterState, followerClusterState, followedIndices);
final List<Index> leaderIndicesToFollow = getLeaderIndicesToFollow(clusterAlias, autoFollowPattern,
leaderClusterState, followerClusterState, followedIndices);
if (leaderIndicesToFollow.isEmpty()) {
finalise(slot, new AutoFollowResult(clusterAlias));
} else {
Expand Down Expand Up @@ -337,12 +338,21 @@ private void finalise(int slot, AutoFollowResult result) {
}
}

static List<Index> getLeaderIndicesToFollow(AutoFollowPattern autoFollowPattern,
static List<Index> getLeaderIndicesToFollow(String clusterAlias,
AutoFollowPattern autoFollowPattern,
ClusterState leaderClusterState,
ClusterState followerClusterState,
List<String> followedIndexUUIDs) {
List<Index> leaderIndicesToFollow = new ArrayList<>();
for (IndexMetaData leaderIndexMetaData : leaderClusterState.getMetaData()) {
// If an auto follow pattern has been set up for the local cluster then
// we should not automatically follow a leader index that is also a follow index because
// this can result into an index creation explosion.
if (leaderIndexMetaData.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY) != null &&
clusterAlias.equals("_local_")) {
continue;
}

if (autoFollowPattern.match(leaderIndexMetaData.getIndex().getName())) {
if (followedIndexUUIDs.contains(leaderIndexMetaData.getIndex().getUUID()) == false) {
// TODO: iterate over the indices in the followerClusterState and check whether a IndexMetaData
Expand Down
34 changes: 32 additions & 2 deletions ...ugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ccr.Ccr;
import org.elasticsearch.xpack.ccr.CcrLicenseChecker;
import org.elasticsearch.xpack.ccr.action.AutoFollowCoordinator.AutoFollower;
import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
Expand Down Expand Up @@ -318,7 +319,8 @@ public void testGetLeaderIndicesToFollow() {
.metaData(imdBuilder)
.build();

List<Index> result = AutoFollower.getLeaderIndicesToFollow(autoFollowPattern, leaderState, followerState, Collections.emptyList());
List<Index> result = AutoFollower.getLeaderIndicesToFollow("remote", autoFollowPattern, leaderState, followerState,
Collections.emptyList());
result.sort(Comparator.comparing(Index::getName));
assertThat(result.size(), equalTo(5));
assertThat(result.get(0).getName(), equalTo("metrics-0"));
Expand All @@ -328,7 +330,7 @@ public void testGetLeaderIndicesToFollow() {
assertThat(result.get(4).getName(), equalTo("metrics-4"));

List<String> followedIndexUUIDs = Collections.singletonList(leaderState.metaData().index("metrics-2").getIndexUUID());
result = AutoFollower.getLeaderIndicesToFollow(autoFollowPattern, leaderState, followerState, followedIndexUUIDs);
result = AutoFollower.getLeaderIndicesToFollow("remote", autoFollowPattern, leaderState, followerState, followedIndexUUIDs);
result.sort(Comparator.comparing(Index::getName));
assertThat(result.size(), equalTo(4));
assertThat(result.get(0).getName(), equalTo("metrics-0"));
Expand All @@ -337,6 +339,34 @@ public void testGetLeaderIndicesToFollow() {
assertThat(result.get(3).getName(), equalTo("metrics-4"));
}

public void testGetLeaderIndicesToFollowDoNotSelectFollowIndicesInTheSameCluster() {
MetaData.Builder imdBuilder = MetaData.builder();
imdBuilder.put(IndexMetaData.builder("metrics-0")
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0));
imdBuilder.put(IndexMetaData.builder("metrics-1")
.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>())
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0));

AutoFollowPattern autoFollowPattern =
new AutoFollowPattern(Collections.singletonList("metrics-*"), null, null, null, null, null, null, null, null);
imdBuilder.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(Collections.singletonMap("remote", autoFollowPattern),
Collections.emptyMap(), Collections.emptyMap()));

ClusterState clusterState = ClusterState.builder(new ClusterName("name"))
.metaData(imdBuilder)
.build();

List<Index> result = AutoFollower.getLeaderIndicesToFollow("_local_", autoFollowPattern, clusterState,
clusterState, Collections.emptyList());
result.sort(Comparator.comparing(Index::getName));
assertThat(result.size(), equalTo(1));
assertThat(result.get(0).getName(), equalTo("metrics-0"));
}

public void testGetFollowerIndexName() {
AutoFollowPattern autoFollowPattern = new AutoFollowPattern(Collections.singletonList("metrics-*"), null, null,
null, null, null, null, null, null);
Expand Down