From 81c6485306f775b6148c0b66a931bb2b3b8a1eae Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 26 Sep 2018 07:34:51 +0200 Subject: [PATCH] [CCR] Don't auto follow follow indices in the same cluster. (#33944) --- .../ccr/action/AutoFollowCoordinator.java | 16 +++++++-- .../action/AutoFollowCoordinatorTests.java | 34 +++++++++++++++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java index b429227207903..f855bc8e1c039 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; @@ -259,8 +260,8 @@ void autoFollowIndices() { if (leaderClusterState != null) { assert e == null; final List followedIndices = autoFollowMetadata.getFollowedLeaderIndexUUIDs().get(clusterAlias); - final List leaderIndicesToFollow = - getLeaderIndicesToFollow(autoFollowPattern, leaderClusterState, followerClusterState, followedIndices); + final List leaderIndicesToFollow = getLeaderIndicesToFollow(clusterAlias, autoFollowPattern, + leaderClusterState, followerClusterState, followedIndices); if (leaderIndicesToFollow.isEmpty()) { finalise(slot, new AutoFollowResult(clusterAlias)); } else { @@ -337,12 +338,21 @@ private void finalise(int slot, AutoFollowResult result) { } } - static List getLeaderIndicesToFollow(AutoFollowPattern autoFollowPattern, + static List getLeaderIndicesToFollow(String clusterAlias, + AutoFollowPattern autoFollowPattern, ClusterState leaderClusterState, ClusterState followerClusterState, List followedIndexUUIDs) { List leaderIndicesToFollow = new ArrayList<>(); for (IndexMetaData leaderIndexMetaData : leaderClusterState.getMetaData()) { + // If an auto follow pattern has been set up for the local cluster then + // we should not automatically follow a leader index that is also a follow index because + // this can result into an index creation explosion. + if (leaderIndexMetaData.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY) != null && + clusterAlias.equals("_local_")) { + continue; + } + if (autoFollowPattern.match(leaderIndexMetaData.getIndex().getName())) { if (followedIndexUUIDs.contains(leaderIndexMetaData.getIndex().getUUID()) == false) { // TODO: iterate over the indices in the followerClusterState and check whether a IndexMetaData diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java index 82f3153603946..3f31f8a2df767 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.action.AutoFollowCoordinator.AutoFollower; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; @@ -318,7 +319,8 @@ public void testGetLeaderIndicesToFollow() { .metaData(imdBuilder) .build(); - List result = AutoFollower.getLeaderIndicesToFollow(autoFollowPattern, leaderState, followerState, Collections.emptyList()); + List result = AutoFollower.getLeaderIndicesToFollow("remote", autoFollowPattern, leaderState, followerState, + Collections.emptyList()); result.sort(Comparator.comparing(Index::getName)); assertThat(result.size(), equalTo(5)); assertThat(result.get(0).getName(), equalTo("metrics-0")); @@ -328,7 +330,7 @@ public void testGetLeaderIndicesToFollow() { assertThat(result.get(4).getName(), equalTo("metrics-4")); List followedIndexUUIDs = Collections.singletonList(leaderState.metaData().index("metrics-2").getIndexUUID()); - result = AutoFollower.getLeaderIndicesToFollow(autoFollowPattern, leaderState, followerState, followedIndexUUIDs); + result = AutoFollower.getLeaderIndicesToFollow("remote", autoFollowPattern, leaderState, followerState, followedIndexUUIDs); result.sort(Comparator.comparing(Index::getName)); assertThat(result.size(), equalTo(4)); assertThat(result.get(0).getName(), equalTo("metrics-0")); @@ -337,6 +339,34 @@ public void testGetLeaderIndicesToFollow() { assertThat(result.get(3).getName(), equalTo("metrics-4")); } + public void testGetLeaderIndicesToFollowDoNotSelectFollowIndicesInTheSameCluster() { + MetaData.Builder imdBuilder = MetaData.builder(); + imdBuilder.put(IndexMetaData.builder("metrics-0") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0)); + imdBuilder.put(IndexMetaData.builder("metrics-1") + .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>()) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0)); + + AutoFollowPattern autoFollowPattern = + new AutoFollowPattern(Collections.singletonList("metrics-*"), null, null, null, null, null, null, null, null); + imdBuilder.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(Collections.singletonMap("remote", autoFollowPattern), + Collections.emptyMap(), Collections.emptyMap())); + + ClusterState clusterState = ClusterState.builder(new ClusterName("name")) + .metaData(imdBuilder) + .build(); + + List result = AutoFollower.getLeaderIndicesToFollow("_local_", autoFollowPattern, clusterState, + clusterState, Collections.emptyList()); + result.sort(Comparator.comparing(Index::getName)); + assertThat(result.size(), equalTo(1)); + assertThat(result.get(0).getName(), equalTo("metrics-0")); + } + public void testGetFollowerIndexName() { AutoFollowPattern autoFollowPattern = new AutoFollowPattern(Collections.singletonList("metrics-*"), null, null, null, null, null, null, null, null);