Fix sporadic Elasticsearch test failures (#444)

open-telemetry · May 27, 2020 · 5c7e321 · 5c7e321
1 parent d4a14f6
commit 5c7e321
Show file tree

Hide file tree

Showing 18 changed files with 128 additions and 0 deletions.
diff --git a/...earch/elasticsearch-rest-5.0/src/latestDepTest/groovy/Elasticsearch6RestClientTest.groovy b/...earch/elasticsearch-rest-5.0/src/latestDepTest/groovy/Elasticsearch6RestClientTest.groovy
@@ -49,6 +49,12 @@ class Elasticsearch6RestClientTest extends AgentTestRunner {
   RestClient client
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/.../elasticsearch/elasticsearch-rest-5.0/src/test/groovy/Elasticsearch5RestClientTest.groovy b/.../elasticsearch/elasticsearch-rest-5.0/src/test/groovy/Elasticsearch5RestClientTest.groovy
@@ -51,6 +51,12 @@ class Elasticsearch5RestClientTest extends AgentTestRunner {
   static RestClient client
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...earch/elasticsearch-rest-6.4/src/latestDepTest/groovy/Elasticsearch6RestClientTest.groovy b/...earch/elasticsearch-rest-6.4/src/latestDepTest/groovy/Elasticsearch6RestClientTest.groovy
@@ -52,6 +52,12 @@ class Elasticsearch6RestClientTest extends AgentTestRunner {
   RestClient client
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/.../elasticsearch/elasticsearch-rest-6.4/src/test/groovy/Elasticsearch6RestClientTest.groovy b/.../elasticsearch/elasticsearch-rest-6.4/src/test/groovy/Elasticsearch6RestClientTest.groovy
@@ -49,6 +49,12 @@ class Elasticsearch6RestClientTest extends AgentTestRunner {
   RestClient client
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/.../elasticsearch-transport-2.0/src/latestDepTest/groovy/Elasticsearch2NodeClientTest.groovy b/.../elasticsearch-transport-2.0/src/latestDepTest/groovy/Elasticsearch2NodeClientTest.groovy
@@ -42,6 +42,12 @@ class Elasticsearch2NodeClientTest extends AgentTestRunner {
   def client = testNode.client()
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...ticsearch-transport-2.0/src/latestDepTest/groovy/Elasticsearch2TransportClientTest.groovy b/...ticsearch-transport-2.0/src/latestDepTest/groovy/Elasticsearch2TransportClientTest.groovy
@@ -47,6 +47,12 @@ class Elasticsearch2TransportClientTest extends AgentTestRunner {
   TransportClient client
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...transport-2.0/src/latestDepTest/groovy/springdata/Elasticsearch2SpringTemplateTest.groovy b/...transport-2.0/src/latestDepTest/groovy/springdata/Elasticsearch2SpringTemplateTest.groovy
@@ -53,6 +53,12 @@ class Elasticsearch2SpringTemplateTest extends AgentTestRunner {
   ElasticsearchTemplate template
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...ticsearch/elasticsearch-transport-2.0/src/test/groovy/Elasticsearch2NodeClientTest.groovy b/...ticsearch/elasticsearch-transport-2.0/src/test/groovy/Elasticsearch2NodeClientTest.groovy
@@ -43,6 +43,12 @@ class Elasticsearch2NodeClientTest extends AgentTestRunner {
   def client = testNode.client()
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...arch/elasticsearch-transport-2.0/src/test/groovy/Elasticsearch2TransportClientTest.groovy b/...arch/elasticsearch-transport-2.0/src/test/groovy/Elasticsearch2TransportClientTest.groovy
@@ -48,6 +48,12 @@ class Elasticsearch2TransportClientTest extends AgentTestRunner {
   TransportClient client
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...icsearch-transport-2.0/src/test/groovy/springdata/Elasticsearch2SpringTemplateTest.groovy b/...icsearch-transport-2.0/src/test/groovy/springdata/Elasticsearch2SpringTemplateTest.groovy
@@ -54,6 +54,12 @@ class Elasticsearch2SpringTemplateTest extends AgentTestRunner {
   ElasticsearchTemplate template
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...ticsearch/elasticsearch-transport-5.0/src/test/groovy/Elasticsearch5NodeClientTest.groovy b/...ticsearch/elasticsearch-transport-5.0/src/test/groovy/Elasticsearch5NodeClientTest.groovy
@@ -45,6 +45,12 @@ class Elasticsearch5NodeClientTest extends AgentTestRunner {
   def client = testNode.client()
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...arch/elasticsearch-transport-5.0/src/test/groovy/Elasticsearch5TransportClientTest.groovy b/...arch/elasticsearch-transport-5.0/src/test/groovy/Elasticsearch5TransportClientTest.groovy
@@ -51,6 +51,12 @@ class Elasticsearch5TransportClientTest extends AgentTestRunner {
   TransportClient client
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...icsearch/elasticsearch-transport-5.3/src/test/groovy/Elasticsearch53NodeClientTest.groovy b/...icsearch/elasticsearch-transport-5.3/src/test/groovy/Elasticsearch53NodeClientTest.groovy
@@ -45,6 +45,12 @@ class Elasticsearch53NodeClientTest extends AgentTestRunner {
   def client = testNode.client()
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...rch/elasticsearch-transport-5.3/src/test/groovy/Elasticsearch53TransportClientTest.groovy b/...rch/elasticsearch-transport-5.3/src/test/groovy/Elasticsearch53TransportClientTest.groovy
@@ -51,6 +51,12 @@ class Elasticsearch53TransportClientTest extends AgentTestRunner {
   TransportClient client
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...csearch-transport-5.3/src/test/groovy/springdata/Elasticsearch53SpringTemplateTest.groovy b/...csearch-transport-5.3/src/test/groovy/springdata/Elasticsearch53SpringTemplateTest.groovy
@@ -63,6 +63,12 @@ class Elasticsearch53SpringTemplateTest extends AgentTestRunner {
   ElasticsearchTemplate template
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...ticsearch/elasticsearch-transport-6.0/src/test/groovy/Elasticsearch6NodeClientTest.groovy b/...ticsearch/elasticsearch-transport-6.0/src/test/groovy/Elasticsearch6NodeClientTest.groovy
@@ -44,6 +44,12 @@ class Elasticsearch6NodeClientTest extends AgentTestRunner {
   def client = testNode.client()
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/...arch/elasticsearch-transport-6.0/src/test/groovy/Elasticsearch6TransportClientTest.groovy b/...arch/elasticsearch-transport-6.0/src/test/groovy/Elasticsearch6TransportClientTest.groovy
@@ -50,6 +50,12 @@ class Elasticsearch6TransportClientTest extends AgentTestRunner {
   TransportClient client
 
   def setupSpec() {
+    withRetryOnBindException({
+      setupSpecUnderRetry()
+    })
+  }
+
+  def setupSpecUnderRetry() {
     httpPort = PortUtils.randomOpenPort()
     tcpPort = PortUtils.randomOpenPort()
 

diff --git a/testing/src/main/groovy/io/opentelemetry/auto/test/AgentTestRunner.java b/testing/src/main/groovy/io/opentelemetry/auto/test/AgentTestRunner.java
@@ -19,6 +19,7 @@
 import ch.qos.logback.classic.Logger;
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
+import com.google.common.base.Throwables;
 import com.google.common.collect.Sets;
 import groovy.lang.Closure;
 import groovy.lang.DelegatesTo;
@@ -35,6 +36,7 @@
 import io.opentelemetry.trace.Tracer;
 import java.lang.instrument.ClassFileTransformer;
 import java.lang.instrument.Instrumentation;
+import java.net.BindException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.ServiceLoader;
@@ -177,6 +179,30 @@ public void cleanUpAfterTests() {
     TEST_LISTENER.deactivateTest(this);
   }
 
+  /**
+   * This is used by setupSpec() methods to auto-retry setup that depends on finding and then using
+   * an available free port, because that kind of setup can fail sporadically with
+   * "java.net.BindException: Address already in use" if the available port gets re-used between
+   * when we find the available port and when we use it.
+   *
+   * @param closure the groovy closure to run with retry
+   */
+  public static void withRetryOnBindException(final Closure<?> closure) {
+    withRetryOnBindException(closure, 3);
+  }
+
+  private static void withRetryOnBindException(final Closure<?> closure, final int numRetries) {
+    try {
+      closure.call();
+    } catch (final Throwable t) {
+      if (numRetries == 0 || !(Throwables.getRootCause(t) instanceof BindException)) {
+        throw t;
+      }
+      log.debug("retrying due to bind exception: {}", t.getMessage(), t);
+      withRetryOnBindException(closure, numRetries - 1);
+    }
+  }
+
   @AfterClass
   public static synchronized void agentCleanup() {
     if (null != activeTransformer) {