Added retries to the job completion handler. (Netflix#361)

* Added retries to the job completion handler.

genie-web/src/main/java/com/netflix/genie/GenieWeb.java

@@ -22,6 +22,7 @@
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
 import org.springframework.boot.autoconfigure.session.SessionAutoConfiguration;
+import org.springframework.retry.annotation.EnableRetry;
 
 import java.util.Map;
 
@@ -31,6 +32,7 @@
  * @author tgianos
  * @since 3.0.0
  */
+@EnableRetry
 @SpringBootApplication(exclude = {SessionAutoConfiguration.class, RedisAutoConfiguration.class})
 public class GenieWeb {
 

genie-web/src/main/java/com/netflix/genie/web/configs/MvcConfig.java

@@ -30,6 +30,9 @@
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.ResourceLoader;
 import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
+import org.springframework.retry.backoff.ExponentialBackOffPolicy;
+import org.springframework.retry.policy.SimpleRetryPolicy;
+import org.springframework.retry.support.RetryTemplate;
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.servlet.config.annotation.PathMatchConfigurer;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter;
@@ -38,6 +41,7 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.Collections;
 
 /**
  * Configuration for Spring MVC.
@@ -103,6 +107,30 @@ public RestTemplate restTemplate(
         return new RestTemplate(factory);
     }
 
+    /**
+     * Get RetryTemplate.
+     *
+     * @param noOfRetries number of retries
+     * @param initialInterval initial interval for the backoff policy
+     * @param maxInterval maximum interval for the backoff policy
+     * @return The retry template to use
+     */
+    @Bean(name = "genieRetryTemplate")
+    public RetryTemplate retryTemplate(
+        @Value("${genie.retry.noOfRetries:5}") final int noOfRetries,
+        @Value("${genie.retry.initialInterval:10000}") final int initialInterval,
+        @Value("${genie.retry.maxInterval:60000}") final int maxInterval
+    ) {
+        final RetryTemplate retryTemplate = new RetryTemplate();
+        retryTemplate.setRetryPolicy(new SimpleRetryPolicy(noOfRetries,
+            Collections.singletonMap(Exception.class, true)));
+        final ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
+        backOffPolicy.setInitialInterval(initialInterval);
+        backOffPolicy.setMaxInterval(maxInterval);
+        retryTemplate.setBackOffPolicy(backOffPolicy);
+        return retryTemplate;
+    }
+
     /**
      * Get the directory writer to use.
      *