Allow remote retry max delay to be user configurable

src/main/java/com/google/devtools/build/lib/remote/RemoteRetrier.java

@@ -158,7 +158,7 @@ public static class ExponentialBackoff implements Backoff {
       Preconditions.checkArgument(jitter >= 0 && jitter <= 1, "jitter must be in the range (0, 1)");
       Preconditions.checkArgument(maxAttempts >= 0, "maxAttempts must be >= 0");
       nextDelayMillis = initial.toMillis();
-      maxMillis = max.toMillis();
+      maxMillis = Math.max(max.toMillis(), nextDelayMillis);
       this.multiplier = multiplier;
       this.jitter = jitter;
       this.maxAttempts = maxAttempts;
@@ -167,7 +167,7 @@ public static class ExponentialBackoff implements Backoff {
     public ExponentialBackoff(RemoteOptions options) {
       this(
           /* initial = */ Duration.ofMillis(100),
-          /* max = */ Duration.ofSeconds(5),
+          /* max = */ options.remoteRetryMaxDelay,
           /* multiplier= */ 2,
           /* jitter= */ 0.1,
           options.remoteMaxRetryAttempts);

src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java

@@ -363,6 +363,19 @@ public RemoteBuildEventUploadModeConverter() {
               + "If set to 0, retries are disabled.")
   public int remoteMaxRetryAttempts;
 
+  @Option(
+      name = "remote_retry_max_delay",
+      defaultValue = "5s",
+      documentationCategory = OptionDocumentationCategory.REMOTE,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      converter = RemoteTimeoutConverter.class,
+      help =
+          "The maximum backoff delay between remote retry attempts. Following units can be used:"
+              + " Days (d), hours (h), minutes (m), seconds (s), and milliseconds (ms). If"
+              + " the unit is omitted, the value is interpreted as seconds."
+          )
+  public Duration remoteRetryMaxDelay;
+
   @Option(
       name = "disk_cache",
       defaultValue = "null",