From 4cdb6da8920cd3346a9d79d67432cef588d93bb0 Mon Sep 17 00:00:00 2001
From: Igor Bernstein <igorbernstein@google.com>
Date: Tue, 21 May 2024 07:55:08 -0400
Subject: [PATCH] fix: retry INTERNAL retriable auth errors (#2239)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix: retry INTERNAL retriable auth errors

Change-Id: I3939a89d40ecd4304bccaf0340fe169d6a083712

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
---
 .../v2/stub/ConvertExceptionCallable.java     | 10 ++++++-
 .../v2/stub/ConvertExceptionCallableTest.java | 26 +++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallable.java
index 7ea1f90b38..31109abb03 100644
--- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallable.java
+++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallable.java
@@ -77,12 +77,20 @@ protected void onCompleteImpl() {
   private Throwable convertException(Throwable t) {
     // Long lived connections sometimes are disconnected via an RST frame or a goaway. These errors
     // are transient and should be retried.
-    if (isRstStreamError(t) || isGoAway(t)) {
+    if (isRstStreamError(t) || isGoAway(t) || isRetriableAuthError(t)) {
       return new InternalException(t, ((InternalException) t).getStatusCode(), true);
     }
     return t;
   }
 
+  private boolean isRetriableAuthError(Throwable t) {
+    if (t instanceof InternalException && t.getMessage() != null) {
+      String error = t.getMessage();
+      return error.contains("Authentication backend internal server error. Please retry");
+    }
+    return false;
+  }
+
   private boolean isRstStreamError(Throwable t) {
     if (t instanceof InternalException && t.getMessage() != null) {
       String error = t.getMessage().toLowerCase();
diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallableTest.java
index 534d341914..c15e20b561 100644
--- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallableTest.java
+++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallableTest.java
@@ -59,6 +59,32 @@ public void rstStreamExceptionConvertedToRetryableTest() {
     assertTrue(actualException.isRetryable());
   }
 
+  @Test
+  public void retriableAuthExceptionConvertedToRetryableTest() {
+    ApiException originalException =
+        new InternalException(
+            new StatusRuntimeException(
+                Status.INTERNAL.withDescription(
+                    "Authentication backend internal server error. Please retry")),
+            GrpcStatusCode.of(Status.Code.INTERNAL),
+            false);
+    assertFalse(originalException.isRetryable());
+    SettableExceptionCallable<String, String> settableExceptionCallable =
+        new SettableExceptionCallable<>(originalException);
+    ConvertExceptionCallable<String, String> convertStreamExceptionCallable =
+        new ConvertExceptionCallable<>(settableExceptionCallable);
+
+    Throwable actualError = null;
+    try {
+      convertStreamExceptionCallable.all().call("fake-request");
+    } catch (Throwable t) {
+      actualError = t;
+    }
+    assert actualError instanceof InternalException;
+    InternalException actualException = (InternalException) actualError;
+    assertTrue(actualException.isRetryable());
+  }
+
   private static final class SettableExceptionCallable<RequestT, ResponseT>
       extends ServerStreamingCallable<RequestT, ResponseT> {
     private final Throwable throwable;