[bug](bdbje) Add retry for reSetupBdbEnvironment() restore.execute() (

apache#18777) * In reSetupBdbEnvironment() `restore.execute()` may throw NullPointerException, add retry for `restore.execute()`
mongo360 · Jun 21, 2023 · 75ed931 · 75ed931
1 parent 056b7a9
commit 75ed931
Showing 1 changed file with 19 additions and 4 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
@@ -356,10 +356,25 @@ private void reSetupBdbEnvironment(InsufficientLogException insufficientLogEx) {
         // ATTN: here we use `getServingEnv()`, because only serving catalog has
         // helper nodes.
         Pair<String, Integer> helperNode = Env.getServingEnv().getHelperNode();
-        NetworkRestore restore = new NetworkRestore();
-        NetworkRestoreConfig config = new NetworkRestoreConfig();
-        config.setRetainLogFiles(false);
-        restore.execute(insufficientLogEx, config);
+
+        for (int i = 0; i < RETRY_TIME; i++) {
+            try {
+                NetworkRestore restore = new NetworkRestore();
+                NetworkRestoreConfig config = new NetworkRestoreConfig();
+                config.setRetainLogFiles(false);
+                restore.execute(insufficientLogEx, config);
+                break;
+            } catch (Exception e) {
+                LOG.warn("retry={}, reSetupBdbEnvironment exception:", i, e);
+                try {
+                    Thread.sleep(5 * 1000);
+                    LOG.warn("after sleep insufficientLogEx:", insufficientLogEx);
+                } catch (InterruptedException e1) {
+                    LOG.warn("InterruptedException", e1);
+                }
+            }
+        }
+
         bdbEnvironment.close();
         bdbEnvironment.setup(new File(environmentPath), selfNodeName, selfNodeHostPort,
                 helperNode.first + ":" + helperNode.second, Env.getServingEnv().isElectable());