From 3eaab3f24476bf2de4237600c0df02ee2ba31ba0 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Thu, 4 Jul 2024 19:54:07 +0800 Subject: [PATCH 1/9] set grpc channel's keepAliveTime and remove proxy on InterruptedException/TimeoutException --- fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java | 2 ++ .../main/java/org/apache/doris/rpc/BackendServiceClient.java | 1 + 2 files changed, 3 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 2c2d4437441c3d..6780fbe5b57c8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -950,6 +950,7 @@ private void waitPipelineRpc(List Date: Thu, 4 Jul 2024 20:13:46 +0800 Subject: [PATCH 2/9] update --- .../main/java/org/apache/doris/rpc/BackendServiceClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java index 0380a3c31453b9..bb51cf96dc44ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java @@ -60,7 +60,7 @@ public BackendServiceClient(TNetworkAddress address, Executor executor) { // Is the underlying channel in a normal state? (That means the RPC call will not fail immediately) public boolean isNormalState() { - ConnectivityState state = channel.getState(false); + ConnectivityState state = channel.getState(true); return state == ConnectivityState.CONNECTING || state == ConnectivityState.IDLE || state == ConnectivityState.READY; @@ -191,7 +191,7 @@ public Future alterVaultSync( public void shutdown() { - ConnectivityState state = channel.getState(false); + ConnectivityState state = channel.getState(true); LOG.warn("shut down backend service client: {}, channel state: {}", address, state); if (!channel.isShutdown()) { channel.shutdown(); From 9d105c67ca3e821d05e067fd8da2768b3da2459c Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Fri, 5 Jul 2024 11:44:29 +0800 Subject: [PATCH 3/9] update --- be/src/service/brpc_service.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/service/brpc_service.cpp b/be/src/service/brpc_service.cpp index 534c7b7b0c63a7..b4f0c6f77d0370 100644 --- a/be/src/service/brpc_service.cpp +++ b/be/src/service/brpc_service.cpp @@ -75,6 +75,7 @@ Status BRpcService::start(int port, int num_threads) { if (num_threads != -1) { options.num_threads = num_threads; } + options.idle_timeout_sec = 10; if (config::enable_https) { auto sslOptions = options.mutable_ssl_options(); From e67eed3a2a53676c515e9814ecb1a5401669fe24 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Fri, 5 Jul 2024 15:51:08 +0800 Subject: [PATCH 4/9] update --- be/src/service/brpc_service.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/service/brpc_service.cpp b/be/src/service/brpc_service.cpp index b4f0c6f77d0370..534c7b7b0c63a7 100644 --- a/be/src/service/brpc_service.cpp +++ b/be/src/service/brpc_service.cpp @@ -75,7 +75,6 @@ Status BRpcService::start(int port, int num_threads) { if (num_threads != -1) { options.num_threads = num_threads; } - options.idle_timeout_sec = 10; if (config::enable_https) { auto sslOptions = options.mutable_ssl_options(); From e903b10acfcb1bf335413fdfba04b07cbfdbb459 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Tue, 9 Jul 2024 11:33:26 +0800 Subject: [PATCH 5/9] update --- .../src/main/java/org/apache/doris/common/Config.java | 6 ++++++ .../java/org/apache/doris/rpc/BackendServiceClient.java | 7 +++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 109a0376a316e8..8840d1ce8e9b9a 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1465,6 +1465,12 @@ public class Config extends ConfigBase { @ConfField public static int grpc_threadmgr_threads_nums = 4096; + /** + * sets the time without read activity before sending a keepalive ping + */ + @ConfField + public static int grpc_keep_alive_second = 10; + /** * Used to set minimal number of replication per tablet. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java index bb51cf96dc44ed..98e460dbcfc502 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java @@ -46,8 +46,7 @@ public class BackendServiceClient { public BackendServiceClient(TNetworkAddress address, Executor executor) { this.address = address; channel = NettyChannelBuilder.forAddress(address.getHostname(), address.getPort()) - .executor(executor) - .keepAliveTime(5, TimeUnit.SECONDS) + .executor(executor).keepAliveTime(Config.grpc_keep_alive_second, TimeUnit.SECONDS) .flowControlWindow(Config.grpc_max_message_size_bytes) .keepAliveWithoutCalls(true) .maxInboundMessageSize(Config.grpc_max_message_size_bytes).enableRetry().maxRetryAttempts(MAX_RETRY_NUM) @@ -60,7 +59,7 @@ public BackendServiceClient(TNetworkAddress address, Executor executor) { // Is the underlying channel in a normal state? (That means the RPC call will not fail immediately) public boolean isNormalState() { - ConnectivityState state = channel.getState(true); + ConnectivityState state = channel.getState(false); return state == ConnectivityState.CONNECTING || state == ConnectivityState.IDLE || state == ConnectivityState.READY; @@ -191,7 +190,7 @@ public Future alterVaultSync( public void shutdown() { - ConnectivityState state = channel.getState(true); + ConnectivityState state = channel.getState(false); LOG.warn("shut down backend service client: {}, channel state: {}", address, state); if (!channel.isShutdown()) { channel.shutdown(); From 31b450ccef44bcdb99ad087b0692c537be107a03 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Tue, 9 Jul 2024 15:24:20 +0800 Subject: [PATCH 6/9] add comment --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 8840d1ce8e9b9a..a6439fbe2e6329 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1467,6 +1467,7 @@ public class Config extends ConfigBase { /** * sets the time without read activity before sending a keepalive ping + * the smaller the value, the sooner the channel is unavailable, but it will increase network io */ @ConfField public static int grpc_keep_alive_second = 10; From e4ca8664e30c8c8ca6f79829b6a11dcb62f6e051 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Wed, 10 Jul 2024 11:18:52 +0800 Subject: [PATCH 7/9] format --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index a6439fbe2e6329..16c2c7394e4a42 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1467,7 +1467,7 @@ public class Config extends ConfigBase { /** * sets the time without read activity before sending a keepalive ping - * the smaller the value, the sooner the channel is unavailable, but it will increase network io + * the smaller the value, the sooner the channel is unavailable, but it will increase network io */ @ConfField public static int grpc_keep_alive_second = 10; From e9aedddbfc3d69e07d72bc368333d87c0966b680 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Tue, 16 Jul 2024 13:23:21 +0800 Subject: [PATCH 8/9] update --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 16c2c7394e4a42..f2dc97e81fa212 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1469,7 +1469,8 @@ public class Config extends ConfigBase { * sets the time without read activity before sending a keepalive ping * the smaller the value, the sooner the channel is unavailable, but it will increase network io */ - @ConfField + @ConfField(description = { "设置grpc连接发送 keepalive ping 之前没有数据传输的时间。", + "The time without grpc read activity before sending a keepalive ping" }) public static int grpc_keep_alive_second = 10; /** From 7f5fe64c24aa2a5620aa3fff79ac0a34ed5219ac Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Tue, 16 Jul 2024 13:27:32 +0800 Subject: [PATCH 9/9] format --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index f2dc97e81fa212..c0e0c27e23e770 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1470,7 +1470,7 @@ public class Config extends ConfigBase { * the smaller the value, the sooner the channel is unavailable, but it will increase network io */ @ConfField(description = { "设置grpc连接发送 keepalive ping 之前没有数据传输的时间。", - "The time without grpc read activity before sending a keepalive ping" }) + "The time without grpc read activity before sending a keepalive ping" }) public static int grpc_keep_alive_second = 10; /**