From 5fcc3843d4e9768317dd8b397314819ad875518b Mon Sep 17 00:00:00 2001 From: Goson Zhang <4675739@qq.com> Date: Sun, 19 Jan 2025 19:23:06 +0800 Subject: [PATCH] [INLONG-11678][SDK] Optimize the ProxyClientConfig class (#11679) Co-authored-by: gosonzhang --- .../inlong/agent/core/HeartbeatManager.java | 10 +- .../sinks/filecollect/SenderManager.java | 12 +- .../sdk/dataproxy/DefaultMessageSender.java | 38 +- .../sdk/dataproxy/ProxyClientConfig.java | 552 ------------------ .../sdk/dataproxy/TcpMsgSenderConfig.java | 350 +++++++++++ .../sdk/dataproxy/codec/ProtocolEncoder.java | 6 +- .../sdk/dataproxy/common/HttpContentType.java | 27 + .../dataproxy/common/ProxyClientConfig.java | 527 +++++++++++++++++ .../sdk/dataproxy/common/ReportProtocol.java | 29 + .../SdkConsts.java} | 73 ++- .../dataproxy/config/ProxyConfigManager.java | 194 +++--- .../dataproxy/example/HttpClientExample.java | 16 +- .../dataproxy/example/TcpClientExample.java | 16 +- .../dataproxy/example/UdpClientExample.java | 4 +- .../dataproxy/http/HttpMsgSenderConfig.java | 261 +++++++++ .../dataproxy/http/InternalHttpSender.java | 33 +- .../sdk/dataproxy/metric/MetricConfig.java | 117 ++-- .../sdk/dataproxy/network/ClientMgr.java | 44 +- .../dataproxy/network/HttpProxySender.java | 46 +- .../sdk/dataproxy/network/NettyClient.java | 20 +- .../inlong/sdk/dataproxy/network/Sender.java | 65 +-- .../dataproxy/threads/MetricWorkerThread.java | 18 +- .../dataproxy/threads/TimeoutScanThread.java | 10 +- .../sdk/dataproxy/utils/ProxyUtils.java | 37 +- .../sdk/dataproxy/ProxyClientConfigTest.java | 75 +++ .../sdk/dataproxy/ProxyConfigManagerTest.java | 4 +- .../sdk/dirtydata/InlongSdkDirtySender.java | 9 +- 27 files changed, 1688 insertions(+), 905 deletions(-) delete mode 100644 inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/ProxyClientConfig.java create mode 100644 inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/TcpMsgSenderConfig.java create mode 100644 inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/HttpContentType.java create mode 100644 inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/ProxyClientConfig.java create mode 100644 inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/ReportProtocol.java rename inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/{ConfigConstants.java => common/SdkConsts.java} (57%) create mode 100644 inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/http/HttpMsgSenderConfig.java create mode 100644 inlong-sdk/dataproxy-sdk/src/test/java/org/apache/inlong/sdk/dataproxy/ProxyClientConfigTest.java diff --git a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/HeartbeatManager.java b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/HeartbeatManager.java index ead9439114a..f6c30466628 100644 --- a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/HeartbeatManager.java +++ b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/HeartbeatManager.java @@ -24,13 +24,12 @@ import org.apache.inlong.agent.utils.AgentUtils; import org.apache.inlong.agent.utils.HttpManager; import org.apache.inlong.agent.utils.ThreadUtils; -import org.apache.inlong.common.constant.ProtocolType; import org.apache.inlong.common.enums.ComponentTypeEnum; import org.apache.inlong.common.enums.NodeSrvStatus; import org.apache.inlong.common.heartbeat.AbstractHeartbeatManager; import org.apache.inlong.common.heartbeat.HeartbeatMsg; import org.apache.inlong.sdk.dataproxy.DefaultMessageSender; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.TcpMsgSenderConfig; import io.netty.util.concurrent.DefaultThreadFactory; import org.apache.commons.lang3.StringUtils; @@ -194,13 +193,12 @@ private void createMessageSender() { String managerAddr = conf.get(AGENT_MANAGER_ADDR); String authSecretId = conf.get(AGENT_MANAGER_AUTH_SECRET_ID); String authSecretKey = conf.get(AGENT_MANAGER_AUTH_SECRET_KEY); - ProxyClientConfig proxyClientConfig = null; + TcpMsgSenderConfig proxyClientConfig = null; try { - proxyClientConfig = new ProxyClientConfig(managerAddr, INLONG_AGENT_SYSTEM, authSecretId, authSecretKey); + proxyClientConfig = new TcpMsgSenderConfig(managerAddr, INLONG_AGENT_SYSTEM, authSecretId, authSecretKey); proxyClientConfig.setTotalAsyncCallbackSize(CommonConstants.DEFAULT_PROXY_TOTAL_ASYNC_PROXY_SIZE); proxyClientConfig.setAliveConnections(CommonConstants.DEFAULT_PROXY_ALIVE_CONNECTION_NUM); - proxyClientConfig.setIoThreadNum(CommonConstants.DEFAULT_PROXY_CLIENT_IO_THREAD_NUM); - proxyClientConfig.setProtocolType(ProtocolType.TCP); + proxyClientConfig.setNettyWorkerThreadNum(CommonConstants.DEFAULT_PROXY_CLIENT_IO_THREAD_NUM); proxyClientConfig.setRequestTimeoutMs(30000L); ThreadFactory SHARED_FACTORY = new DefaultThreadFactory("agent-sender-manager-heartbeat", Thread.currentThread().isDaemon()); diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java index 9ef20bdf4bb..aff97981e74 100755 --- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java +++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java @@ -29,10 +29,9 @@ import org.apache.inlong.agent.plugin.message.SequentialID; import org.apache.inlong.agent.utils.AgentUtils; import org.apache.inlong.agent.utils.ThreadUtils; -import org.apache.inlong.common.constant.ProtocolType; import org.apache.inlong.common.metric.MetricRegister; import org.apache.inlong.sdk.dataproxy.DefaultMessageSender; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.TcpMsgSenderConfig; import org.apache.inlong.sdk.dataproxy.common.SendMessageCallback; import org.apache.inlong.sdk.dataproxy.common.SendResult; import org.apache.inlong.sdk.dataproxy.exception.ProxySdkException; @@ -199,15 +198,14 @@ private AgentMetricItem getMetricItem(String groupId, String streamId) { * createMessageSender */ private void createMessageSender() throws Exception { - ProxyClientConfig proxyClientConfig = new ProxyClientConfig(managerAddr, inlongGroupId, authSecretId, - authSecretKey); + TcpMsgSenderConfig proxyClientConfig = new TcpMsgSenderConfig( + managerAddr, inlongGroupId, authSecretId, authSecretKey); proxyClientConfig.setTotalAsyncCallbackSize(totalAsyncBufSize); proxyClientConfig.setAliveConnections(aliveConnectionNum); proxyClientConfig.setRequestTimeoutMs(maxSenderTimeout * 1000L); - proxyClientConfig.setIoThreadNum(ioThreadNum); - proxyClientConfig.setEnableBusyWait(enableBusyWait); - proxyClientConfig.setProtocolType(ProtocolType.TCP); + proxyClientConfig.setNettyWorkerThreadNum(ioThreadNum); + proxyClientConfig.setEnableEpollBusyWait(enableBusyWait); SHARED_FACTORY = new DefaultThreadFactory("agent-sender-manager-" + sourcePath, Thread.currentThread().isDaemon()); diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/DefaultMessageSender.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/DefaultMessageSender.java index 92dec7b125e..b887c399e58 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/DefaultMessageSender.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/DefaultMessageSender.java @@ -17,10 +17,10 @@ package org.apache.inlong.sdk.dataproxy; -import org.apache.inlong.common.constant.ProtocolType; import org.apache.inlong.common.msg.AttributeConstants; import org.apache.inlong.common.util.MessageUtils; import org.apache.inlong.sdk.dataproxy.codec.EncodeObject; +import org.apache.inlong.sdk.dataproxy.common.SdkConsts; import org.apache.inlong.sdk.dataproxy.common.SendMessageCallback; import org.apache.inlong.sdk.dataproxy.common.SendResult; import org.apache.inlong.sdk.dataproxy.config.ProxyConfigEntry; @@ -55,26 +55,26 @@ public class DefaultMessageSender implements MessageSender { /* Store index */ private final Map storeIndex = new ConcurrentHashMap(); private String groupId; - private int msgtype = ConfigConstants.MSG_TYPE; + private int msgtype = SdkConsts.MSG_TYPE; private boolean isCompress = true; private boolean isGroupIdTransfer = false; private boolean isReport = false; private boolean isSupportLF = false; private int maxPacketLength = -1; - private int cpsSize = ConfigConstants.COMPRESS_SIZE; + private int cpsSize = SdkConsts.COMPRESS_SIZE; private final int senderMaxAttempt; - public DefaultMessageSender(ProxyClientConfig configure) throws Exception { + public DefaultMessageSender(TcpMsgSenderConfig configure) throws Exception { this(configure, null); } - public DefaultMessageSender(ProxyClientConfig configure, ThreadFactory selfDefineFactory) throws Exception { + public DefaultMessageSender(TcpMsgSenderConfig configure, ThreadFactory selfDefineFactory) throws Exception { ProxyUtils.validClientConfig(configure); sender = new Sender(configure, selfDefineFactory); sender.start(); groupId = configure.getInlongGroupId(); indexCol = new IndexCollectThread(storeIndex); - senderMaxAttempt = configure.getSenderMaxAttempt(); + senderMaxAttempt = configure.getMaxSyncSendAttempt(); indexCol.start(); } @@ -82,31 +82,27 @@ public DefaultMessageSender(ProxyClientConfig configure, ThreadFactory selfDefin /** * generate by cluster id * - * @param configure - sender + * @param tcpConfig - sender * @return - sender */ public static DefaultMessageSender generateSenderByClusterId( - ProxyClientConfig configure) throws Exception { + TcpMsgSenderConfig tcpConfig) throws Exception { - return generateSenderByClusterId(configure, null); + return generateSenderByClusterId(tcpConfig, null); } /** * generate by cluster id * - * @param configure - sender + * @param tcpConfig - sender * @param selfDefineFactory - sender factory * @return - sender */ - public static DefaultMessageSender generateSenderByClusterId(ProxyClientConfig configure, + public static DefaultMessageSender generateSenderByClusterId(TcpMsgSenderConfig tcpConfig, ThreadFactory selfDefineFactory) throws Exception { - // correct ProtocolType settings - if (!ProtocolType.TCP.equals(configure.getProtocolType())) { - configure.setProtocolType(ProtocolType.TCP); - } - LOGGER.info("Initial tcp sender, configure is {}", configure); + LOGGER.info("Initial tcp sender, configure is {}", tcpConfig); // initial sender object - ProxyConfigManager proxyConfigManager = new ProxyConfigManager(configure); + ProxyConfigManager proxyConfigManager = new ProxyConfigManager(tcpConfig); Tuple2 result = proxyConfigManager.getGroupIdConfigure(true); if (result.getF0() == null) { @@ -117,7 +113,7 @@ public static DefaultMessageSender generateSenderByClusterId(ProxyClientConfig c return sender; } else { DefaultMessageSender tmpMessageSender = - new DefaultMessageSender(configure, selfDefineFactory); + new DefaultMessageSender(tcpConfig, selfDefineFactory); tmpMessageSender.setMaxPacketLength(result.getF0().getMaxPacketLength()); CACHE_SENDER.put(result.getF0().getClusterId(), tmpMessageSender); return tmpMessageSender; @@ -144,8 +140,8 @@ public void close() { shutdownInternalThreads(); } - public ProxyClientConfig getProxyClientConfig() { - return sender.getConfigure(); + public TcpMsgSenderConfig getProxyClientConfig() { + return sender.getTcpConfig(); } public boolean isSupportLF() { @@ -213,7 +209,7 @@ public void setMaxPacketLength(int maxPacketLength) { } public String getSDKVersion() { - return ConfigConstants.PROXY_SDK_VERSION; + return SdkConsts.PROXY_SDK_VERSION; } private SendResult attemptSendMessage(Function sendOperation) { diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/ProxyClientConfig.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/ProxyClientConfig.java deleted file mode 100644 index efb5ac940be..00000000000 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/ProxyClientConfig.java +++ /dev/null @@ -1,552 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.inlong.sdk.dataproxy; - -import org.apache.inlong.sdk.dataproxy.exception.ProxySdkException; -import org.apache.inlong.sdk.dataproxy.metric.MetricConfig; - -import lombok.Data; -import org.apache.commons.lang3.StringUtils; - -@Data -public class ProxyClientConfig { - - private String managerIP = ""; - private int managerPort = 8099; - private boolean visitManagerByHttp = true; - private boolean onlyUseLocalProxyConfig = false; - private int managerConnTimeoutMs = ConfigConstants.VAL_DEF_CONNECT_TIMEOUT_MS; - private int managerSocketTimeoutMs = ConfigConstants.VAL_DEF_SOCKET_TIMEOUT_MS; - private long managerConfigSyncInrMs = - ConfigConstants.VAL_DEF_CONFIG_SYNC_INTERVAL_MIN * ConfigConstants.VAL_UNIT_MIN_TO_MS; - private int configSyncMaxRetryIfFail = ConfigConstants.VAL_DEF_RETRY_IF_CONFIG_SYNC_FAIL; - private String configStoreBasePath = System.getProperty("user.dir"); - // max expired time for config cache. - private long configCacheExpiredMs = ConfigConstants.VAL_DEF_CACHE_CONFIG_EXPIRED_MS; - // max expired time for config query failure status - private long configFailStatusExpiredMs = ConfigConstants.VAL_DEF_CONFIG_FAIL_STATUS_EXPIRED_MS; - // nodes force choose interval ms - private long forceReChooseInrMs = ConfigConstants.VAL_DEF_FORCE_CHOOSE_INR_MS; - private boolean enableAuthentication = false; - private String authSecretId = ""; - private String authSecretKey = ""; - private String inlongGroupId; - private String regionName = ConfigConstants.VAL_DEF_REGION_NAME; - private int aliveConnections = ConfigConstants.VAL_DEF_ALIVE_CONNECTIONS; - // data encrypt info - private boolean enableDataEncrypt = false; - private String rsaPubKeyUrl = ""; - private String userName = ""; - - private int syncThreadPoolSize; - private int asyncCallbackSize; - - private String tlsServerCertFilePathAndName; - private String tlsServerKey; - private String tlsVersion = "TLSv1.2"; - private int maxTimeoutCnt = ConfigConstants.MAX_TIMEOUT_CNT; - private String protocolType; - - // metric configure - private MetricConfig metricConfig = new MetricConfig(); - - // connect timeout in milliseconds - private long connectTimeoutMs = ConfigConstants.VAL_DEF_CONNECT_TIMEOUT_MS; - // request timeout in milliseconds - private long requestTimeoutMs = ConfigConstants.VAL_DEF_REQUEST_TIMEOUT_MS; - // connect close wait period in milliseconds - private long conCloseWaitPeriodMs = - ConfigConstants.VAL_DEF_REQUEST_TIMEOUT_MS + ConfigConstants.VAL_DEF_CONNECT_CLOSE_DELAY_MS; - // client reconnect wait period in ms - private long reConnectWaitMs = ConfigConstants.VAL_DEF_RECONNECT_WAIT_MS; - // socket receive buffer - private int recvBufferSize = ConfigConstants.DEFAULT_RECEIVE_BUFFER_SIZE; - // socket send buffer - private int sendBufferSize = ConfigConstants.DEFAULT_SEND_BUFFER_SIZE; - // max message count per connection - private long maxMsgInFlightPerConn = ConfigConstants.MAX_INFLIGHT_MSG_COUNT_PER_CONNECTION; - - // configuration for http client - // whether discard old metric when cache is full. - private boolean discardOldMessage = false; - private int proxyHttpUpdateIntervalMinutes; - // thread number for async sending data. - private int asyncWorkerNumber = 3; - // interval for async worker in microseconds. - private int asyncWorkerInterval = 500; - private boolean cleanHttpCacheWhenClosing = false; - - private int ioThreadNum = Runtime.getRuntime().availableProcessors(); - private boolean enableBusyWait = false; - - private int senderMaxAttempt = ConfigConstants.DEFAULT_SENDER_MAX_ATTEMPT; - - /* pay attention to the last url parameter ip */ - public ProxyClientConfig(boolean visitManagerByHttp, String managerIp, - int managerPort, String inlongGroupId, String authSecretId, String authSecretKey) throws ProxySdkException { - if (StringUtils.isBlank(managerIp)) { - throw new ProxySdkException("managerIp is Blank!"); - } - if (managerPort <= 0) { - throw new ProxySdkException("managerPort <= 0!"); - } - if (StringUtils.isBlank(inlongGroupId)) { - throw new ProxySdkException("groupId is blank!"); - } - this.inlongGroupId = inlongGroupId.trim(); - this.visitManagerByHttp = visitManagerByHttp; - this.managerPort = managerPort; - this.managerIP = managerIp; - this.syncThreadPoolSize = ConfigConstants.SYNC_THREAD_POOL_SIZE; - this.asyncCallbackSize = ConfigConstants.ASYNC_CALLBACK_SIZE; - this.proxyHttpUpdateIntervalMinutes = ConfigConstants.PROXY_HTTP_UPDATE_INTERVAL_MINUTES; - this.authSecretId = authSecretId; - this.authSecretKey = authSecretKey; - } - - /* pay attention to the last url parameter ip */ - public ProxyClientConfig(String managerAddress, - String inlongGroupId, String authSecretId, String authSecretKey) throws ProxySdkException { - checkAndParseAddress(managerAddress); - if (StringUtils.isBlank(inlongGroupId)) { - throw new ProxySdkException("groupId is blank!"); - } - this.inlongGroupId = inlongGroupId.trim(); - this.syncThreadPoolSize = ConfigConstants.SYNC_THREAD_POOL_SIZE; - this.asyncCallbackSize = ConfigConstants.ASYNC_CALLBACK_SIZE; - this.proxyHttpUpdateIntervalMinutes = ConfigConstants.PROXY_HTTP_UPDATE_INTERVAL_MINUTES; - this.authSecretId = authSecretId; - this.authSecretKey = authSecretKey; - } - - public String getManagerIP() { - return managerIP; - } - - public int getManagerPort() { - return managerPort; - } - - public boolean isVisitManagerByHttp() { - return visitManagerByHttp; - } - - public boolean isOnlyUseLocalProxyConfig() { - return onlyUseLocalProxyConfig; - } - - public void setOnlyUseLocalProxyConfig(boolean onlyUseLocalProxyConfig) { - this.onlyUseLocalProxyConfig = onlyUseLocalProxyConfig; - } - - public boolean isEnableAuthentication() { - return this.enableAuthentication; - } - - public String getAuthSecretId() { - return authSecretId; - } - - public String getAuthSecretKey() { - return authSecretKey; - } - - public void setAuthenticationInfo(boolean needAuthentication, String secretId, String secretKey) { - this.enableAuthentication = needAuthentication; - if (!this.enableAuthentication) { - return; - } - if (StringUtils.isBlank(secretId)) { - throw new IllegalArgumentException("secretId is Blank!"); - } - if (StringUtils.isBlank(secretKey)) { - throw new IllegalArgumentException("secretKey is Blank!"); - } - this.authSecretId = secretId.trim(); - this.authSecretKey = secretKey.trim(); - } - - public long getManagerConfigSyncInrMs() { - return managerConfigSyncInrMs; - } - - public void setManagerConfigSyncInrMin(int managerConfigSyncInrMin) { - int tmpValue = - Math.min(ConfigConstants.VAL_MAX_CONFIG_SYNC_INTERVAL_MIN, - Math.max(ConfigConstants.VAL_MIN_CONFIG_SYNC_INTERVAL_MIN, managerConfigSyncInrMin)); - this.managerConfigSyncInrMs = tmpValue * ConfigConstants.VAL_UNIT_MIN_TO_MS; - } - - public int getManagerConnTimeoutMs() { - return managerConnTimeoutMs; - } - - public void setManagerConnTimeoutMs(int managerConnTimeoutMs) { - this.managerConnTimeoutMs = - Math.min(ConfigConstants.VAL_MAX_CONNECT_TIMEOUT_MS, - Math.max(ConfigConstants.VAL_MIN_CONNECT_TIMEOUT_MS, managerConnTimeoutMs)); - } - - public int getManagerSocketTimeoutMs() { - return managerSocketTimeoutMs; - } - - public void setManagerSocketTimeoutMs(int managerSocketTimeoutMs) { - this.managerSocketTimeoutMs = - Math.min(ConfigConstants.VAL_MAX_SOCKET_TIMEOUT_MS, - Math.max(ConfigConstants.VAL_MIN_SOCKET_TIMEOUT_MS, managerSocketTimeoutMs)); - } - - public int getConfigSyncMaxRetryIfFail() { - return configSyncMaxRetryIfFail; - } - - public void setConfigSyncMaxRetryIfFail(int configSyncMaxRetryIfFail) { - this.configSyncMaxRetryIfFail = - Math.min(configSyncMaxRetryIfFail, ConfigConstants.VAL_MAX_RETRY_IF_CONFIG_SYNC_FAIL); - } - - public String getConfigStoreBasePath() { - return configStoreBasePath; - } - - public void setConfigStoreBasePath(String configStoreBasePath) { - if (StringUtils.isBlank(configStoreBasePath)) { - return; - } - this.configStoreBasePath = configStoreBasePath.trim(); - } - - public long getConfigCacheExpiredMs() { - return configCacheExpiredMs; - } - - public void setConfigCacheExpiredMs(long configCacheExpiredMs) { - this.configCacheExpiredMs = configCacheExpiredMs; - } - - public long getConfigFailStatusExpiredMs() { - return configFailStatusExpiredMs; - } - - public void setConfigFailStatusExpiredMs(long configFailStatusExpiredMs) { - this.configFailStatusExpiredMs = - Math.min(configFailStatusExpiredMs, ConfigConstants.VAL_MAX_CONFIG_FAIL_STATUS_EXPIRED_MS); - } - - public long getForceReChooseInrMs() { - return forceReChooseInrMs; - } - - public void setForceReChooseInrMs(long forceReChooseInrMs) { - this.forceReChooseInrMs = - Math.max(ConfigConstants.VAL_MIN_FORCE_CHOOSE_INR_MS, forceReChooseInrMs); - } - - public String getInlongGroupId() { - return inlongGroupId; - } - - public String getRegionName() { - return regionName; - } - - public void setRegionName(String regionName) { - if (StringUtils.isNotBlank(regionName)) { - this.regionName = regionName.trim(); - } - } - - public int getAliveConnections() { - return this.aliveConnections; - } - - public void setAliveConnections(int aliveConnections) { - this.aliveConnections = - Math.max(ConfigConstants.VAL_MIN_ALIVE_CONNECTIONS, aliveConnections); - } - - public boolean isEnableDataEncrypt() { - return enableDataEncrypt; - } - - public String getRsaPubKeyUrl() { - return rsaPubKeyUrl; - } - - public String getUserName() { - return userName; - } - - public void enableDataEncrypt(boolean needDataEncrypt, String userName, String rsaPubKeyUrl) { - this.enableDataEncrypt = needDataEncrypt; - if (!this.enableDataEncrypt) { - return; - } - if (StringUtils.isBlank(userName)) { - throw new IllegalArgumentException("userName is Blank!"); - } - if (StringUtils.isBlank(rsaPubKeyUrl)) { - throw new IllegalArgumentException("rsaPubKeyUrl is Blank!"); - } - this.userName = userName.trim(); - this.rsaPubKeyUrl = rsaPubKeyUrl.trim(); - } - - public String getTlsServerCertFilePathAndName() { - return tlsServerCertFilePathAndName; - } - - public String getTlsServerKey() { - return tlsServerKey; - } - - public int getSyncThreadPoolSize() { - return syncThreadPoolSize; - } - - public void setSyncThreadPoolSize(int syncThreadPoolSize) { - if (syncThreadPoolSize > ConfigConstants.MAX_SYNC_THREAD_POOL_SIZE) { - throw new IllegalArgumentException(""); - } - this.syncThreadPoolSize = syncThreadPoolSize; - } - - public int getTotalAsyncCallbackSize() { - return asyncCallbackSize; - } - - public void setTotalAsyncCallbackSize(int asyncCallbackSize) { - this.asyncCallbackSize = asyncCallbackSize; - } - - public int getMaxTimeoutCnt() { - return maxTimeoutCnt; - } - - public void setMaxTimeoutCnt(int maxTimeoutCnt) { - if (maxTimeoutCnt < 0) { - throw new IllegalArgumentException("maxTimeoutCnt must bigger than 0"); - } - this.maxTimeoutCnt = maxTimeoutCnt; - } - - public long getConnectTimeoutMs() { - return connectTimeoutMs; - } - - public void setConnectTimeoutMs(long connectTimeoutMs) { - if (connectTimeoutMs >= ConfigConstants.VAL_MIN_CONNECT_TIMEOUT_MS) { - this.connectTimeoutMs = connectTimeoutMs; - } - } - - public long getRequestTimeoutMs() { - return requestTimeoutMs; - } - - public void setRequestTimeoutMs(long requestTimeoutMs) { - if (requestTimeoutMs >= ConfigConstants.VAL_MIN_REQUEST_TIMEOUT_MS) { - this.requestTimeoutMs = requestTimeoutMs; - this.conCloseWaitPeriodMs = - this.requestTimeoutMs + ConfigConstants.VAL_DEF_CONNECT_CLOSE_DELAY_MS; - } - } - - public long getConCloseWaitPeriodMs() { - return conCloseWaitPeriodMs; - } - - public void setConCloseWaitPeriodMs(long conCloseWaitPeriodMs) { - if (conCloseWaitPeriodMs >= 0) { - this.conCloseWaitPeriodMs = conCloseWaitPeriodMs; - } - } - - public long getReConnectWaitMs() { - return reConnectWaitMs; - } - - public void setReConnectWaitMs(long reConnectWaitMs) { - if (reConnectWaitMs > ConfigConstants.VAL_MAX_RECONNECT_WAIT_MS) { - this.reConnectWaitMs = ConfigConstants.VAL_MAX_RECONNECT_WAIT_MS; - } - } - - public int getRecvBufferSize() { - return recvBufferSize; - } - - public void setRecvBufferSize(int recvBufferSize) { - if (recvBufferSize > 0 && recvBufferSize < Integer.MAX_VALUE) { - this.recvBufferSize = recvBufferSize; - } - } - - public int getSendBufferSize() { - return sendBufferSize; - } - - public void setSendBufferSize(int sendBufferSize) { - if (sendBufferSize > 0 && sendBufferSize < Integer.MAX_VALUE) { - this.sendBufferSize = sendBufferSize; - } - } - - public long getMaxMsgInFlightPerConn() { - return maxMsgInFlightPerConn; - } - - public void setMaxMsgInFlightPerConn(long maxMsgInFlightPerConn) { - this.maxMsgInFlightPerConn = maxMsgInFlightPerConn; - } - - public void setHttpsInfo(String tlsServerCertFilePathAndName, String tlsServerKey) { - if (StringUtils.isBlank(tlsServerCertFilePathAndName)) { - throw new IllegalArgumentException("tlsServerCertFilePathAndName is Blank!"); - } - if (StringUtils.isBlank(tlsServerKey)) { - throw new IllegalArgumentException("tlsServerKey is Blank!"); - } - this.tlsServerKey = tlsServerKey; - this.tlsServerCertFilePathAndName = tlsServerCertFilePathAndName; - } - - public String getTlsVersion() { - return tlsVersion; - } - - public void setTlsVersion(String tlsVersion) { - this.tlsVersion = tlsVersion; - } - - public int getProxyHttpUpdateIntervalMinutes() { - return proxyHttpUpdateIntervalMinutes; - } - - public void setProxyHttpUpdateIntervalMinutes(int proxyHttpUpdateIntervalMinutes) { - this.proxyHttpUpdateIntervalMinutes = proxyHttpUpdateIntervalMinutes; - } - - public boolean isDiscardOldMessage() { - return discardOldMessage; - } - - public void setDiscardOldMessage(boolean discardOldMessage) { - this.discardOldMessage = discardOldMessage; - } - - public int getAsyncWorkerNumber() { - return asyncWorkerNumber; - } - - public void setAsyncWorkerNumber(int asyncWorkerNumber) { - this.asyncWorkerNumber = asyncWorkerNumber; - } - - public int getAsyncWorkerInterval() { - return asyncWorkerInterval; - } - - public void setAsyncWorkerInterval(int asyncWorkerInterval) { - this.asyncWorkerInterval = asyncWorkerInterval; - } - - public boolean isCleanHttpCacheWhenClosing() { - return cleanHttpCacheWhenClosing; - } - - public void setCleanHttpCacheWhenClosing(boolean cleanHttpCacheWhenClosing) { - this.cleanHttpCacheWhenClosing = cleanHttpCacheWhenClosing; - } - - public MetricConfig getMetricConfig() { - return metricConfig; - } - - public boolean isEnableMetric() { - return metricConfig.isEnableMetric(); - } - - public void setMetricConfig(MetricConfig metricConfig) { - if (metricConfig == null) { - return; - } - this.metricConfig = metricConfig; - } - - public int getIoThreadNum() { - return ioThreadNum; - } - - public void setIoThreadNum(int ioThreadNum) { - this.ioThreadNum = ioThreadNum; - } - - public boolean isEnableBusyWait() { - return enableBusyWait; - } - - public void setEnableBusyWait(boolean enableBusyWait) { - this.enableBusyWait = enableBusyWait; - } - - public int getSenderMaxAttempt() { - return senderMaxAttempt; - } - - public void setSenderAttempt(int senderMaxAttempt) { - this.senderMaxAttempt = senderMaxAttempt; - } - - private void checkAndParseAddress(String managerAddress) throws ProxySdkException { - if (StringUtils.isBlank(managerAddress) - || (!managerAddress.startsWith(ConfigConstants.HTTP) - && !managerAddress.startsWith(ConfigConstants.HTTPS))) { - throw new ProxySdkException("managerAddress is blank or missing http/https protocol"); - } - String hostPortInfo; - if (managerAddress.startsWith(ConfigConstants.HTTPS)) { - this.visitManagerByHttp = false; - hostPortInfo = managerAddress.substring(ConfigConstants.HTTPS.length()); - } else { - hostPortInfo = managerAddress.substring(ConfigConstants.HTTP.length()); - } - if (StringUtils.isBlank(hostPortInfo)) { - throw new ProxySdkException("managerAddress must include host:port info!"); - } - String[] fields = hostPortInfo.split(":"); - if (fields.length == 1) { - throw new ProxySdkException("managerAddress must include port info!"); - } else if (fields.length > 2) { - throw new ProxySdkException("managerAddress must only include host:port info!"); - } - if (StringUtils.isBlank(fields[0])) { - throw new ProxySdkException("managerAddress's host is blank!"); - } - this.managerIP = fields[0].trim(); - if (StringUtils.isBlank(fields[1])) { - throw new ProxySdkException("managerAddress's port is blank!"); - } - try { - this.managerPort = Integer.parseInt(fields[1]); - } catch (Throwable ex) { - throw new ProxySdkException("managerAddress's port must be number!"); - } - } -} diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/TcpMsgSenderConfig.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/TcpMsgSenderConfig.java new file mode 100644 index 00000000000..b9054c4084a --- /dev/null +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/TcpMsgSenderConfig.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.dataproxy; + +import org.apache.inlong.common.msg.MsgType; +import org.apache.inlong.sdk.dataproxy.common.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.common.ReportProtocol; +import org.apache.inlong.sdk.dataproxy.common.SdkConsts; +import org.apache.inlong.sdk.dataproxy.exception.ProxySdkException; +import org.apache.inlong.sdk.dataproxy.utils.ProxyUtils; + +import java.util.Objects; + +/** + * TCP Message Sender configuration: + * + * Used to define the setting related to the sender reporting data using the TCP protocol, including + * the message type, request timeout, whether to enable data compression, + * the number of netty worker threads, and other configuration settings + */ +public class TcpMsgSenderConfig extends ProxyClientConfig implements Cloneable { + + // msg report type + private MsgType sdkMsgType = MsgType.MSG_BIN_MULTI_BODY; + // whether separate event by line feed + private boolean separateEventByLF = true; + // whether enable data compress + private boolean enableDataCompress = true; + // min compress enable data length + private int minCompEnableLength = SdkConsts.VAL_DEF_COMPRESS_ENABLE_SIZE; + // whether enable epoll busy wait + private boolean enableEpollBusyWait = false; + // netty worker thread num + private int nettyWorkerThreadNum = SdkConsts.VAL_DEF_TCP_NETTY_WORKER_THREAD_NUM; + // socket receive buffer + private int rcvBufferSize = SdkConsts.DEFAULT_RECEIVE_BUFFER_SIZE; + // socket send buffer + private int sendBufferSize = SdkConsts.DEFAULT_SEND_BUFFER_SIZE; + // connect timeout in milliseconds + private int connectTimeoutMs = SdkConsts.VAL_DEF_TCP_CONNECT_TIMEOUT_MS; + // request timeout in milliseconds + private long requestTimeoutMs = SdkConsts.VAL_DEF_REQUEST_TIMEOUT_MS; + // connect close wait period in milliseconds + private long conCloseWaitPeriodMs = + SdkConsts.VAL_DEF_REQUEST_TIMEOUT_MS + SdkConsts.VAL_DEF_CONNECT_CLOSE_DELAY_MS; + // max message count per connection + private int maxMsgInFlightPerConn = SdkConsts.MAX_INFLIGHT_MSG_COUNT_PER_CONNECTION; + // client reconnect wait period in ms + private long frozenReconnectWaitMs = SdkConsts.VAL_DEF_FROZEN_RECONNECT_WAIT_MS; + private long busyReconnectWaitMs = SdkConsts.VAL_DEF_BUSY_RECONNECT_WAIT_MS; + private long reconFailWaitMs = SdkConsts.VAL_DEF_RECONNECT_FAIL_WAIT_MS; + // the maximum allowed synchronization message timeout + private int maxAllowedSyncMsgTimeoutCnt = SdkConsts.VAL_DEF_SYNC_MSG_TIMEOUT_CNT; + // the synchronization message timeout check duration ms + private long syncMsgTimeoutChkDurMs = SdkConsts.VAL_DEF_SYNC_TIMEOUT_CHK_DUR_MS; + // max sync send attempt + private int maxSyncSendAttempt = SdkConsts.DEFAULT_SENDER_MAX_ATTEMPT; + + // async callback size + private int asyncCallbackSize = SdkConsts.ASYNC_CALLBACK_SIZE; + + public TcpMsgSenderConfig(boolean visitMgrByHttps, + String managerIP, int managerPort, String groupId) throws ProxySdkException { + super(visitMgrByHttps, managerIP, managerPort, groupId, ReportProtocol.TCP, null); + } + + public TcpMsgSenderConfig(String managerAddress, String groupId) throws ProxySdkException { + super(managerAddress, groupId, ReportProtocol.TCP, null); + } + + public TcpMsgSenderConfig(boolean visitMgrByHttps, String managerIP, int managerPort, + String groupId, String mgrAuthSecretId, String mgrAuthSecretKey) throws ProxySdkException { + super(visitMgrByHttps, managerIP, managerPort, groupId, ReportProtocol.TCP, null); + this.setMgrAuthzInfo(true, mgrAuthSecretId, mgrAuthSecretKey); + } + + public TcpMsgSenderConfig(String managerAddress, + String groupId, String mgrAuthSecretId, String mgrAuthSecretKey) throws ProxySdkException { + super(managerAddress, groupId, ReportProtocol.TCP, null); + this.setMgrAuthzInfo(true, mgrAuthSecretId, mgrAuthSecretKey); + } + + public MsgType getSdkMsgType() { + return sdkMsgType; + } + + public void setSdkMsgType(MsgType sdkMsgType) { + if (!ProxyUtils.SdkAllowedMsgType.contains(sdkMsgType)) { + throw new IllegalArgumentException( + "Only allowed msgType:" + ProxyUtils.SdkAllowedMsgType); + } + this.sdkMsgType = sdkMsgType; + } + + public boolean isSeparateEventByLF() { + return separateEventByLF; + } + + public void setSeparateEventByLF(boolean separateEventByLF) { + this.separateEventByLF = separateEventByLF; + } + + public boolean isEnableDataCompress() { + return enableDataCompress; + } + + public void setEnableDataCompress(boolean enableDataCompress) { + this.enableDataCompress = enableDataCompress; + } + + public int getMinCompEnableLength() { + return minCompEnableLength; + } + + public void setMinCompEnableLength(int minCompEnableLength) { + this.minCompEnableLength = Math.max( + SdkConsts.VAL_MIN_COMPRESS_ENABLE_SIZE, minCompEnableLength); + } + + public boolean isEnableEpollBusyWait() { + return enableEpollBusyWait; + } + + public void setEnableEpollBusyWait(boolean enableEpollBusyWait) { + this.enableEpollBusyWait = enableEpollBusyWait; + } + + public int getNettyWorkerThreadNum() { + return nettyWorkerThreadNum; + } + + public void setNettyWorkerThreadNum(int nettyWorkerThreadNum) { + this.nettyWorkerThreadNum = Math.max( + SdkConsts.VAL_MIN_TCP_NETTY_WORKER_THREAD_NUM, nettyWorkerThreadNum); + } + + public int getRcvBufferSize() { + return rcvBufferSize; + } + + public void setRcvBufferSize(int rcvBufferSize) { + this.rcvBufferSize = Math.min(rcvBufferSize, Integer.MAX_VALUE - 1); + } + + public int getSendBufferSize() { + return sendBufferSize; + } + + public void setSendBufferSize(int sendBufferSize) { + this.sendBufferSize = Math.min(sendBufferSize, Integer.MAX_VALUE - 1); + } + + public int getConnectTimeoutMs() { + return connectTimeoutMs; + } + + public void setConnectTimeoutMs(int connectTimeoutMs) { + if (connectTimeoutMs >= SdkConsts.VAL_MIN_CONNECT_TIMEOUT_MS) { + this.connectTimeoutMs = connectTimeoutMs; + } + } + + public long getRequestTimeoutMs() { + return requestTimeoutMs; + } + + public void setRequestTimeoutMs(long requestTimeoutMs) { + if (requestTimeoutMs >= SdkConsts.VAL_MIN_REQUEST_TIMEOUT_MS) { + this.requestTimeoutMs = requestTimeoutMs; + this.conCloseWaitPeriodMs = + this.requestTimeoutMs + SdkConsts.VAL_DEF_CONNECT_CLOSE_DELAY_MS; + } + } + + public long getConCloseWaitPeriodMs() { + return conCloseWaitPeriodMs; + } + + public void setConCloseWaitPeriodMs(long conCloseWaitPeriodMs) { + if (conCloseWaitPeriodMs >= 0) { + this.conCloseWaitPeriodMs = conCloseWaitPeriodMs; + } + } + + public int getMaxMsgInFlightPerConn() { + return maxMsgInFlightPerConn; + } + + public void setMaxMsgInFlightPerConn(int maxMsgInFlightPerConn) { + this.maxMsgInFlightPerConn = maxMsgInFlightPerConn; + } + + public long getFrozenReconnectWaitMs() { + return frozenReconnectWaitMs; + } + + public void setFrozenReconnectWaitMs(long frozenReconnectWaitMs) { + if (frozenReconnectWaitMs > SdkConsts.VAL_MAX_RECONNECT_WAIT_MS) { + this.frozenReconnectWaitMs = SdkConsts.VAL_MAX_RECONNECT_WAIT_MS; + } + } + + public long getBusyReconnectWaitMs() { + return busyReconnectWaitMs; + } + + public void setBusyReconnectWaitMs(long busyReconnectWaitMs) { + if (busyReconnectWaitMs > SdkConsts.VAL_MAX_RECONNECT_WAIT_MS) { + this.busyReconnectWaitMs = SdkConsts.VAL_MAX_RECONNECT_WAIT_MS; + } + } + + public long getReconFailWaitMs() { + return reconFailWaitMs; + } + + public void setReconFailWaitMs(long reconFailWaitMs) { + if (reconFailWaitMs > SdkConsts.VAL_MAX_RECONNECT_WAIT_MS) { + this.reconFailWaitMs = SdkConsts.VAL_MAX_RECONNECT_WAIT_MS; + } + } + + public int getMaxAllowedSyncMsgTimeoutCnt() { + return maxAllowedSyncMsgTimeoutCnt; + } + + public void setMaxAllowedSyncMsgTimeoutCnt(int maxAllowedSyncMsgTimeoutCnt) { + this.maxAllowedSyncMsgTimeoutCnt = maxAllowedSyncMsgTimeoutCnt; + } + + public long getSyncMsgTimeoutChkDurMs() { + return syncMsgTimeoutChkDurMs; + } + + public void setSyncMsgTimeoutChkDurMs(long syncMsgTimeoutChkDurMs) { + this.syncMsgTimeoutChkDurMs = Math.max( + SdkConsts.VAL_MIN_SYNC_TIMEOUT_CHK_DUR_MS, syncMsgTimeoutChkDurMs); + } + + public int getMaxSyncSendAttempt() { + return maxSyncSendAttempt; + } + + public void setMaxSyncSendAttempt(int maxSyncSendAttempt) { + this.maxSyncSendAttempt = maxSyncSendAttempt; + } + + public int getTotalAsyncCallbackSize() { + return asyncCallbackSize; + } + + public void setTotalAsyncCallbackSize(int asyncCallbackSize) { + this.asyncCallbackSize = asyncCallbackSize; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + if (!super.equals(o)) + return false; + TcpMsgSenderConfig config = (TcpMsgSenderConfig) o; + return separateEventByLF == config.separateEventByLF + && enableDataCompress == config.enableDataCompress + && minCompEnableLength == config.minCompEnableLength + && enableEpollBusyWait == config.enableEpollBusyWait + && nettyWorkerThreadNum == config.nettyWorkerThreadNum + && rcvBufferSize == config.rcvBufferSize + && sendBufferSize == config.sendBufferSize + && connectTimeoutMs == config.connectTimeoutMs + && requestTimeoutMs == config.requestTimeoutMs + && conCloseWaitPeriodMs == config.conCloseWaitPeriodMs + && maxMsgInFlightPerConn == config.maxMsgInFlightPerConn + && frozenReconnectWaitMs == config.frozenReconnectWaitMs + && busyReconnectWaitMs == config.busyReconnectWaitMs + && reconFailWaitMs == config.reconFailWaitMs + && maxAllowedSyncMsgTimeoutCnt == config.maxAllowedSyncMsgTimeoutCnt + && syncMsgTimeoutChkDurMs == config.syncMsgTimeoutChkDurMs + && sdkMsgType == config.sdkMsgType + && maxSyncSendAttempt == config.maxSyncSendAttempt + && asyncCallbackSize == config.asyncCallbackSize; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), sdkMsgType, separateEventByLF, + enableDataCompress, minCompEnableLength, enableEpollBusyWait, + nettyWorkerThreadNum, rcvBufferSize, sendBufferSize, connectTimeoutMs, + requestTimeoutMs, conCloseWaitPeriodMs, maxMsgInFlightPerConn, + frozenReconnectWaitMs, busyReconnectWaitMs, reconFailWaitMs, + maxAllowedSyncMsgTimeoutCnt, maxSyncSendAttempt, syncMsgTimeoutChkDurMs, + asyncCallbackSize); + } + + @Override + public TcpMsgSenderConfig clone() { + try { + TcpMsgSenderConfig copy = (TcpMsgSenderConfig) super.clone(); + if (copy != null) { + copy.sdkMsgType = this.sdkMsgType; + } + return copy; + } catch (Throwable ex) { + logger.warn("Failed to clone TcpMsgSenderConfig", ex); + return null; + } + } + + @Override + public String toString() { + final StringBuilder strBuff = + new StringBuilder("TcpMsgSenderConfig{sdkMsgType=").append(sdkMsgType) + .append(", separateEventByLF=").append(separateEventByLF) + .append(", enableDataCompress=").append(enableDataCompress) + .append(", minCompEnableLength=").append(minCompEnableLength) + .append(", enableEpollBusyWait=").append(enableEpollBusyWait) + .append(", nettyWorkerThreadNum=").append(nettyWorkerThreadNum) + .append(", rcvBufferSize=").append(rcvBufferSize) + .append(", sendBufferSize=").append(sendBufferSize) + .append(", connectTimeoutMs=").append(connectTimeoutMs) + .append(", requestTimeoutMs=").append(requestTimeoutMs) + .append(", conCloseWaitPeriodMs=").append(conCloseWaitPeriodMs) + .append(", maxMsgInFlightPerConn=").append(maxMsgInFlightPerConn) + .append(", frozenReconnectWaitMs=").append(frozenReconnectWaitMs) + .append(", busyReconnectWaitMs=").append(busyReconnectWaitMs) + .append(", reconFailWaitMs=").append(reconFailWaitMs) + .append(", maxAllowedSyncMsgTimeoutCnt=").append(maxAllowedSyncMsgTimeoutCnt) + .append(", maxSyncSendAttempt=").append(maxSyncSendAttempt) + .append(", syncMsgTimeoutChkDurMs=").append(syncMsgTimeoutChkDurMs) + .append(", asyncCallbackSize=").append(asyncCallbackSize); + return super.getSetting(strBuff); + } +} diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/codec/ProtocolEncoder.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/codec/ProtocolEncoder.java index bf1cc995027..66594623e0b 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/codec/ProtocolEncoder.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/codec/ProtocolEncoder.java @@ -41,9 +41,9 @@ import java.security.SecureRandom; import java.util.List; -import static org.apache.inlong.sdk.dataproxy.ConfigConstants.FLAG_ALLOW_AUTH; -import static org.apache.inlong.sdk.dataproxy.ConfigConstants.FLAG_ALLOW_COMPRESS; -import static org.apache.inlong.sdk.dataproxy.ConfigConstants.FLAG_ALLOW_ENCRYPT; +import static org.apache.inlong.sdk.dataproxy.common.SdkConsts.FLAG_ALLOW_AUTH; +import static org.apache.inlong.sdk.dataproxy.common.SdkConsts.FLAG_ALLOW_COMPRESS; +import static org.apache.inlong.sdk.dataproxy.common.SdkConsts.FLAG_ALLOW_ENCRYPT; public class ProtocolEncoder extends MessageToMessageEncoder { diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/HttpContentType.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/HttpContentType.java new file mode 100644 index 00000000000..0db9261c70a --- /dev/null +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/HttpContentType.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.dataproxy.common; + +/** + * HTTP Report Content Type enum + * + * This enumeration defines of HTTP reporting supported content types. + */ +public enum HttpContentType { + APPLICATION_X_WWW_FORM_URLENCODED +} diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/ProxyClientConfig.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/ProxyClientConfig.java new file mode 100644 index 00000000000..48039c7c65e --- /dev/null +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/ProxyClientConfig.java @@ -0,0 +1,527 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.dataproxy.common; + +import org.apache.inlong.sdk.dataproxy.exception.ProxySdkException; +import org.apache.inlong.sdk.dataproxy.metric.MetricConfig; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; + +/** + * Configuration settings related to Manager metadata synchronization: + * 1. Manager address, visit protocol, authentication info + * 2. requested groupId, username info + * 3. meta cache setting info + * 4. other setting info + */ +public class ProxyClientConfig implements Cloneable { + + protected static final Logger logger = LoggerFactory.getLogger(ProxyClientConfig.class); + // whether to access the manager via https + private boolean visitMgrByHttps = false; + private String tlsVersion = "TLSv1.2"; + // manager ip + private String managerIP; + // manager port + private int managerPort; + // whether enable authentication + private boolean enableMgrAuthz = false; + // auth secret id + private String mgrAuthSecretId = ""; + // auth secret key + private String mgrAuthSecretKey = ""; + // sdk report group Id + private final String inlongGroupId; + // sdk report protocol + private final String dataRptProtocol; + // sdk report region name + private String regionName = SdkConsts.VAL_DEF_REGION_NAME; + // group meta config key + private String groupMetaConfigKey; + // manager socket timeout ms + private int mgrSocketTimeoutMs = SdkConsts.VAL_DEF_MGR_SOCKET_TIMEOUT_MS; + // manager connect timeout ms + private int mgrConnTimeoutMs = SdkConsts.VAL_DEF_TCP_CONNECT_TIMEOUT_MS; + // whether to start using local proxy configure + private boolean onlyUseLocalProxyConfig = false; + // max retry count if meta query + private int metaQryMaxRetryIfFail = SdkConsts.VAL_DEF_RETRY_IF_CONFIG_SYNC_FAIL; + // meta sync interval ms + private long mgrMetaSyncInrMs = + SdkConsts.VAL_DEF_CONFIG_SYNC_INTERVAL_MIN * SdkConsts.VAL_UNIT_MIN_TO_MS; + // max retry count if meta query failure + private int metaQueryMaxRetryIfFail = SdkConsts.VAL_DEF_RETRY_IF_CONFIG_QUERY_FAIL; + // query wait ms if meta query failure + private int metaQueryWaitMsIfFail = SdkConsts.VAL_DEF_WAIT_MS_IF_CONFIG_QUERY_FAIL; + // max retry count if meta sync failure + private int metaSyncMaxRetryIfFail = SdkConsts.VAL_DEF_RETRY_IF_CONFIG_SYNC_FAIL; + // sync wait ms if meta sync failure + private int metaSyncWaitMsIfFail = SdkConsts.VAL_DEF_WAIT_MS_IF_CONFIG_SYNC_FAIL; + // meta cache storage base path + private String metaStoreBasePath = System.getProperty("user.dir"); + // max expired time for meta cache. + private long metaCacheExpiredMs = SdkConsts.VAL_DEF_CACHE_CONFIG_EXPIRED_MS; + // max expired time for meta query failure status + private long metaQryFailCacheExpiredMs = SdkConsts.VAL_DEF_CONFIG_FAIL_STATUS_EXPIRED_MS; + // maximum number of concurrent nodes + private int aliveConnections = SdkConsts.VAL_DEF_ALIVE_CONNECTIONS; + // node forced selection interval ms + private long forceReChooseInrMs = SdkConsts.VAL_DEF_FORCE_CHOOSE_INR_MS; + // metric setting + private MetricConfig metricConfig = new MetricConfig(); + // report setting + private boolean enableReportAuthz = false; + private boolean enableReportEncrypt = false; + private String rptRsaPubKeyUrl = ""; + private String rptUserName = ""; + private String rptSecretKey = ""; + + protected ProxyClientConfig(boolean visitMgrByHttps, String managerIP, int managerPort, + String groupId, ReportProtocol rptProtocol, String regionName) throws ProxySdkException { + this.visitMgrByHttps = visitMgrByHttps; + if (StringUtils.isBlank(managerIP)) { + throw new ProxySdkException("managerIP is Blank!"); + } + this.managerIP = managerIP.trim(); + if (managerPort <= 0) { + throw new ProxySdkException("managerPort <= 0!"); + } + this.managerPort = managerPort; + if (StringUtils.isBlank(groupId)) { + throw new ProxySdkException("groupId is Blank!"); + } + this.inlongGroupId = groupId.trim(); + this.dataRptProtocol = rptProtocol.name(); + this.setRegionName(regionName); + this.groupMetaConfigKey = this.buildMgrConfigKey(); + } + + protected ProxyClientConfig(String managerAddress, + String groupId, ReportProtocol rptProtocol, String regionName) throws ProxySdkException { + this.checkAndParseAddress(managerAddress); + if (StringUtils.isBlank(groupId)) { + throw new ProxySdkException("groupId is Blank!"); + } + this.inlongGroupId = groupId.trim(); + this.dataRptProtocol = rptProtocol.name(); + this.setRegionName(regionName); + this.groupMetaConfigKey = this.buildMgrConfigKey(); + } + + public void setMgrAuthzInfo(boolean needMgrAuthz, + String mgrAuthSecretId, String mgrAuthSecretKey) throws ProxySdkException { + this.enableMgrAuthz = needMgrAuthz; + if (!this.enableMgrAuthz) { + return; + } + if (StringUtils.isBlank(mgrAuthSecretId)) { + throw new ProxySdkException("mgrAuthSecretId is Blank!"); + } + if (StringUtils.isBlank(mgrAuthSecretKey)) { + throw new ProxySdkException("mgrAuthSecretKey is Blank!"); + } + this.mgrAuthSecretId = mgrAuthSecretId.trim(); + this.mgrAuthSecretKey = mgrAuthSecretKey.trim(); + } + + public void setRptAuthzAndEncryptInfo(Boolean enableRptAuthz, Boolean enableRptEncrypt, + String rptUserName, String rptSecretKey, String rptRsaPubKeyUrl) throws ProxySdkException { + if (enableRptAuthz != null) { + this.enableReportAuthz = enableRptAuthz; + } + if (enableRptEncrypt != null) { + this.enableReportEncrypt = enableRptEncrypt; + } + if (this.enableReportAuthz) { + if (StringUtils.isBlank(rptUserName)) { + throw new ProxySdkException("rptUserName is Blank!"); + } + if (StringUtils.isBlank(rptSecretKey)) { + throw new ProxySdkException("rptSecretKey is Blank!"); + } + this.rptUserName = rptUserName.trim(); + this.rptSecretKey = rptSecretKey.trim(); + } + if (this.enableReportEncrypt) { + if (StringUtils.isBlank(rptUserName)) { + throw new ProxySdkException("rptUserName is Blank!"); + } + if (StringUtils.isBlank(rptRsaPubKeyUrl)) { + throw new ProxySdkException("rptRsaPubKeyUrl is Blank!"); + } + this.rptUserName = rptUserName.trim(); + this.rptRsaPubKeyUrl = rptRsaPubKeyUrl.trim(); + } + } + + public boolean isVisitMgrByHttps() { + return visitMgrByHttps; + } + + public String getTlsVersion() { + return tlsVersion; + } + + public void setTlsVersion(String tlsVersion) { + if (StringUtils.isNotBlank(tlsVersion)) { + this.tlsVersion = tlsVersion.trim(); + } + } + + public String getManagerIP() { + return managerIP; + } + + public int getManagerPort() { + return managerPort; + } + + public boolean isEnableMgrAuthz() { + return enableMgrAuthz; + } + + public String getMgrAuthSecretId() { + return mgrAuthSecretId; + } + + public String getMgrAuthSecretKey() { + return mgrAuthSecretKey; + } + + public String getInlongGroupId() { + return inlongGroupId; + } + + public String getDataRptProtocol() { + return dataRptProtocol; + } + + public String getRegionName() { + return regionName; + } + + public String getGroupMetaConfigKey() { + return groupMetaConfigKey; + } + + public void setRegionName(String regionName) { + if (StringUtils.isNotBlank(regionName)) { + this.regionName = regionName.trim().toLowerCase(); + this.groupMetaConfigKey = this.buildMgrConfigKey(); + } + } + + public int getMgrSocketTimeoutMs() { + return mgrSocketTimeoutMs; + } + + public void setMgrSocketTimeoutMs(int mgrSocketTimeoutMs) { + this.mgrSocketTimeoutMs = + Math.min(SdkConsts.VAL_MAX_SOCKET_TIMEOUT_MS, + Math.max(SdkConsts.VAL_MIN_SOCKET_TIMEOUT_MS, mgrSocketTimeoutMs)); + } + + public int getMgrConnTimeoutMs() { + return mgrConnTimeoutMs; + } + + public void setMgrConnTimeoutMs(int mgrConnTimeoutMs) { + this.mgrConnTimeoutMs = + Math.min(SdkConsts.VAL_MAX_CONNECT_TIMEOUT_MS, + Math.max(SdkConsts.VAL_MIN_CONNECT_TIMEOUT_MS, mgrConnTimeoutMs)); + } + + public boolean isOnlyUseLocalProxyConfig() { + return onlyUseLocalProxyConfig; + } + + public void setOnlyUseLocalProxyConfig(boolean onlyUseLocalProxyConfig) { + this.onlyUseLocalProxyConfig = onlyUseLocalProxyConfig; + } + + public long getMgrMetaSyncInrMs() { + return mgrMetaSyncInrMs; + } + + public void setMgrMetaSyncInrMin(int mgrMetaSyncInrMin) { + int tmpValue = + Math.min(SdkConsts.VAL_MAX_CONFIG_SYNC_INTERVAL_MIN, + Math.max(SdkConsts.VAL_MIN_CONFIG_SYNC_INTERVAL_MIN, mgrMetaSyncInrMin)); + this.mgrMetaSyncInrMs = tmpValue * SdkConsts.VAL_UNIT_MIN_TO_MS; + } + + public int getMetaQueryMaxRetryIfFail() { + return metaQueryMaxRetryIfFail; + } + + public void setMetaQueryMaxRetryIfFail(int metaQueryMaxRetryIfFail) { + this.metaQueryMaxRetryIfFail = metaQueryMaxRetryIfFail; + } + + public int getMetaQueryWaitMsIfFail() { + return metaQueryWaitMsIfFail; + } + + public void setMetaQueryWaitMsIfFail(int metaQueryWaitMsIfFail) { + this.metaQueryWaitMsIfFail = Math.min(SdkConsts.VAL_MAX_WAIT_MS_IF_CONFIG_REQ_FAIL, + Math.max(SdkConsts.VAL_MIN_WAIT_MS_IF_CONFIG_REQ_FAIL, metaQueryWaitMsIfFail)); + } + + public int getMetaSyncMaxRetryIfFail() { + return metaSyncMaxRetryIfFail; + } + + public void setMetaSyncMaxRetryIfFail(int metaSyncMaxRetryIfFail) { + this.metaSyncMaxRetryIfFail = + Math.min(metaSyncMaxRetryIfFail, SdkConsts.VAL_MAX_RETRY_IF_CONFIG_SYNC_FAIL); + } + + public int getMetaSyncWaitMsIfFail() { + return metaSyncWaitMsIfFail; + } + + public void setMetaSyncWaitMsIfFail(int metaSyncWaitMsIfFail) { + this.metaSyncWaitMsIfFail = Math.min(SdkConsts.VAL_MAX_WAIT_MS_IF_CONFIG_REQ_FAIL, + Math.max(SdkConsts.VAL_MIN_WAIT_MS_IF_CONFIG_REQ_FAIL, metaSyncWaitMsIfFail)); + + } + + public String getMetaStoreBasePath() { + return metaStoreBasePath; + } + + public void setMetaStoreBasePath(String metaStoreBasePath) { + if (StringUtils.isBlank(metaStoreBasePath)) { + return; + } + this.metaStoreBasePath = metaStoreBasePath.trim(); + } + + public long getMetaCacheExpiredMs() { + return metaCacheExpiredMs; + } + + public void setMetaCacheExpiredMs(long metaCacheExpiredMs) { + this.metaCacheExpiredMs = metaCacheExpiredMs; + } + + public long getMetaQryFailCacheExpiredMs() { + return metaQryFailCacheExpiredMs; + } + + public void setMetaQryFailCacheExpiredMs(long metaQryFailCacheExpiredMs) { + this.metaQryFailCacheExpiredMs = + Math.min(metaQryFailCacheExpiredMs, SdkConsts.VAL_MAX_CONFIG_FAIL_STATUS_EXPIRED_MS); + } + + public int getAliveConnections() { + return aliveConnections; + } + + public void setAliveConnections(int aliveConnections) { + this.aliveConnections = + Math.max(SdkConsts.VAL_MIN_ALIVE_CONNECTIONS, aliveConnections); + } + + public long getForceReChooseInrMs() { + return forceReChooseInrMs; + } + + public void setForceReChooseInrMs(long forceReChooseInrMs) { + this.forceReChooseInrMs = + Math.max(SdkConsts.VAL_MIN_FORCE_CHOOSE_INR_MS, forceReChooseInrMs); + } + + public boolean isEnableReportAuthz() { + return enableReportAuthz; + } + + public boolean isEnableReportEncrypt() { + return enableReportEncrypt; + } + + public String getRptRsaPubKeyUrl() { + return rptRsaPubKeyUrl; + } + + public String getRptUserName() { + return rptUserName; + } + + public String getRptSecretKey() { + return rptSecretKey; + } + + public boolean isEnableMetric() { + return metricConfig.isEnableMetric(); + } + + public void setMetricConfig(MetricConfig metricConfig) { + if (metricConfig == null) { + throw new IllegalArgumentException("metricConfig is null"); + } + this.metricConfig = metricConfig; + } + + public MetricConfig getMetricConfig() { + return metricConfig; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ProxyClientConfig that = (ProxyClientConfig) o; + return visitMgrByHttps == that.visitMgrByHttps + && managerPort == that.managerPort + && enableMgrAuthz == that.enableMgrAuthz + && mgrSocketTimeoutMs == that.mgrSocketTimeoutMs + && mgrConnTimeoutMs == that.mgrConnTimeoutMs + && onlyUseLocalProxyConfig == that.onlyUseLocalProxyConfig + && mgrMetaSyncInrMs == that.mgrMetaSyncInrMs + && metaSyncMaxRetryIfFail == that.metaSyncMaxRetryIfFail + && metaCacheExpiredMs == that.metaCacheExpiredMs + && metaQryFailCacheExpiredMs == that.metaQryFailCacheExpiredMs + && aliveConnections == that.aliveConnections + && forceReChooseInrMs == that.forceReChooseInrMs + && enableReportAuthz == that.enableReportAuthz + && enableReportEncrypt == that.enableReportEncrypt + && Objects.equals(tlsVersion, that.tlsVersion) + && Objects.equals(managerIP, that.managerIP) + && Objects.equals(mgrAuthSecretId, that.mgrAuthSecretId) + && Objects.equals(mgrAuthSecretKey, that.mgrAuthSecretKey) + && Objects.equals(inlongGroupId, that.inlongGroupId) + && Objects.equals(dataRptProtocol, that.dataRptProtocol) + && Objects.equals(regionName, that.regionName) + && Objects.equals(groupMetaConfigKey, that.groupMetaConfigKey) + && Objects.equals(metaStoreBasePath, that.metaStoreBasePath) + && Objects.equals(metricConfig, that.metricConfig) + && Objects.equals(rptRsaPubKeyUrl, that.rptRsaPubKeyUrl) + && Objects.equals(rptUserName, that.rptUserName) + && Objects.equals(rptSecretKey, that.rptSecretKey); + } + + @Override + public int hashCode() { + return Objects.hash(visitMgrByHttps, tlsVersion, + managerIP, managerPort, enableMgrAuthz, + mgrAuthSecretId, mgrAuthSecretKey, inlongGroupId, + dataRptProtocol, regionName, groupMetaConfigKey, + mgrSocketTimeoutMs, mgrConnTimeoutMs, onlyUseLocalProxyConfig, + mgrMetaSyncInrMs, metaSyncMaxRetryIfFail, metaStoreBasePath, + metaCacheExpiredMs, metaQryFailCacheExpiredMs, aliveConnections, + forceReChooseInrMs, metricConfig, enableReportAuthz, enableReportEncrypt, + rptRsaPubKeyUrl, rptUserName, rptSecretKey); + } + + @Override + public ProxyClientConfig clone() { + try { + return (ProxyClientConfig) super.clone(); + } catch (Throwable ex) { + logger.warn("Failed to clone ManagerConfig", ex); + return null; + } + } + + protected String getSetting(StringBuilder strBuff) { + strBuff.append(", visitMgrByHttps=").append(visitMgrByHttps) + .append(", tlsVersion='").append(tlsVersion) + .append("', managerIP='").append(managerIP) + .append("', managerPort=").append(managerPort) + .append(", enableMgrAuthz=").append(enableMgrAuthz) + .append(", mgrAuthSecretId='").append(mgrAuthSecretId) + .append("', mgrAuthSecretKey='").append(mgrAuthSecretKey) + .append("', inlongGroupId='").append(inlongGroupId) + .append("', dataRptProtocol='").append(dataRptProtocol) + .append("', regionName='").append(regionName) + .append("', groupMetaConfigKey='").append(groupMetaConfigKey) + .append("', mgrSocketTimeoutMs=").append(mgrSocketTimeoutMs) + .append(", mgrConnTimeoutMs=").append(mgrConnTimeoutMs) + .append(", onlyUseLocalProxyConfig=").append(onlyUseLocalProxyConfig) + .append(", mgrMetaSyncInrMs=").append(mgrMetaSyncInrMs) + .append(", metaSyncMaxRetryIfFail=").append(metaSyncMaxRetryIfFail) + .append(", metaStoreBasePath='").append(metaStoreBasePath) + .append("', metaCacheExpiredMs=").append(metaCacheExpiredMs) + .append(", metaQryFailCacheExpiredMs=").append(metaQryFailCacheExpiredMs) + .append(", aliveConnections=").append(aliveConnections) + .append(", forceReChooseInrMs=").append(forceReChooseInrMs) + .append(", enableReportAuthz=").append(enableReportAuthz) + .append(", enableReportEncrypt=").append(enableReportEncrypt) + .append(", rptRsaPubKeyUrl='").append(rptRsaPubKeyUrl) + .append("', rptUserName='").append(rptUserName) + .append("', rptSecretKey='").append(rptSecretKey) + .append("', metricConfig="); + metricConfig.getSetting(strBuff); + return strBuff.append("}").toString(); + } + + private void checkAndParseAddress(String managerAddress) throws ProxySdkException { + if (StringUtils.isBlank(managerAddress) + || (!managerAddress.startsWith(SdkConsts.PREFIX_HTTP) + && !managerAddress.startsWith(SdkConsts.PREFIX_HTTPS))) { + throw new ProxySdkException("managerAddress is blank or missing http/https protocol"); + } + String hostPortInfo; + if (managerAddress.startsWith(SdkConsts.PREFIX_HTTPS)) { + this.visitMgrByHttps = true; + hostPortInfo = managerAddress.substring(SdkConsts.PREFIX_HTTPS.length()); + } else { + hostPortInfo = managerAddress.substring(SdkConsts.PREFIX_HTTP.length()); + } + if (StringUtils.isBlank(hostPortInfo)) { + throw new ProxySdkException("managerAddress must include host:port info!"); + } + // parse ip and port + String[] fields = hostPortInfo.split(":"); + if (fields.length == 1) { + throw new ProxySdkException("managerAddress must include port info!"); + } else if (fields.length > 2) { + throw new ProxySdkException("managerAddress must only include host:port info!"); + } + // get manager ip + if (StringUtils.isBlank(fields[0])) { + throw new ProxySdkException("managerAddress's host is blank!"); + } + this.managerIP = fields[0].trim(); + // get manager port + if (StringUtils.isBlank(fields[1])) { + throw new ProxySdkException("managerAddress's port is blank!"); + } + int tmValue; + try { + tmValue = Integer.parseInt(fields[1].trim()); + } catch (Throwable ex) { + throw new ProxySdkException("managerAddress's port must be number!"); + } + if (tmValue <= 0) { + throw new ProxySdkException("managerAddress's port must be > 0!"); + } + this.managerPort = tmValue; + } + + private String buildMgrConfigKey() { + return this.inlongGroupId + ":" + this.regionName + ":" + this.dataRptProtocol; + } +} diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/ReportProtocol.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/ReportProtocol.java new file mode 100644 index 00000000000..572f5d8c70d --- /dev/null +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/ReportProtocol.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.dataproxy.common; + +/** + * Report Protocol enum + * + * This enumeration defines the protocol types allowed for the SDK to report data. + * Currently only TCP and HTTP(S) are supported. + */ +public enum ReportProtocol { + TCP, + HTTP +} diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/ConfigConstants.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/SdkConsts.java similarity index 57% rename from inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/ConfigConstants.java rename to inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/SdkConsts.java index 0216b77c2c9..8ae882e21b8 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/ConfigConstants.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/common/SdkConsts.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.inlong.sdk.dataproxy; +package org.apache.inlong.sdk.dataproxy.common; -public class ConfigConstants { +public class SdkConsts { public static final String PROXY_SDK_VERSION = "1.2.11"; - public static String HTTP = "http://"; - public static String HTTPS = "https://"; + public static String PREFIX_HTTP = "http://"; + public static String PREFIX_HTTPS = "https://"; // dataproxy node config public static final String MANAGER_DATAPROXY_API = "/inlong/manager/openapi/dataproxy/getIpList/"; @@ -40,12 +40,19 @@ public class ConfigConstants { public static final int VAL_MAX_CONFIG_SYNC_INTERVAL_MIN = 30; public static final long VAL_UNIT_MIN_TO_MS = 60 * 1000L; // config info sync max retry if failure + public static final int VAL_DEF_RETRY_IF_CONFIG_QUERY_FAIL = 3; public static final int VAL_DEF_RETRY_IF_CONFIG_SYNC_FAIL = 2; public static final int VAL_MAX_RETRY_IF_CONFIG_SYNC_FAIL = 5; + // config wait ms if retry failure + public static final int VAL_DEF_WAIT_MS_IF_CONFIG_QUERY_FAIL = 500; + public static final int VAL_DEF_WAIT_MS_IF_CONFIG_SYNC_FAIL = 800; + public static final int VAL_MAX_WAIT_MS_IF_CONFIG_REQ_FAIL = 20000; + public static final int VAL_MIN_WAIT_MS_IF_CONFIG_REQ_FAIL = 100; + // cache config expired time in ms public static final long VAL_DEF_CACHE_CONFIG_EXPIRED_MS = 20 * 60 * 1000L; // cache config fail status expired time in ms - public static final long VAL_DEF_CONFIG_FAIL_STATUS_EXPIRED_MS = 1000L; + public static final long VAL_DEF_CONFIG_FAIL_STATUS_EXPIRED_MS = 400L; public static final long VAL_MAX_CONFIG_FAIL_STATUS_EXPIRED_MS = 3 * 60 * 1000L; // node force choose interval in ms @@ -53,28 +60,65 @@ public class ConfigConstants { public static final long VAL_MIN_FORCE_CHOOSE_INR_MS = 30 * 1000L; // connection timeout in milliseconds - public static final int VAL_DEF_CONNECT_TIMEOUT_MS = 8000; + public static final int VAL_DEF_TCP_CONNECT_TIMEOUT_MS = 8000; + public static final int VAL_DEF_HTTP_CONNECT_TIMEOUT_MS = 8000; public static final int VAL_MIN_CONNECT_TIMEOUT_MS = 2000; public static final int VAL_MAX_CONNECT_TIMEOUT_MS = 60000; public static final int VAL_DEF_CONNECT_CLOSE_DELAY_MS = 500; // socket timeout in milliseconds - public static final int VAL_DEF_SOCKET_TIMEOUT_MS = 20000; + public static final int VAL_DEF_MGR_SOCKET_TIMEOUT_MS = 15000; + public static final int VAL_DEF_NODE_SOCKET_TIMEOUT_MS = 10000; public static final int VAL_MIN_SOCKET_TIMEOUT_MS = 2000; public static final int VAL_MAX_SOCKET_TIMEOUT_MS = 60000; // active connects - public static final int VAL_DEF_ALIVE_CONNECTIONS = 6; + public static final int VAL_DEF_ALIVE_CONNECTIONS = 7; public static final int VAL_MIN_ALIVE_CONNECTIONS = 1; // request timeout in milliseconds public static final long VAL_DEF_REQUEST_TIMEOUT_MS = 10000L; public static final long VAL_MIN_REQUEST_TIMEOUT_MS = 500L; // reconnect wait ms - public static final long VAL_DEF_RECONNECT_WAIT_MS = 1000L; - public static final long VAL_MAX_RECONNECT_WAIT_MS = 180000L; + public static final long VAL_DEF_FROZEN_RECONNECT_WAIT_MS = 30000L; + public static final long VAL_DEF_BUSY_RECONNECT_WAIT_MS = 20000L; + public static final long VAL_DEF_RECONNECT_FAIL_WAIT_MS = 120000L; + public static final long VAL_MAX_RECONNECT_WAIT_MS = 600000L; // socket buffer size - public static final int DEFAULT_SEND_BUFFER_SIZE = 16777216; - public static final int DEFAULT_RECEIVE_BUFFER_SIZE = 16777216; + public static final int DEFAULT_SEND_BUFFER_SIZE = 16 * 1024 * 1024; // 16M + public static final int DEFAULT_RECEIVE_BUFFER_SIZE = 16 * 1024 * 1024; // 16M // max inflight msg count per connection - public static final long MAX_INFLIGHT_MSG_COUNT_PER_CONNECTION = 10000L; + public static final int MAX_INFLIGHT_MSG_COUNT_PER_CONNECTION = 4000; + + // data compress enable size + public static final int VAL_DEF_COMPRESS_ENABLE_SIZE = 120; + public static final int VAL_MIN_COMPRESS_ENABLE_SIZE = 1; + // TCP netty worker thread num + public static final int VAL_DEF_TCP_NETTY_WORKER_THREAD_NUM = + Runtime.getRuntime().availableProcessors(); + public static final int VAL_MIN_TCP_NETTY_WORKER_THREAD_NUM = 1; + // sync message timeout allowed count + public static final int VAL_DEF_SYNC_MSG_TIMEOUT_CNT = 10; + // sync message timeout check duration ms + public static final long VAL_DEF_SYNC_TIMEOUT_CHK_DUR_MS = 3 * 60 * 1000L; + public static final long VAL_MIN_SYNC_TIMEOUT_CHK_DUR_MS = 10 * 1000L; + + // HTTP sdk close wait period ms + public static final long VAL_DEF_HTTP_SDK_CLOSE_WAIT_MS = 20000L; + public static final long VAL_MAX_HTTP_SDK_CLOSE_WAIT_MS = 90000L; + public static final long VAL_MIN_HTTP_SDK_CLOSE_WAIT_MS = 100L; + // HTTP node reuse wait ms if failed + public static final long VAL_DEF_HTTP_REUSE_FAIL_WAIT_MS = 20000L; + public static final long VAL_MAX_HTTP_REUSE_FAIL_WAIT_MS = 300000L; + public static final long VAL_MIN_HTTP_REUSE_FAIL_WAIT_MS = 1000L; + // HTTP async report request cache size + public static final int VAL_DEF_HTTP_ASYNC_RPT_CACHE_SIZE = 2000; + public static final int VAL_MIN_HTTP_ASYNC_RPT_CACHE_SIZE = 1; + // HTTP async report worker thread count + public static final int VAL_DEF_HTTP_ASYNC_RPT_WORKER_NUM = + Runtime.getRuntime().availableProcessors(); + public static final int VAL_MIN_HTTP_ASYNC_RPT_WORKER_NUM = 1; + // HTTP async report worker thread sleep ms if idle + public static final int VAL_DEF_HTTP_ASYNC_WORKER_IDLE_WAIT_MS = 300; + public static final int VAL_MAX_HTTP_ASYNC_WORKER_IDLE_WAIT_MS = 3000; + public static final int VAL_MIN_HTTP_ASYNC_WORKER_IDLE_WAIT_MS = 10; public static final int MAX_TIMEOUT_CNT = 10; public static final int LOAD_THRESHOLD = 0; @@ -109,6 +153,5 @@ public class ConfigConstants { public static int DEFAULT_SENDER_MAX_ATTEMPT = 1; /* Reserved attribute data size(bytes). */ - public static int RESERVED_ATTRIBUTE_LENGTH = 10000; - + public static int RESERVED_ATTRIBUTE_LENGTH = 256; } diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/config/ProxyConfigManager.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/config/ProxyConfigManager.java index 27cc4ee8ef0..36c8f3ec463 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/config/ProxyConfigManager.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/config/ProxyConfigManager.java @@ -20,8 +20,8 @@ import org.apache.inlong.common.pojo.dataproxy.DataProxyNodeInfo; import org.apache.inlong.common.pojo.dataproxy.DataProxyNodeResponse; import org.apache.inlong.common.util.BasicAuth; -import org.apache.inlong.sdk.dataproxy.ConfigConstants; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.common.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.common.SdkConsts; import org.apache.inlong.sdk.dataproxy.network.ClientMgr; import org.apache.inlong.sdk.dataproxy.utils.LogCounter; import org.apache.inlong.sdk.dataproxy.utils.ProxyUtils; @@ -103,7 +103,7 @@ public class ProxyConfigManager extends Thread { private final ThreadLocalRandom random = ThreadLocalRandom.current(); private final AtomicBoolean shutDown = new AtomicBoolean(false); // proxy configure info - private ProxyClientConfig clientConfig = null; + private ProxyClientConfig mgrConfig = null; private String localProxyConfigStoreFile; private String proxyConfigVisitUrl; private String proxyQueryFailKey; @@ -131,7 +131,7 @@ public ProxyConfigManager(String callerId, ProxyClientConfig configure, ClientMg if (this.clientManager != null) { this.setName("ConfigManager-" + this.callerId); logger.info("ConfigManager({}) started, groupId={}", - this.callerId, clientConfig.getInlongGroupId()); + this.callerId, mgrConfig.getInlongGroupId()); } } @@ -162,7 +162,7 @@ public void shutDown() { if (shutDown.compareAndSet(false, true)) { this.interrupt(); logger.info("ConfigManager({}) begin to shutdown, groupId={}!", - this.callerId, clientConfig.getInlongGroupId()); + this.callerId, mgrConfig.getInlongGroupId()); } } @@ -176,10 +176,10 @@ public Tuple2 getGroupIdConfigure(boolean needRetry) t if (shutDown.get()) { return new Tuple2<>(null, "SDK has shutdown!"); } - if (clientConfig == null) { + if (mgrConfig == null) { return new Tuple2<>(null, "Configure not initialized!"); } - if (clientConfig.isOnlyUseLocalProxyConfig()) { + if (mgrConfig.isOnlyUseLocalProxyConfig()) { return getLocalProxyListFromFile(this.localProxyConfigStoreFile); } else { boolean readFromRmt = false; @@ -196,8 +196,8 @@ public Tuple2 getGroupIdConfigure(boolean needRetry) t break; } // sleep then retry - Thread.sleep(500L); - } while (++retryCount < clientConfig.getConfigSyncMaxRetryIfFail()); + ProxyUtils.sleepSomeTime(mgrConfig.getMetaQueryWaitMsIfFail()); + } while (++retryCount < mgrConfig.getMetaQueryMaxRetryIfFail()); } if (shutDown.get()) { return new Tuple2<>(null, "SDK has shutdown!"); @@ -218,13 +218,13 @@ public Tuple2 getGroupIdConfigure(boolean needRetry) t * @throws Exception ex */ public Tuple2 getEncryptConfigure(boolean needRetry) throws Exception { - if (!clientConfig.isEnableDataEncrypt()) { + if (!mgrConfig.isEnableReportEncrypt()) { return new Tuple2<>(null, "Not need data encrypt!"); } if (shutDown.get()) { return new Tuple2<>(null, "SDK has shutdown!"); } - if (clientConfig == null) { + if (mgrConfig == null) { return new Tuple2<>(null, "Configure not initialized!"); } EncryptConfigEntry encryptEntry = this.userEncryptConfigEntry; @@ -244,8 +244,8 @@ public Tuple2 getEncryptConfigure(boolean needRetry) break; } // sleep then retry - Thread.sleep(500L); - } while (++retryCount < clientConfig.getConfigSyncMaxRetryIfFail()); + ProxyUtils.sleepSomeTime(mgrConfig.getMetaQueryWaitMsIfFail()); + } while (++retryCount < mgrConfig.getMetaQueryMaxRetryIfFail()); } if (shutDown.get()) { return new Tuple2<>(null, "SDK has shutdown!"); @@ -262,7 +262,7 @@ public Tuple2 getEncryptConfigure(boolean needRetry) @Override public void run() { logger.info("ConfigManager({}) thread start, groupId={}", - this.callerId, clientConfig.getInlongGroupId()); + this.callerId, mgrConfig.getInlongGroupId()); while (!shutDown.get()) { // update proxy nodes meta configures try { @@ -270,17 +270,17 @@ public void run() { } catch (Throwable ex) { if (exptCounter.shouldPrint()) { logger.warn("ConfigManager({}) refresh proxy configure exception, groupId={}", - this.callerId, clientConfig.getInlongGroupId(), ex); + this.callerId, mgrConfig.getInlongGroupId(), ex); } } // update encrypt configure - if (clientConfig.isEnableDataEncrypt()) { + if (mgrConfig.isEnableReportEncrypt()) { try { doEncryptConfigEntryQueryWork(); } catch (Throwable ex) { if (exptCounter.shouldPrint()) { logger.warn("ConfigManager({}) refresh encrypt info exception, groupId={}", - this.callerId, clientConfig.getInlongGroupId(), ex); + this.callerId, mgrConfig.getInlongGroupId(), ex); } } } @@ -288,14 +288,10 @@ public void run() { break; } // sleep some time - try { - Thread.sleep(clientConfig.getManagerConfigSyncInrMs() + random.nextInt(100) * 100); - } catch (Throwable e2) { - // - } + ProxyUtils.sleepSomeTime(mgrConfig.getMgrMetaSyncInrMs() + random.nextInt(100) * 100); } logger.info("ConfigManager({}) worker existed, groupId={}", - this.callerId, this.clientConfig.getInlongGroupId()); + this.callerId, this.mgrConfig.getInlongGroupId()); } /** @@ -312,7 +308,7 @@ public void doProxyEntryQueryWork() throws Exception { localMd5 = calcHostInfoMd5(proxyInfoList); } Tuple2 result; - if (clientConfig.isOnlyUseLocalProxyConfig()) { + if (mgrConfig.isOnlyUseLocalProxyConfig()) { result = getLocalProxyListFromFile(this.localProxyConfigStoreFile); } else { int retryCnt = 0; @@ -322,8 +318,8 @@ public void doProxyEntryQueryWork() throws Exception { break; } // sleep then retry. - Thread.sleep(2000L); - } while (++retryCnt < this.clientConfig.getConfigSyncMaxRetryIfFail() && !shutDown.get()); + ProxyUtils.sleepSomeTime(mgrConfig.getMetaSyncWaitMsIfFail()); + } while (++retryCnt < this.mgrConfig.getMetaSyncMaxRetryIfFail() && !shutDown.get()); if (shutDown.get()) { return; } @@ -334,19 +330,19 @@ public void doProxyEntryQueryWork() throws Exception { if (localMd5 == null && result.getF0() == null) { if (exptCounter.shouldPrint()) { logger.warn("ConfigManager({}) connect manager({}) failure, get cached configure, groupId={}", - this.callerId, this.proxyConfigVisitUrl, this.clientConfig.getInlongGroupId()); + this.callerId, this.proxyConfigVisitUrl, this.mgrConfig.getInlongGroupId()); } result = tryToReadCacheProxyEntry(); } if (localMd5 != null && result.getF0() == null && proxyInfoList != null) { if (exptCounter.shouldPrint()) { logger.warn("ConfigManager({}) connect manager({}) failure, using the last configure, groupId={}", - this.callerId, this.proxyConfigVisitUrl, this.clientConfig.getInlongGroupId()); + this.callerId, this.proxyConfigVisitUrl, this.mgrConfig.getInlongGroupId()); } } } if (localMd5 == null && result.getF0() == null && proxyInfoList == null) { - if (clientConfig.isOnlyUseLocalProxyConfig()) { + if (mgrConfig.isOnlyUseLocalProxyConfig()) { throw new Exception("Read local proxy configure failure, please check first!"); } else { throw new Exception("Connect Manager failure, please check first!"); @@ -367,15 +363,15 @@ private void doEncryptConfigEntryQueryWork() throws Exception { break; } // sleep then retry - Thread.sleep(500L); - } while (++retryCount < clientConfig.getConfigSyncMaxRetryIfFail()); + ProxyUtils.sleepSomeTime(mgrConfig.getMetaSyncWaitMsIfFail()); + } while (++retryCount < mgrConfig.getMetaSyncMaxRetryIfFail()); if (shutDown.get()) { return; } if (result.getF0() == null) { if (this.userEncryptConfigEntry != null) { logger.warn("ConfigManager({}) connect manager({}) failure, using the last pubKey, userName={}", - this.callerId, this.encryptConfigVisitUrl, this.clientConfig.getUserName()); + this.callerId, this.encryptConfigVisitUrl, this.mgrConfig.getRptUserName()); return; } throw new Exception("Visit manager error:" + result.getF1()); @@ -416,7 +412,7 @@ private Tuple2 requestProxyEntryQuietly() { } // parse result logger.debug("ConfigManager({}) received configure, from manager({}), groupId={}, result={}", - callerId, proxyConfigVisitUrl, clientConfig.getInlongGroupId(), queryResult.getF1()); + callerId, proxyConfigVisitUrl, mgrConfig.getInlongGroupId(), queryResult.getF1()); try { Tuple2 parseResult = getProxyConfigEntry(true, queryResult.getF1()); @@ -429,7 +425,7 @@ private Tuple2 requestProxyEntryQuietly() { } catch (Throwable ex) { if (exptCounter.shouldPrint()) { logger.warn("ConfigManager({}) parse failure, from manager({}), groupId={}, result={}", - callerId, proxyConfigVisitUrl, clientConfig.getInlongGroupId(), queryResult.getF1(), ex); + callerId, proxyConfigVisitUrl, mgrConfig.getInlongGroupId(), queryResult.getF1(), ex); } bookManagerQryFailStatus(true, ex.getMessage()); return new Tuple2<>(null, ex.getMessage()); @@ -462,28 +458,28 @@ private String calcHostInfoMd5(List hostInfoList) { private void compareAndUpdateProxyList(ProxyConfigEntry proxyEntry) { if ((proxyEntry == null || proxyEntry.isNodesEmpty()) && (proxyInfoList.isEmpty() - || (System.currentTimeMillis() - lstUpdateTime) < clientConfig.getForceReChooseInrMs())) { + || (System.currentTimeMillis() - lstUpdateTime) < mgrConfig.getForceReChooseInrMs())) { return; } int newSwitchStat; - List newBusInfoList; + List newProxyNodeList; if (proxyEntry == null || proxyEntry.isNodesEmpty()) { newSwitchStat = oldStat; - newBusInfoList = new ArrayList<>(proxyInfoList.size()); - newBusInfoList.addAll(proxyInfoList); + newProxyNodeList = new ArrayList<>(proxyInfoList.size()); + newProxyNodeList.addAll(proxyInfoList); } else { newSwitchStat = proxyEntry.getSwitchStat(); - newBusInfoList = new ArrayList<>(proxyEntry.getSize()); + newProxyNodeList = new ArrayList<>(proxyEntry.getSize()); for (Map.Entry entry : proxyEntry.getHostMap().entrySet()) { - newBusInfoList.add(entry.getValue()); + newProxyNodeList.add(entry.getValue()); } } - String newMd5 = calcHostInfoMd5(newBusInfoList); + String newMd5 = calcHostInfoMd5(newProxyNodeList); String oldMd5 = calcHostInfoMd5(proxyInfoList); boolean nodeChanged = newMd5 != null && !newMd5.equals(oldMd5); if (nodeChanged || newSwitchStat != oldStat - || (System.currentTimeMillis() - lstUpdateTime) >= clientConfig.getForceReChooseInrMs()) { - proxyInfoList = newBusInfoList; + || (System.currentTimeMillis() - lstUpdateTime) >= mgrConfig.getForceReChooseInrMs()) { + proxyInfoList = newProxyNodeList; clientManager.updateProxyInfoList(nodeChanged, proxyInfoList); lstUpdateTime = System.currentTimeMillis(); oldStat = newSwitchStat; @@ -507,7 +503,7 @@ private void tryToWriteCacheProxyEntry(ProxyConfigEntry entry) { } catch (Throwable ex) { if (exptCounter.shouldPrint()) { logger.warn("ConfigManager({}) write cache file({}) exception, groupId={}, data={}", - this.callerId, this.clientConfig.getInlongGroupId(), + this.callerId, this.mgrConfig.getInlongGroupId(), this.proxyConfigCacheFile, entry.toString(), ex); } } finally { @@ -526,8 +522,8 @@ private Tuple2 tryToReadCacheProxyEntry() { File file = new File(this.proxyConfigCacheFile); if (file.exists()) { long diffTime = System.currentTimeMillis() - file.lastModified(); - if (clientConfig.getConfigCacheExpiredMs() > 0 - && diffTime < clientConfig.getConfigCacheExpiredMs()) { + if (mgrConfig.getMetaCacheExpiredMs() > 0 + && diffTime < mgrConfig.getMetaCacheExpiredMs()) { JsonReader reader = new JsonReader(new FileReader(this.proxyConfigCacheFile)); ProxyConfigEntry proxyConfigEntry = gson.fromJson(reader, ProxyConfigEntry.class); return new Tuple2<>(proxyConfigEntry, "Ok"); @@ -539,7 +535,7 @@ private Tuple2 tryToReadCacheProxyEntry() { } catch (Throwable ex) { if (exptCounter.shouldPrint()) { logger.warn("ConfigManager({}) read cache file({}) exception, groupId={}", - this.callerId, this.proxyConfigCacheFile, this.clientConfig.getInlongGroupId(), ex); + this.callerId, this.proxyConfigCacheFile, this.mgrConfig.getInlongGroupId(), ex); } return new Tuple2<>(null, "read cache configure failure:" + ex.getMessage()); } finally { @@ -571,7 +567,7 @@ private Tuple2 requestPubKeyFromManager() { } catch (Throwable ex) { if (parseCounter.shouldPrint()) { logger.warn("ConfigManager({}) parse failure, userName={}, config={}!", - this.callerId, this.clientConfig.getUserName(), queryResult.getF1()); + this.callerId, this.mgrConfig.getRptUserName(), queryResult.getF1()); } errorMsg = "parse pubkey failure:" + ex.getMessage(); bookManagerQryFailStatus(false, errorMsg); @@ -586,7 +582,7 @@ private Tuple2 requestPubKeyFromManager() { if (!pubKeyConf.has("resultCode")) { if (parseCounter.shouldPrint()) { logger.warn("ConfigManager({}) config failure: resultCode field not exist, userName={}, config={}!", - this.callerId, this.clientConfig.getUserName(), queryResult.getF1()); + this.callerId, this.mgrConfig.getRptUserName(), queryResult.getF1()); } throw new Exception("resultCode field not exist"); } @@ -594,14 +590,14 @@ private Tuple2 requestPubKeyFromManager() { if (resultCode != 0) { if (parseCounter.shouldPrint()) { logger.warn("ConfigManager({}) config failure: resultCode != 0, userName={}, config={}!", - this.callerId, this.clientConfig.getUserName(), queryResult.getF1()); + this.callerId, this.mgrConfig.getRptUserName(), queryResult.getF1()); } throw new Exception("resultCode != 0!"); } if (!pubKeyConf.has("resultData")) { if (parseCounter.shouldPrint()) { logger.warn("ConfigManager({}) config failure: resultData field not exist, userName={}, config={}!", - this.callerId, this.clientConfig.getUserName(), queryResult.getF1()); + this.callerId, this.mgrConfig.getRptUserName(), queryResult.getF1()); } throw new Exception("resultData field not exist"); } @@ -611,7 +607,7 @@ private Tuple2 requestPubKeyFromManager() { if (StringUtils.isBlank(publicKey)) { if (parseCounter.shouldPrint()) { logger.warn("ConfigManager({}) config failure: publicKey is blank, userName={}, config={}!", - this.callerId, this.clientConfig.getUserName(), queryResult.getF1()); + this.callerId, this.mgrConfig.getRptUserName(), queryResult.getF1()); } throw new Exception("publicKey is blank!"); } @@ -619,7 +615,7 @@ private Tuple2 requestPubKeyFromManager() { if (StringUtils.isBlank(username)) { if (parseCounter.shouldPrint()) { logger.warn("ConfigManager({}) config failure: username is blank, userName={}, config={}!", - this.callerId, this.clientConfig.getUserName(), queryResult.getF1()); + this.callerId, this.mgrConfig.getRptUserName(), queryResult.getF1()); } throw new Exception("username is blank!"); } @@ -627,7 +623,7 @@ private Tuple2 requestPubKeyFromManager() { if (StringUtils.isBlank(versionStr)) { if (parseCounter.shouldPrint()) { logger.warn("ConfigManager({}) config failure: version is blank, userName={}, config={}!", - this.callerId, this.clientConfig.getUserName(), queryResult.getF1()); + this.callerId, this.mgrConfig.getRptUserName(), queryResult.getF1()); } throw new Exception("version is blank!"); } @@ -655,8 +651,8 @@ private Tuple2 readCachedPubKeyEntry() { File file = new File(this.encryptConfigCacheFile); if (file.exists()) { long diffTime = System.currentTimeMillis() - file.lastModified(); - if (clientConfig.getConfigCacheExpiredMs() > 0 - && diffTime < clientConfig.getConfigCacheExpiredMs()) { + if (mgrConfig.getMetaCacheExpiredMs() > 0 + && diffTime < mgrConfig.getMetaCacheExpiredMs()) { fis = new FileInputStream(file); is = new ObjectInputStream(fis); entry = (EncryptConfigEntry) is.readObject(); @@ -671,7 +667,7 @@ private Tuple2 readCachedPubKeyEntry() { } catch (Throwable ex) { if (exptCounter.shouldPrint()) { logger.warn("ConfigManager({}) read({}) file exception, userName={}", - callerId, encryptConfigCacheFile, clientConfig.getUserName(), ex); + callerId, encryptConfigCacheFile, mgrConfig.getRptUserName(), ex); } return new Tuple2<>(null, "read PubKeyEntry file failure:" + ex.getMessage()); } finally { @@ -706,7 +702,7 @@ private void writeCachePubKeyEntryFile(EncryptConfigEntry entry) { } catch (Throwable ex) { if (exptCounter.shouldPrint()) { logger.warn("ConfigManager({}) write file({}) exception, userName={}, content={}", - callerId, encryptConfigCacheFile, clientConfig.getUserName(), entry.toString(), ex); + callerId, encryptConfigCacheFile, mgrConfig.getRptUserName(), entry.toString(), ex); } } finally { if (fos != null) { @@ -724,15 +720,15 @@ private void writeCachePubKeyEntryFile(EncryptConfigEntry entry) { private Tuple2 requestConfiguration( boolean queryProxyInfo, String url, List params) { HttpParams myParams = new BasicHttpParams(); - HttpConnectionParams.setConnectionTimeout(myParams, clientConfig.getManagerConnTimeoutMs()); - HttpConnectionParams.setSoTimeout(myParams, clientConfig.getManagerSocketTimeoutMs()); + HttpConnectionParams.setConnectionTimeout(myParams, mgrConfig.getMgrConnTimeoutMs()); + HttpConnectionParams.setSoTimeout(myParams, mgrConfig.getMgrSocketTimeoutMs()); CloseableHttpClient httpClient; // build http(s) client try { - if (this.clientConfig.isVisitManagerByHttp()) { - httpClient = new DefaultHttpClient(myParams); - } else { + if (this.mgrConfig.isVisitMgrByHttps()) { httpClient = getCloseableHttpClient(params); + } else { + httpClient = new DefaultHttpClient(myParams); } } catch (Throwable eHttp) { if (exptCounter.shouldPrint()) { @@ -790,11 +786,11 @@ private CloseableHttpClient getCloseableHttpClient(List para headers.add(new BasicHeader(paramItem.getName(), paramItem.getValue())); } RequestConfig requestConfig = RequestConfig.custom() - .setConnectTimeout(clientConfig.getManagerConnTimeoutMs()) - .setSocketTimeout(clientConfig.getManagerSocketTimeoutMs()).build(); + .setConnectTimeout(mgrConfig.getMgrConnTimeoutMs()) + .setSocketTimeout(mgrConfig.getMgrSocketTimeoutMs()).build(); SSLContext sslContext = SSLContexts.custom().build(); SSLConnectionSocketFactory sslSf = new SSLConnectionSocketFactory(sslContext, - new String[]{clientConfig.getTlsVersion()}, null, + new String[]{mgrConfig.getTlsVersion()}, null, SSLConnectionSocketFactory.getDefaultHostnameVerifier()); httpClient = HttpClients.custom().setDefaultHeaders(headers).setDefaultRequestConfig(requestConfig) .setSSLSocketFactory(sslSf).build(); @@ -802,63 +798,63 @@ private CloseableHttpClient getCloseableHttpClient(List para } private void storeAndBuildMetaConfigure(ProxyClientConfig config) { - this.clientConfig = config; + this.mgrConfig = config; StringBuilder strBuff = new StringBuilder(512); this.proxyConfigVisitUrl = strBuff - .append(clientConfig.isVisitManagerByHttp() ? ConfigConstants.HTTP : ConfigConstants.HTTPS) - .append(clientConfig.getManagerIP()).append(":").append(clientConfig.getManagerPort()) - .append(ConfigConstants.MANAGER_DATAPROXY_API).append(clientConfig.getInlongGroupId()) + .append(mgrConfig.isVisitMgrByHttps() ? SdkConsts.PREFIX_HTTPS : SdkConsts.PREFIX_HTTP) + .append(mgrConfig.getManagerIP()).append(":").append(mgrConfig.getManagerPort()) + .append(SdkConsts.MANAGER_DATAPROXY_API).append(mgrConfig.getInlongGroupId()) .toString(); strBuff.delete(0, strBuff.length()); this.proxyQueryFailKey = strBuff - .append("proxy:").append(clientConfig.getInlongGroupId()) - .append("#").append(clientConfig.getRegionName()) - .append("#").append(clientConfig.getProtocolType()).toString(); + .append("proxy:").append(mgrConfig.getInlongGroupId()) + .append("#").append(mgrConfig.getRegionName()) + .append("#").append(mgrConfig.getDataRptProtocol()).toString(); strBuff.delete(0, strBuff.length()); this.localProxyConfigStoreFile = strBuff - .append(clientConfig.getConfigStoreBasePath()) - .append(ConfigConstants.META_STORE_SUB_DIR) - .append(clientConfig.getInlongGroupId()) - .append(ConfigConstants.LOCAL_DP_CONFIG_FILE_SUFFIX) + .append(mgrConfig.getMetaStoreBasePath()) + .append(SdkConsts.META_STORE_SUB_DIR) + .append(mgrConfig.getInlongGroupId()) + .append(SdkConsts.LOCAL_DP_CONFIG_FILE_SUFFIX) .toString(); strBuff.delete(0, strBuff.length()); this.proxyConfigCacheFile = strBuff - .append(clientConfig.getConfigStoreBasePath()) - .append(ConfigConstants.META_STORE_SUB_DIR) - .append(clientConfig.getInlongGroupId()) - .append(ConfigConstants.REMOTE_DP_CACHE_FILE_SUFFIX) + .append(mgrConfig.getMetaStoreBasePath()) + .append(SdkConsts.META_STORE_SUB_DIR) + .append(mgrConfig.getInlongGroupId()) + .append(SdkConsts.REMOTE_DP_CACHE_FILE_SUFFIX) .toString(); strBuff.delete(0, strBuff.length()); - this.encryptConfigVisitUrl = clientConfig.getRsaPubKeyUrl(); + this.encryptConfigVisitUrl = mgrConfig.getRptRsaPubKeyUrl(); this.encryptQueryFailKey = strBuff - .append("encrypt:").append(clientConfig.getUserName()).toString(); + .append("encrypt:").append(mgrConfig.getRptUserName()).toString(); strBuff.delete(0, strBuff.length()); this.encryptConfigCacheFile = strBuff - .append(clientConfig.getConfigStoreBasePath()) - .append(ConfigConstants.META_STORE_SUB_DIR) - .append(clientConfig.getUserName()) - .append(ConfigConstants.REMOTE_ENCRYPT_CACHE_FILE_SUFFIX) + .append(mgrConfig.getMetaStoreBasePath()) + .append(SdkConsts.META_STORE_SUB_DIR) + .append(mgrConfig.getRptUserName()) + .append(SdkConsts.REMOTE_ENCRYPT_CACHE_FILE_SUFFIX) .toString(); strBuff.delete(0, strBuff.length()); } private void addAuthorizationInfo(HttpPost httpPost) { httpPost.addHeader(BasicAuth.BASIC_AUTH_HEADER, - BasicAuth.genBasicAuthCredential(clientConfig.getAuthSecretId(), - clientConfig.getAuthSecretKey())); + BasicAuth.genBasicAuthCredential(mgrConfig.getMgrAuthSecretId(), + mgrConfig.getMgrAuthSecretKey())); } private List buildProxyNodeQueryParams() { ArrayList params = new ArrayList<>(); params.add(new BasicNameValuePair("ip", ProxyUtils.getLocalIp())); - params.add(new BasicNameValuePair("protocolType", clientConfig.getProtocolType())); + params.add(new BasicNameValuePair("protocolType", mgrConfig.getDataRptProtocol())); return params; } private List buildPubKeyQueryParams() { List params = new ArrayList<>(); params.add(new BasicNameValuePair("operation", "query")); - params.add(new BasicNameValuePair("username", clientConfig.getUserName())); + params.add(new BasicNameValuePair("username", mgrConfig.getRptUserName())); return params; } @@ -881,7 +877,7 @@ private void rmvManagerQryFailStatus(boolean proxyQry) { } private String getManagerQryResultInFailStatus(boolean proxyQry) { - if (clientConfig.getConfigFailStatusExpiredMs() <= 0) { + if (mgrConfig.getMetaQryFailCacheExpiredMs() <= 0) { return null; } Tuple2 queryResult; @@ -891,8 +887,8 @@ private String getManagerQryResultInFailStatus(boolean proxyQry) { queryResult = fetchFailEncryptMap.get(encryptQueryFailKey); } if (queryResult != null - && (System.currentTimeMillis() - queryResult.getF0().get() < clientConfig - .getConfigFailStatusExpiredMs())) { + && (System.currentTimeMillis() - queryResult.getF0().get() < mgrConfig + .getMetaQryFailCacheExpiredMs())) { return queryResult.getF1(); } return null; @@ -907,7 +903,7 @@ private Tuple2 getProxyConfigEntry(boolean fromManager } catch (Throwable ex) { if (parseCounter.shouldPrint()) { logger.warn("ConfigManager({}) parse exception, groupId={}, config={}", - this.callerId, clientConfig.getInlongGroupId(), strRet, ex); + this.callerId, mgrConfig.getInlongGroupId(), strRet, ex); } return new Tuple2<>(null, "parse failure:" + ex.getMessage()); } @@ -927,7 +923,7 @@ private Tuple2 getProxyConfigEntry(boolean fromManager } catch (Throwable ex) { if (parseCounter.shouldPrint()) { logger.warn("ConfigManager({}) parse local file exception, groupId={}, config={}", - this.callerId, clientConfig.getInlongGroupId(), strRet, ex); + this.callerId, mgrConfig.getInlongGroupId(), strRet, ex); } return new Tuple2<>(null, "parse file failure:" + ex.getMessage()); } @@ -949,7 +945,7 @@ private Tuple2 getProxyConfigEntry(boolean fromManager || proxy.getPort() < 0) { if (exptCounter.shouldPrint()) { logger.warn("Invalid proxy node: groupId={}, id={}, ip={}, port={}", - clientConfig.getInlongGroupId(), proxy.getId(), proxy.getIp(), proxy.getPort()); + mgrConfig.getInlongGroupId(), proxy.getId(), proxy.getIp(), proxy.getPort()); } continue; } @@ -965,7 +961,7 @@ private Tuple2 getProxyConfigEntry(boolean fromManager clusterId = proxyNodeConfig.getClusterId(); } // parse load - int load = ConfigConstants.LOAD_THRESHOLD; + int load = SdkConsts.LOAD_THRESHOLD; if (ObjectUtils.isNotEmpty(proxyNodeConfig.getLoad())) { load = proxyNodeConfig.getLoad() > 200 ? 200 : (Math.max(proxyNodeConfig.getLoad(), 0)); } @@ -982,7 +978,7 @@ private Tuple2 getProxyConfigEntry(boolean fromManager // build ProxyConfigEntry ProxyConfigEntry proxyEntry = new ProxyConfigEntry(); proxyEntry.setClusterId(clusterId); - proxyEntry.setGroupId(clientConfig.getInlongGroupId()); + proxyEntry.setGroupId(mgrConfig.getInlongGroupId()); proxyEntry.setInterVisit(isIntranet); proxyEntry.setHostMap(hostMap); proxyEntry.setSwitchStat(isSwitch); diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/HttpClientExample.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/HttpClientExample.java index d370623f8de..8db8fdab6c4 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/HttpClientExample.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/HttpClientExample.java @@ -17,9 +17,8 @@ package org.apache.inlong.sdk.dataproxy.example; -import org.apache.inlong.common.constant.ProtocolType; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; import org.apache.inlong.sdk.dataproxy.exception.ProxySdkException; +import org.apache.inlong.sdk.dataproxy.http.HttpMsgSenderConfig; import org.apache.inlong.sdk.dataproxy.network.HttpProxySender; import java.util.ArrayList; @@ -47,17 +46,16 @@ public static HttpProxySender getMessageSender(String inLongManagerAddr, String inLongManagerPort, String inlongGroupId, boolean requestByHttp, boolean isReadProxyIPFromLocal, String configBasePath) { - ProxyClientConfig proxyConfig = null; + HttpMsgSenderConfig httpConfig = null; HttpProxySender sender = null; try { - proxyConfig = new ProxyClientConfig(requestByHttp, inLongManagerAddr, + httpConfig = new HttpMsgSenderConfig(requestByHttp, inLongManagerAddr, Integer.valueOf(inLongManagerPort), inlongGroupId, "admin", "inlong");// user and password of manager - proxyConfig.setConfigStoreBasePath(configBasePath); - proxyConfig.setOnlyUseLocalProxyConfig(isReadProxyIPFromLocal); - proxyConfig.setDiscardOldMessage(true); - proxyConfig.setProtocolType(ProtocolType.HTTP); - sender = new HttpProxySender(proxyConfig); + httpConfig.setMetaStoreBasePath(configBasePath); + httpConfig.setOnlyUseLocalProxyConfig(isReadProxyIPFromLocal); + httpConfig.setDiscardHttpCacheWhenClosing(true); + sender = new HttpProxySender(httpConfig); } catch (ProxySdkException e) { e.printStackTrace(); } catch (Exception e) { diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/TcpClientExample.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/TcpClientExample.java index a85086ac38f..098ae515368 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/TcpClientExample.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/TcpClientExample.java @@ -17,9 +17,8 @@ package org.apache.inlong.sdk.dataproxy.example; -import org.apache.inlong.common.constant.ProtocolType; import org.apache.inlong.sdk.dataproxy.DefaultMessageSender; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.TcpMsgSenderConfig; import org.apache.inlong.sdk.dataproxy.common.SendResult; import org.apache.commons.lang3.StringUtils; @@ -62,18 +61,17 @@ public static void main(String[] args) throws InterruptedException { public DefaultMessageSender getMessageSender(String inLongManagerAddr, String inLongManagerPort, String inlongGroupId, boolean requestByHttp, boolean isReadProxyIPFromLocal, String configBasePath, int msgType) { - ProxyClientConfig dataProxyConfig = null; + TcpMsgSenderConfig tcpConfig = null; DefaultMessageSender messageSender = null; try { - dataProxyConfig = new ProxyClientConfig(requestByHttp, inLongManagerAddr, + tcpConfig = new TcpMsgSenderConfig(requestByHttp, inLongManagerAddr, Integer.valueOf(inLongManagerPort), inlongGroupId, "admin", "inlong"); if (StringUtils.isNotEmpty(configBasePath)) { - dataProxyConfig.setConfigStoreBasePath(configBasePath); + tcpConfig.setMetaStoreBasePath(configBasePath); } - dataProxyConfig.setOnlyUseLocalProxyConfig(isReadProxyIPFromLocal); - dataProxyConfig.setProtocolType(ProtocolType.TCP); - dataProxyConfig.setRequestTimeoutMs(20000L); - messageSender = DefaultMessageSender.generateSenderByClusterId(dataProxyConfig); + tcpConfig.setOnlyUseLocalProxyConfig(isReadProxyIPFromLocal); + tcpConfig.setRequestTimeoutMs(20000L); + messageSender = DefaultMessageSender.generateSenderByClusterId(tcpConfig); messageSender.setMsgtype(msgType); } catch (Exception e) { logger.error("getMessageSender has exception e = {}", e); diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/UdpClientExample.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/UdpClientExample.java index 5a564b96cb3..abc661348c8 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/UdpClientExample.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/UdpClientExample.java @@ -48,8 +48,8 @@ import java.util.Collections; import java.util.concurrent.TimeUnit; -import static org.apache.inlong.sdk.dataproxy.ConfigConstants.FLAG_ALLOW_COMPRESS; -import static org.apache.inlong.sdk.dataproxy.ConfigConstants.FLAG_ALLOW_ENCRYPT; +import static org.apache.inlong.sdk.dataproxy.common.SdkConsts.FLAG_ALLOW_COMPRESS; +import static org.apache.inlong.sdk.dataproxy.common.SdkConsts.FLAG_ALLOW_ENCRYPT; public class UdpClientExample { diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/http/HttpMsgSenderConfig.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/http/HttpMsgSenderConfig.java new file mode 100644 index 00000000000..4833968d186 --- /dev/null +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/http/HttpMsgSenderConfig.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.dataproxy.http; + +import org.apache.inlong.common.msg.AttributeConstants; +import org.apache.inlong.sdk.dataproxy.common.HttpContentType; +import org.apache.inlong.sdk.dataproxy.common.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.common.ReportProtocol; +import org.apache.inlong.sdk.dataproxy.common.SdkConsts; +import org.apache.inlong.sdk.dataproxy.exception.ProxySdkException; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Objects; + +/** + * HTTP Message Sender configuration: + * + * Used to define the setting related to the sender reporting data using the HTTP protocol, including + * whether to use HTTPS for reporting, the separator character between message bodies when + * reporting multiple messages, HTTP timeout setting, message queue during HTTP asynchronous sending, + * and the number of sending threads, etc. + */ +public class HttpMsgSenderConfig extends ProxyClientConfig implements Cloneable { + + // whether report data over https + private boolean rptDataByHttps = false; + // report data content type + private HttpContentType httpContentType = HttpContentType.APPLICATION_X_WWW_FORM_URLENCODED; + // http events separator + private String httpEventsSeparator = AttributeConstants.LINE_FEED_SEP; + // whether separate event by line feed + private boolean sepEventByLF = true; + // connect timeout in milliseconds + private int httpConTimeoutMs = SdkConsts.VAL_DEF_HTTP_CONNECT_TIMEOUT_MS; + // socket timeout in milliseconds + private int httpSocketTimeoutMs = SdkConsts.VAL_DEF_NODE_SOCKET_TIMEOUT_MS; + // discard cache msg when closing + private boolean discardHttpCacheWhenClosing = false; + // http close wait period ms + private long httpCloseWaitPeriodMs = SdkConsts.VAL_DEF_HTTP_SDK_CLOSE_WAIT_MS; + // node reuse freezing time + private long httpNodeReuseWaitIfFailMs = SdkConsts.VAL_DEF_HTTP_REUSE_FAIL_WAIT_MS; + // message cache size for async report data. + private int httpAsyncRptCacheSize = SdkConsts.VAL_DEF_HTTP_ASYNC_RPT_CACHE_SIZE; + // thread number for async report data. + private int httpAsyncRptWorkerNum = SdkConsts.VAL_DEF_HTTP_ASYNC_RPT_WORKER_NUM; + // interval for async worker in microseconds. + private int httpAsyncWorkerIdleWaitMs = SdkConsts.VAL_DEF_HTTP_ASYNC_WORKER_IDLE_WAIT_MS; + + public HttpMsgSenderConfig(boolean visitMgrByHttps, + String managerIP, int managerPort, String groupId) throws ProxySdkException { + super(visitMgrByHttps, managerIP, managerPort, groupId, ReportProtocol.HTTP, null); + } + + public HttpMsgSenderConfig(String managerAddress, String groupId) throws ProxySdkException { + super(managerAddress, groupId, ReportProtocol.HTTP, null); + } + + public HttpMsgSenderConfig(boolean visitMgrByHttps, String managerIP, int managerPort, + String groupId, String mgrAuthSecretId, String mgrAuthSecretKey) throws ProxySdkException { + super(visitMgrByHttps, managerIP, managerPort, groupId, ReportProtocol.HTTP, null); + this.setMgrAuthzInfo(true, mgrAuthSecretId, mgrAuthSecretKey); + } + + public HttpMsgSenderConfig(String managerAddress, + String groupId, String mgrAuthSecretId, String mgrAuthSecretKey) throws ProxySdkException { + super(managerAddress, groupId, ReportProtocol.HTTP, null); + this.setMgrAuthzInfo(true, mgrAuthSecretId, mgrAuthSecretKey); + } + + public boolean isRptDataByHttps() { + return rptDataByHttps; + } + + public void setRptDataByHttps(boolean rptDataByHttps) { + this.rptDataByHttps = rptDataByHttps; + } + + public HttpContentType getContentType() { + return httpContentType; + } + + public void setContentType(HttpContentType httpContentType) { + if (httpContentType == null) { + return; + } + this.httpContentType = httpContentType; + } + + public String getHttpEventsSeparator() { + return httpEventsSeparator; + } + + public boolean isSepEventByLF() { + return sepEventByLF; + } + + public void setHttpEventsSeparator(String httpEventsSeparator) { + if (StringUtils.isBlank(httpEventsSeparator)) { + throw new IllegalArgumentException("eventSeparator cannot be blank"); + } + String tmpValue = httpEventsSeparator.trim(); + if (tmpValue.length() != 1) { + throw new IllegalArgumentException("eventSeparator must be a single character"); + } + this.httpEventsSeparator = tmpValue; + this.sepEventByLF = this.httpEventsSeparator.equals(AttributeConstants.LINE_FEED_SEP); + } + + public int getHttpConTimeoutMs() { + return httpConTimeoutMs; + } + + public void setHttpConTimeoutMs(int httpConTimeoutMs) { + this.httpConTimeoutMs = Math.max( + SdkConsts.VAL_MIN_CONNECT_TIMEOUT_MS, httpConTimeoutMs); + } + + public int getHttpSocketTimeoutMs() { + return httpSocketTimeoutMs; + } + + public void setHttpSocketTimeoutMs(int httpSocketTimeoutMs) { + this.httpSocketTimeoutMs = + Math.min(SdkConsts.VAL_MAX_SOCKET_TIMEOUT_MS, + Math.max(SdkConsts.VAL_MIN_SOCKET_TIMEOUT_MS, httpSocketTimeoutMs)); + } + + public boolean isDiscardHttpCacheWhenClosing() { + return discardHttpCacheWhenClosing; + } + + public void setDiscardHttpCacheWhenClosing(boolean discardHttpCacheWhenClosing) { + this.discardHttpCacheWhenClosing = discardHttpCacheWhenClosing; + } + + public long getHttpCloseWaitPeriodMs() { + return httpCloseWaitPeriodMs; + } + + public void setHttpCloseWaitPeriodMs(long httpCloseWaitPeriodMs) { + this.httpCloseWaitPeriodMs = httpCloseWaitPeriodMs; + this.httpCloseWaitPeriodMs = + Math.min(SdkConsts.VAL_MAX_HTTP_SDK_CLOSE_WAIT_MS, + Math.max(SdkConsts.VAL_MIN_HTTP_SDK_CLOSE_WAIT_MS, httpCloseWaitPeriodMs)); + } + + public long getHttpNodeReuseWaitIfFailMs() { + return httpNodeReuseWaitIfFailMs; + } + + public void setHttpNodeReuseWaitIfFailMs(long httpNodeReuseWaitIfFailMs) { + this.httpNodeReuseWaitIfFailMs = + Math.min(SdkConsts.VAL_MAX_HTTP_REUSE_FAIL_WAIT_MS, + Math.max(SdkConsts.VAL_MIN_HTTP_REUSE_FAIL_WAIT_MS, httpNodeReuseWaitIfFailMs)); + } + + public int getHttpAsyncRptCacheSize() { + return httpAsyncRptCacheSize; + } + + public int getHttpAsyncRptWorkerNum() { + return httpAsyncRptWorkerNum; + } + + public void setHttpAsyncRptPoolConfig(int httpAsyncRptCacheSize, int httpAsyncRptWorkerNum) { + this.httpAsyncRptCacheSize = + Math.max(SdkConsts.VAL_MIN_HTTP_ASYNC_RPT_CACHE_SIZE, httpAsyncRptCacheSize); + this.httpAsyncRptWorkerNum = + Math.max(SdkConsts.VAL_MIN_HTTP_ASYNC_RPT_WORKER_NUM, httpAsyncRptWorkerNum); + } + + public int getHttpAsyncWorkerIdleWaitMs() { + return httpAsyncWorkerIdleWaitMs; + } + + public void setHttpAsyncWorkerIdleWaitMs(int httpAsyncWorkerIdleWaitMs) { + this.httpAsyncWorkerIdleWaitMs = + Math.min(SdkConsts.VAL_MAX_HTTP_ASYNC_WORKER_IDLE_WAIT_MS, + Math.max(SdkConsts.VAL_MIN_HTTP_ASYNC_WORKER_IDLE_WAIT_MS, httpAsyncWorkerIdleWaitMs)); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + if (!super.equals(o)) + return false; + HttpMsgSenderConfig that = (HttpMsgSenderConfig) o; + return rptDataByHttps == that.rptDataByHttps + && sepEventByLF == that.sepEventByLF + && httpConTimeoutMs == that.httpConTimeoutMs + && httpSocketTimeoutMs == that.httpSocketTimeoutMs + && discardHttpCacheWhenClosing == that.discardHttpCacheWhenClosing + && httpCloseWaitPeriodMs == that.httpCloseWaitPeriodMs + && httpNodeReuseWaitIfFailMs == that.httpNodeReuseWaitIfFailMs + && httpAsyncRptCacheSize == that.httpAsyncRptCacheSize + && httpAsyncRptWorkerNum == that.httpAsyncRptWorkerNum + && httpAsyncWorkerIdleWaitMs == that.httpAsyncWorkerIdleWaitMs + && httpContentType == that.httpContentType + && Objects.equals(httpEventsSeparator, that.httpEventsSeparator); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), rptDataByHttps, httpContentType, + httpEventsSeparator, sepEventByLF, httpConTimeoutMs, httpSocketTimeoutMs, + discardHttpCacheWhenClosing, httpCloseWaitPeriodMs, httpNodeReuseWaitIfFailMs, + httpAsyncRptCacheSize, httpAsyncRptWorkerNum, httpAsyncWorkerIdleWaitMs); + } + + @Override + public HttpMsgSenderConfig clone() { + try { + HttpMsgSenderConfig copy = (HttpMsgSenderConfig) super.clone(); + if (copy != null) { + copy.httpContentType = this.httpContentType; + } + return copy; + } catch (Throwable ex) { + logger.warn("Failed to clone HttpMsgSenderConfig", ex); + return null; + } + } + + @Override + public String toString() { + final StringBuilder strBuff = + new StringBuilder("HttpMsgSenderConfig{rptDataByHttps=").append(rptDataByHttps) + .append(", httpContentType=").append(httpContentType) + .append(", httpEventsSeparator='").append(httpEventsSeparator) + .append("', sepEventByLF=").append(sepEventByLF) + .append(", httpConTimeoutMs=").append(httpConTimeoutMs) + .append(", httpSocketTimeoutMs=").append(httpSocketTimeoutMs) + .append(", discardHttpCacheWhenClosing=").append(discardHttpCacheWhenClosing) + .append(", httpCloseWaitPeriodMs=").append(httpCloseWaitPeriodMs) + .append(", httpNodeReuseWaitIfFailMs=").append(httpNodeReuseWaitIfFailMs) + .append(", httpAsyncRptCacheSize=").append(httpAsyncRptCacheSize) + .append(", httpAsyncRptWorkerNum=").append(httpAsyncRptWorkerNum) + .append(", httpAsyncWorkerIdleWaitMs=").append(httpAsyncWorkerIdleWaitMs); + return super.getSetting(strBuff); + } +} diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/http/InternalHttpSender.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/http/InternalHttpSender.java index 7c2738e37d2..8a558ddc131 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/http/InternalHttpSender.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/http/InternalHttpSender.java @@ -18,11 +18,11 @@ package org.apache.inlong.sdk.dataproxy.http; import org.apache.inlong.common.enums.DataProxyErrCode; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; import org.apache.inlong.sdk.dataproxy.common.SendResult; import org.apache.inlong.sdk.dataproxy.config.HostInfo; import org.apache.inlong.sdk.dataproxy.network.HttpMessage; import org.apache.inlong.sdk.dataproxy.utils.ConcurrentHashSet; +import org.apache.inlong.sdk.dataproxy.utils.ProxyUtils; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -58,7 +58,7 @@ public class InternalHttpSender { private static final Logger logger = LoggerFactory.getLogger(InternalHttpSender.class); - private final ProxyClientConfig proxyClientConfig; + private final HttpMsgSenderConfig httpConfig; private final ConcurrentHashSet hostList; private final LinkedBlockingQueue messageCache; @@ -66,17 +66,16 @@ public class InternalHttpSender { private CloseableHttpClient httpClient; private boolean bShutDown = false; - public InternalHttpSender(ProxyClientConfig proxyClientConfig, - ConcurrentHashSet hostList, - LinkedBlockingQueue messageCache) { - this.proxyClientConfig = proxyClientConfig; + public InternalHttpSender(HttpMsgSenderConfig httpConfig, + ConcurrentHashSet hostList, LinkedBlockingQueue messageCache) { + this.httpConfig = httpConfig; this.hostList = hostList; this.messageCache = messageCache; submitWorkThread(); } private void submitWorkThread() { - for (int i = 0; i < proxyClientConfig.getAsyncWorkerNumber(); i++) { + for (int i = 0; i < httpConfig.getHttpAsyncRptWorkerNum(); i++) { workerServices.execute(new WorkerRunner()); } } @@ -132,7 +131,7 @@ public void run() { httpMessage.getCallback().onMessageAck(result); } } - TimeUnit.MILLISECONDS.sleep(proxyClientConfig.getAsyncWorkerInterval()); + TimeUnit.MILLISECONDS.sleep(httpConfig.getHttpAsyncWorkerIdleWaitMs()); } catch (Exception exception) { logger.error("exception caught", exception); } @@ -149,7 +148,7 @@ public List getRandomHostInfo() { List tmpHostList = new ArrayList<>(hostList); Collections.shuffle(tmpHostList); // respect alive connection - int maxIndex = Math.min(proxyClientConfig.getAliveConnections(), tmpHostList.size()); + int maxIndex = Math.min(httpConfig.getAliveConnections(), tmpHostList.size()); return tmpHostList.subList(0, maxIndex); } @@ -231,8 +230,20 @@ public SendResult sendMessageWithHostInfo(List bodies, String groupId, S */ public void close() throws Exception { bShutDown = true; - if (proxyClientConfig.isCleanHttpCacheWhenClosing()) { - messageCache.clear(); + if (!messageCache.isEmpty()) { + if (httpConfig.isDiscardHttpCacheWhenClosing()) { + messageCache.clear(); + } else { + long curTime = System.currentTimeMillis(); + while (!messageCache.isEmpty() + || (System.currentTimeMillis() - curTime) < httpConfig.getHttpCloseWaitPeriodMs()) { + ProxyUtils.sleepSomeTime(100); + } + if (!messageCache.isEmpty()) { + logger.warn("Close httpClient, remain {} messages", messageCache.size()); + messageCache.clear(); + } + } } if (httpClient != null) { httpClient.close(); diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/metric/MetricConfig.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/metric/MetricConfig.java index 29ea696c7b0..6c4f7e2dae0 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/metric/MetricConfig.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/metric/MetricConfig.java @@ -18,27 +18,30 @@ package org.apache.inlong.sdk.dataproxy.metric; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; public class MetricConfig { - private static final long DEF_METRIC_REPORT_INTVL_MS = 60000L; - private static final long MIN_METRIC_REPORT_INTVL_MS = 30000L; + private static final Logger logger = LoggerFactory.getLogger(MetricConfig.class); + private static final long DEF_METRIC_OUTPUT_INTVL_MS = 60000L; + private static final long MIN_METRIC_OUTPUT_INTVL_MS = 10000L; + private static final long DEF_METRIC_OUTPUT_WARN_INT_MS = 10000L; private static final long DEF_METRIC_DATE_FORMAT_MS = 60000L; - private static final long MIN_METRIC_DATE_FORMAT_MS = 1L; private static final String DEF_METRIC_REPORT_GROUP_ID = "inlong_sla_metric"; // metric enable - private boolean enableMetric = false; + private boolean enableMetric = true; // whether use groupId as key for metric, default is true - private boolean useGroupIdAsKey = true; - // whether use StreamId as key for metric, default is true - private boolean useStreamIdAsKey = true; - // whether use localIp as key for metric, default is true - private boolean useLocalIpAsKey = true; + private boolean maskGroupId = false; + // whether mask StreamId for metric, default is false + private boolean maskStreamId = false; // metric report interval, default is 1 mins in milliseconds. - private long metricRptIntvlMs = DEF_METRIC_REPORT_INTVL_MS; + private long metricOutIntvlMs = DEF_METRIC_OUTPUT_INTVL_MS; + private long metricOutWarnIntMs = DEF_METRIC_OUTPUT_WARN_INT_MS; // metric date format private long dateFormatIntvlMs = DEF_METRIC_DATE_FORMAT_MS; - // metric groupId private String metricGroupId = DEF_METRIC_REPORT_GROUP_ID; public MetricConfig() { @@ -53,38 +56,36 @@ public boolean isEnableMetric() { return enableMetric; } - public void setMetricKeyBuildParams( - boolean useGroupIdAsKey, boolean useStreamIdAsKey, boolean useLocalIpAsKey) { - this.useGroupIdAsKey = useGroupIdAsKey; - this.useStreamIdAsKey = useStreamIdAsKey; - this.useLocalIpAsKey = useLocalIpAsKey; + public void setMetricKeyMaskInfos(boolean maskGroupId, boolean maskStreamId) { + this.maskGroupId = maskGroupId; + this.maskStreamId = maskStreamId; } - public boolean isUseGroupIdAsKey() { - return useGroupIdAsKey; + public boolean isMaskGroupId() { + return maskGroupId; } - public boolean isUseStreamIdAsKey() { - return useStreamIdAsKey; + public boolean isMaskStreamId() { + return maskStreamId; } - public boolean isUseLocalIpAsKey() { - return useLocalIpAsKey; + public void setMetricOutIntvlMs(long metricOutIntvlMs) { + if (metricOutIntvlMs >= MIN_METRIC_OUTPUT_INTVL_MS) { + this.metricOutIntvlMs = metricOutIntvlMs; + } } - public void setMetricRptIntvlMs(long metricRptIntvlMs) { - if (metricRptIntvlMs >= MIN_METRIC_REPORT_INTVL_MS) { - this.metricRptIntvlMs = metricRptIntvlMs; - } + public long getMetricOutIntvlMs() { + return metricOutIntvlMs; } - public long getMetricRptIntvlMs() { - return metricRptIntvlMs; + public long getMetricOutWarnIntMs() { + return metricOutWarnIntMs; } - public void setDateFormatIntvlMs(long dateFormatIntvlMs) { - if (dateFormatIntvlMs >= MIN_METRIC_DATE_FORMAT_MS) { - this.dateFormatIntvlMs = dateFormatIntvlMs; + public void setMetricOutWarnIntMs(long metricOutWarnIntMs) { + if (metricOutWarnIntMs >= MIN_METRIC_OUTPUT_INTVL_MS) { + this.metricOutWarnIntMs = metricOutWarnIntMs; } } @@ -98,21 +99,51 @@ public String getMetricGroupId() { public void setMetricGroupId(String metricGroupId) { if (StringUtils.isNotBlank(metricGroupId)) { - this.metricGroupId = metricGroupId; + this.metricGroupId = metricGroupId.trim(); } } @Override - public String toString() { - final StringBuilder sb = new StringBuilder("MetricConfig{"); - sb.append("enableMetric=").append(enableMetric); - sb.append(", useGroupIdAsKey=").append(useGroupIdAsKey); - sb.append(", useStreamIdAsKey=").append(useStreamIdAsKey); - sb.append(", useLocalIpAsKey=").append(useLocalIpAsKey); - sb.append(", metricRptIntvlMs=").append(metricRptIntvlMs); - sb.append(", dateFormatIntvlMs=").append(dateFormatIntvlMs); - sb.append(", metricGroupId='").append(metricGroupId).append('\''); - sb.append('}'); - return sb.toString(); + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + MetricConfig that = (MetricConfig) o; + return enableMetric == that.enableMetric + && maskGroupId == that.maskGroupId + && maskStreamId == that.maskStreamId + && metricOutIntvlMs == that.metricOutIntvlMs + && metricOutWarnIntMs == that.metricOutWarnIntMs + && dateFormatIntvlMs == that.dateFormatIntvlMs + && Objects.equals(metricGroupId, that.metricGroupId); + } + + @Override + public int hashCode() { + return Objects.hash(enableMetric, maskGroupId, maskStreamId, + metricOutIntvlMs, dateFormatIntvlMs, metricOutWarnIntMs, + metricGroupId); + } + + @Override + public MetricConfig clone() { + try { + return (MetricConfig) super.clone(); + } catch (Throwable ex) { + logger.warn("Failed to clone MetricConfig", ex); + return null; + } + } + + public void getSetting(StringBuilder strBuff) { + strBuff.append("{enableMetric=").append(enableMetric) + .append(", maskGroupId=").append(maskGroupId) + .append(", maskStreamId=").append(maskStreamId) + .append(", metricRptIntvlMs=").append(metricOutIntvlMs) + .append(", metricOutWarnIntMs=").append(metricOutWarnIntMs) + .append(", dateFormatIntvlMs=").append(dateFormatIntvlMs) + .append(", metricGroupId='").append(metricGroupId) + .append("'}"); } } diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/ClientMgr.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/ClientMgr.java index 608fcf67d4f..b9eec027996 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/ClientMgr.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/ClientMgr.java @@ -17,7 +17,7 @@ package org.apache.inlong.sdk.dataproxy.network; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.TcpMsgSenderConfig; import org.apache.inlong.sdk.dataproxy.codec.EncodeObject; import org.apache.inlong.sdk.dataproxy.common.SendResult; import org.apache.inlong.sdk.dataproxy.config.EncryptConfigEntry; @@ -59,7 +59,7 @@ public class ClientMgr { private static final byte[] hbMsgBody = ProxyUtils.getLocalIp().getBytes(StandardCharsets.UTF_8); private final Sender sender; - private final ProxyClientConfig configure; + private final TcpMsgSenderConfig tcpConfig; private final Bootstrap bootstrap; private final ProxyConfigManager configManager; private final SendHBThread sendHBThread; @@ -82,21 +82,21 @@ public class ClientMgr { /** * Build up the connection between the server and client. */ - public ClientMgr(ProxyClientConfig configure, Sender sender) { - this(configure, sender, null); + public ClientMgr(TcpMsgSenderConfig tcpConfig, Sender sender) { + this(tcpConfig, sender, null); } /** * Build up the connection between the server and client. */ - public ClientMgr(ProxyClientConfig configure, Sender sender, ThreadFactory selfDefineFactory) { - this.configure = configure; + public ClientMgr(TcpMsgSenderConfig tcpConfig, Sender sender, ThreadFactory selfDefineFactory) { + this.tcpConfig = tcpConfig; this.sender = sender; // Initialize the bootstrap - this.bootstrap = buildBootstrap(this.configure, this.sender, selfDefineFactory); + this.bootstrap = buildBootstrap(this.tcpConfig, this.sender, selfDefineFactory); // initial configure manager this.configManager = new ProxyConfigManager( - sender.getInstanceId(), configure, this); + sender.getInstanceId(), tcpConfig, this); this.sendHBThread = new SendHBThread(); } @@ -109,7 +109,7 @@ public void start() { } catch (Throwable ex) { if (exptCounter.shouldPrint()) { logger.error("ClientMgr({}) query {} exception", - sender.getInstanceId(), configure.getInlongGroupId(), ex); + sender.getInstanceId(), tcpConfig.getInlongGroupId(), ex); } } this.configManager.setDaemon(true); @@ -313,7 +313,7 @@ public void updateProxyInfoList(boolean nodeChanged, List newNodes) { Collections.sort(candidateNodes); Collections.shuffle(candidateNodes); int curTotalCnt = candidateNodes.size(); - int needActiveCnt = Math.min(this.configure.getAliveConnections(), curTotalCnt); + int needActiveCnt = Math.min(this.tcpConfig.getAliveConnections(), curTotalCnt); // build next step nodes NettyClient client; int maxCycleCnt = 3; @@ -328,7 +328,7 @@ public void updateProxyInfoList(boolean nodeChanged, List newNodes) { } try { client = new NettyClient(this.sender.getInstanceId(), - this.bootstrap, hostInfo, this.configure); + this.bootstrap, hostInfo, this.tcpConfig); if (!client.connect(true)) { this.connFailNodeMap.put(hostInfo.getReferenceName(), hostInfo); client.close(false); @@ -385,7 +385,7 @@ public void updateProxyInfoList(boolean nodeChanged, List newNodes) { } public String getGroupId() { - return configure.getInlongGroupId(); + return tcpConfig.getInlongGroupId(); } public boolean isIdTransNum() { @@ -449,18 +449,22 @@ private void updateProxyNodes(List newProxyNodes) { } } - private Bootstrap buildBootstrap(ProxyClientConfig config, Sender sender, ThreadFactory selfFactory) { + private Bootstrap buildBootstrap(TcpMsgSenderConfig tcpConfig, Sender sender, ThreadFactory selfFactory) { if (selfFactory == null) { selfFactory = new DefaultThreadFactory( "sdk-netty-workers", Thread.currentThread().isDaemon()); } EventLoopGroup eventLoopGroup = EventLoopUtil.newEventLoopGroup( - config.getIoThreadNum(), config.isEnableBusyWait(), selfFactory); + tcpConfig.getNettyWorkerThreadNum(), tcpConfig.isEnableEpollBusyWait(), selfFactory); Bootstrap tmpBootstrap = new Bootstrap(); tmpBootstrap.group(eventLoopGroup); tmpBootstrap.channel(EventLoopUtil.getClientSocketChannelClass(eventLoopGroup)); - tmpBootstrap.option(ChannelOption.SO_RCVBUF, config.getRecvBufferSize()); - tmpBootstrap.option(ChannelOption.SO_SNDBUF, config.getSendBufferSize()); + if (tcpConfig.getRcvBufferSize() > 0) { + tmpBootstrap.option(ChannelOption.SO_RCVBUF, tcpConfig.getRcvBufferSize()); + } + if (tcpConfig.getSendBufferSize() > 0) { + tmpBootstrap.option(ChannelOption.SO_SNDBUF, tcpConfig.getSendBufferSize()); + } tmpBootstrap.handler(new ClientPipelineFactory(this, sender)); return tmpBootstrap; } @@ -482,9 +486,9 @@ private void sendHeartBeatMsg(NettyClient client) { Collections.singletonList(hbMsgBody), 8, false, false, false, System.currentTimeMillis() / 1000, 1, "", "", ""); try { - if (configure.isEnableAuthentication()) { - encodeObject.setAuth(configure.isEnableAuthentication(), - configure.getAuthSecretId(), configure.getAuthSecretKey()); + if (tcpConfig.isEnableReportAuthz()) { + encodeObject.setAuth(tcpConfig.isEnableReportAuthz(), + tcpConfig.getRptUserName(), tcpConfig.getRptSecretKey()); } client.write(encodeObject); } catch (Throwable ex) { @@ -519,7 +523,7 @@ public void run() { curTime = System.currentTimeMillis(); if (deletingClientMaps != null && !deletingClientMaps.isEmpty()) { if (lastUpdateTime > 0 - && curTime - lastUpdateTime > configure.getConCloseWaitPeriodMs()) { + && curTime - lastUpdateTime > tcpConfig.getConCloseWaitPeriodMs()) { for (NettyClient client : deletingClientMaps.values()) { if (client == null) { continue; diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/HttpProxySender.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/HttpProxySender.java index 1cf9939ff98..9da6305b498 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/HttpProxySender.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/HttpProxySender.java @@ -17,13 +17,12 @@ package org.apache.inlong.sdk.dataproxy.network; -import org.apache.inlong.common.constant.ProtocolType; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; import org.apache.inlong.sdk.dataproxy.common.SendMessageCallback; import org.apache.inlong.sdk.dataproxy.common.SendResult; import org.apache.inlong.sdk.dataproxy.config.HostInfo; import org.apache.inlong.sdk.dataproxy.config.ProxyConfigEntry; import org.apache.inlong.sdk.dataproxy.config.ProxyConfigManager; +import org.apache.inlong.sdk.dataproxy.http.HttpMsgSenderConfig; import org.apache.inlong.sdk.dataproxy.http.InternalHttpSender; import org.apache.inlong.sdk.dataproxy.utils.ConcurrentHashSet; import org.apache.inlong.sdk.dataproxy.utils.Tuple2; @@ -47,7 +46,7 @@ public class HttpProxySender extends Thread { private final ConcurrentHashSet hostList = new ConcurrentHashSet<>(); - private final ProxyClientConfig proxyClientConfig; + private final HttpMsgSenderConfig proxyClientConfig; private ProxyConfigManager proxyConfigManager; private boolean bShutDown = false; @@ -55,35 +54,31 @@ public class HttpProxySender extends Thread { private final InternalHttpSender internalHttpSender; private final LinkedBlockingQueue messageCache; - public HttpProxySender(ProxyClientConfig configure) throws Exception { - // correct ProtocolType settings - if (!ProtocolType.HTTP.equals(configure.getProtocolType())) { - configure.setProtocolType(ProtocolType.HTTP); - } - logger.info("Initial http sender, configure is {}", configure); - this.proxyClientConfig = configure; - initTDMClientAndRequest(configure); - this.messageCache = new LinkedBlockingQueue<>(configure.getTotalAsyncCallbackSize()); - internalHttpSender = new InternalHttpSender(configure, hostList, messageCache); + public HttpProxySender(HttpMsgSenderConfig httpConfig) throws Exception { + logger.info("Initial http sender, configure is {}", httpConfig); + this.proxyClientConfig = httpConfig; + initTDMClientAndRequest(httpConfig); + this.messageCache = new LinkedBlockingQueue<>(httpConfig.getHttpAsyncRptCacheSize()); + internalHttpSender = new InternalHttpSender(httpConfig, hostList, messageCache); } /** * get proxy list * - * @param configure + * @param httpConfig * @throws Exception */ - private void initTDMClientAndRequest(ProxyClientConfig configure) throws Exception { + private void initTDMClientAndRequest(HttpMsgSenderConfig httpConfig) throws Exception { try { - proxyConfigManager = new ProxyConfigManager(configure); + proxyConfigManager = new ProxyConfigManager(httpConfig); ProxyConfigEntry proxyConfigEntry = retryGettingProxyConfig(); hostList.addAll(proxyConfigEntry.getHostMap().values()); this.setDaemon(true); this.start(); } catch (Throwable e) { - if (configure.isOnlyUseLocalProxyConfig()) { + if (httpConfig.isOnlyUseLocalProxyConfig()) { throw new Exception("Get local proxy configure failure! e = {}", e); } else { throw new Exception("Visit TDManager error! e = {}", e); @@ -111,8 +106,8 @@ public void run() { while (!bShutDown) { try { int rand = ThreadLocalRandom.current().nextInt(0, 600); - int randSleepTime = proxyClientConfig.getProxyHttpUpdateIntervalMinutes() * 60 + rand; - TimeUnit.MILLISECONDS.sleep(randSleepTime * 1000); + long randSleepTime = proxyClientConfig.getMgrMetaSyncInrMs() + rand; + TimeUnit.MILLISECONDS.sleep(randSleepTime); if (proxyConfigManager != null) { Tuple2 result = proxyConfigManager.getGroupIdConfigure(false); @@ -190,18 +185,7 @@ public void asyncSendMessage(List bodies, String groupId, String streamI timeout, timeUnit, callback); try { if (!messageCache.offer(httpMessage)) { - if (!proxyClientConfig.isDiscardOldMessage()) { - // put and wait for capacity available. - messageCache.put(httpMessage); - } else { - // discard old message and use new message instead. - logger.debug("discard old message and use new message instead"); - HttpMessage oldMessage = messageCache.poll(); - if (oldMessage != null) { - oldMessage.getCallback().onMessageAck(SendResult.TIMEOUT); - } - messageCache.offer(httpMessage); - } + callback.onMessageAck(SendResult.ASYNC_CALLBACK_BUFFER_FULL); } } catch (Exception exception) { logger.error("error async sending data", exception); diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/NettyClient.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/NettyClient.java index bdbf12f4444..30cb4c65671 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/NettyClient.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/NettyClient.java @@ -17,7 +17,7 @@ package org.apache.inlong.sdk.dataproxy.network; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.TcpMsgSenderConfig; import org.apache.inlong.sdk.dataproxy.codec.EncodeObject; import org.apache.inlong.sdk.dataproxy.config.HostInfo; import org.apache.inlong.sdk.dataproxy.utils.LogCounter; @@ -49,7 +49,7 @@ public class NettyClient { private final static int CLIENT_STATUS_BUSY = 3; private final String callerId; - private final ProxyClientConfig configure; + private final TcpMsgSenderConfig tcpConfig; private final Bootstrap bootstrap; private final HostInfo hostInfo; private Channel channel = null; @@ -60,11 +60,11 @@ public class NettyClient { private final AtomicLong lstReConTime = new AtomicLong(0); public NettyClient(String callerId, - Bootstrap bootstrap, HostInfo hostInfo, ProxyClientConfig configure) { + Bootstrap bootstrap, HostInfo hostInfo, TcpMsgSenderConfig tcpConfig) { this.callerId = callerId; this.bootstrap = bootstrap; this.hostInfo = hostInfo; - this.configure = configure; + this.tcpConfig = tcpConfig; setState(CLIENT_STATUS_INIT); } @@ -84,7 +84,7 @@ public void operationComplete(ChannelFuture arg0) throws Exception { }); try { // Wait until the connection is built. - awaitLatch.await(configure.getConnectTimeoutMs(), TimeUnit.MILLISECONDS); + awaitLatch.await(tcpConfig.getConnectTimeoutMs(), TimeUnit.MILLISECONDS); } catch (Throwable ex) { if (conExptCnt.shouldPrint()) { logger.warn("NettyClient({}) connect to {} exception", @@ -129,7 +129,7 @@ public void operationComplete(ChannelFuture arg0) throws Exception { } }); // Wait until the connection is close. - awaitLatch.await(configure.getConCloseWaitPeriodMs(), TimeUnit.MILLISECONDS); + awaitLatch.await(tcpConfig.getConCloseWaitPeriodMs(), TimeUnit.MILLISECONDS); // Return if close this connection fail. if (!future.isSuccess()) { ret = false; @@ -172,7 +172,7 @@ public ChannelFuture write(EncodeObject encodeObject) { public boolean reconnect(boolean needPrint) { if (this.isActive() || this.msgInFlight.get() > 0 - || (System.currentTimeMillis() - this.lstReConTime.get()) < this.configure.getReConnectWaitMs()) { + || (System.currentTimeMillis() - this.lstReConTime.get()) < this.tcpConfig.getReconFailWaitMs()) { return false; } if (reconSemaphore.tryAcquire()) { @@ -237,8 +237,8 @@ public boolean isIdleClient(long curTime) { } public boolean tryIncMsgInFlight() { - if (configure.getMaxMsgInFlightPerConn() > 0) { - if (msgInFlight.getAndIncrement() > configure.getMaxMsgInFlightPerConn()) { + if (tcpConfig.getMaxMsgInFlightPerConn() > 0) { + if (msgInFlight.getAndIncrement() > tcpConfig.getMaxMsgInFlightPerConn()) { msgInFlight.decrementAndGet(); return false; } @@ -247,7 +247,7 @@ public boolean tryIncMsgInFlight() { } public void decMsgInFlight() { - if (configure.getMaxMsgInFlightPerConn() > 0) { + if (tcpConfig.getMaxMsgInFlightPerConn() > 0) { if (msgInFlight.decrementAndGet() < 0L) { logger.warn("NettyClient({}) dec inflight({}) value < 0", callerId, hostInfo.getReferenceName()); } diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/Sender.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/Sender.java index 4ba988b19f0..0d8115dc4fe 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/Sender.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/network/Sender.java @@ -17,7 +17,7 @@ package org.apache.inlong.sdk.dataproxy.network; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.TcpMsgSenderConfig; import org.apache.inlong.sdk.dataproxy.codec.EncodeObject; import org.apache.inlong.sdk.dataproxy.common.SendMessageCallback; import org.apache.inlong.sdk.dataproxy.common.SendResult; @@ -53,7 +53,6 @@ public class Sender { private static final LogCounter exptCnt = new LogCounter(10, 100000, 60 * 1000L); private static final LogCounter unwritableExptCnt = new LogCounter(10, 100000, 60 * 1000L); private static final LogCounter reqChkLoggCount = new LogCounter(10, 100000, 60 * 1000L); - private static final AtomicLong senderIdGen = new AtomicLong(0L); /* Store the callback used by asynchronously message sending. */ private final ConcurrentHashMap> callbacks = @@ -67,26 +66,26 @@ public class Sender { private final AtomicBoolean started = new AtomicBoolean(false); private final ClientMgr clientMgr; private final String instanceId; - private final ProxyClientConfig configure; + private final TcpMsgSenderConfig tcpConfig; private MetricWorkerThread metricWorker = null; private int clusterId = -1; - public Sender(ProxyClientConfig configure) throws Exception { - this(configure, null); + public Sender(TcpMsgSenderConfig tcpConfig) throws Exception { + this(tcpConfig, null); } /** - * Constructor of sender takes two arguments {@link ProxyClientConfig} and {@link ThreadFactory} + * Constructor of sender takes two arguments {@link TcpMsgSenderConfig} and {@link ThreadFactory} */ - public Sender(ProxyClientConfig configure, ThreadFactory selfDefineFactory) throws Exception { - this.configure = configure; + public Sender(TcpMsgSenderConfig tcpConfig, ThreadFactory selfDefineFactory) throws Exception { + this.tcpConfig = tcpConfig; this.instanceId = "sender-" + senderIdGen.incrementAndGet(); - this.asyncCallbackMaxSize = configure.getTotalAsyncCallbackSize(); + this.asyncCallbackMaxSize = tcpConfig.getTotalAsyncCallbackSize(); this.threadPool = Executors.newCachedThreadPool(); - this.clientMgr = new ClientMgr(configure, this, selfDefineFactory); - this.scanThread = new TimeoutScanThread(this, configure); - if (configure.isEnableMetric()) { - metricWorker = new MetricWorkerThread(configure, this); + this.clientMgr = new ClientMgr(tcpConfig, this, selfDefineFactory); + this.scanThread = new TimeoutScanThread(this, tcpConfig); + if (tcpConfig.isEnableMetric()) { + metricWorker = new MetricWorkerThread(tcpConfig, this); } logger.info("Sender({}) instance initialized!", this.instanceId); } @@ -102,21 +101,21 @@ public void start() throws Exception { proxyConfigEntry = this.clientMgr.getGroupIdConfigure(); setClusterId(proxyConfigEntry.getClusterId()); } catch (Throwable ex) { - if (configure.isOnlyUseLocalProxyConfig()) { + if (tcpConfig.isOnlyUseLocalProxyConfig()) { throw new Exception("Get local proxy configure failure!", ex); } else { throw new Exception("Visit manager error!", ex); } } if (!proxyConfigEntry.isInterVisit()) { - if (!configure.isEnableAuthentication()) { + if (!tcpConfig.isEnableReportAuthz()) { throw new Exception("In OutNetwork isNeedAuthentication must be true!"); } - if (!configure.isEnableDataEncrypt()) { + if (!tcpConfig.isEnableReportEncrypt()) { throw new Exception("In OutNetwork isNeedDataEncry must be true!"); } } - if (this.configure.isEnableMetric()) { + if (this.tcpConfig.isEnableMetric()) { this.metricWorker.start(); } logger.info("Sender({}) instance started!", this.instanceId); @@ -130,7 +129,7 @@ public void close() { scanThread.shutDown(); clientMgr.shutDown(); threadPool.shutdown(); - if (configure.isEnableMetric()) { + if (tcpConfig.isEnableMetric()) { metricWorker.close(); } logger.info("Sender({}) instance stopped!", this.instanceId); @@ -166,7 +165,7 @@ public SendResult syncSendMessage(EncodeObject encodeObject, String msgUUID) { if (!started.get()) { return SendResult.SENDER_CLOSED; } - if (configure.isEnableMetric()) { + if (tcpConfig.isEnableMetric()) { metricWorker.recordNumByKey(encodeObject.getMessageId(), encodeObject.getGroupId(), encodeObject.getStreamId(), ProxyUtils.getLocalIp(), encodeObject.getDt(), encodeObject.getPackageTime(), encodeObject.getRealCnt()); @@ -203,16 +202,16 @@ public SendResult syncSendMessage(EncodeObject encodeObject, String msgUUID) { clientMgr.getStreamIdNum(encodeObject.getStreamId())); } } - if (this.configure.isEnableDataEncrypt()) { + if (this.tcpConfig.isEnableReportEncrypt()) { encodeObject.setEncryptEntry(true, - configure.getAuthSecretId(), clientMgr.getEncryptConfigureInfo()); + tcpConfig.getRptUserName(), clientMgr.getEncryptConfigureInfo()); } encodeObject.setMsgUUID(msgUUID); SyncMessageCallable callable = new SyncMessageCallable( - clientResult.getF1(), encodeObject, configure.getRequestTimeoutMs()); + clientResult.getF1(), encodeObject, tcpConfig.getRequestTimeoutMs()); syncCallables.put(encodeObject.getMessageId(), callable); Future future = threadPool.submit(callable); - message = future.get(configure.getRequestTimeoutMs(), TimeUnit.MILLISECONDS); + message = future.get(tcpConfig.getRequestTimeoutMs(), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { syncCallables.remove(encodeObject.getMessageId()); return SendResult.THREAD_INTERRUPT; @@ -255,7 +254,7 @@ public SendResult syncSendMessage(EncodeObject encodeObject, String msgUUID) { } scanThread.resetTimeoutChannel(clientResult.getF1().getChannel()); if (message == SendResult.OK) { - if (configure.isEnableMetric()) { + if (tcpConfig.isEnableMetric()) { metricWorker.recordSuccessByMessageId(encodeObject.getMessageId()); } } @@ -319,7 +318,7 @@ public void asyncSendMessage(EncodeObject encodeObject, throw new ProxySdkException(SendResult.SENDER_CLOSED.toString()); } } - if (configure.isEnableMetric()) { + if (tcpConfig.isEnableMetric()) { metricWorker.recordNumByKey(encodeObject.getMessageId(), encodeObject.getGroupId(), encodeObject.getStreamId(), ProxyUtils.getLocalIp(), encodeObject.getPackageTime(), encodeObject.getDt(), encodeObject.getRealCnt()); @@ -394,7 +393,7 @@ public void asyncSendMessage(EncodeObject encodeObject, callbacks.computeIfAbsent(clientResult.getF1().getChannel(), (k) -> new ConcurrentHashMap<>()); QueueObject queueObject = msgQueueMap.putIfAbsent(encodeObject.getMessageId(), new QueueObject(clientResult.getF1(), System.currentTimeMillis(), callback, - size, configure.getRequestTimeoutMs())); + size, tcpConfig.getRequestTimeoutMs())); if (queueObject != null) { if (reqChkLoggCount.shouldPrint()) { logger.warn("Sender({}) found message id {} has existed.", @@ -408,9 +407,9 @@ public void asyncSendMessage(EncodeObject encodeObject, clientMgr.getStreamIdNum(encodeObject.getStreamId())); } } - if (this.configure.isEnableDataEncrypt()) { + if (this.tcpConfig.isEnableReportEncrypt()) { encodeObject.setEncryptEntry(true, - configure.getAuthSecretId(), clientMgr.getEncryptConfigureInfo()); + tcpConfig.getRptUserName(), clientMgr.getEncryptConfigureInfo()); } encodeObject.setMsgUUID(msgUUID); clientResult.getF1().write(encodeObject); @@ -422,7 +421,7 @@ public void notifyFeedback(Channel channel, EncodeObject response) { SyncMessageCallable callable = syncCallables.remove(messageId); SendResult result = response.getSendResult(); if (result == SendResult.OK) { - if (configure.isEnableMetric()) { + if (tcpConfig.isEnableMetric()) { metricWorker.recordSuccessByMessageId(messageId); } } else { @@ -512,7 +511,7 @@ public void waitForAckForChannel(Channel channel) { } try { while (!queueObjMap.isEmpty()) { - if (System.currentTimeMillis() - startTime >= configure.getConCloseWaitPeriodMs()) { + if (System.currentTimeMillis() - startTime >= tcpConfig.getConCloseWaitPeriodMs()) { break; } try { @@ -558,8 +557,8 @@ public ClientMgr getClientMgr() { return clientMgr; } - public ProxyClientConfig getConfigure() { - return configure; + public TcpMsgSenderConfig getTcpConfig() { + return tcpConfig; } private void checkCallbackList() { @@ -567,7 +566,7 @@ private void checkCallbackList() { try { long startTime = System.currentTimeMillis(); while (currentBufferSize.get() > 0 - && System.currentTimeMillis() - startTime < configure.getConCloseWaitPeriodMs()) { + && System.currentTimeMillis() - startTime < tcpConfig.getConCloseWaitPeriodMs()) { Thread.sleep(300L); } if (currentBufferSize.get() > 0) { diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/threads/MetricWorkerThread.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/threads/MetricWorkerThread.java index a974adc803b..6ed3f55c914 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/threads/MetricWorkerThread.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/threads/MetricWorkerThread.java @@ -17,7 +17,7 @@ package org.apache.inlong.sdk.dataproxy.threads; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.TcpMsgSenderConfig; import org.apache.inlong.sdk.dataproxy.codec.EncodeObject; import org.apache.inlong.sdk.dataproxy.common.SendMessageCallback; import org.apache.inlong.sdk.dataproxy.common.SendResult; @@ -57,8 +57,8 @@ public class MetricWorkerThread extends Thread implements Closeable { private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); private volatile boolean bShutdown = false; - public MetricWorkerThread(ProxyClientConfig proxyClientConfig, Sender sender) { - this.metricConfig = proxyClientConfig.getMetricConfig(); + public MetricWorkerThread(TcpMsgSenderConfig clientConfig, Sender sender) { + this.metricConfig = clientConfig.getMetricConfig(); this.sender = sender; this.setDaemon(true); this.setName("MetricWorkerThread"); @@ -73,13 +73,11 @@ public long getFormatKeyTime(long keyTime) { */ private String getKeyStringByConfig(String groupId, String streamId, String localIp, long keyTime) { StringBuilder builder = new StringBuilder(); - String groupIdStr = metricConfig.isUseGroupIdAsKey() ? groupId : DEFAULT_KEY_ITEM; - String streamIdStr = metricConfig.isUseStreamIdAsKey() ? streamId : DEFAULT_KEY_ITEM; - String localIpStr = metricConfig.isUseLocalIpAsKey() ? localIp : DEFAULT_KEY_ITEM; + String groupIdStr = metricConfig.isMaskGroupId() ? DEFAULT_KEY_ITEM : groupId; + String streamIdStr = metricConfig.isMaskStreamId() ? DEFAULT_KEY_ITEM : streamId; builder.append(groupIdStr).append(DEFAULT_KEY_SPLITTER) .append(streamIdStr).append(DEFAULT_KEY_SPLITTER) - .append(localIpStr).append(DEFAULT_KEY_SPLITTER) .append(keyTime); return builder.toString(); } @@ -181,7 +179,7 @@ public void run() { try { checkCacheRecords(); flushMetric(false); - TimeUnit.MILLISECONDS.sleep(metricConfig.getMetricRptIntvlMs()); + TimeUnit.MILLISECONDS.sleep(metricConfig.getMetricOutIntvlMs()); } catch (Throwable ex) { // exception happens } @@ -216,7 +214,7 @@ private void flushMapRecords(boolean isClosing, ConcurrentHashMap metricConfig.getMetricRptIntvlMs())) { + + delayTime > metricConfig.getMetricOutIntvlMs())) { summary = cacheMap.remove(keyName); if (summary != null) { long metricDtTime = summary.getStartCalculateTime() / 1000; @@ -247,7 +245,7 @@ private void flushRecords(boolean isClosing) { private void checkCacheRecords() { for (String msgId : metricValueCache.keySet()) { MessageRecord record = metricValueCache.get(msgId); - if (record != null && record.getMessageTime() + delayTime > metricConfig.getMetricRptIntvlMs()) { + if (record != null && record.getMessageTime() + delayTime > metricConfig.getMetricOutIntvlMs()) { recordFailedByMessageId(msgId); } } diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/threads/TimeoutScanThread.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/threads/TimeoutScanThread.java index 5211ba99077..f7d4064896e 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/threads/TimeoutScanThread.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/threads/TimeoutScanThread.java @@ -17,7 +17,7 @@ package org.apache.inlong.sdk.dataproxy.threads; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.TcpMsgSenderConfig; import org.apache.inlong.sdk.dataproxy.common.SendResult; import org.apache.inlong.sdk.dataproxy.network.QueueObject; import org.apache.inlong.sdk.dataproxy.network.Sender; @@ -41,13 +41,13 @@ public class TimeoutScanThread extends Thread { private static final LogCounter exptCnt = new LogCounter(10, 100000, 60 * 1000L); private volatile boolean bShutDown = false; private long printCount = 0; - private final ProxyClientConfig config; + private final TcpMsgSenderConfig tcpConfig; private final Sender sender; private final ConcurrentHashMap timeoutChannelStat = new ConcurrentHashMap<>(); - public TimeoutScanThread(Sender sender, ProxyClientConfig config) { + public TimeoutScanThread(Sender sender, TcpMsgSenderConfig tcpConfig) { this.bShutDown = false; - this.config = config; + this.tcpConfig = tcpConfig; this.sender = sender; this.setDaemon(true); this.setName("TimeoutScanThread-" + this.sender.getInstanceId()); @@ -111,7 +111,7 @@ private void checkTimeoutChannel() { if (System.currentTimeMillis() - timeScanObject.getTime() > MAX_CHANNEL_TIMEOUT) { timeoutChannelStat.remove(tmpChannel); } else { - if (timeScanObject.getCurTimeoutCount() > config.getMaxTimeoutCnt()) { + if (timeScanObject.getCurTimeoutCount() > tcpConfig.getMaxAllowedSyncMsgTimeoutCnt()) { timeoutChannelStat.remove(tmpChannel); if (tmpChannel.isOpen() && tmpChannel.isActive()) { sender.getClientMgr().setConnectionBusy(tmpChannel); diff --git a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/utils/ProxyUtils.java b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/utils/ProxyUtils.java index 6da97d47506..c74aea445e7 100644 --- a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/utils/ProxyUtils.java +++ b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/utils/ProxyUtils.java @@ -18,8 +18,9 @@ package org.apache.inlong.sdk.dataproxy.utils; import org.apache.inlong.common.msg.AttributeConstants; -import org.apache.inlong.sdk.dataproxy.ConfigConstants; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.common.msg.MsgType; +import org.apache.inlong.sdk.dataproxy.TcpMsgSenderConfig; +import org.apache.inlong.sdk.dataproxy.common.SdkConsts; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -40,7 +41,7 @@ public class ProxyUtils { private static final int TIME_LENGTH = 13; private static final Set invalidAttr = new HashSet<>(); - + public static final Set SdkAllowedMsgType = new HashSet<>(); private static String localHost; static { @@ -49,6 +50,9 @@ public class ProxyUtils { "cnt", "mt", "m", "sid", "t", "NodeIP", "messageId", "_file_status_check", "_secretId", "_signature", "_timeStamp", "_nonce", "_userName", "_clientIP", "_encyVersion", "_encyAesKey", "proxySend", "errMsg", "errCode", AttributeConstants.MSG_RPT_TIME); + + Collections.addAll(SdkAllowedMsgType, + MsgType.MSG_ACK_SERVICE, MsgType.MSG_MULTI_BODY, MsgType.MSG_BIN_MULTI_BODY); } public static String getLocalIp() { @@ -68,6 +72,15 @@ public static String getLocalIp() { return ip; } + public static boolean sleepSomeTime(long sleepTimeMs) { + try { + Thread.sleep(sleepTimeMs); + return true; + } catch (Throwable ex) { + return false; + } + } + public static boolean isAttrKeysValid(Map attrsMap) { if (attrsMap == null || attrsMap.size() == 0) { return false; @@ -129,9 +142,9 @@ public static boolean isBodyLengthValid(byte[] body, int maxLen) { return true; } // Reserve space for attribute - if (body.length > maxLen - ConfigConstants.RESERVED_ATTRIBUTE_LENGTH) { + if (body.length > maxLen - SdkConsts.RESERVED_ATTRIBUTE_LENGTH) { logger.debug("body length({}) > max length({}) - fixed attribute length({})", - body.length, maxLen, ConfigConstants.RESERVED_ATTRIBUTE_LENGTH); + body.length, maxLen, SdkConsts.RESERVED_ATTRIBUTE_LENGTH); return false; } return true; @@ -153,9 +166,9 @@ public static boolean isBodyLengthValid(List bodyList, int maxLen) { size += body.length; } // Reserve space for attribute - if (size > maxLen - ConfigConstants.RESERVED_ATTRIBUTE_LENGTH) { + if (size > maxLen - SdkConsts.RESERVED_ATTRIBUTE_LENGTH) { logger.debug("bodyList length({}) > max length({}) - fixed attribute length({})", - size, maxLen, ConfigConstants.RESERVED_ATTRIBUTE_LENGTH); + size, maxLen, SdkConsts.RESERVED_ATTRIBUTE_LENGTH); return false; } return true; @@ -171,14 +184,14 @@ public static long covertZeroDt(long dt) { /** * valid client config * - * @param clientConfig + * @param tcpConfig */ - public static void validClientConfig(ProxyClientConfig clientConfig) { - if (clientConfig.isEnableAuthentication()) { - if (StringUtils.isBlank(clientConfig.getAuthSecretId())) { + public static void validClientConfig(TcpMsgSenderConfig tcpConfig) { + if (tcpConfig.isEnableMgrAuthz()) { + if (StringUtils.isBlank(tcpConfig.getMgrAuthSecretId())) { throw new IllegalArgumentException("Authentication require secretId not Blank!"); } - if (StringUtils.isBlank(clientConfig.getAuthSecretKey())) { + if (StringUtils.isBlank(tcpConfig.getMgrAuthSecretKey())) { throw new IllegalArgumentException("Authentication require secretKey not Blank!"); } } diff --git a/inlong-sdk/dataproxy-sdk/src/test/java/org/apache/inlong/sdk/dataproxy/ProxyClientConfigTest.java b/inlong-sdk/dataproxy-sdk/src/test/java/org/apache/inlong/sdk/dataproxy/ProxyClientConfigTest.java new file mode 100644 index 00000000000..7165b2c8ad7 --- /dev/null +++ b/inlong-sdk/dataproxy-sdk/src/test/java/org/apache/inlong/sdk/dataproxy/ProxyClientConfigTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.dataproxy; + +import org.apache.inlong.sdk.dataproxy.common.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.http.HttpMsgSenderConfig; + +import org.junit.Assert; +import org.junit.Test; + +public class ProxyClientConfigTest { + + @Test + public void testManagerConfig() throws Exception { + HttpMsgSenderConfig httpConfig = new HttpMsgSenderConfig( + "http://127.0.0.1:800", "test_id", "secretId", "secretKey"); + HttpMsgSenderConfig httpConfig1 = httpConfig.clone(); + Assert.assertEquals(httpConfig, httpConfig1); + httpConfig1.setRegionName("sz"); + httpConfig1.setHttpAsyncRptPoolConfig(50, 10); + Assert.assertNotEquals(httpConfig1.getRegionName(), httpConfig.getRegionName()); + Assert.assertNotEquals(httpConfig1.getHttpAsyncRptCacheSize(), httpConfig.getHttpAsyncRptCacheSize()); + Assert.assertNotEquals(httpConfig1.getHttpAsyncRptWorkerNum(), httpConfig.getHttpAsyncRptWorkerNum()); + httpConfig.setRptDataByHttps(true); + httpConfig.setMetaCacheExpiredMs(30000); + Assert.assertNotEquals(httpConfig1.isRptDataByHttps(), httpConfig.isRptDataByHttps()); + Assert.assertNotEquals(httpConfig1.getMetaCacheExpiredMs(), httpConfig.getMetaCacheExpiredMs()); + + SubHttpClass subHttpClass = new SubHttpClass(httpConfig); + subHttpClass.getHttpConfig().setRptDataByHttps(false); + subHttpClass.getHttpConfig().setMetaCacheExpiredMs(99999); + Assert.assertNotEquals( + subHttpClass.getHttpConfig().isRptDataByHttps(), httpConfig.isRptDataByHttps()); + Assert.assertNotEquals( + subHttpClass.getHttpConfig().getMetaCacheExpiredMs(), httpConfig.getMetaCacheExpiredMs()); + } + + public static class BaseClass { + + protected final ProxyClientConfig proxyConfig; + + public BaseClass(ProxyClientConfig configure) { + proxyConfig = configure.clone(); + } + } + + public static class SubHttpClass extends BaseClass { + + private final HttpMsgSenderConfig httpConfig; + + public SubHttpClass(HttpMsgSenderConfig configure) { + super(configure); + this.httpConfig = (HttpMsgSenderConfig) proxyConfig; + } + + public HttpMsgSenderConfig getHttpConfig() { + return httpConfig; + } + } +} diff --git a/inlong-sdk/dataproxy-sdk/src/test/java/org/apache/inlong/sdk/dataproxy/ProxyConfigManagerTest.java b/inlong-sdk/dataproxy-sdk/src/test/java/org/apache/inlong/sdk/dataproxy/ProxyConfigManagerTest.java index 40fec3b57d4..c7e3f773e2c 100644 --- a/inlong-sdk/dataproxy-sdk/src/test/java/org/apache/inlong/sdk/dataproxy/ProxyConfigManagerTest.java +++ b/inlong-sdk/dataproxy-sdk/src/test/java/org/apache/inlong/sdk/dataproxy/ProxyConfigManagerTest.java @@ -35,12 +35,12 @@ public class ProxyConfigManagerTest { private final String localFile = Paths.get( Objects.requireNonNull(this.getClass().getClassLoader().getResource("proxylist.json")).toURI()) .toString(); - private final ProxyClientConfig clientConfig = PowerMockito.mock(ProxyClientConfig.class); + private final TcpMsgSenderConfig clientConfig = PowerMockito.mock(TcpMsgSenderConfig.class); private final ClientMgr clientMgr = PowerMockito.mock(ClientMgr.class); private final ProxyConfigManager proxyConfigManager; public ProxyConfigManagerTest() throws URISyntaxException { - clientConfig.setConfigStoreBasePath(localFile); + clientConfig.setMetaStoreBasePath(localFile); proxyConfigManager = new ProxyConfigManager("test", clientConfig, clientMgr); } diff --git a/inlong-sdk/dirty-data-sdk/src/main/java/org/apache/inlong/sdk/dirtydata/InlongSdkDirtySender.java b/inlong-sdk/dirty-data-sdk/src/main/java/org/apache/inlong/sdk/dirtydata/InlongSdkDirtySender.java index b029392a497..9bf6be20143 100644 --- a/inlong-sdk/dirty-data-sdk/src/main/java/org/apache/inlong/sdk/dirtydata/InlongSdkDirtySender.java +++ b/inlong-sdk/dirty-data-sdk/src/main/java/org/apache/inlong/sdk/dirtydata/InlongSdkDirtySender.java @@ -18,7 +18,7 @@ package org.apache.inlong.sdk.dirtydata; import org.apache.inlong.sdk.dataproxy.DefaultMessageSender; -import org.apache.inlong.sdk.dataproxy.ProxyClientConfig; +import org.apache.inlong.sdk.dataproxy.TcpMsgSenderConfig; import org.apache.inlong.sdk.dataproxy.common.SendMessageCallback; import org.apache.inlong.sdk.dataproxy.common.SendResult; @@ -58,13 +58,12 @@ public void init() throws Exception { Preconditions.checkNotNull(authId, "authId cannot be null"); Preconditions.checkNotNull(authKey, "authKey cannot be null"); - ProxyClientConfig proxyClientConfig = - new ProxyClientConfig(true, + TcpMsgSenderConfig proxyClientConfig = + new TcpMsgSenderConfig(true, inlongManagerAddr, inlongManagerPort, inlongGroupId, authId, authKey); proxyClientConfig.setOnlyUseLocalProxyConfig(false); - proxyClientConfig.setAsyncCallbackSize(maxCallbackSize); + proxyClientConfig.setTotalAsyncCallbackSize(maxCallbackSize); this.sender = DefaultMessageSender.generateSenderByClusterId(proxyClientConfig); - this.sender.setMsgtype(7); this.dirtyDataQueue = new LinkedBlockingQueue<>(maxCallbackSize); this.executor = Executors.newSingleThreadExecutor();