From 5ce3ff04af90310e665039beb3e5e8c5187e9faa Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 8 Aug 2023 11:37:26 +0800 Subject: [PATCH 1/8] fix(db): fix getHeadBlockId abnormal return problem --- chainbase/src/main/java/org/tron/core/ChainBaseManager.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java index adf66527499..dad76041777 100644 --- a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java +++ b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java @@ -316,9 +316,7 @@ public BlockCapsule getHead() throws HeaderNotFound { } public synchronized BlockId getHeadBlockId() { - return new BlockId( - dynamicPropertiesStore.getLatestBlockHeaderHash(), - dynamicPropertiesStore.getLatestBlockHeaderNumber()); + return new BlockId(dynamicPropertiesStore.getLatestBlockHeaderHash()); } public long getHeadBlockNum() { From a9c4f4324c895f2f3c74eb7d7fb13add719419fe Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Tue, 15 Aug 2023 11:23:32 +0800 Subject: [PATCH 2/8] feat(all): tune single Thread into SingleThreadExecutor (#5410) --- .../common/es/ExecutorServiceManager.java | 55 +++++++++++++++++++ .../org/tron/consensus/dpos/DposTask.java | 20 +++---- .../common/backup/socket/BackupServer.java | 12 +++- .../tron/core/config/TronLogShutdownHook.java | 4 +- .../main/java/org/tron/core/db/Manager.java | 25 ++++++--- .../nodepersist/NodePersistService.java | 30 ++++------ .../tron/common/backup/BackupServerTest.java | 44 +++++++++++++++ 7 files changed, 150 insertions(+), 40 deletions(-) create mode 100644 common/src/main/java/org/tron/common/es/ExecutorServiceManager.java create mode 100644 framework/src/test/java/org/tron/common/backup/BackupServerTest.java diff --git a/common/src/main/java/org/tron/common/es/ExecutorServiceManager.java b/common/src/main/java/org/tron/common/es/ExecutorServiceManager.java new file mode 100644 index 00000000000..f1e60fdcfbc --- /dev/null +++ b/common/src/main/java/org/tron/common/es/ExecutorServiceManager.java @@ -0,0 +1,55 @@ +package org.tron.common.es; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import lombok.extern.slf4j.Slf4j; + +@Slf4j(topic = "common") +public class ExecutorServiceManager { + + public static ExecutorService newSingleThreadExecutor(String name) { + return newSingleThreadExecutor(name, false); + } + + public static ExecutorService newSingleThreadExecutor(String name, boolean isDaemon) { + return Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder().setNameFormat(name).setDaemon(isDaemon).build()); + } + + + public static ScheduledExecutorService newSingleThreadScheduledExecutor(String name) { + return newSingleThreadScheduledExecutor(name, false); + } + + public static ScheduledExecutorService newSingleThreadScheduledExecutor(String name, + boolean isDaemon) { + return Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat(name).setDaemon(isDaemon).build()); + } + + public static void shutdownAndAwaitTermination(ExecutorService pool, String name) { + if (pool == null) { + return; + } + logger.info("Pool {} shutdown...", name); + pool.shutdown(); // Disable new tasks from being submitted + try { + // Wait a while for existing tasks to terminate + if (!pool.awaitTermination(60, java.util.concurrent.TimeUnit.SECONDS)) { + pool.shutdownNow(); // Cancel currently executing tasks + // Wait a while for tasks to respond to being cancelled + if (!pool.awaitTermination(60, java.util.concurrent.TimeUnit.SECONDS)) { + logger.warn("Pool {} did not terminate", name); + } + } + } catch (InterruptedException ie) { + // (Re-)Cancel if current thread also interrupted + pool.shutdownNow(); + // Preserve interrupt status + Thread.currentThread().interrupt(); + } + logger.info("Pool {} shutdown done", name); + } +} diff --git a/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java b/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java index dcfa85ca5f3..537fe49ae65 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java @@ -3,12 +3,14 @@ import static org.tron.core.config.Parameter.ChainConstant.BLOCK_PRODUCED_INTERVAL; import com.google.protobuf.ByteString; +import java.util.concurrent.ExecutorService; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.joda.time.DateTime; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; +import org.springframework.util.ObjectUtils; +import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; import org.tron.common.utils.Sha256Hash; @@ -34,16 +36,18 @@ public class DposTask { @Setter private DposService dposService; - private Thread produceThread; + private ExecutorService produceExecutor; + + private final String name = "DPosMiner"; private volatile boolean isRunning = true; public void init() { - if (!dposService.isEnable() || StringUtils.isEmpty(dposService.getMiners())) { + if (!dposService.isEnable() || ObjectUtils.isEmpty(dposService.getMiners())) { return; } - + produceExecutor = ExecutorServiceManager.newSingleThreadExecutor(name); Runnable runnable = () -> { while (isRunning) { try { @@ -67,17 +71,13 @@ public void init() { } } }; - produceThread = new Thread(runnable, "DPosMiner"); - produceThread.start(); + produceExecutor.submit(runnable); logger.info("DPoS task started."); } public void stop() { isRunning = false; - if (produceThread != null) { - produceThread.interrupt(); - } - logger.info("DPoS task stopped."); + ExecutorServiceManager.shutdownAndAwaitTermination(produceExecutor, name); } private State produceBlock() { diff --git a/framework/src/main/java/org/tron/common/backup/socket/BackupServer.java b/framework/src/main/java/org/tron/common/backup/socket/BackupServer.java index e3b1de31736..fa2c0947852 100644 --- a/framework/src/main/java/org/tron/common/backup/socket/BackupServer.java +++ b/framework/src/main/java/org/tron/common/backup/socket/BackupServer.java @@ -7,11 +7,13 @@ import io.netty.channel.socket.nio.NioDatagramChannel; import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.tron.common.backup.BackupManager; +import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; import org.tron.p2p.stats.TrafficStats; @@ -29,6 +31,9 @@ public class BackupServer { private volatile boolean shutdown = false; + private final String name = "BackupServer"; + private ExecutorService executor; + @Autowired public BackupServer(final BackupManager backupManager) { this.backupManager = backupManager; @@ -36,13 +41,14 @@ public BackupServer(final BackupManager backupManager) { public void initServer() { if (port > 0 && commonParameter.getBackupMembers().size() > 0) { - new Thread(() -> { + executor = ExecutorServiceManager.newSingleThreadExecutor(name); + executor.submit(() -> { try { start(); } catch (Exception e) { logger.error("Start backup server failed, {}", e); } - }, "BackupServer").start(); + }); } } @@ -88,6 +94,7 @@ public void initChannel(NioDatagramChannel ch) public void close() { logger.info("Closing backup server..."); shutdown = true; + ExecutorServiceManager.shutdownAndAwaitTermination(executor, name); if (channel != null) { try { channel.close().await(10, TimeUnit.SECONDS); @@ -95,5 +102,6 @@ public void close() { logger.warn("Closing backup server failed.", e); } } + logger.info("Backup server closed."); } } diff --git a/framework/src/main/java/org/tron/core/config/TronLogShutdownHook.java b/framework/src/main/java/org/tron/core/config/TronLogShutdownHook.java index 880aa7e3090..f873b88ca44 100644 --- a/framework/src/main/java/org/tron/core/config/TronLogShutdownHook.java +++ b/framework/src/main/java/org/tron/core/config/TronLogShutdownHook.java @@ -16,9 +16,9 @@ public class TronLogShutdownHook extends ShutdownHookBase { private static final Duration CHECK_SHUTDOWN_DELAY = Duration.buildByMilliseconds(100); /** - * The check times before shutdown. default is 50 + * The check times before shutdown. default is 60000/100 = 600 times. */ - private Integer check_times = 50; + private final long check_times = 60 * 1000 / CHECK_SHUTDOWN_DELAY.getMilliseconds(); public TronLogShutdownHook() { } diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 3928f657e44..3a18dce32a9 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -49,6 +49,7 @@ import org.tron.api.GrpcAPI.TransactionInfoList; import org.tron.common.args.GenesisBlock; import org.tron.common.bloom.Bloom; +import org.tron.common.es.ExecutorServiceManager; import org.tron.common.logsfilter.EventPluginLoader; import org.tron.common.logsfilter.FilterQuery; import org.tron.common.logsfilter.capsule.BlockFilterCapsule; @@ -253,6 +254,13 @@ public class Manager { private AtomicInteger blockWaitLock = new AtomicInteger(0); private Object transactionLock = new Object(); + private ExecutorService rePushEs; + private static final String rePushEsName = "repush"; + private ExecutorService triggerEs; + private static final String triggerEsName = "event-trigger"; + private ExecutorService filterEs; + private static final String filterEsName = "filter"; + /** * Cycle thread to rePush Transactions */ @@ -429,14 +437,17 @@ public BlockingQueue getRePushTransactions() { public void stopRePushThread() { isRunRePushThread = false; + ExecutorServiceManager.shutdownAndAwaitTermination(rePushEs, rePushEsName); } public void stopRePushTriggerThread() { isRunTriggerCapsuleProcessThread = false; + ExecutorServiceManager.shutdownAndAwaitTermination(triggerEs, triggerEsName); } public void stopFilterProcessThread() { isRunFilterProcessThread = false; + ExecutorServiceManager.shutdownAndAwaitTermination(filterEs, filterEsName); } @PostConstruct @@ -524,21 +535,19 @@ public void init() { revokingStore.enable(); validateSignService = Executors .newFixedThreadPool(Args.getInstance().getValidateSignThreadNum()); - Thread rePushThread = new Thread(rePushLoop); - rePushThread.setDaemon(true); - rePushThread.start(); + rePushEs = ExecutorServiceManager.newSingleThreadExecutor(rePushEsName, true); + rePushEs.submit(rePushLoop); // add contract event listener for subscribing if (Args.getInstance().isEventSubscribe()) { startEventSubscribing(); - Thread triggerCapsuleProcessThread = new Thread(triggerCapsuleProcessLoop); - triggerCapsuleProcessThread.setDaemon(true); - triggerCapsuleProcessThread.start(); + triggerEs = ExecutorServiceManager.newSingleThreadExecutor(triggerEsName, true); + triggerEs.submit(triggerCapsuleProcessLoop); } // start json rpc filter process if (CommonParameter.getInstance().isJsonRpcFilterEnabled()) { - Thread filterProcessThread = new Thread(filterProcessLoop); - filterProcessThread.start(); + filterEs = ExecutorServiceManager.newSingleThreadExecutor(filterEsName); + filterEs.submit(filterProcessLoop); } //initStoreFactory diff --git a/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java b/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java index 457fe7c55cb..2445a7d64e5 100644 --- a/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java +++ b/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java @@ -4,12 +4,12 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.List; -import java.util.Objects; -import java.util.Timer; -import java.util.TimerTask; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; import org.tron.common.utils.JsonUtil; @@ -27,28 +27,22 @@ public class NodePersistService { private final boolean isNodePersist = CommonParameter.getInstance().isNodeDiscoveryPersist(); @Autowired private CommonStore commonStore; - private Timer nodePersistTaskTimer; + + private ScheduledExecutorService nodePersistExecutor; + + private final String name = "NodePersistTask"; public void init() { if (isNodePersist) { - nodePersistTaskTimer = new Timer("NodePersistTaskTimer"); - nodePersistTaskTimer.scheduleAtFixedRate(new TimerTask() { - @Override - public void run() { - dbWrite(); - } - }, DB_COMMIT_RATE, DB_COMMIT_RATE); + nodePersistExecutor = ExecutorServiceManager.newSingleThreadScheduledExecutor(name); + nodePersistExecutor.scheduleAtFixedRate(this::dbWrite, DB_COMMIT_RATE, DB_COMMIT_RATE, + TimeUnit.MILLISECONDS); } } public void close() { - if (Objects.isNull(nodePersistTaskTimer)) { - return; - } - try { - nodePersistTaskTimer.cancel(); - } catch (Exception e) { - logger.error("Close nodePersistTaskTimer failed", e); + if (isNodePersist) { + ExecutorServiceManager.shutdownAndAwaitTermination(nodePersistExecutor, name); } } diff --git a/framework/src/test/java/org/tron/common/backup/BackupServerTest.java b/framework/src/test/java/org/tron/common/backup/BackupServerTest.java new file mode 100644 index 00000000000..34b17ec186f --- /dev/null +++ b/framework/src/test/java/org/tron/common/backup/BackupServerTest.java @@ -0,0 +1,44 @@ +package org.tron.common.backup; + +import java.util.ArrayList; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.tron.common.backup.socket.BackupServer; +import org.tron.common.parameter.CommonParameter; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + + +public class BackupServerTest { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private BackupServer backupServer; + + @Before + public void setUp() throws Exception { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); + CommonParameter.getInstance().setBackupPort(80); + List members = new ArrayList<>(); + members.add("127.0.0.2"); + CommonParameter.getInstance().setBackupMembers(members); + BackupManager backupManager = new BackupManager(); + backupManager.init(); + backupServer = new BackupServer(backupManager); + } + + @After + public void tearDown() { + backupServer.close(); + Args.clearParam(); + } + + @Test + public void test() throws InterruptedException { + backupServer.initServer(); + } +} From 83151aaf4d66911533347b6250bb135a6c0883a9 Mon Sep 17 00:00:00 2001 From: chengtx01 <56535423+chengtx01@users.noreply.github.com> Date: Thu, 17 Aug 2023 19:09:51 +0800 Subject: [PATCH 3/8] feat(net):improve chain inventory generating logic (#5393) --- .../SyncBlockChainMsgHandler.java | 20 +++++++++++++++++-- .../SyncBlockChainMsgHandlerTest.java | 16 +++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java index 9027034ccc7..3c83bc51e47 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java @@ -87,6 +87,18 @@ private boolean check(PeerConnection peer, SyncBlockChainMessage msg) throws P2p private LinkedList getLostBlockIds(List blockIds) throws P2pException { + BlockId unForkId = getUnForkId(blockIds); + LinkedList ids = getBlockIds(unForkId.getNum()); + + if (ids.isEmpty() || !unForkId.equals(ids.peekFirst())) { + unForkId = getUnForkId(blockIds); + ids = getBlockIds(unForkId.getNum()); + } + + return ids; + } + + private BlockId getUnForkId(List blockIds) throws P2pException { BlockId unForkId = null; for (int i = blockIds.size() - 1; i >= 0; i--) { if (tronNetDelegate.containBlockInMainChain(blockIds.get(i))) { @@ -99,13 +111,17 @@ private LinkedList getLostBlockIds(List blockIds) throws P2pEx throw new P2pException(TypeEnum.SYNC_FAILED, "unForkId is null"); } + return unForkId; + } + + private LinkedList getBlockIds(Long unForkNum) throws P2pException { BlockId headID = tronNetDelegate.getHeadBlockId(); long headNum = headID.getNum(); - long len = Math.min(headNum, unForkId.getNum() + NetConstants.SYNC_FETCH_BATCH_NUM); + long len = Math.min(headNum, unForkNum + NetConstants.SYNC_FETCH_BATCH_NUM); LinkedList ids = new LinkedList<>(); - for (long i = unForkId.getNum(); i <= len; i++) { + for (long i = unForkNum; i <= len; i++) { if (i == headNum) { ids.add(headID); } else { diff --git a/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java b/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java index 2dbad09c655..53375ec7815 100644 --- a/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java +++ b/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java @@ -2,6 +2,7 @@ import java.io.File; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -64,6 +65,21 @@ public void testProcessMessage() throws Exception { method.setAccessible(true); boolean f = (boolean)method.invoke(handler, peer, message); Assert.assertTrue(!f); + + Method method1 = handler.getClass().getDeclaredMethod( + "getLostBlockIds", List.class); + method1.setAccessible(true); + try { + method1.invoke(handler, blockIds); + } catch (InvocationTargetException e) { + Assert.assertEquals("unForkId is null", e.getTargetException().getMessage()); + } + + Method method2 = handler.getClass().getDeclaredMethod( + "getBlockIds", Long.class); + method2.setAccessible(true); + List list = (List) method2.invoke(handler, 0L); + Assert.assertEquals(1, list.size()); } @After From f18864c710f373fc6069ca0ca70dee026c02896a Mon Sep 17 00:00:00 2001 From: Kayle Date: Fri, 18 Aug 2023 13:56:07 +0800 Subject: [PATCH 4/8] feat(http): optimize getAddress method (#5367) --- framework/build.gradle | 1 + .../org/tron/core/services/http/Util.java | 59 +++++++-- .../http/GetBrokerageServletTest.java | 91 +++++++++++++ .../services/http/GetRewardServletTest.java | 120 ++++++++++++++++++ 4 files changed, 260 insertions(+), 11 deletions(-) create mode 100644 framework/src/test/java/org/tron/core/services/http/GetBrokerageServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/GetRewardServletTest.java diff --git a/framework/build.gradle b/framework/build.gradle index 063d320d477..9bc93cacf33 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -86,6 +86,7 @@ dependencies { compile group: 'org.pf4j', name: 'pf4j', version: '2.5.0' testImplementation group: 'org.springframework', name: 'spring-test', version: '5.2.0.RELEASE' + testImplementation group: 'org.springframework', name: 'spring-web', version: '5.2.0.RELEASE' compile group: 'org.zeromq', name: 'jeromq', version: '0.5.3' compile project(":chainbase") diff --git a/framework/src/main/java/org/tron/core/services/http/Util.java b/framework/src/main/java/org/tron/core/services/http/Util.java index 2cbf5d2e718..06c4f8ef26a 100644 --- a/framework/src/main/java/org/tron/core/services/http/Util.java +++ b/framework/src/main/java/org/tron/core/services/http/Util.java @@ -11,19 +11,22 @@ import com.google.protobuf.GeneratedMessageV3; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; + +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; import java.lang.reflect.Constructor; import java.math.BigDecimal; import java.nio.charset.Charset; import java.security.InvalidParameterException; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.encoders.Hex; +import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.util.StringUtil; import org.tron.api.GrpcAPI; import org.tron.api.GrpcAPI.BlockList; @@ -70,6 +73,8 @@ public class Util { public static final String FUNCTION_SELECTOR = "function_selector"; public static final String FUNCTION_PARAMETER = "parameter"; public static final String CALL_DATA = "data"; + public static final String APPLICATION_FORM_URLENCODED = "application/x-www-form-urlencoded"; + public static final String APPLICATION_JSON = "application/json"; public static String printTransactionFee(String transactionFee) { JSONObject jsonObject = new JSONObject(); @@ -498,16 +503,7 @@ public static void printAccount(Account reply, HttpServletResponse response, Boo public static byte[] getAddress(HttpServletRequest request) throws Exception { byte[] address = null; String addressParam = "address"; - String addressStr = request.getParameter(addressParam); - if (StringUtils.isBlank(addressStr)) { - String input = request.getReader().lines() - .collect(Collectors.joining(System.lineSeparator())); - Util.checkBodySize(input); - JSONObject jsonObject = JSON.parseObject(input); - if (jsonObject != null) { - addressStr = jsonObject.getString(addressParam); - } - } + String addressStr = checkGetParam(request, addressParam); if (StringUtils.isNotBlank(addressStr)) { if (StringUtils.startsWith(addressStr, Constant.ADD_PRE_FIX_STRING_MAINNET)) { address = Hex.decode(addressStr); @@ -518,6 +514,47 @@ public static byte[] getAddress(HttpServletRequest request) throws Exception { return address; } + private static String checkGetParam(HttpServletRequest request, String key) throws Exception { + String method = request.getMethod(); + String value = null; + + if (HttpMethod.GET.toString().toUpperCase().equalsIgnoreCase(method)) { + return request.getParameter(key); + } + if (HttpMethod.POST.toString().toUpperCase().equals(method)) { + String contentType = request.getContentType(); + if (StringUtils.isBlank(contentType)) { + return null; + } + if (APPLICATION_JSON.toLowerCase().contains(contentType)) { + value = getRequestValue(request); + if (StringUtils.isBlank(value)) { + return null; + } + + JSONObject jsonObject = JSON.parseObject(value); + if (jsonObject != null) { + return jsonObject.getString(key); + } + } else if (APPLICATION_FORM_URLENCODED.toLowerCase().contains(contentType)) { + return request.getParameter(key); + } else { + return null; + } + } + return value; + } + + public static String getRequestValue(HttpServletRequest request) throws IOException { + BufferedReader reader = new BufferedReader(new InputStreamReader(request.getInputStream())); + String line; + StringBuilder sb = new StringBuilder(); + while ((line = reader.readLine()) != null) { + sb.append(line); + } + return sb.toString(); + } + public static List convertLogAddressToTronAddress(TransactionInfo transactionInfo) { List newLogList = new ArrayList<>(); diff --git a/framework/src/test/java/org/tron/core/services/http/GetBrokerageServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetBrokerageServletTest.java new file mode 100644 index 00000000000..898108d0dcd --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/GetBrokerageServletTest.java @@ -0,0 +1,91 @@ +package org.tron.core.services.http; + +import com.alibaba.fastjson.JSONObject; + +import java.io.UnsupportedEncodingException; +import javax.annotation.Resource; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class GetBrokerageServletTest extends BaseTest { + + @Resource + private GetBrokerageServlet getBrokerageServlet; + + static { + dbPath = "db_GetBrokerageServlet_test"; + Args.setParam( + new String[]{ + "--output-directory", dbPath, + }, Constant.TEST_CONF + ); + } + + public MockHttpServletRequest createRequest(String contentType) { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("POST"); + request.setContentType(contentType); + request.setCharacterEncoding("UTF-8"); + return request; + } + + @Test + public void getBrokerageValueByJsonTest() { + int expect = 20; + String jsonParam = "{\"address\": \"27VZHn9PFZwNh7o2EporxmLkpe157iWZVkh\"}"; + MockHttpServletRequest request = createRequest("application/json"); + request.setContent(jsonParam.getBytes()); + MockHttpServletResponse response = new MockHttpServletResponse(); + getBrokerageServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + int brokerage = (int)result.get("brokerage"); + Assert.assertEquals(expect, brokerage); + } catch (UnsupportedEncodingException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void getBrokerageValueTest() { + int expect = 20; + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + request.addParameter("address", "27VZHn9PFZwNh7o2EporxmLkpe157iWZVkh"); + MockHttpServletResponse response = new MockHttpServletResponse(); + getBrokerageServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + int brokerage = (int)result.get("brokerage"); + Assert.assertEquals(expect, brokerage); + } catch (UnsupportedEncodingException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void getByBlankParamTest() { + int expect = 0; + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + request.addParameter("address", ""); + MockHttpServletResponse response = new MockHttpServletResponse(); + getBrokerageServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + int brokerage = (int)result.get("brokerage"); + Assert.assertEquals(expect, brokerage); + String content = (String) result.get("Error"); + Assert.assertNull(content); + } catch (UnsupportedEncodingException e) { + Assert.fail(e.getMessage()); + } + } +} diff --git a/framework/src/test/java/org/tron/core/services/http/GetRewardServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetRewardServletTest.java new file mode 100644 index 00000000000..47e563f01f9 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/GetRewardServletTest.java @@ -0,0 +1,120 @@ +package org.tron.core.services.http; + +import static org.tron.common.utils.Commons.decodeFromBase58Check; + +import com.alibaba.fastjson.JSONObject; + +import java.io.File; +import java.io.UnsupportedEncodingException; +import javax.annotation.Resource; + +import org.apache.commons.lang3.StringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.common.utils.FileUtil; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; +import org.tron.core.db.Manager; +import org.tron.core.service.MortgageService; +import org.tron.core.store.DelegationStore; + +public class GetRewardServletTest extends BaseTest { + + @Resource + private Manager manager; + + @Resource + private MortgageService mortgageService; + + @Resource + private DelegationStore delegationStore; + + @Resource + GetRewardServlet getRewardServlet; + + static { + dbPath = "db_GetRewardServlet_test"; + Args.setParam( + new String[]{ + "--output-directory", dbPath, + }, Constant.TEST_CONF + ); + } + + public MockHttpServletRequest createRequest(String contentType) { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("POST"); + request.setContentType(contentType); + request.setCharacterEncoding("UTF-8"); + return request; + } + + public void init() { + manager.getDynamicPropertiesStore().saveChangeDelegation(1); + byte[] sr = decodeFromBase58Check("27VZHn9PFZwNh7o2EporxmLkpe157iWZVkh"); + delegationStore.setBrokerage(0, sr, 10); + delegationStore.setWitnessVote(0, sr, 100000000); + } + + @Test + public void getRewardValueByJsonTest() { + init(); + int expect = 138181; + String jsonParam = "{\"address\": \"27VZHn9PFZwNh7o2EporxmLkpe157iWZVkh\"}"; + MockHttpServletRequest request = createRequest("application/json"); + MockHttpServletResponse response = new MockHttpServletResponse(); + request.setContent(jsonParam.getBytes()); + try { + getRewardServlet.doPost(request, response); + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + int reward = (int)result.get("reward"); + Assert.assertEquals(expect, reward); + } catch (UnsupportedEncodingException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void getRewardValueTest() { + init(); + int expect = 138181; + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + MockHttpServletResponse response = new MockHttpServletResponse(); + mortgageService.payStandbyWitness(); + request.addParameter("address", "27VZHn9PFZwNh7o2EporxmLkpe157iWZVkh"); + getRewardServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + int reward = (int)result.get("reward"); + Assert.assertEquals(expect, reward); + } catch (UnsupportedEncodingException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void getByBlankParamTest() { + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + MockHttpServletResponse response = new MockHttpServletResponse(); + request.addParameter("address", ""); + GetRewardServlet getRewardServlet = new GetRewardServlet(); + getRewardServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + int reward = (int)result.get("reward"); + Assert.assertEquals(0, reward); + String content = (String) result.get("Error"); + Assert.assertNull(content); + } catch (UnsupportedEncodingException e) { + Assert.fail(e.getMessage()); + } + } + +} From f20e11b893f5735ae5256c92bf9793224387c9e2 Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Mon, 21 Aug 2023 18:53:22 +0800 Subject: [PATCH 5/8] feat(db): optimize for bloomFilter initialization (#5394) --- .../java/org/tron/core/ChainBaseManager.java | 5 + .../org/tron/core/db/TransactionCache.java | 0 .../org/tron/core/db2/common/TxCacheDB.java | 149 +++++++++++++++++- .../org/tron/core/db/TxCacheDBInitTest.java | 90 +++++++++++ 4 files changed, 241 insertions(+), 3 deletions(-) rename {framework => chainbase}/src/main/java/org/tron/core/db/TransactionCache.java (100%) create mode 100644 framework/src/test/java/org/tron/core/db/TxCacheDBInitTest.java diff --git a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java index dad76041777..939d95dc584 100644 --- a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java +++ b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java @@ -28,6 +28,7 @@ import org.tron.core.db.PbftSignDataStore; import org.tron.core.db.RecentBlockStore; import org.tron.core.db.RecentTransactionStore; +import org.tron.core.db.TransactionCache; import org.tron.core.db.TransactionStore; import org.tron.core.db2.core.ITronChainBase; import org.tron.core.exception.BadItemException; @@ -237,6 +238,9 @@ public class ChainBaseManager { @Autowired private DbStatService dbStatService; + @Autowired + private TransactionCache transactionCache; + @Getter @Setter private NodeType nodeType; @@ -291,6 +295,7 @@ public void closeAllStore() { closeOneStore(pbftSignDataStore); closeOneStore(sectionBloomStore); closeOneStore(accountAssetStore); + closeOneStore(transactionCache); } // for test only diff --git a/framework/src/main/java/org/tron/core/db/TransactionCache.java b/chainbase/src/main/java/org/tron/core/db/TransactionCache.java similarity index 100% rename from framework/src/main/java/org/tron/core/db/TransactionCache.java rename to chainbase/src/main/java/org/tron/core/db/TransactionCache.java diff --git a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java index 71cf361b06a..9923a876cad 100644 --- a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java +++ b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java @@ -3,10 +3,24 @@ import com.google.common.hash.BloomFilter; import com.google.common.hash.Funnels; import com.google.common.primitives.Longs; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Reader; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; +import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; import org.bouncycastle.util.encoders.Hex; @@ -17,6 +31,7 @@ import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; import org.tron.common.utils.ByteArray; +import org.tron.common.utils.FileUtil; import org.tron.common.utils.JsonUtil; import org.tron.common.utils.StorageUtils; import org.tron.core.capsule.BytesCapsule; @@ -42,6 +57,7 @@ public class TxCacheDB implements DB, Flusher { private BloomFilter[] bloomFilters = new BloomFilter[2]; // filterStartBlock record the start block of the active filter private volatile long filterStartBlock = INVALID_BLOCK; + private volatile long currentBlockNum = INVALID_BLOCK; // currentFilterIndex records the index of the active filter private volatile int currentFilterIndex = 0; @@ -57,6 +73,12 @@ public class TxCacheDB implements DB, Flusher { // replace persistentStore and optimizes startup performance private RecentTransactionStore recentTransactionStore; + private final Path cacheFile0; + private final Path cacheFile1; + private final Path cacheProperties; + private final Path cacheDir; + private AtomicBoolean isValid = new AtomicBoolean(false); + public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) { this.name = name; this.TRANSACTION_COUNT = @@ -85,6 +107,10 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) { MAX_BLOCK_SIZE * TRANSACTION_COUNT); this.bloomFilters[1] = BloomFilter.create(Funnels.byteArrayFunnel(), MAX_BLOCK_SIZE * TRANSACTION_COUNT); + cacheDir = Paths.get(CommonParameter.getInstance().getOutputDirectory(), ".cache"); + this.cacheFile0 = Paths.get(cacheDir.toString(), "bloomFilters_0"); + this.cacheFile1 = Paths.get(cacheDir.toString(), "bloomFilters_1"); + this.cacheProperties = Paths.get(cacheDir.toString(), "txCache.properties"); } @@ -110,6 +136,10 @@ private void initCache() { } public void init() { + if (recovery()) { + isValid.set(true); + return; + } long size = recentTransactionStore.size(); if (size != MAX_BLOCK_SIZE) { // 0. load from persistentStore @@ -129,6 +159,7 @@ public void init() { logger.info("Load cache from recentTransactionStore, filter: {}, filter-fpp: {}, cost: {} ms.", bloomFilters[1].approximateElementCount(), bloomFilters[1].expectedFpp(), System.currentTimeMillis() - start); + isValid.set(true); } @Override @@ -172,7 +203,7 @@ public void put(byte[] key, byte[] value) { MAX_BLOCK_SIZE * TRANSACTION_COUNT); } bloomFilters[currentFilterIndex].put(key); - + currentBlockNum = blockNum; if (lastMetricBlock != blockNum) { lastMetricBlock = blockNum; Metrics.gaugeSet(MetricKeys.Gauge.TX_CACHE, @@ -208,13 +239,15 @@ public Iterator> iterator() { } @Override - public void flush(Map batch) { + public synchronized void flush(Map batch) { + isValid.set(false); batch.forEach((k, v) -> this.put(k.getBytes(), v.getBytes())); + isValid.set(true); } @Override public void close() { - reset(); + dump(); bloomFilters[0] = null; bloomFilters[1] = null; persistentStore.close(); @@ -224,6 +257,116 @@ public void close() { public void reset() { } + private boolean recovery() { + FileUtil.createDirIfNotExists(this.cacheDir.toString()); + logger.info("recovery bloomFilters start."); + CompletableFuture loadProperties = CompletableFuture.supplyAsync(this::loadProperties); + CompletableFuture tk0 = loadProperties.thenApplyAsync( + v -> recovery(0, this.cacheFile0)); + CompletableFuture tk1 = loadProperties.thenApplyAsync( + v -> recovery(1, this.cacheFile1)); + + return CompletableFuture.allOf(tk0, tk1).thenApply(v -> { + logger.info("recovery bloomFilters success."); + return true; + }).exceptionally(this::handleException).join(); + } + + private boolean recovery(int index, Path file) { + try (InputStream in = new BufferedInputStream(Files.newInputStream(file, + StandardOpenOption.READ, StandardOpenOption.DELETE_ON_CLOSE))) { + logger.info("recovery bloomFilter[{}] from file.", index); + long start = System.currentTimeMillis(); + bloomFilters[index] = BloomFilter.readFrom(in, Funnels.byteArrayFunnel()); + logger.info("recovery bloomFilter[{}] from file done,filter: {}, filter-fpp: {}, cost {} ms.", + index, bloomFilters[index].approximateElementCount(), bloomFilters[index].expectedFpp(), + System.currentTimeMillis() - start); + return true; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private boolean handleException(Throwable e) { + bloomFilters[0] = BloomFilter.create(Funnels.byteArrayFunnel(), + MAX_BLOCK_SIZE * TRANSACTION_COUNT); + bloomFilters[1] = BloomFilter.create(Funnels.byteArrayFunnel(), + MAX_BLOCK_SIZE * TRANSACTION_COUNT); + try { + Files.deleteIfExists(this.cacheFile0); + Files.deleteIfExists(this.cacheFile1); + } catch (Exception ignored) { + + } + logger.info("recovery bloomFilters failed. {}", e.getMessage()); + logger.info("rollback to previous mode."); + return false; + } + + private void dump() { + if (!isValid.get()) { + logger.info("bloomFilters is not valid."); + } + FileUtil.createDirIfNotExists(this.cacheDir.toString()); + logger.info("dump bloomFilters start."); + CompletableFuture task0 = CompletableFuture.runAsync( + () -> dump(0, this.cacheFile0)); + CompletableFuture task1 = CompletableFuture.runAsync( + () -> dump(1, this.cacheFile1)); + CompletableFuture.allOf(task0, task1).thenRun(() -> { + writeProperties(); + logger.info("dump bloomFilters done."); + + }).exceptionally(e -> { + logger.info("dump bloomFilters to file failed. {}", e.getMessage()); + return null; + }).join(); + } + + private void dump(int index, Path file) { + try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(file))) { + logger.info("dump bloomFilters[{}] to file.", index); + long start = System.currentTimeMillis(); + bloomFilters[index].writeTo(out); + logger.info("dump bloomFilters[{}] to file done,filter: {}, filter-fpp: {}, cost {} ms.", + index, bloomFilters[index].approximateElementCount(), bloomFilters[index].expectedFpp(), + System.currentTimeMillis() - start); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private boolean loadProperties() { + try (Reader r = new InputStreamReader(new BufferedInputStream(Files.newInputStream( + this.cacheProperties, StandardOpenOption.READ, StandardOpenOption.DELETE_ON_CLOSE)), + StandardCharsets.UTF_8)) { + Properties properties = new Properties(); + properties.load(r); + filterStartBlock = Long.parseLong(properties.getProperty("filterStartBlock")); + currentBlockNum = Long.parseLong(properties.getProperty("currentBlockNum")); + currentFilterIndex = Integer.parseInt(properties.getProperty("currentFilterIndex")); + logger.info("filterStartBlock: {}, currentBlockNum: {}, currentFilterIndex: {}, load done.", + filterStartBlock, currentBlockNum, currentFilterIndex); + return true; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void writeProperties() { + try (Writer w = Files.newBufferedWriter(this.cacheProperties, StandardCharsets.UTF_8)) { + Properties properties = new Properties(); + properties.setProperty("filterStartBlock", String.valueOf(filterStartBlock)); + properties.setProperty("currentBlockNum", String.valueOf(currentBlockNum)); + properties.setProperty("currentFilterIndex", String.valueOf(currentFilterIndex)); + properties.store(w, "Generated by the application. PLEASE DO NOT EDIT! "); + logger.info("filterStartBlock: {}, currentBlockNum: {}, currentFilterIndex: {}, write done.", + filterStartBlock, currentBlockNum, currentFilterIndex); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @Override public TxCacheDB newInstance() { return new TxCacheDB(name, recentTransactionStore); diff --git a/framework/src/test/java/org/tron/core/db/TxCacheDBInitTest.java b/framework/src/test/java/org/tron/core/db/TxCacheDBInitTest.java new file mode 100644 index 00000000000..c3cb7cb2eb6 --- /dev/null +++ b/framework/src/test/java/org/tron/core/db/TxCacheDBInitTest.java @@ -0,0 +1,90 @@ +package org.tron.core.db; + +import java.io.IOException; +import lombok.extern.slf4j.Slf4j; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.springframework.beans.factory.support.DefaultListableBeanFactory; +import org.tron.common.application.TronApplicationContext; +import org.tron.common.utils.ByteArray; +import org.tron.core.Constant; +import org.tron.core.capsule.BytesCapsule; +import org.tron.core.config.DefaultConfig; +import org.tron.core.config.args.Args; +import org.tron.keystore.Wallet; + +@Slf4j +public class TxCacheDBInitTest { + + private static TronApplicationContext context; + + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private final byte[][] hash = new byte[140000][64]; + + @AfterClass + public static void destroy() { + context.destroy(); + Args.clearParam(); + } + + /** + * Init data. + */ + @BeforeClass + public static void init() throws IOException { + Args.setParam(new String[]{"--output-directory", temporaryFolder.newFolder().toString(), + "--p2p-disable", "true"}, Constant.TEST_CONF); + context = new TronApplicationContext(DefaultConfig.class); + } + + @Test + public void reload() { + TransactionCache db = context.getBean(TransactionCache.class); + db.initCache(); + putTransaction(); + DefaultListableBeanFactory defaultListableBeanFactory = + (DefaultListableBeanFactory) context.getAutowireCapableBeanFactory(); + queryTransaction(); + defaultListableBeanFactory.destroySingleton("transactionCache"); + TransactionCache transactionCache = new TransactionCache("transactionCache", + context.getBean(RecentTransactionStore.class)); + transactionCache.initCache(); + defaultListableBeanFactory.registerSingleton("transactionCache",transactionCache); + queryTransaction(); + } + + private void putTransaction() { + TransactionCache db = context.getBean(TransactionCache.class); + for (int i = 1; i < 140000; i++) { + hash[i] = Wallet.generateRandomBytes(64); + db.put(hash[i], new BytesCapsule(ByteArray.fromLong(i))); + } + } + + private void queryTransaction() { + TransactionCache db = context.getBean(TransactionCache.class); + // [1,65537] are expired + for (int i = 1; i < 65538; i++) { + try { + Assert.assertFalse("index = " + i, db.has(hash[i])); + } catch (Exception e) { + Assert.fail("transaction should be expired index = " + i); + } + } + // [65538,140000] are in cache + for (int i = 65538; i < 140000; i++) { + try { + Assert.assertTrue("index = " + i, db.has(hash[i])); + } catch (Exception e) { + Assert.fail("transaction should not be expired index = " + i); + } + } + } + +} \ No newline at end of file From ac90c45be4aa5f177184c742ab2b01f963ca6e8e Mon Sep 17 00:00:00 2001 From: lxcmyf Date: Tue, 22 Aug 2023 15:48:58 +0800 Subject: [PATCH 6/8] feat(api): optimize compatibility (#5408) --- .../tron/core/capsule/TransactionCapsule.java | 4 + .../http/GetAssetIssueListServlet.java | 33 +++++-- .../GetAvailableUnfreezeCountServlet.java | 3 + .../services/http/GetBlockByNumServlet.java | 9 +- .../http/GetCanDelegatedMaxSizeServlet.java | 8 +- .../GetCanWithdrawUnfreezeAmountServlet.java | 14 ++- .../services/http/GetNowBlockServlet.java | 36 ++++++-- .../core/services/http/ListNodesServlet.java | 33 +++++-- .../services/http/ListProposalsServlet.java | 33 +++++-- .../tron/core/services/http/PostParams.java | 6 ++ .../org/tron/core/services/http/Util.java | 30 ++++++- .../http/GetAssetIssueListServletTest.java | 83 ++++++++++++++++++ .../services/http/GetNowBlockServletTest.java | 86 +++++++++++++++++++ .../services/http/ListNodesServletTest.java | 83 ++++++++++++++++++ .../http/ListProposalsServletTest.java | 83 ++++++++++++++++++ 15 files changed, 504 insertions(+), 40 deletions(-) create mode 100644 framework/src/test/java/org/tron/core/services/http/GetAssetIssueListServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/GetNowBlockServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/ListNodesServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/ListProposalsServletTest.java diff --git a/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java b/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java index a33f445c15f..1edb5b114fb 100755 --- a/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java +++ b/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java @@ -346,6 +346,10 @@ public static byte[] getOwner(Transaction.Contract contract) { } } return owner.toByteArray(); + } catch (InvalidProtocolBufferException invalidProtocolBufferException) { + logger.warn("InvalidProtocolBufferException occurred because {}, please verify the interface " + + "input parameters", invalidProtocolBufferException.getMessage()); + return new byte[0]; } catch (Exception ex) { logger.error(ex.getMessage()); return new byte[0]; diff --git a/framework/src/main/java/org/tron/core/services/http/GetAssetIssueListServlet.java b/framework/src/main/java/org/tron/core/services/http/GetAssetIssueListServlet.java index 7757ecd8722..01ec00dbef3 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetAssetIssueListServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetAssetIssueListServlet.java @@ -1,5 +1,8 @@ package org.tron.core.services.http; +import static org.tron.core.services.http.Util.existVisible; + +import java.io.IOException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; @@ -8,7 +11,6 @@ import org.tron.api.GrpcAPI.AssetIssueList; import org.tron.core.Wallet; - @Component @Slf4j(topic = "API") public class GetAssetIssueListServlet extends RateLimiterServlet { @@ -16,21 +18,36 @@ public class GetAssetIssueListServlet extends RateLimiterServlet { @Autowired private Wallet wallet; + @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) { try { boolean visible = Util.getVisible(request); - AssetIssueList reply = wallet.getAssetIssueList(); - if (reply != null) { - response.getWriter().println(JsonFormat.printToString(reply, visible)); - } else { - response.getWriter().println("{}"); - } + response(response, visible); } catch (Exception e) { Util.processError(e, response); } } + @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) { - doGet(request, response); + try { + PostParams params = PostParams.getPostParams(request); + boolean visible = Util.getVisible(request); + if (!existVisible(request)) { + visible = params.isVisible(); + } + response(response, visible); + } catch (Exception e) { + Util.processError(e, response); + } + } + + private void response(HttpServletResponse response, boolean visible) throws IOException { + AssetIssueList reply = wallet.getAssetIssueList(); + if (reply != null) { + response.getWriter().println(JsonFormat.printToString(reply, visible)); + } else { + response.getWriter().println("{}"); + } } } diff --git a/framework/src/main/java/org/tron/core/services/http/GetAvailableUnfreezeCountServlet.java b/framework/src/main/java/org/tron/core/services/http/GetAvailableUnfreezeCountServlet.java index 28fec9f3ee3..51f78fc4390 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetAvailableUnfreezeCountServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetAvailableUnfreezeCountServlet.java @@ -23,6 +23,9 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { try { boolean visible = Util.getVisible(request); String ownerAddress = request.getParameter("ownerAddress"); + if (ownerAddress == null) { + ownerAddress = request.getParameter("owner_address"); + } if (visible) { ownerAddress = Util.getHexAddress(ownerAddress); } diff --git a/framework/src/main/java/org/tron/core/services/http/GetBlockByNumServlet.java b/framework/src/main/java/org/tron/core/services/http/GetBlockByNumServlet.java index 44babc69551..800b421ace0 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetBlockByNumServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetBlockByNumServlet.java @@ -18,14 +18,21 @@ public class GetBlockByNumServlet extends RateLimiterServlet { @Autowired private Wallet wallet; + @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) { try { - fillResponse(Util.getVisible(request), Long.parseLong(request.getParameter("num")), response); + long num = 0; + String numStr = request.getParameter("num"); + if (numStr != null) { + num = Long.parseLong(numStr); + } + fillResponse(Util.getVisible(request), num, response); } catch (Exception e) { Util.processError(e, response); } } + @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) { try { PostParams params = PostParams.getPostParams(request); diff --git a/framework/src/main/java/org/tron/core/services/http/GetCanDelegatedMaxSizeServlet.java b/framework/src/main/java/org/tron/core/services/http/GetCanDelegatedMaxSizeServlet.java index 81dd52400b5..924306a6a3f 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetCanDelegatedMaxSizeServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetCanDelegatedMaxSizeServlet.java @@ -18,10 +18,15 @@ public class GetCanDelegatedMaxSizeServlet extends RateLimiterServlet { @Autowired private Wallet wallet; + @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) { try { boolean visible = Util.getVisible(request); - int type = Integer.parseInt(request.getParameter("type")); + int type = 0; + String typeStr = request.getParameter("type"); + if (typeStr != null) { + type = Integer.parseInt(typeStr); + } String ownerAddress = request.getParameter("owner_address"); if (visible) { ownerAddress = Util.getHexAddress(ownerAddress); @@ -34,6 +39,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { } } + @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) { try { PostParams params = PostParams.getPostParams(request); diff --git a/framework/src/main/java/org/tron/core/services/http/GetCanWithdrawUnfreezeAmountServlet.java b/framework/src/main/java/org/tron/core/services/http/GetCanWithdrawUnfreezeAmountServlet.java index 4c1bbaf4f99..435cca9e5fb 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetCanWithdrawUnfreezeAmountServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetCanWithdrawUnfreezeAmountServlet.java @@ -18,23 +18,29 @@ public class GetCanWithdrawUnfreezeAmountServlet extends RateLimiterServlet { @Autowired private Wallet wallet; + @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) { try { boolean visible = Util.getVisible(request); String ownerAddress = request.getParameter("owner_address"); - long timestamp = Long.valueOf(request.getParameter("timestamp")); + long timestamp = 0; + String timestampStr = request.getParameter("timestamp"); + if (timestampStr != null) { + timestamp = Long.parseLong(timestampStr); + } if (visible) { ownerAddress = Util.getHexAddress(ownerAddress); } fillResponse(visible, - ByteString.copyFrom(ByteArray.fromHexString(ownerAddress)), - timestamp, - response); + ByteString.copyFrom(ByteArray.fromHexString(ownerAddress)), + timestamp, + response); } catch (Exception e) { Util.processError(e, response); } } + @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) { try { PostParams params = PostParams.getPostParams(request); diff --git a/framework/src/main/java/org/tron/core/services/http/GetNowBlockServlet.java b/framework/src/main/java/org/tron/core/services/http/GetNowBlockServlet.java index f7d9700aeb9..78e7178cfe3 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetNowBlockServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetNowBlockServlet.java @@ -1,5 +1,9 @@ package org.tron.core.services.http; +import static org.tron.core.services.http.Util.existVisible; +import static org.tron.core.services.http.Util.getVisible; + +import java.io.IOException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; @@ -8,7 +12,6 @@ import org.tron.core.Wallet; import org.tron.protos.Protocol.Block; - @Component @Slf4j(topic = "API") public class GetNowBlockServlet extends RateLimiterServlet { @@ -16,21 +19,36 @@ public class GetNowBlockServlet extends RateLimiterServlet { @Autowired private Wallet wallet; + @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) { try { - boolean visible = Util.getVisible(request); - Block reply = wallet.getNowBlock(); - if (reply != null) { - response.getWriter().println(Util.printBlock(reply, visible)); - } else { - response.getWriter().println("{}"); - } + boolean visible = getVisible(request); + response(response, visible); } catch (Exception e) { Util.processError(e, response); } } + @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) { - doGet(request, response); + try { + PostParams params = PostParams.getPostParams(request); + boolean visible = getVisible(request); + if (!existVisible(request)) { + visible = params.isVisible(); + } + response(response, visible); + } catch (Exception e) { + Util.processError(e, response); + } + } + + private void response(HttpServletResponse response, boolean visible) throws IOException { + Block reply = wallet.getNowBlock(); + if (reply != null) { + response.getWriter().println(Util.printBlock(reply, visible)); + } else { + response.getWriter().println("{}"); + } } } \ No newline at end of file diff --git a/framework/src/main/java/org/tron/core/services/http/ListNodesServlet.java b/framework/src/main/java/org/tron/core/services/http/ListNodesServlet.java index 94fd568a3f4..fb0b877832b 100644 --- a/framework/src/main/java/org/tron/core/services/http/ListNodesServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/ListNodesServlet.java @@ -1,5 +1,8 @@ package org.tron.core.services.http; +import static org.tron.core.services.http.Util.existVisible; + +import java.io.IOException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; @@ -8,7 +11,6 @@ import org.tron.api.GrpcAPI.NodeList; import org.tron.core.Wallet; - @Component @Slf4j(topic = "API") public class ListNodesServlet extends RateLimiterServlet { @@ -16,21 +18,36 @@ public class ListNodesServlet extends RateLimiterServlet { @Autowired private Wallet wallet; + @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) { try { boolean visible = Util.getVisible(request); - NodeList reply = wallet.listNodes(); - if (reply != null) { - response.getWriter().println(JsonFormat.printToString(reply, visible)); - } else { - response.getWriter().println("{}"); - } + response(response, visible); } catch (Exception e) { Util.processError(e, response); } } + @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) { - doGet(request, response); + try { + PostParams params = PostParams.getPostParams(request); + boolean visible = Util.getVisible(request); + if (!existVisible(request)) { + visible = params.isVisible(); + } + response(response, visible); + } catch (Exception e) { + Util.processError(e, response); + } + } + + private void response(HttpServletResponse response, boolean visible) throws IOException { + NodeList reply = wallet.listNodes(); + if (reply != null) { + response.getWriter().println(JsonFormat.printToString(reply, visible)); + } else { + response.getWriter().println("{}"); + } } } diff --git a/framework/src/main/java/org/tron/core/services/http/ListProposalsServlet.java b/framework/src/main/java/org/tron/core/services/http/ListProposalsServlet.java index f8c67395b1b..e309963edda 100644 --- a/framework/src/main/java/org/tron/core/services/http/ListProposalsServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/ListProposalsServlet.java @@ -1,5 +1,8 @@ package org.tron.core.services.http; +import static org.tron.core.services.http.Util.existVisible; + +import java.io.IOException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; @@ -8,7 +11,6 @@ import org.tron.api.GrpcAPI.ProposalList; import org.tron.core.Wallet; - @Component @Slf4j(topic = "API") public class ListProposalsServlet extends RateLimiterServlet { @@ -16,21 +18,36 @@ public class ListProposalsServlet extends RateLimiterServlet { @Autowired private Wallet wallet; + @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) { try { boolean visible = Util.getVisible(request); - ProposalList reply = wallet.getProposalList(); - if (reply != null) { - response.getWriter().println(JsonFormat.printToString(reply, visible)); - } else { - response.getWriter().println("{}"); - } + response(response, visible); } catch (Exception e) { Util.processError(e, response); } } + @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) { - doGet(request, response); + try { + PostParams params = PostParams.getPostParams(request); + boolean visible = Util.getVisible(request); + if (!existVisible(request)) { + visible = params.isVisible(); + } + response(response, visible); + } catch (Exception e) { + Util.processError(e, response); + } + } + + private void response(HttpServletResponse response, boolean visible) throws IOException { + ProposalList reply = wallet.getProposalList(); + if (reply != null) { + response.getWriter().println(JsonFormat.printToString(reply, visible)); + } else { + response.getWriter().println("{}"); + } } } diff --git a/framework/src/main/java/org/tron/core/services/http/PostParams.java b/framework/src/main/java/org/tron/core/services/http/PostParams.java index a2077fb3c78..7dcb0be6ae3 100644 --- a/framework/src/main/java/org/tron/core/services/http/PostParams.java +++ b/framework/src/main/java/org/tron/core/services/http/PostParams.java @@ -1,5 +1,8 @@ package org.tron.core.services.http; +import static org.apache.http.entity.ContentType.APPLICATION_FORM_URLENCODED; +import static org.tron.core.services.http.Util.getJsonString; + import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; import lombok.Getter; @@ -21,6 +24,9 @@ public PostParams(String params, boolean visible) { public static PostParams getPostParams(HttpServletRequest request) throws Exception { String input = request.getReader().lines().collect(Collectors.joining(System.lineSeparator())); Util.checkBodySize(input); + if (APPLICATION_FORM_URLENCODED.getMimeType().equals(request.getContentType())) { + input = getJsonString(input); + } boolean visible = Util.getVisiblePost(input); return new PostParams(input, visible); } diff --git a/framework/src/main/java/org/tron/core/services/http/Util.java b/framework/src/main/java/org/tron/core/services/http/Util.java index 06c4f8ef26a..186f77cf3e8 100644 --- a/framework/src/main/java/org/tron/core/services/http/Util.java +++ b/framework/src/main/java/org/tron/core/services/http/Util.java @@ -1,5 +1,6 @@ package org.tron.core.services.http; +import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.tron.common.utils.Commons.decodeFromBase58Check; import com.alibaba.fastjson.JSON; @@ -21,13 +22,17 @@ import java.security.InvalidParameterException; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Objects; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.encoders.Hex; import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.util.MultiMap; import org.eclipse.jetty.util.StringUtil; +import org.eclipse.jetty.util.UrlEncoded; import org.tron.api.GrpcAPI; import org.tron.api.GrpcAPI.BlockList; import org.tron.api.GrpcAPI.TransactionApprovedList; @@ -338,12 +343,16 @@ public static boolean getVisible(final HttpServletRequest request) { return visible; } + public static boolean existVisible(final HttpServletRequest request) { + return Objects.nonNull(request.getParameter(VISIBLE)); + } + public static boolean getVisiblePost(final String input) { boolean visible = false; if (StringUtil.isNotBlank(input)) { JSONObject jsonObject = JSON.parseObject(input); if (jsonObject.containsKey(VISIBLE)) { - visible = jsonObject.getBoolean(VISIBLE); + visible = Boolean.parseBoolean(jsonObject.getString(VISIBLE)); } } @@ -606,4 +615,23 @@ public static void validateParameter(String contract) throws InvalidParameterExc } } + public static String getJsonString(String str) { + if (StringUtils.isEmpty(str)) { + return EMPTY; + } + MultiMap params = new MultiMap<>(); + UrlEncoded.decodeUtf8To(str, params); + JSONObject json = new JSONObject(); + for (Map.Entry> entry : params.entrySet()) { + String key = entry.getKey(); + List values = entry.getValue(); + if (values.size() == 1) { + json.put(key, values.get(0)); + } else { + json.put(key, values); + } + } + return json.toString(); + } + } diff --git a/framework/src/test/java/org/tron/core/services/http/GetAssetIssueListServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetAssetIssueListServletTest.java new file mode 100644 index 00000000000..141c349e252 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/GetAssetIssueListServletTest.java @@ -0,0 +1,83 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; + +import com.alibaba.fastjson.JSONObject; +import java.io.UnsupportedEncodingException; +import javax.annotation.Resource; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class GetAssetIssueListServletTest extends BaseTest { + + @Resource + private GetAssetIssueListServlet getAssetIssueListServlet; + + static { + dbPath = "db_GetNowBlockServlet_test"; + Args.setParam( + new String[]{ + "--output-directory", dbPath, + }, Constant.TEST_CONF + ); + } + + public MockHttpServletRequest createRequest(String contentType) { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("POST"); + request.setContentType(contentType); + request.setCharacterEncoding("UTF-8"); + return request; + } + + @Test + public void testGetAssetIssueListByJson() { + String jsonParam = "{\"visible\": true}"; + MockHttpServletRequest request = createRequest("application/json"); + request.setContent(jsonParam.getBytes()); + MockHttpServletResponse response = new MockHttpServletResponse(); + getAssetIssueListServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject.parseObject(contentAsString); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + + @Test + public void testGetAssetIssueListValue() { + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + try { + String params = "visible=true"; + request.setContent(params.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + getAssetIssueListServlet.doPost(request, response); + String contentAsString = response.getContentAsString(); + JSONObject.parseObject(contentAsString); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + + @Test + public void testGetAssetIssueListEmptyParam() { + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + String params = "visible="; + request.setContent(params.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + getAssetIssueListServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject.parseObject(contentAsString); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + +} diff --git a/framework/src/test/java/org/tron/core/services/http/GetNowBlockServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetNowBlockServletTest.java new file mode 100644 index 00000000000..682f780b249 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/GetNowBlockServletTest.java @@ -0,0 +1,86 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.alibaba.fastjson.JSONObject; +import java.io.UnsupportedEncodingException; +import javax.annotation.Resource; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class GetNowBlockServletTest extends BaseTest { + + @Resource + private GetNowBlockServlet getNowBlockServlet; + + static { + dbPath = "db_GetNowBlockServlet_test"; + Args.setParam( + new String[]{ + "--output-directory", dbPath, + }, Constant.TEST_CONF + ); + } + + public MockHttpServletRequest createRequest(String contentType) { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("POST"); + request.setContentType(contentType); + request.setCharacterEncoding("UTF-8"); + return request; + } + + @Test + public void testGetNowBlockByJson() { + String jsonParam = "{\"visible\": true}"; + MockHttpServletRequest request = createRequest("application/json"); + request.setContent(jsonParam.getBytes()); + MockHttpServletResponse response = new MockHttpServletResponse(); + getNowBlockServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + assertTrue(result.containsKey("blockID")); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + + @Test + public void testGetNowBlockValue() { + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + try { + String params = "visible=true"; + request.setContent(params.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + getNowBlockServlet.doPost(request, response); + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + assertTrue(result.containsKey("blockID")); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + + @Test + public void testGetNowBlockEmptyParam() { + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + String params = "visible="; + request.setContent(params.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + getNowBlockServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + assertTrue(result.containsKey("blockID")); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } +} diff --git a/framework/src/test/java/org/tron/core/services/http/ListNodesServletTest.java b/framework/src/test/java/org/tron/core/services/http/ListNodesServletTest.java new file mode 100644 index 00000000000..e2b8e7e9b6e --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/ListNodesServletTest.java @@ -0,0 +1,83 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.io.UnsupportedEncodingException; +import javax.annotation.Resource; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class ListNodesServletTest extends BaseTest { + + @Resource + private ListNodesServlet listNodesServlet; + + static { + dbPath = "db_GetNowBlockServlet_test"; + Args.setParam( + new String[]{ + "--output-directory", dbPath, + }, Constant.TEST_CONF + ); + } + + public MockHttpServletRequest createRequest(String contentType) { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("POST"); + request.setContentType(contentType); + request.setCharacterEncoding("UTF-8"); + return request; + } + + @Test + public void testListNodesByJson() { + String jsonParam = "{\"visible\": true}"; + MockHttpServletRequest request = createRequest("application/json"); + request.setContent(jsonParam.getBytes()); + MockHttpServletResponse response = new MockHttpServletResponse(); + listNodesServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + assertNotNull(contentAsString); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + + @Test + public void testListNodesValue() { + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + try { + String params = "visible=true"; + request.setContent(params.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + listNodesServlet.doPost(request, response); + String contentAsString = response.getContentAsString(); + assertNotNull(contentAsString); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + + @Test + public void testListNodesEmptyParam() { + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + String params = "visible="; + request.setContent(params.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + listNodesServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + assertNotNull(contentAsString); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + +} diff --git a/framework/src/test/java/org/tron/core/services/http/ListProposalsServletTest.java b/framework/src/test/java/org/tron/core/services/http/ListProposalsServletTest.java new file mode 100644 index 00000000000..0879d02004b --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/ListProposalsServletTest.java @@ -0,0 +1,83 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; + +import com.alibaba.fastjson.JSONObject; +import java.io.UnsupportedEncodingException; +import javax.annotation.Resource; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class ListProposalsServletTest extends BaseTest { + + @Resource + private ListProposalsServlet listProposalsServlet; + + static { + dbPath = "db_GetNowBlockServlet_test"; + Args.setParam( + new String[]{ + "--output-directory", dbPath, + }, Constant.TEST_CONF + ); + } + + public MockHttpServletRequest createRequest(String contentType) { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("POST"); + request.setContentType(contentType); + request.setCharacterEncoding("UTF-8"); + return request; + } + + @Test + public void testListProposalsByJson() { + String jsonParam = "{\"visible\": true}"; + MockHttpServletRequest request = createRequest("application/json"); + request.setContent(jsonParam.getBytes()); + MockHttpServletResponse response = new MockHttpServletResponse(); + listProposalsServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject.parseObject(contentAsString); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + + @Test + public void testListProposalsValue() { + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + try { + String params = "visible=true"; + request.setContent(params.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + listProposalsServlet.doPost(request, response); + String contentAsString = response.getContentAsString(); + JSONObject.parseObject(contentAsString); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + + @Test + public void testListProposalsEmptyParam() { + MockHttpServletRequest request = createRequest("application/x-www-form-urlencoded"); + String params = "visible="; + request.setContent(params.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + listProposalsServlet.doPost(request, response); + try { + String contentAsString = response.getContentAsString(); + JSONObject.parseObject(contentAsString); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + +} From e1d096be16bd8dcda1673cbfc9cab7f8490573e9 Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Wed, 23 Aug 2023 10:57:05 +0800 Subject: [PATCH 7/8] feat(all):make @PreDestroy work (#5421) --- .../common/application/ApplicationImpl.java | 3 - .../application/TronApplicationContext.java | 19 ++-- .../main/java/org/tron/program/FullNode.java | 7 +- .../java/org/tron/program/SolidityNode.java | 5 +- .../LiteFnQueryGrpcInterceptorTest.java | 91 ++++++++++--------- .../filter/RpcApiAccessInterceptorTest.java | 23 ++--- .../tron/program/LiteFullNodeToolTest.java | 5 +- 7 files changed, 69 insertions(+), 84 deletions(-) diff --git a/framework/src/main/java/org/tron/common/application/ApplicationImpl.java b/framework/src/main/java/org/tron/common/application/ApplicationImpl.java index 0e38e97baaf..26200abec2d 100644 --- a/framework/src/main/java/org/tron/common/application/ApplicationImpl.java +++ b/framework/src/main/java/org/tron/common/application/ApplicationImpl.java @@ -12,8 +12,6 @@ import org.tron.core.db.Manager; import org.tron.core.metrics.MetricsUtil; import org.tron.core.net.TronNetService; -import org.tron.program.FullNode; -import org.tron.program.SolidityNode; @Slf4j(topic = "app") @Component @@ -87,7 +85,6 @@ public void shutdown() { dbManager.stopFilterProcessThread(); dynamicArgs.close(); logger.info("******** end to shutdown ********"); - FullNode.shutDownSign = true; } @Override diff --git a/framework/src/main/java/org/tron/common/application/TronApplicationContext.java b/framework/src/main/java/org/tron/common/application/TronApplicationContext.java index 7f0aea813a3..482e9e6219d 100644 --- a/framework/src/main/java/org/tron/common/application/TronApplicationContext.java +++ b/framework/src/main/java/org/tron/common/application/TronApplicationContext.java @@ -2,8 +2,7 @@ import org.springframework.beans.factory.support.DefaultListableBeanFactory; import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.tron.core.db.Manager; -import org.tron.core.net.TronNetService; +import org.tron.program.FullNode; public class TronApplicationContext extends AnnotationConfigApplicationContext { @@ -23,19 +22,13 @@ public TronApplicationContext(String... basePackages) { } @Override - public void destroy() { - + public void doClose() { + logger.info("******** start to close ********"); Application appT = ApplicationFactory.create(this); appT.shutdownServices(); appT.shutdown(); - - TronNetService tronNetService = getBean(TronNetService.class); - tronNetService.close(); - - Manager dbManager = getBean(Manager.class); - dbManager.stopRePushThread(); - dbManager.stopRePushTriggerThread(); - dbManager.stopFilterProcessThread(); - super.destroy(); + super.doClose(); + logger.info("******** close end ********"); + FullNode.shutDownSign = true; } } diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index 62732dd5e60..5ebf70e0d7a 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -81,7 +81,7 @@ public static void main(String[] args) { context.register(DefaultConfig.class); context.refresh(); Application appT = ApplicationFactory.create(context); - shutdown(appT); + context.registerShutdownHook(); // grpc api server RpcApiService rpcApiService = context.getBean(RpcApiService.class); @@ -138,9 +138,4 @@ public static void main(String[] args) { rpcApiService.blockUntilShutdown(); } - - public static void shutdown(final Application app) { - logger.info("********register application shutdown hook********"); - Runtime.getRuntime().addShutdownHook(new Thread(app::shutdown)); - } } diff --git a/framework/src/main/java/org/tron/program/SolidityNode.java b/framework/src/main/java/org/tron/program/SolidityNode.java index 0ca001da7bb..6101db8068c 100644 --- a/framework/src/main/java/org/tron/program/SolidityNode.java +++ b/framework/src/main/java/org/tron/program/SolidityNode.java @@ -71,7 +71,8 @@ public static void main(String[] args) { } parameter.setSolidityNode(true); - ApplicationContext context = new TronApplicationContext(DefaultConfig.class); + TronApplicationContext context = new TronApplicationContext(DefaultConfig.class); + context.registerShutdownHook(); if (parameter.isHelp()) { logger.info("Here is the help message."); @@ -81,8 +82,6 @@ public static void main(String[] args) { Metrics.init(); Application appT = ApplicationFactory.create(context); - FullNode.shutdown(appT); - RpcApiService rpcApiService = context.getBean(RpcApiService.class); appT.addService(rpcApiService); //http diff --git a/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java b/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java index 4ee8fd051d0..439bf1f718f 100644 --- a/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java +++ b/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java @@ -3,23 +3,23 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; -import java.io.File; +import java.io.IOException; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; - -import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; import org.tron.api.GrpcAPI; import org.tron.api.WalletGrpc; import org.tron.api.WalletSolidityGrpc; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.utils.FileUtil; import org.tron.common.utils.PublicMethod; import org.tron.core.ChainBaseManager; import org.tron.core.Constant; @@ -32,54 +32,58 @@ @Slf4j public class LiteFnQueryGrpcInterceptorTest { - private TronApplicationContext context; - private ManagedChannel channelFull = null; - private ManagedChannel channelpBFT = null; - private WalletGrpc.WalletBlockingStub blockingStubFull = null; - private WalletSolidityGrpc.WalletSolidityBlockingStub blockingStubSolidity = null; - private WalletSolidityGrpc.WalletSolidityBlockingStub blockingStubpBFT = null; - private RpcApiService rpcApiService; - private RpcApiServiceOnSolidity rpcApiServiceOnSolidity; - private RpcApiServiceOnPBFT rpcApiServiceOnPBFT; - private Application appTest; - private ChainBaseManager chainBaseManager; - - private String dbPath = "output_grpc_interceptor_test"; + private static TronApplicationContext context; + private static ManagedChannel channelFull = null; + private static ManagedChannel channelSolidity = null; + private static ManagedChannel channelpBFT = null; + private static WalletGrpc.WalletBlockingStub blockingStubFull = null; + private static WalletSolidityGrpc.WalletSolidityBlockingStub blockingStubSolidity = null; + private static WalletSolidityGrpc.WalletSolidityBlockingStub blockingStubpBFT = null; + private static ChainBaseManager chainBaseManager; + private static final String ERROR_MSG = + "UNAVAILABLE: this API is closed because this node is a lite fullnode"; @Rule public ExpectedException thrown = ExpectedException.none(); + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + /** * init logic. */ - @Before - public void init() { - Args.setParam(new String[]{"-d", dbPath}, Constant.TEST_CONF); + @BeforeClass + public static void init() throws IOException { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort()); Args.getInstance().setRpcOnSolidityPort(PublicMethod.chooseRandomPort()); Args.getInstance().setRpcOnPBFTPort(PublicMethod.chooseRandomPort()); String fullnode = String.format("%s:%d", Args.getInstance().getNodeDiscoveryBindIp(), Args.getInstance().getRpcPort()); + String solidityNode = String.format("%s:%d", Args.getInstance().getNodeDiscoveryBindIp(), + Args.getInstance().getRpcOnSolidityPort()); String pBFTNode = String.format("%s:%d", Args.getInstance().getNodeDiscoveryBindIp(), - Args.getInstance().getRpcOnPBFTPort()); + Args.getInstance().getRpcOnPBFTPort()); channelFull = ManagedChannelBuilder.forTarget(fullnode) .usePlaintext() .build(); + channelSolidity = ManagedChannelBuilder.forTarget(solidityNode) + .usePlaintext() + .build(); channelpBFT = ManagedChannelBuilder.forTarget(pBFTNode) .usePlaintext() .build(); context = new TronApplicationContext(DefaultConfig.class); blockingStubFull = WalletGrpc.newBlockingStub(channelFull); - blockingStubSolidity = WalletSolidityGrpc.newBlockingStub(channelFull); + blockingStubSolidity = WalletSolidityGrpc.newBlockingStub(channelSolidity); blockingStubpBFT = WalletSolidityGrpc.newBlockingStub(channelpBFT); - blockingStubSolidity = WalletSolidityGrpc.newBlockingStub(channelFull); - rpcApiService = context.getBean(RpcApiService.class); - rpcApiServiceOnSolidity = context.getBean(RpcApiServiceOnSolidity.class); - rpcApiServiceOnPBFT = context.getBean(RpcApiServiceOnPBFT.class); + RpcApiService rpcApiService = context.getBean(RpcApiService.class); + RpcApiServiceOnSolidity rpcOnSolidity = context.getBean(RpcApiServiceOnSolidity.class); + RpcApiServiceOnPBFT rpcApiServiceOnPBFT = context.getBean(RpcApiServiceOnPBFT.class); chainBaseManager = context.getBean(ChainBaseManager.class); - appTest = ApplicationFactory.create(context); + Application appTest = ApplicationFactory.create(context); appTest.addService(rpcApiService); - appTest.addService(rpcApiServiceOnSolidity); + appTest.addService(rpcOnSolidity); appTest.addService(rpcApiServiceOnPBFT); appTest.initServices(Args.getInstance()); appTest.startServices(); @@ -89,23 +93,19 @@ public void init() { /** * destroy the context. */ - @After - public void destroy() throws InterruptedException { + @AfterClass + public static void destroy() throws InterruptedException { if (channelFull != null) { channelFull.shutdown().awaitTermination(5, TimeUnit.SECONDS); } + if (channelSolidity != null) { + channelSolidity.shutdown().awaitTermination(5, TimeUnit.SECONDS); + } if (channelpBFT != null) { channelpBFT.shutdown().awaitTermination(5, TimeUnit.SECONDS); } + context.close(); Args.clearParam(); - appTest.shutdownServices(); - appTest.shutdown(); - context.destroy(); - if (FileUtil.deleteDir(new File(dbPath))) { - logger.info("Release resources successful."); - } else { - logger.info("Release resources failure."); - } } @Test @@ -113,16 +113,25 @@ public void testGrpcApiThrowStatusRuntimeException() { final GrpcAPI.NumberMessage message = GrpcAPI.NumberMessage.newBuilder().setNum(0).build(); chainBaseManager.setNodeType(ChainBaseManager.NodeType.LITE); thrown.expect(StatusRuntimeException.class); - thrown.expectMessage("UNAVAILABLE: this API is closed because this node is a lite fullnode"); + thrown.expectMessage(ERROR_MSG); blockingStubFull.getBlockByNum(message); } + @Test + public void testGrpcSolidityThrowStatusRuntimeException() { + final GrpcAPI.NumberMessage message = GrpcAPI.NumberMessage.newBuilder().setNum(0).build(); + chainBaseManager.setNodeType(ChainBaseManager.NodeType.LITE); + thrown.expect(StatusRuntimeException.class); + thrown.expectMessage(ERROR_MSG); + blockingStubSolidity.getBlockByNum(message); + } + @Test public void testpBFTGrpcApiThrowStatusRuntimeException() { final GrpcAPI.NumberMessage message = GrpcAPI.NumberMessage.newBuilder().setNum(0).build(); chainBaseManager.setNodeType(ChainBaseManager.NodeType.LITE); thrown.expect(StatusRuntimeException.class); - thrown.expectMessage("UNAVAILABLE: this API is closed because this node is a lite fullnode"); + thrown.expectMessage(ERROR_MSG); blockingStubpBFT.getBlockByNum(message); } diff --git a/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java b/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java index edd15fc19de..ac1f21f6160 100644 --- a/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java +++ b/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java @@ -4,7 +4,7 @@ import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; import io.grpc.stub.ServerCallStreamObserver; -import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -13,16 +13,17 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; import org.tron.api.GrpcAPI; import org.tron.api.WalletGrpc; import org.tron.api.WalletSolidityGrpc; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.utils.FileUtil; import org.tron.common.utils.PublicMethod; import org.tron.core.Constant; import org.tron.core.config.DefaultConfig; @@ -41,17 +42,18 @@ public class RpcApiAccessInterceptorTest { private static WalletSolidityGrpc.WalletSolidityBlockingStub blockingStubPBFT = null; private static Application appTest; - private static String dbPath = "output_rpc_api_access_interceptor_test"; - @Rule public ExpectedException thrown = ExpectedException.none(); + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + /** * init logic. */ @BeforeClass - public static void init() { - Args.setParam(new String[] {"-d", dbPath}, Constant.TEST_CONF); + public static void init() throws IOException { + Args.setParam(new String[] {"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort()); Args.getInstance().setRpcOnSolidityPort(PublicMethod.chooseRandomPort()); Args.getInstance().setRpcOnPBFTPort(PublicMethod.chooseRandomPort()); @@ -97,15 +99,8 @@ public static void init() { */ @AfterClass public static void destroy() { + context.close(); Args.clearParam(); - appTest.shutdownServices(); - appTest.shutdown(); - context.destroy(); - if (FileUtil.deleteDir(new File(dbPath))) { - logger.info("Release resources successful."); - } else { - logger.info("Release resources failure."); - } } @Test diff --git a/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java b/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java index e7e164e8e3b..205b2fbd30d 100644 --- a/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java +++ b/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java @@ -5,7 +5,6 @@ import java.io.File; import java.nio.file.Paths; import java.util.concurrent.TimeUnit; - import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Test; @@ -76,9 +75,7 @@ public void shutdown() throws InterruptedException { if (channelFull != null) { channelFull.shutdown().awaitTermination(5, TimeUnit.SECONDS); } - appTest.shutdownServices(); - appTest.shutdown(); - context.destroy(); + context.close(); } public void init() { From a4100b6ac250071c50c7ce5e0067a6016c283d05 Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Wed, 23 Aug 2023 11:07:29 +0800 Subject: [PATCH 8/8] feat(db):tune the databases closure (#5429) --- .../java/org/tron/core/ChainBaseManager.java | 58 +--------- .../java/org/tron/core/db/TronDatabase.java | 9 +- .../tron/core/db/TronStoreWithRevoking.java | 9 +- .../org/tron/core/db2/common/TxCacheDB.java | 12 +++ .../tron/core/db2/core/SnapshotManager.java | 4 - .../common/application/ApplicationImpl.java | 7 +- .../main/java/org/tron/core/db/Manager.java | 23 +--- .../org/tron/core/db/TronDatabaseTest.java | 101 ++++++++++++++++++ .../org/tron/core/db/TxCacheDBInitTest.java | 1 + 9 files changed, 140 insertions(+), 84 deletions(-) create mode 100644 framework/src/test/java/org/tron/core/db/TronDatabaseTest.java diff --git a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java index 939d95dc584..e43d442534a 100644 --- a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java +++ b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java @@ -28,9 +28,7 @@ import org.tron.core.db.PbftSignDataStore; import org.tron.core.db.RecentBlockStore; import org.tron.core.db.RecentTransactionStore; -import org.tron.core.db.TransactionCache; import org.tron.core.db.TransactionStore; -import org.tron.core.db2.core.ITronChainBase; import org.tron.core.exception.BadItemException; import org.tron.core.exception.HeaderNotFound; import org.tron.core.exception.ItemNotFoundException; @@ -238,9 +236,6 @@ public class ChainBaseManager { @Autowired private DbStatService dbStatService; - @Autowired - private TransactionCache transactionCache; - @Getter @Setter private NodeType nodeType; @@ -249,55 +244,6 @@ public class ChainBaseManager { @Setter private long lowestBlockNum = -1; // except num = 0. - public void closeOneStore(ITronChainBase database) { - logger.info("******** Begin to close {}. ********", database.getName()); - try { - database.close(); - } catch (Exception e) { - logger.info("Failed to close {}.", database.getName(), e); - } finally { - logger.info("******** End to close {}. ********", database.getName()); - } - } - - public void closeAllStore() { - dbStatService.shutdown(); - closeOneStore(transactionRetStore); - closeOneStore(recentBlockStore); - closeOneStore(transactionHistoryStore); - closeOneStore(transactionStore); - closeOneStore(accountStore); - closeOneStore(blockStore); - closeOneStore(blockIndexStore); - closeOneStore(accountIdIndexStore); - closeOneStore(accountIndexStore); - closeOneStore(witnessScheduleStore); - closeOneStore(assetIssueStore); - closeOneStore(dynamicPropertiesStore); - closeOneStore(abiStore); - closeOneStore(codeStore); - closeOneStore(contractStore); - closeOneStore(contractStateStore); - closeOneStore(storageRowStore); - closeOneStore(exchangeStore); - closeOneStore(proposalStore); - closeOneStore(votesStore); - closeOneStore(delegatedResourceStore); - closeOneStore(delegatedResourceAccountIndexStore); - closeOneStore(assetIssueV2Store); - closeOneStore(exchangeV2Store); - closeOneStore(nullifierStore); - closeOneStore(merkleTreeStore); - closeOneStore(delegationStore); - closeOneStore(proofStore); - closeOneStore(commonStore); - closeOneStore(commonDataBase); - closeOneStore(pbftSignDataStore); - closeOneStore(sectionBloomStore); - closeOneStore(accountAssetStore); - closeOneStore(transactionCache); - } - // for test only public List getWitnesses() { return witnessScheduleStore.getActiveWitnesses(); @@ -437,6 +383,10 @@ private void init() { this.nodeType = getLowestBlockNum() > 1 ? NodeType.LITE : NodeType.FULL; } + public void shutdown() { + dbStatService.shutdown(); + } + public boolean isLiteNode() { return getNodeType() == NodeType.LITE; } diff --git a/chainbase/src/main/java/org/tron/core/db/TronDatabase.java b/chainbase/src/main/java/org/tron/core/db/TronDatabase.java index b96d7543302..d791e189eb4 100644 --- a/chainbase/src/main/java/org/tron/core/db/TronDatabase.java +++ b/chainbase/src/main/java/org/tron/core/db/TronDatabase.java @@ -94,7 +94,14 @@ public void reset() { */ @Override public void close() { - dbSource.closeDB(); + logger.info("******** Begin to close {}. ********", getName()); + try { + dbSource.closeDB(); + } catch (Exception e) { + logger.warn("Failed to close {}.", getName(), e); + } finally { + logger.info("******** End to close {}. ********", getName()); + } } public abstract void put(byte[] key, T item); diff --git a/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java b/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java index c1da54d84f4..4b75ddee3a4 100644 --- a/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java +++ b/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java @@ -182,7 +182,14 @@ public String getName() { @Override public void close() { - revokingDB.close(); + logger.info("******** Begin to close {}. ********", getName()); + try { + revokingDB.close(); + } catch (Exception e) { + logger.warn("Failed to close {}.", getName(), e); + } finally { + logger.info("******** End to close {}. ********", getName()); + } } @Override diff --git a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java index 9923a876cad..34f6720a3a1 100644 --- a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java +++ b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java @@ -21,6 +21,8 @@ import java.util.Properties; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; import org.bouncycastle.util.encoders.Hex; @@ -79,6 +81,10 @@ public class TxCacheDB implements DB, Flusher { private final Path cacheDir; private AtomicBoolean isValid = new AtomicBoolean(false); + @Getter + @Setter + private volatile boolean alive; + public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) { this.name = name; this.TRANSACTION_COUNT = @@ -138,6 +144,7 @@ private void initCache() { public void init() { if (recovery()) { isValid.set(true); + setAlive(true); return; } long size = recentTransactionStore.size(); @@ -160,6 +167,7 @@ public void init() { bloomFilters[1].approximateElementCount(), bloomFilters[1].expectedFpp(), System.currentTimeMillis() - start); isValid.set(true); + setAlive(true); } @Override @@ -247,10 +255,14 @@ public synchronized void flush(Map batch) { @Override public void close() { + if (!isAlive()) { + return; + } dump(); bloomFilters[0] = null; bloomFilters[1] = null; persistentStore.close(); + setAlive(false); } @Override diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java index 4eed2c06733..96796b3c460 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java @@ -284,10 +284,6 @@ public synchronized void disable() { @Override public void shutdown() { - logger.info("******** Begin to pop revokingDb. ********"); - logger.info("******** Before revokingDb size: {}.", size); - checkTmpStore.close(); - logger.info("******** End to pop revokingDb. ********"); if (pruneCheckpointThread != null) { pruneCheckpointThread.shutdown(); } diff --git a/framework/src/main/java/org/tron/common/application/ApplicationImpl.java b/framework/src/main/java/org/tron/common/application/ApplicationImpl.java index 26200abec2d..b6bcd670a03 100644 --- a/framework/src/main/java/org/tron/common/application/ApplicationImpl.java +++ b/framework/src/main/java/org/tron/common/application/ApplicationImpl.java @@ -77,12 +77,13 @@ public void shutdown() { synchronized (dbManager.getRevokingStore()) { dbManager.getSession().reset(); closeRevokingStore(); - closeAllStore(); } dbManager.stopRePushThread(); dbManager.stopRePushTriggerThread(); EventPluginLoader.getInstance().stopPlugin(); dbManager.stopFilterProcessThread(); + dbManager.stopValidateSignThread(); + getChainBaseManager().shutdown(); dynamicArgs.close(); logger.info("******** end to shutdown ********"); } @@ -112,8 +113,4 @@ private void closeRevokingStore() { dbManager.getRevokingStore().shutdown(); } - private void closeAllStore() { - dbManager.closeAllStore(); - } - } diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 3a18dce32a9..c50e8c900a1 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -105,7 +105,6 @@ import org.tron.core.db.api.MoveAbiHelper; import org.tron.core.db2.ISession; import org.tron.core.db2.core.Chainbase; -import org.tron.core.db2.core.ITronChainBase; import org.tron.core.db2.core.SnapshotManager; import org.tron.core.exception.AccountResourceInsufficientException; import org.tron.core.exception.BadBlockException; @@ -450,6 +449,10 @@ public void stopFilterProcessThread() { ExecutorServiceManager.shutdownAndAwaitTermination(filterEs, filterEsName); } + public void stopValidateSignThread() { + ExecutorServiceManager.shutdownAndAwaitTermination(validateSignService, "validate-sign"); + } + @PostConstruct public void init() { ChainBaseManager.init(chainBaseManager); @@ -1926,24 +1929,6 @@ public NullifierStore getNullifierStore() { return chainBaseManager.getNullifierStore(); } - public void closeAllStore() { - logger.info("******** Begin to close db. ********"); - chainBaseManager.closeAllStore(); - validateSignService.shutdown(); - logger.info("******** End to close db. ********"); - } - - public void closeOneStore(ITronChainBase database) { - logger.info("******** Begin to close {}. ********", database.getName()); - try { - database.close(); - } catch (Exception e) { - logger.info("Failed to close {}.", database.getName(), e); - } finally { - logger.info("******** End to close {}. ********", database.getName()); - } - } - public boolean isTooManyPending() { return getPendingTransactions().size() + getRePushTransactions().size() > maxTransactionPendingSize; diff --git a/framework/src/test/java/org/tron/core/db/TronDatabaseTest.java b/framework/src/test/java/org/tron/core/db/TronDatabaseTest.java new file mode 100644 index 00000000000..f38f55df64d --- /dev/null +++ b/framework/src/test/java/org/tron/core/db/TronDatabaseTest.java @@ -0,0 +1,101 @@ +package org.tron.core.db; + +import com.google.protobuf.InvalidProtocolBufferException; +import java.io.IOException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDB; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; +import org.tron.core.exception.BadItemException; +import org.tron.core.exception.ItemNotFoundException; + +public class TronDatabaseTest extends TronDatabase { + + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + static { + RocksDB.loadLibrary(); + } + + @BeforeClass + public static void initArgs() throws IOException { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); + } + + @AfterClass + public static void destroy() { + Args.clearParam(); + } + + @Override + public void put(byte[] key, String item) { + + } + + @Override + public void delete(byte[] key) { + + } + + @Override + public String get(byte[] key) { + return "test"; + } + + @Override + public boolean has(byte[] key) { + return false; + } + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void TestInit() { + TronDatabaseTest db = new TronDatabaseTest(); + Assert.assertNull(db.getDbSource()); + Assert.assertNull(db.getDbName()); + } + + @Test + public void TestIterator() { + TronDatabaseTest db = new TronDatabaseTest(); + thrown.expect(UnsupportedOperationException.class); + db.iterator(); + } + + @Test + public void TestIsNotEmpty() { + TronDatabaseTest db = new TronDatabaseTest(); + thrown.expect(UnsupportedOperationException.class); + db.isNotEmpty(); + } + + @Test + public void TestGetUnchecked() { + TronDatabaseTest db = new TronDatabaseTest(); + Assert.assertNull(db.getUnchecked("test".getBytes())); + } + + @Test + public void TestClose() { + TronDatabaseTest db = new TronDatabaseTest(); + db.close(); + } + + @Test + public void TestGetFromRoot() throws + InvalidProtocolBufferException, BadItemException, ItemNotFoundException { + TronDatabaseTest db = new TronDatabaseTest(); + Assert.assertEquals(db.getFromRoot("test".getBytes()), + "test"); + } +} diff --git a/framework/src/test/java/org/tron/core/db/TxCacheDBInitTest.java b/framework/src/test/java/org/tron/core/db/TxCacheDBInitTest.java index c3cb7cb2eb6..e415476d739 100644 --- a/framework/src/test/java/org/tron/core/db/TxCacheDBInitTest.java +++ b/framework/src/test/java/org/tron/core/db/TxCacheDBInitTest.java @@ -51,6 +51,7 @@ public void reload() { DefaultListableBeanFactory defaultListableBeanFactory = (DefaultListableBeanFactory) context.getAutowireCapableBeanFactory(); queryTransaction(); + db.close(); defaultListableBeanFactory.destroySingleton("transactionCache"); TransactionCache transactionCache = new TransactionCache("transactionCache", context.getBean(RecentTransactionStore.class));