From 586601f929d4fa95a62531356ef0954d860d3ff4 Mon Sep 17 00:00:00 2001 From: Jorge <46056498+jorgectf@users.noreply.github.com> Date: Wed, 19 Jul 2023 08:00:26 +0200 Subject: [PATCH 01/15] CI:Add CodeQL workflow (#5331) * Add CodeQL workflow * Clean `on.push.branches` --- .github/workflows/codeql.yml | 64 ++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000..4ab98d5cf3c --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,64 @@ +name: "CodeQL" + +on: + push: + branches: [ 'develop', 'master', 'release_**' ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ 'develop' ] + schedule: + - cron: '6 10 * * 0' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'java' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" From 040ea41b9aa866740ac0118bdaebfdfbae7b49f5 Mon Sep 17 00:00:00 2001 From: Kayle Date: Wed, 19 Jul 2023 14:05:00 +0800 Subject: [PATCH 02/15] feat(issue): Add labels to issue templates (#5358) --- .github/ISSUE_TEMPLATE/ask-a-question.md | 2 +- .github/ISSUE_TEMPLATE/report-a-bug.md | 2 +- .github/ISSUE_TEMPLATE/request-a-feature.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/ask-a-question.md b/.github/ISSUE_TEMPLATE/ask-a-question.md index a92239b39eb..96d3d783684 100644 --- a/.github/ISSUE_TEMPLATE/ask-a-question.md +++ b/.github/ISSUE_TEMPLATE/ask-a-question.md @@ -2,7 +2,7 @@ name: Ask a question about: Something is unclear title: '' -labels: '' +labels: 'type:docs' assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/report-a-bug.md b/.github/ISSUE_TEMPLATE/report-a-bug.md index 649d7e97ed1..cfadd364c21 100644 --- a/.github/ISSUE_TEMPLATE/report-a-bug.md +++ b/.github/ISSUE_TEMPLATE/report-a-bug.md @@ -2,7 +2,7 @@ name: Report a bug about: Create a report to help us improve title: '' -labels: '' +labels: 'type:bug' assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/request-a-feature.md b/.github/ISSUE_TEMPLATE/request-a-feature.md index 9047b7e6a87..5099a72008b 100644 --- a/.github/ISSUE_TEMPLATE/request-a-feature.md +++ b/.github/ISSUE_TEMPLATE/request-a-feature.md @@ -2,7 +2,7 @@ name: Request a feature about: Suggest an idea for this project title: '' -labels: '' +labels: 'type:feature' assignees: '' --- From a1968e5ac474bdc71e0447285154fd7284e47cb5 Mon Sep 17 00:00:00 2001 From: Kayle Date: Thu, 20 Jul 2023 12:16:47 +0800 Subject: [PATCH 03/15] feat(issue): Optimize ask-a-question.md (#5361) 1. Updates imported to social apps for communication 2. Update the description, choose one of the two social apps --- .github/ISSUE_TEMPLATE/ask-a-question.md | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/ask-a-question.md b/.github/ISSUE_TEMPLATE/ask-a-question.md index 96d3d783684..14bf73d9de8 100644 --- a/.github/ISSUE_TEMPLATE/ask-a-question.md +++ b/.github/ISSUE_TEMPLATE/ask-a-question.md @@ -6,23 +6,5 @@ labels: 'type:docs' assignees: '' --- - - - -### System information - -java-tron version: `java -jar FullNode.jar -v` -OS & Version: Windows/Linux/OSX - - -### 1. What did you do? - - - -### 2. What did you expect to see? - - - -### 3. What did you see instead? - +This should only be used in very rare cases e.g. if you are not 100% sure if something is a bug or asking a question that leads to improving the documentation. For general questions please use [discord](https://discord.gg/5gyhdvG9) or [Telegram](https://t.me/TronOfficialDevelopersGroupEn). From 3ad08d17fbf1bc769037041c28e2fcff7cd8f8f6 Mon Sep 17 00:00:00 2001 From: Kayle Date: Thu, 20 Jul 2023 14:15:29 +0800 Subject: [PATCH 04/15] feat(config): Update seed node ip list in config.conf (#5360) --- framework/src/main/resources/config.conf | 37 ++++++++++++------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index 67aa374eec3..c049bbbebc7 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -16,7 +16,7 @@ storage { # block_KDB, peers, properties, recent-block, trans, # utxo, votes, witness, witness_schedule. - # Otherwise, db configs will remain defualt and data will be stored in + # Otherwise, db configs will remain default and data will be stored in # the path of "output-directory" or which is set by "-d" ("--output-directory"). # setting can impove leveldb performance .... start @@ -359,23 +359,24 @@ seed.node = { "34.220.77.106:18888", "13.127.47.162:18888", "13.124.62.58:18888", - "13.229.128.108:18888", - "35.182.37.246:18888", - "34.200.228.125:18888", - "18.220.232.201:18888", - "13.57.30.186:18888", - "35.165.103.105:18888", - "18.184.238.21:18888", - "34.250.140.143:18888", - "35.176.192.130:18888", - "52.47.197.188:18888", - "52.62.210.100:18888", - "13.231.4.243:18888", - "18.231.76.29:18888", - "35.154.90.144:18888", - "13.125.210.234:18888", - "13.250.40.82:18888", - "35.183.101.48:18888" + "54.151.226.240:18888", + "35.174.93.198:18888", + "18.210.241.149:18888", + "54.177.115.127:18888", + "54.254.131.82:18888", + "18.167.171.167:18888", + "54.167.11.177:18888", + "35.74.7.196:18888", + "52.196.244.176:18888", + "54.248.129.19:18888", + "43.198.142.160:18888", + "3.0.214.7:18888", + "54.153.59.116:18888", + "54.153.94.160:18888", + "54.82.161.39:18888", + "54.179.207.68:18888", + "18.142.82.44:18888", + "18.163.230.203:18888" ] } From f15b2d94820841f9df688874ee2f42c2e91554ae Mon Sep 17 00:00:00 2001 From: Kayle Date: Thu, 20 Jul 2023 16:09:47 +0800 Subject: [PATCH 05/15] feat(issue): Add SECURITY.md (#5363) --- SECURITY.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..b3125ce6af1 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +## Supported Versions +Please see [Releases](https://github.com/tronprotocol/java-tron/releases). We recommend using the [most recently released version](https://github.com/tronprotocol/java-tron/releases/latest). + +## Reporting a Vulnerability +**Please do not file a public ticket** mentioning the vulnerability. +To find out how to report a vulnerability in TRON, visit [https://hackerone.com/tron_dao](https://hackerone.com/tron_dao?type=team) or email [bounty@tron.network](mailto:bounty@tron.network). +Please read the [disclosure policy](https://www.hackerone.com/disclosure-guidelines) for more information about publicly disclosed security vulnerabilities. From 57d118596cd76ab3ba83b0e8a43f7241b0cac3a9 Mon Sep 17 00:00:00 2001 From: Kayle Date: Thu, 20 Jul 2023 17:02:19 +0800 Subject: [PATCH 06/15] fix(docs): fix doc links for README (#5364) --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3e51097916f..2e0050ad84c 100644 --- a/README.md +++ b/README.md @@ -42,15 +42,14 @@

## Table of Contents -- [What’s TRON?](#What’s-TRON) -- [Building the Source Code](#Building-the-source) - - [Getting the Source Code](#Getting-the-Source-Code) - - [Build](#Build) -- [Running java-tron](#Running-java-tron) -- [Community](#Community) -- [Contribution](#Contribution) -- [Resources](#Resources) -- [License](#License) +- [What’s TRON?](#whats-tron) +- [Building the Source Code](#building-the-source) +- [Running java-tron](#running-java-tron) +- [Community](#community) +- [Contribution](#contribution) +- [Resources](#resources) +- [Integrity Check](#integrity-Check) +- [License](#license) ## What's TRON? From 9a65bb648022e85ea78bffb8bd5c6a78253e3952 Mon Sep 17 00:00:00 2001 From: Kayle Date: Thu, 20 Jul 2023 18:25:05 +0800 Subject: [PATCH 07/15] docs(README.md): fix catalogue links (#5365) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e0050ad84c..4d1d7b0b3b6 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ - [Community](#community) - [Contribution](#contribution) - [Resources](#resources) -- [Integrity Check](#integrity-Check) +- [Integrity Check](#integrity-check) - [License](#license) ## What's TRON? From 10223ccf44124b0ac2429354fc693b6d0d5f9563 Mon Sep 17 00:00:00 2001 From: Jake151987 <127825539+Jake151987@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:29:26 +0800 Subject: [PATCH 08/15] docs(ISSUE_TEMPLATE): fix invitation URL for Discord (#5387) --- .github/ISSUE_TEMPLATE/ask-a-question.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/ask-a-question.md b/.github/ISSUE_TEMPLATE/ask-a-question.md index 14bf73d9de8..e503998b477 100644 --- a/.github/ISSUE_TEMPLATE/ask-a-question.md +++ b/.github/ISSUE_TEMPLATE/ask-a-question.md @@ -6,5 +6,5 @@ labels: 'type:docs' assignees: '' --- -This should only be used in very rare cases e.g. if you are not 100% sure if something is a bug or asking a question that leads to improving the documentation. For general questions please use [discord](https://discord.gg/5gyhdvG9) or [Telegram](https://t.me/TronOfficialDevelopersGroupEn). +This should only be used in very rare cases e.g. if you are not 100% sure if something is a bug or asking a question that leads to improving the documentation. For general questions please use [Discord](https://discord.gg/cGKSsRVCGm) or [Telegram](https://t.me/TronOfficialDevelopersGroupEn). From 5ce3ff04af90310e665039beb3e5e8c5187e9faa Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 8 Aug 2023 11:37:26 +0800 Subject: [PATCH 09/15] fix(db): fix getHeadBlockId abnormal return problem --- chainbase/src/main/java/org/tron/core/ChainBaseManager.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java index adf66527499..dad76041777 100644 --- a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java +++ b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java @@ -316,9 +316,7 @@ public BlockCapsule getHead() throws HeaderNotFound { } public synchronized BlockId getHeadBlockId() { - return new BlockId( - dynamicPropertiesStore.getLatestBlockHeaderHash(), - dynamicPropertiesStore.getLatestBlockHeaderNumber()); + return new BlockId(dynamicPropertiesStore.getLatestBlockHeaderHash()); } public long getHeadBlockNum() { From 56efc4ee5c7eb3eab3619971fdf6dd21b6899391 Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:46:01 +0800 Subject: [PATCH 10/15] feat(log): optimize checkpointV2 check tips (#5383) --- .../src/main/java/org/tron/core/db2/core/SnapshotManager.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java index 230a812e093..4eed2c06733 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java @@ -493,7 +493,9 @@ public void check() { if (!isV2Open()) { List cpList = getCheckpointList(); if (cpList != null && cpList.size() != 0) { - logger.error("checkpoint check failed, can't convert checkpoint from v2 to v1"); + logger.error("checkpoint check failed, the checkpoint version of database not match your " + + "config file, please set storage.checkpoint.version = 2 in your config file " + + "and restart the node."); System.exit(-1); } checkV1(); From 2bbc0fddb77e5d0df53990db704fe5b37f8a3298 Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:49:58 +0800 Subject: [PATCH 11/15] feat(sign): optimize default signature verification efficiency (#5396) --- framework/src/main/java/org/tron/core/config/args/Args.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 50e4dacf6a9..040a06c6a02 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -824,7 +824,7 @@ public static void setParam(final String[] args, final String confFileName) { PARAMETER.validateSignThreadNum = config.hasPath(Constant.NODE_VALIDATE_SIGN_THREAD_NUM) ? config .getInt(Constant.NODE_VALIDATE_SIGN_THREAD_NUM) - : (Runtime.getRuntime().availableProcessors() + 1) / 2; + : Runtime.getRuntime().availableProcessors(); PARAMETER.walletExtensionApi = config.hasPath(Constant.NODE_WALLET_EXTENSION_API) From 1a33965c569c3b9ba568410270fbda79dd9fff76 Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:38:34 +0800 Subject: [PATCH 12/15] feat(log): optimize error log output for Wallet.java (#5366) --- framework/src/main/java/org/tron/core/Wallet.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/framework/src/main/java/org/tron/core/Wallet.java b/framework/src/main/java/org/tron/core/Wallet.java index b866ab54001..3560a7032bc 100755 --- a/framework/src/main/java/org/tron/core/Wallet.java +++ b/framework/src/main/java/org/tron/core/Wallet.java @@ -2058,7 +2058,7 @@ public IncrementalMerkleTree getMerkleTreeOfBlock(long blockNum) throws ZksnarkE .parseFrom(chainBaseManager.getMerkleTreeIndexStore().get(blockNum)); } } catch (Exception ex) { - logger.error(ex.getMessage()); + logger.error("GetMerkleTreeOfBlock failed, blockNum:{}", blockNum, ex); } return null; @@ -4304,7 +4304,7 @@ public long getEnergyFee(long timestamp) { return energyFee; } catch (Exception e) { - logger.error("getEnergyFee timestamp={} failed, error is {}", timestamp, e.getMessage()); + logger.error("GetEnergyFee timestamp={} failed", timestamp, e); return getEnergyFee(); } } @@ -4313,7 +4313,7 @@ public String getEnergyPrices() { try { return chainBaseManager.getDynamicPropertiesStore().getEnergyPriceHistory(); } catch (Exception e) { - logger.error("getEnergyPrices failed, error is {}", e.getMessage()); + logger.error("GetEnergyPrices failed", e); } return null; @@ -4323,7 +4323,7 @@ public String getBandwidthPrices() { try { return chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistory(); } catch (Exception e) { - logger.error("getBandwidthPrices failed, error is {}", e.getMessage()); + logger.error("GetBandwidthPrices failed", e); } return null; @@ -4444,7 +4444,7 @@ public String getMemoFeePrices() { try { return chainBaseManager.getDynamicPropertiesStore().getMemoFeeHistory(); } catch (Exception e) { - logger.error("getMemoFeePrices failed, error is {}", e.getMessage()); + logger.error("GetMemoFeePrices failed", e); } return null; } From 61c61773a97979ec2967eeaa9b580338d320f492 Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Thu, 10 Aug 2023 14:27:27 +0800 Subject: [PATCH 13/15] feat(log): optimize error tips for db open(#5385) --- .../leveldb/LevelDbDataSourceImpl.java | 21 ++++++++++++---- .../rocksdb/RocksDbDataSourceImpl.java | 10 ++++++-- framework/build.gradle | 1 + .../leveldb/LevelDbDataSourceImplTest.java | 25 +++++++++++++++++++ .../leveldb/RocksDbDataSourceImplTest.java | 24 ++++++++++++++++++ 5 files changed, 74 insertions(+), 7 deletions(-) diff --git a/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java b/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java index 2292f974c4b..43a24ff4416 100644 --- a/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java +++ b/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java @@ -142,11 +142,22 @@ private void openDatabase(Options dbOptions) throws IOException { if (!Files.isSymbolicLink(dbPath.getParent())) { Files.createDirectories(dbPath.getParent()); } - database = factory.open(dbPath.toFile(), dbOptions); - if (!this.getDBName().startsWith("checkpoint")) { - logger.info("DB {} open success with writeBufferSize {} M, cacheSize {} M, maxOpenFiles {}.", - this.getDBName(), dbOptions.writeBufferSize() / 1024 / 1024, - dbOptions.cacheSize() / 1024 / 1024, dbOptions.maxOpenFiles()); + try { + database = factory.open(dbPath.toFile(), dbOptions); + if (!this.getDBName().startsWith("checkpoint")) { + logger + .info("DB {} open success with writeBufferSize {} M, cacheSize {} M, maxOpenFiles {}.", + this.getDBName(), dbOptions.writeBufferSize() / 1024 / 1024, + dbOptions.cacheSize() / 1024 / 1024, dbOptions.maxOpenFiles()); + } + } catch (IOException e) { + if (e.getMessage().contains("Corruption:")) { + logger.error("Database {} corrupted, please delete database directory({}) and restart.", + dataBaseName, parentPath, e); + } else { + logger.error("Open Database {} failed", dataBaseName, e); + } + System.exit(1); } } diff --git a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java index a561ed1fa11..6c5d8018487 100644 --- a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java +++ b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java @@ -32,6 +32,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.Statistics; +import org.rocksdb.Status; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; import org.slf4j.LoggerFactory; @@ -265,8 +266,13 @@ protected void log(InfoLogLevel infoLogLevel, String logMsg) { try { database = RocksDB.open(options, dbPath.toString()); } catch (RocksDBException e) { - throw new RuntimeException( - String.format("failed to open database: %s", dataBaseName), e); + if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) { + logger.error("Database {} corrupted, please delete database directory({}) " + + "and restart.", dataBaseName, parentPath, e); + } else { + logger.error("Open Database {} failed", dataBaseName, e); + } + System.exit(1); } alive = true; diff --git a/framework/build.gradle b/framework/build.gradle index d3e13ac0a7e..063d320d477 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -43,6 +43,7 @@ dependencies { testCompile group: 'junit', name: 'junit', version: '4.13.2' testCompile group: 'org.mockito', name: 'mockito-core', version: '2.13.0' testCompile group: 'org.hamcrest', name: 'hamcrest-junit', version: '1.0.0.1' + testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.16.0' compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' diff --git a/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java b/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java index eed995f12b5..b4ac2b4aebf 100644 --- a/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java +++ b/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java @@ -37,7 +37,9 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.contrib.java.lang.system.ExpectedSystemExit; import org.tron.common.utils.ByteArray; import org.tron.common.utils.FileUtil; import org.tron.common.utils.PublicMethod; @@ -65,6 +67,9 @@ public class LevelDbDataSourceImplTest { private byte[] key5 = "00000005aa".getBytes(); private byte[] key6 = "00000006aa".getBytes(); + @Rule + public final ExpectedSystemExit exit = ExpectedSystemExit.none(); + /** * Release resources. */ @@ -332,4 +337,24 @@ public void prefixQueryTest() { dataSource.resetDb(); dataSource.closeDB(); } + + @Test + public void initDbTest() { + exit.expectSystemExitWithStatus(1); + makeExceptionDb("test_initDb"); + LevelDbDataSourceImpl dataSource = new LevelDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_initDb"); + dataSource.initDB(); + dataSource.closeDB(); + } + + private void makeExceptionDb(String dbName) { + LevelDbDataSourceImpl dataSource = new LevelDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_initDb"); + dataSource.initDB(); + dataSource.closeDB(); + FileUtil.saveData(dataSource.getDbPath().toString() + "/CURRENT", + "...", Boolean.FALSE); + } + } diff --git a/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java b/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java index f0ceb7fcac4..62630014b23 100644 --- a/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java +++ b/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java @@ -20,7 +20,9 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.contrib.java.lang.system.ExpectedSystemExit; import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; import org.tron.common.utils.ByteArray; import org.tron.common.utils.FileUtil; @@ -48,6 +50,9 @@ public class RocksDbDataSourceImplTest { private byte[] key5 = "00000005aa".getBytes(); private byte[] key6 = "00000006aa".getBytes(); + @Rule + public final ExpectedSystemExit exit = ExpectedSystemExit.none(); + /** * Release resources. */ @@ -382,4 +387,23 @@ public void prefixQueryTest() { dataSource.resetDb(); dataSource.closeDB(); } + + @Test + public void initDbTest() { + exit.expectSystemExitWithStatus(1); + makeExceptionDb("test_initDb"); + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_initDb"); + dataSource.initDB(); + dataSource.closeDB(); + } + + private void makeExceptionDb(String dbName) { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_initDb"); + dataSource.initDB(); + dataSource.closeDB(); + FileUtil.saveData(dataSource.getDbPath().toString() + "/CURRENT", + "...", Boolean.FALSE); + } } From a9c4f4324c895f2f3c74eb7d7fb13add719419fe Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Tue, 15 Aug 2023 11:23:32 +0800 Subject: [PATCH 14/15] feat(all): tune single Thread into SingleThreadExecutor (#5410) --- .../common/es/ExecutorServiceManager.java | 55 +++++++++++++++++++ .../org/tron/consensus/dpos/DposTask.java | 20 +++---- .../common/backup/socket/BackupServer.java | 12 +++- .../tron/core/config/TronLogShutdownHook.java | 4 +- .../main/java/org/tron/core/db/Manager.java | 25 ++++++--- .../nodepersist/NodePersistService.java | 30 ++++------ .../tron/common/backup/BackupServerTest.java | 44 +++++++++++++++ 7 files changed, 150 insertions(+), 40 deletions(-) create mode 100644 common/src/main/java/org/tron/common/es/ExecutorServiceManager.java create mode 100644 framework/src/test/java/org/tron/common/backup/BackupServerTest.java diff --git a/common/src/main/java/org/tron/common/es/ExecutorServiceManager.java b/common/src/main/java/org/tron/common/es/ExecutorServiceManager.java new file mode 100644 index 00000000000..f1e60fdcfbc --- /dev/null +++ b/common/src/main/java/org/tron/common/es/ExecutorServiceManager.java @@ -0,0 +1,55 @@ +package org.tron.common.es; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import lombok.extern.slf4j.Slf4j; + +@Slf4j(topic = "common") +public class ExecutorServiceManager { + + public static ExecutorService newSingleThreadExecutor(String name) { + return newSingleThreadExecutor(name, false); + } + + public static ExecutorService newSingleThreadExecutor(String name, boolean isDaemon) { + return Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder().setNameFormat(name).setDaemon(isDaemon).build()); + } + + + public static ScheduledExecutorService newSingleThreadScheduledExecutor(String name) { + return newSingleThreadScheduledExecutor(name, false); + } + + public static ScheduledExecutorService newSingleThreadScheduledExecutor(String name, + boolean isDaemon) { + return Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat(name).setDaemon(isDaemon).build()); + } + + public static void shutdownAndAwaitTermination(ExecutorService pool, String name) { + if (pool == null) { + return; + } + logger.info("Pool {} shutdown...", name); + pool.shutdown(); // Disable new tasks from being submitted + try { + // Wait a while for existing tasks to terminate + if (!pool.awaitTermination(60, java.util.concurrent.TimeUnit.SECONDS)) { + pool.shutdownNow(); // Cancel currently executing tasks + // Wait a while for tasks to respond to being cancelled + if (!pool.awaitTermination(60, java.util.concurrent.TimeUnit.SECONDS)) { + logger.warn("Pool {} did not terminate", name); + } + } + } catch (InterruptedException ie) { + // (Re-)Cancel if current thread also interrupted + pool.shutdownNow(); + // Preserve interrupt status + Thread.currentThread().interrupt(); + } + logger.info("Pool {} shutdown done", name); + } +} diff --git a/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java b/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java index dcfa85ca5f3..537fe49ae65 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java @@ -3,12 +3,14 @@ import static org.tron.core.config.Parameter.ChainConstant.BLOCK_PRODUCED_INTERVAL; import com.google.protobuf.ByteString; +import java.util.concurrent.ExecutorService; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.joda.time.DateTime; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; +import org.springframework.util.ObjectUtils; +import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; import org.tron.common.utils.Sha256Hash; @@ -34,16 +36,18 @@ public class DposTask { @Setter private DposService dposService; - private Thread produceThread; + private ExecutorService produceExecutor; + + private final String name = "DPosMiner"; private volatile boolean isRunning = true; public void init() { - if (!dposService.isEnable() || StringUtils.isEmpty(dposService.getMiners())) { + if (!dposService.isEnable() || ObjectUtils.isEmpty(dposService.getMiners())) { return; } - + produceExecutor = ExecutorServiceManager.newSingleThreadExecutor(name); Runnable runnable = () -> { while (isRunning) { try { @@ -67,17 +71,13 @@ public void init() { } } }; - produceThread = new Thread(runnable, "DPosMiner"); - produceThread.start(); + produceExecutor.submit(runnable); logger.info("DPoS task started."); } public void stop() { isRunning = false; - if (produceThread != null) { - produceThread.interrupt(); - } - logger.info("DPoS task stopped."); + ExecutorServiceManager.shutdownAndAwaitTermination(produceExecutor, name); } private State produceBlock() { diff --git a/framework/src/main/java/org/tron/common/backup/socket/BackupServer.java b/framework/src/main/java/org/tron/common/backup/socket/BackupServer.java index e3b1de31736..fa2c0947852 100644 --- a/framework/src/main/java/org/tron/common/backup/socket/BackupServer.java +++ b/framework/src/main/java/org/tron/common/backup/socket/BackupServer.java @@ -7,11 +7,13 @@ import io.netty.channel.socket.nio.NioDatagramChannel; import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.tron.common.backup.BackupManager; +import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; import org.tron.p2p.stats.TrafficStats; @@ -29,6 +31,9 @@ public class BackupServer { private volatile boolean shutdown = false; + private final String name = "BackupServer"; + private ExecutorService executor; + @Autowired public BackupServer(final BackupManager backupManager) { this.backupManager = backupManager; @@ -36,13 +41,14 @@ public BackupServer(final BackupManager backupManager) { public void initServer() { if (port > 0 && commonParameter.getBackupMembers().size() > 0) { - new Thread(() -> { + executor = ExecutorServiceManager.newSingleThreadExecutor(name); + executor.submit(() -> { try { start(); } catch (Exception e) { logger.error("Start backup server failed, {}", e); } - }, "BackupServer").start(); + }); } } @@ -88,6 +94,7 @@ public void initChannel(NioDatagramChannel ch) public void close() { logger.info("Closing backup server..."); shutdown = true; + ExecutorServiceManager.shutdownAndAwaitTermination(executor, name); if (channel != null) { try { channel.close().await(10, TimeUnit.SECONDS); @@ -95,5 +102,6 @@ public void close() { logger.warn("Closing backup server failed.", e); } } + logger.info("Backup server closed."); } } diff --git a/framework/src/main/java/org/tron/core/config/TronLogShutdownHook.java b/framework/src/main/java/org/tron/core/config/TronLogShutdownHook.java index 880aa7e3090..f873b88ca44 100644 --- a/framework/src/main/java/org/tron/core/config/TronLogShutdownHook.java +++ b/framework/src/main/java/org/tron/core/config/TronLogShutdownHook.java @@ -16,9 +16,9 @@ public class TronLogShutdownHook extends ShutdownHookBase { private static final Duration CHECK_SHUTDOWN_DELAY = Duration.buildByMilliseconds(100); /** - * The check times before shutdown. default is 50 + * The check times before shutdown. default is 60000/100 = 600 times. */ - private Integer check_times = 50; + private final long check_times = 60 * 1000 / CHECK_SHUTDOWN_DELAY.getMilliseconds(); public TronLogShutdownHook() { } diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 3928f657e44..3a18dce32a9 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -49,6 +49,7 @@ import org.tron.api.GrpcAPI.TransactionInfoList; import org.tron.common.args.GenesisBlock; import org.tron.common.bloom.Bloom; +import org.tron.common.es.ExecutorServiceManager; import org.tron.common.logsfilter.EventPluginLoader; import org.tron.common.logsfilter.FilterQuery; import org.tron.common.logsfilter.capsule.BlockFilterCapsule; @@ -253,6 +254,13 @@ public class Manager { private AtomicInteger blockWaitLock = new AtomicInteger(0); private Object transactionLock = new Object(); + private ExecutorService rePushEs; + private static final String rePushEsName = "repush"; + private ExecutorService triggerEs; + private static final String triggerEsName = "event-trigger"; + private ExecutorService filterEs; + private static final String filterEsName = "filter"; + /** * Cycle thread to rePush Transactions */ @@ -429,14 +437,17 @@ public BlockingQueue getRePushTransactions() { public void stopRePushThread() { isRunRePushThread = false; + ExecutorServiceManager.shutdownAndAwaitTermination(rePushEs, rePushEsName); } public void stopRePushTriggerThread() { isRunTriggerCapsuleProcessThread = false; + ExecutorServiceManager.shutdownAndAwaitTermination(triggerEs, triggerEsName); } public void stopFilterProcessThread() { isRunFilterProcessThread = false; + ExecutorServiceManager.shutdownAndAwaitTermination(filterEs, filterEsName); } @PostConstruct @@ -524,21 +535,19 @@ public void init() { revokingStore.enable(); validateSignService = Executors .newFixedThreadPool(Args.getInstance().getValidateSignThreadNum()); - Thread rePushThread = new Thread(rePushLoop); - rePushThread.setDaemon(true); - rePushThread.start(); + rePushEs = ExecutorServiceManager.newSingleThreadExecutor(rePushEsName, true); + rePushEs.submit(rePushLoop); // add contract event listener for subscribing if (Args.getInstance().isEventSubscribe()) { startEventSubscribing(); - Thread triggerCapsuleProcessThread = new Thread(triggerCapsuleProcessLoop); - triggerCapsuleProcessThread.setDaemon(true); - triggerCapsuleProcessThread.start(); + triggerEs = ExecutorServiceManager.newSingleThreadExecutor(triggerEsName, true); + triggerEs.submit(triggerCapsuleProcessLoop); } // start json rpc filter process if (CommonParameter.getInstance().isJsonRpcFilterEnabled()) { - Thread filterProcessThread = new Thread(filterProcessLoop); - filterProcessThread.start(); + filterEs = ExecutorServiceManager.newSingleThreadExecutor(filterEsName); + filterEs.submit(filterProcessLoop); } //initStoreFactory diff --git a/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java b/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java index 457fe7c55cb..2445a7d64e5 100644 --- a/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java +++ b/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java @@ -4,12 +4,12 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.List; -import java.util.Objects; -import java.util.Timer; -import java.util.TimerTask; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; import org.tron.common.utils.JsonUtil; @@ -27,28 +27,22 @@ public class NodePersistService { private final boolean isNodePersist = CommonParameter.getInstance().isNodeDiscoveryPersist(); @Autowired private CommonStore commonStore; - private Timer nodePersistTaskTimer; + + private ScheduledExecutorService nodePersistExecutor; + + private final String name = "NodePersistTask"; public void init() { if (isNodePersist) { - nodePersistTaskTimer = new Timer("NodePersistTaskTimer"); - nodePersistTaskTimer.scheduleAtFixedRate(new TimerTask() { - @Override - public void run() { - dbWrite(); - } - }, DB_COMMIT_RATE, DB_COMMIT_RATE); + nodePersistExecutor = ExecutorServiceManager.newSingleThreadScheduledExecutor(name); + nodePersistExecutor.scheduleAtFixedRate(this::dbWrite, DB_COMMIT_RATE, DB_COMMIT_RATE, + TimeUnit.MILLISECONDS); } } public void close() { - if (Objects.isNull(nodePersistTaskTimer)) { - return; - } - try { - nodePersistTaskTimer.cancel(); - } catch (Exception e) { - logger.error("Close nodePersistTaskTimer failed", e); + if (isNodePersist) { + ExecutorServiceManager.shutdownAndAwaitTermination(nodePersistExecutor, name); } } diff --git a/framework/src/test/java/org/tron/common/backup/BackupServerTest.java b/framework/src/test/java/org/tron/common/backup/BackupServerTest.java new file mode 100644 index 00000000000..34b17ec186f --- /dev/null +++ b/framework/src/test/java/org/tron/common/backup/BackupServerTest.java @@ -0,0 +1,44 @@ +package org.tron.common.backup; + +import java.util.ArrayList; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.tron.common.backup.socket.BackupServer; +import org.tron.common.parameter.CommonParameter; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + + +public class BackupServerTest { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private BackupServer backupServer; + + @Before + public void setUp() throws Exception { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); + CommonParameter.getInstance().setBackupPort(80); + List members = new ArrayList<>(); + members.add("127.0.0.2"); + CommonParameter.getInstance().setBackupMembers(members); + BackupManager backupManager = new BackupManager(); + backupManager.init(); + backupServer = new BackupServer(backupManager); + } + + @After + public void tearDown() { + backupServer.close(); + Args.clearParam(); + } + + @Test + public void test() throws InterruptedException { + backupServer.initServer(); + } +} From 83151aaf4d66911533347b6250bb135a6c0883a9 Mon Sep 17 00:00:00 2001 From: chengtx01 <56535423+chengtx01@users.noreply.github.com> Date: Thu, 17 Aug 2023 19:09:51 +0800 Subject: [PATCH 15/15] feat(net):improve chain inventory generating logic (#5393) --- .../SyncBlockChainMsgHandler.java | 20 +++++++++++++++++-- .../SyncBlockChainMsgHandlerTest.java | 16 +++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java index 9027034ccc7..3c83bc51e47 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java @@ -87,6 +87,18 @@ private boolean check(PeerConnection peer, SyncBlockChainMessage msg) throws P2p private LinkedList getLostBlockIds(List blockIds) throws P2pException { + BlockId unForkId = getUnForkId(blockIds); + LinkedList ids = getBlockIds(unForkId.getNum()); + + if (ids.isEmpty() || !unForkId.equals(ids.peekFirst())) { + unForkId = getUnForkId(blockIds); + ids = getBlockIds(unForkId.getNum()); + } + + return ids; + } + + private BlockId getUnForkId(List blockIds) throws P2pException { BlockId unForkId = null; for (int i = blockIds.size() - 1; i >= 0; i--) { if (tronNetDelegate.containBlockInMainChain(blockIds.get(i))) { @@ -99,13 +111,17 @@ private LinkedList getLostBlockIds(List blockIds) throws P2pEx throw new P2pException(TypeEnum.SYNC_FAILED, "unForkId is null"); } + return unForkId; + } + + private LinkedList getBlockIds(Long unForkNum) throws P2pException { BlockId headID = tronNetDelegate.getHeadBlockId(); long headNum = headID.getNum(); - long len = Math.min(headNum, unForkId.getNum() + NetConstants.SYNC_FETCH_BATCH_NUM); + long len = Math.min(headNum, unForkNum + NetConstants.SYNC_FETCH_BATCH_NUM); LinkedList ids = new LinkedList<>(); - for (long i = unForkId.getNum(); i <= len; i++) { + for (long i = unForkNum; i <= len; i++) { if (i == headNum) { ids.add(headID); } else { diff --git a/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java b/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java index 2dbad09c655..53375ec7815 100644 --- a/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java +++ b/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java @@ -2,6 +2,7 @@ import java.io.File; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -64,6 +65,21 @@ public void testProcessMessage() throws Exception { method.setAccessible(true); boolean f = (boolean)method.invoke(handler, peer, message); Assert.assertTrue(!f); + + Method method1 = handler.getClass().getDeclaredMethod( + "getLostBlockIds", List.class); + method1.setAccessible(true); + try { + method1.invoke(handler, blockIds); + } catch (InvocationTargetException e) { + Assert.assertEquals("unForkId is null", e.getTargetException().getMessage()); + } + + Method method2 = handler.getClass().getDeclaredMethod( + "getBlockIds", Long.class); + method2.setAccessible(true); + List list = (List) method2.invoke(handler, 0L); + Assert.assertEquals(1, list.size()); } @After