Merge remote-tracking branch 'upstream/develop' into feat/fix_servlet

.github/ISSUE_TEMPLATE/ask-a-question.md

@@ -2,27 +2,9 @@
 name: Ask a question
 about: Something is unclear
 title: ''
-labels: ''
+labels: 'type:docs'
 assignees: ''
 
 ---
-
-<!-- Please answer these questions before submitting your issue. Thanks! -->
-
-### System information
-
-java-tron version: `java -jar FullNode.jar -v`
-OS & Version: Windows/Linux/OSX
-
-
-### 1. What did you do?
-<!--If possible, provide a recipe for reproducing the error. -->
-
-
-### 2. What did you expect to see?
-
-
-
-### 3. What did you see instead?
-
+This should only be used in very rare cases e.g. if you are not 100% sure if something is a bug or asking a question that leads to improving the documentation. For general questions please use [Discord](https://discord.gg/cGKSsRVCGm) or [Telegram](https://t.me/TronOfficialDevelopersGroupEn).
 

.github/ISSUE_TEMPLATE/report-a-bug.md

@@ -2,7 +2,7 @@
 name: Report a bug
 about: Create a report to help us improve
 title: ''
-labels: ''
+labels: 'type:bug'
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/request-a-feature.md

@@ -2,7 +2,7 @@
 name: Request a feature
 about: Suggest an idea for this project
 title: ''
-labels: ''
+labels: 'type:feature'
 assignees: ''
 
 ---

.github/workflows/codeql.yml

@@ -0,0 +1,64 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ 'develop', 'master', 'release_**' ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ 'develop' ]
+  schedule:
+    - cron: '6 10 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Use only 'java' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

README.md

@@ -42,15 +42,14 @@
 </p>
 
 ## Table of Contents
-- [What’s TRON?](#What’s-TRON)
-- [Building the Source Code](#Building-the-source)
-  - [Getting the Source Code](#Getting-the-Source-Code)
-  - [Build](#Build)
-- [Running java-tron](#Running-java-tron)
-- [Community](#Community)
-- [Contribution](#Contribution)
-- [Resources](#Resources)
-- [License](#License)
+- [What’s TRON?](#whats-tron)
+- [Building the Source Code](#building-the-source)
+- [Running java-tron](#running-java-tron)
+- [Community](#community)
+- [Contribution](#contribution)
+- [Resources](#resources)
+- [Integrity Check](#integrity-check)
+- [License](#license)
 
 ## What's TRON?
 

SECURITY.md

@@ -0,0 +1,7 @@
+## Supported Versions
+Please see [Releases](https://github.com/tronprotocol/java-tron/releases). We recommend using the [most recently released version](https://github.com/tronprotocol/java-tron/releases/latest).
+
+## Reporting a Vulnerability
+**Please do not file a public ticket** mentioning the vulnerability.
+To find out how to report a vulnerability in TRON, visit [https://hackerone.com/tron_dao](https://hackerone.com/tron_dao?type=team) or email [bounty@tron.network](mailto:bounty@tron.network).  
+Please read the [disclosure policy](https://www.hackerone.com/disclosure-guidelines) for more information about publicly disclosed security vulnerabilities.

chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java

@@ -142,11 +142,22 @@ private void openDatabase(Options dbOptions) throws IOException {
     if (!Files.isSymbolicLink(dbPath.getParent())) {
       Files.createDirectories(dbPath.getParent());
     }
-    database = factory.open(dbPath.toFile(), dbOptions);
-    if (!this.getDBName().startsWith("checkpoint")) {
-      logger.info("DB {} open success with writeBufferSize {} M, cacheSize {} M, maxOpenFiles {}.",
-          this.getDBName(), dbOptions.writeBufferSize() / 1024 / 1024,
-          dbOptions.cacheSize() / 1024 / 1024, dbOptions.maxOpenFiles());
+    try {
+      database = factory.open(dbPath.toFile(), dbOptions);
+      if (!this.getDBName().startsWith("checkpoint")) {
+        logger
+            .info("DB {} open success with writeBufferSize {} M, cacheSize {} M, maxOpenFiles {}.",
+                this.getDBName(), dbOptions.writeBufferSize() / 1024 / 1024,
+                dbOptions.cacheSize() / 1024 / 1024, dbOptions.maxOpenFiles());
+      }
+    } catch (IOException e) {
+      if (e.getMessage().contains("Corruption:")) {
+        logger.error("Database {} corrupted, please delete database directory({}) and restart.",
+            dataBaseName, parentPath, e);
+      } else {
+        logger.error("Open Database {} failed", dataBaseName, e);
+      }
+      System.exit(1);
     }
   }
 

chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java

@@ -32,6 +32,7 @@
 import org.rocksdb.RocksDBException;
 import org.rocksdb.RocksIterator;
 import org.rocksdb.Statistics;
+import org.rocksdb.Status;
 import org.rocksdb.WriteBatch;
 import org.rocksdb.WriteOptions;
 import org.slf4j.LoggerFactory;
@@ -265,8 +266,13 @@ protected void log(InfoLogLevel infoLogLevel, String logMsg) {
           try {
             database = RocksDB.open(options, dbPath.toString());
           } catch (RocksDBException e) {
-            throw new RuntimeException(
-                String.format("failed to open database: %s", dataBaseName), e);
+            if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) {
+              logger.error("Database {} corrupted, please delete database directory({}) " +
+                      "and restart.", dataBaseName, parentPath, e);
+            } else {
+              logger.error("Open Database {} failed", dataBaseName, e);
+            }
+            System.exit(1);
           }
 
           alive = true;

chainbase/src/main/java/org/tron/core/ChainBaseManager.java

@@ -316,9 +316,7 @@ public BlockCapsule getHead() throws HeaderNotFound {
   }
 
   public synchronized BlockId getHeadBlockId() {
-    return new BlockId(
-        dynamicPropertiesStore.getLatestBlockHeaderHash(),
-        dynamicPropertiesStore.getLatestBlockHeaderNumber());
+    return new BlockId(dynamicPropertiesStore.getLatestBlockHeaderHash());
   }
 
   public long getHeadBlockNum() {

chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java

@@ -493,7 +493,9 @@ public void check() {
     if (!isV2Open()) {
       List<String> cpList = getCheckpointList();
       if (cpList != null && cpList.size() != 0) {
-        logger.error("checkpoint check failed, can't convert checkpoint from v2 to v1");
+        logger.error("checkpoint check failed, the checkpoint version of database not match your " +
+            "config file, please set storage.checkpoint.version = 2 in your config file " +
+            "and restart the node.");
         System.exit(-1);
       }
       checkV1();

common/src/main/java/org/tron/common/es/ExecutorServiceManager.java

@@ -0,0 +1,55 @@
+package org.tron.common.es;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j(topic = "common")
+public class ExecutorServiceManager {
+
+  public static ExecutorService newSingleThreadExecutor(String name) {
+    return newSingleThreadExecutor(name, false);
+  }
+
+  public static ExecutorService newSingleThreadExecutor(String name, boolean isDaemon) {
+    return Executors.newSingleThreadExecutor(
+        new ThreadFactoryBuilder().setNameFormat(name).setDaemon(isDaemon).build());
+  }
+
+
+  public static ScheduledExecutorService newSingleThreadScheduledExecutor(String name) {
+    return newSingleThreadScheduledExecutor(name, false);
+  }
+
+  public static ScheduledExecutorService newSingleThreadScheduledExecutor(String name,
+                                                                          boolean isDaemon) {
+    return Executors.newSingleThreadScheduledExecutor(
+        new ThreadFactoryBuilder().setNameFormat(name).setDaemon(isDaemon).build());
+  }
+
+  public static void shutdownAndAwaitTermination(ExecutorService pool, String name) {
+    if (pool == null) {
+      return;
+    }
+    logger.info("Pool {} shutdown...", name);
+    pool.shutdown(); // Disable new tasks from being submitted
+    try {
+      // Wait a while for existing tasks to terminate
+      if (!pool.awaitTermination(60, java.util.concurrent.TimeUnit.SECONDS)) {
+        pool.shutdownNow(); // Cancel currently executing tasks
+        // Wait a while for tasks to respond to being cancelled
+        if (!pool.awaitTermination(60, java.util.concurrent.TimeUnit.SECONDS)) {
+          logger.warn("Pool {} did not terminate", name);
+        }
+      }
+    } catch (InterruptedException ie) {
+      // (Re-)Cancel if current thread also interrupted
+      pool.shutdownNow();
+      // Preserve interrupt status
+      Thread.currentThread().interrupt();
+    }
+    logger.info("Pool {} shutdown done", name);
+  }
+}

consensus/src/main/java/org/tron/consensus/dpos/DposTask.java

@@ -3,12 +3,14 @@
 import static org.tron.core.config.Parameter.ChainConstant.BLOCK_PRODUCED_INTERVAL;
 
 import com.google.protobuf.ByteString;
+import java.util.concurrent.ExecutorService;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.joda.time.DateTime;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
-import org.springframework.util.StringUtils;
+import org.springframework.util.ObjectUtils;
+import org.tron.common.es.ExecutorServiceManager;
 import org.tron.common.parameter.CommonParameter;
 import org.tron.common.utils.ByteArray;
 import org.tron.common.utils.Sha256Hash;
@@ -34,16 +36,18 @@ public class DposTask {
   @Setter
   private DposService dposService;
 
-  private Thread produceThread;
+  private ExecutorService produceExecutor;
+
+  private final String name = "DPosMiner";
 
   private volatile boolean isRunning = true;
 
   public void init() {
 
-    if (!dposService.isEnable() || StringUtils.isEmpty(dposService.getMiners())) {
+    if (!dposService.isEnable() || ObjectUtils.isEmpty(dposService.getMiners())) {
       return;
     }
-
+    produceExecutor = ExecutorServiceManager.newSingleThreadExecutor(name);
     Runnable runnable = () -> {
       while (isRunning) {
         try {
@@ -67,17 +71,13 @@ public void init() {
         }
       }
     };
-    produceThread = new Thread(runnable, "DPosMiner");
-    produceThread.start();
+    produceExecutor.submit(runnable);
     logger.info("DPoS task started.");
   }
 
   public void stop() {
     isRunning = false;
-    if (produceThread != null) {
-      produceThread.interrupt();
-    }
-    logger.info("DPoS task stopped.");
+    ExecutorServiceManager.shutdownAndAwaitTermination(produceExecutor, name);
   }
 
   private State produceBlock() {

framework/build.gradle

@@ -43,6 +43,7 @@ dependencies {
     testCompile group: 'junit', name: 'junit', version: '4.13.2'
     testCompile group: 'org.mockito', name: 'mockito-core', version: '2.13.0'
     testCompile group: 'org.hamcrest', name: 'hamcrest-junit', version: '1.0.0.1'
+    testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.16.0'
 
     compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69'
 

framework/src/main/java/org/tron/common/backup/socket/BackupServer.java

@@ -7,11 +7,13 @@
 import io.netty.channel.socket.nio.NioDatagramChannel;
 import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder;
 import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.tron.common.backup.BackupManager;
+import org.tron.common.es.ExecutorServiceManager;
 import org.tron.common.parameter.CommonParameter;
 import org.tron.p2p.stats.TrafficStats;
 
@@ -29,20 +31,24 @@ public class BackupServer {
 
   private volatile boolean shutdown = false;
 
+  private final String name = "BackupServer";
+  private ExecutorService executor;
+
   @Autowired
   public BackupServer(final BackupManager backupManager) {
     this.backupManager = backupManager;
   }
 
   public void initServer() {
     if (port > 0 && commonParameter.getBackupMembers().size() > 0) {
-      new Thread(() -> {
+      executor = ExecutorServiceManager.newSingleThreadExecutor(name);
+      executor.submit(() -> {
         try {
           start();
         } catch (Exception e) {
           logger.error("Start backup server failed, {}", e);
         }
-      }, "BackupServer").start();
+      });
     }
   }
 
@@ -88,12 +94,14 @@ public void initChannel(NioDatagramChannel ch)
   public void close() {
     logger.info("Closing backup server...");
     shutdown = true;
+    ExecutorServiceManager.shutdownAndAwaitTermination(executor, name);
     if (channel != null) {
       try {
         channel.close().await(10, TimeUnit.SECONDS);
       } catch (Exception e) {
         logger.warn("Closing backup server failed.", e);
       }
     }
+    logger.info("Backup server closed.");
   }
 }