feat(toolkit): add db conversion and data spliting extensions

  1. convert leveldb to rocksdb
  2. split lite full node data
  3. optimize gradle build

framework/build.gradle

@@ -267,14 +267,28 @@ createScript(project, 'org.tron.program.DBConvert', 'DBConvert')
 createScript(project, 'org.tron.tool.litefullnode.LiteFullNodeTool', 'LiteFullNodeTool')
 
 def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true'
+def skipSolidity = hasProperty('skipSolidity') ? true : false
+def skipKeystore = hasProperty('skipKeystore') ? true : false
+def skipConvert = hasProperty('skipConvert') ? true : false
+def skipLite = hasProperty('skipLite') ? true : false
+def skipAll = hasProperty('skipAll') ? true : false
 if (releaseBinary == 'true') {
-    artifacts {
-        archives(binaryRelease('buildSolidityNodeJar', 'SolidityNode', 'org.tron.program.SolidityNode'),
-                binaryRelease('buildFullNodeJar', 'FullNode', 'org.tron.program.FullNode'),
-                binaryRelease('buildKeystoreFactoryJar', 'KeystoreFactory', 'org.tron.program.KeystoreFactory'),
-                binaryRelease('buildDBConvertJar', 'DBConvert', 'org.tron.program.DBConvert'),
-                binaryRelease('buildLiteFullNodeToolJar', 'LiteFullNodeTool', 'org.tron.tool.litefullnode.LiteFullNodeTool'))
+   binaryRelease('buildFullNodeJar', 'FullNode', 'org.tron.program.FullNode')
+    if (!skipAll) {
+        if (!skipSolidity) {
+            binaryRelease('buildSolidityNodeJar', 'SolidityNode', 'org.tron.program.SolidityNode')
+        }
+        if (!skipKeystore) {
+            binaryRelease('buildKeystoreFactoryJar', 'KeystoreFactory', 'org.tron.program.KeystoreFactory')
+        }
+        if (!skipConvert) {
+            binaryRelease('buildDBConvertJar', 'DBConvert', 'org.tron.program.DBConvert')
+        }
+        if (!skipLite) {
+            binaryRelease('buildLiteFullNodeToolJar', 'LiteFullNodeTool', 'org.tron.tool.litefullnode.LiteFullNodeTool')
+        }
     }
+
 }
 
 task copyToParent(type: Copy) {

framework/src/main/java/org/tron/core/config/args/Args.java

@@ -1437,7 +1437,7 @@ public static boolean checkIsLiteFullNode() {
         PARAMETER.storage.getDbDirectory(), Constant.INFO_FILE_NAME).toString();
     if (FileUtil.isExists(infoFile)) {
       String value = PropUtil.readProperty(infoFile, Constant.SPLIT_BLOCK_NUM);
-      return !"".equals(value) && Long.parseLong(value) > 0;
+      return !"".equals(value) && Long.parseLong(value) > 1;
     }
     return false;
   }

framework/src/main/java/org/tron/tool/litefullnode/DbTool.java

@@ -27,7 +27,7 @@
 public class DbTool {
 
   private static final String KEY_ENGINE = "ENGINE";
-  private static final String ENGINE_FILE = "engine.properties";
+  public static final String ENGINE_FILE = "engine.properties";
   private static final String FILE_SEPARATOR = File.separator;
   private static final String ROCKSDB = "ROCKSDB";
 

framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java

@@ -14,6 +14,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.List;
@@ -331,6 +332,17 @@ private void fillSnapshotBlockAndTransDb(String sourceDir, String snapshotDir)
     DBInterface destCommonDb = DbTool.getDB(snapshotDir, COMMON_DB_NAME);
     destCommonDb.put(DB_KEY_NODE_TYPE, ByteArray.fromInt(Constant.NODE_TYPE_LIGHT_NODE));
     destCommonDb.put(DB_KEY_LOWEST_BLOCK_NUM, ByteArray.fromLong(startIndex));
+    // copy engine.properties for block、block-index、trans from source if exist
+    copyEngineIfExist(sourceDir, snapshotDir, BLOCK_DB_NAME, BLOCK_INDEX_DB_NAME, TRANS_DB_NAME);
+  }
+
+  private void copyEngineIfExist(String source, String dest, String... dbNames) {
+    for (String dbName : dbNames) {
+      Path ori = Paths.get(source, dbName, DbTool.ENGINE_FILE);
+      if (ori.toFile().exists()) {
+        Util.copy(ori, Paths.get(dest, dbName, DbTool.ENGINE_FILE));
+      }
+    }
   }
 
   private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDBException {

framework/src/main/java/org/tron/tool/litefullnode/Util.java

@@ -43,7 +43,7 @@ public static void copyDatabases(Path src, Path dest, List<String> subDirs)
     });
   }
 
-  private static void copy(Path source, Path dest) {
+  public static void copy(Path source, Path dest) {
     try {
       // create hard link when file is .sst
       if (source.toString().endsWith(".sst")) {

plugins/build.gradle

@@ -33,10 +33,11 @@ dependencies {
     compile group: 'info.picocli', name: 'picocli', version: '4.6.3'
     compile group: 'com.typesafe', name: 'config', version: '1.3.2'
     compile group: 'me.tongfei', name: 'progressbar', version: '0.9.3'
-
-
-    compile 'com.github.halibobo1205.leveldb-java:leveldb:v0.12.5'
-    compile 'com.github.halibobo1205.leveldb-java:leveldb-api:v0.12.5'
+    compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69'
+    compile group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10'
+    compile 'com.halibobor:leveldbjni-all:1.18.2'
+    compile 'com.halibobor:leveldb:1.18.2'
+    compile project(":protocol")
 }
 
 check.dependsOn 'lint'
@@ -104,6 +105,11 @@ def binaryRelease(taskName, jarName, mainClass) {
             }
         }
 
+        // exclude these files for bouncycastle
+        exclude "META-INF/*.SF"
+        exclude "META-INF/*.DSA"
+        exclude "META-INF/*.RSA"
+
         manifest {
             attributes "Main-Class": "${mainClass}"
         }
@@ -132,11 +138,14 @@ createScript(project, 'org.tron.plugins.ArchiveManifest', 'ArchiveManifest')
 createScript(project, 'org.tron.plugins.Toolkit', 'Toolkit')
 
 def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true'
+def skipArchive = hasProperty('skipArchive') ? true : false
+def skipAll = hasProperty('skipAll') ? true : false
 if (releaseBinary == 'true') {
-    artifacts {
-        archives(
-                binaryRelease('buildArchiveManifestJar', 'ArchiveManifest', 'org.tron.plugins.ArchiveManifest'),
-                binaryRelease('buildToolkitJar', 'Toolkit', 'org.tron.plugins.Toolkit'))
+    binaryRelease('buildToolkitJar', 'Toolkit', 'org.tron.plugins.Toolkit')
+    if (!skipAll) {
+        if (!skipArchive) {
+            binaryRelease('buildArchiveManifestJar', 'ArchiveManifest', 'org.tron.plugins.ArchiveManifest')
+        }
     }
 }
 

plugins/src/main/java/org/tron/plugins/ArchiveManifest.java

@@ -82,7 +82,6 @@ public static org.iq80.leveldb.Options newDefaultLevelDbOptions() {
     dbOptions.maxOpenFiles(1000);
     dbOptions.maxBatchSize(64_000);
     dbOptions.maxManifestSize(128);
-    dbOptions.fast(false);
     return dbOptions;
   }
 

plugins/src/main/java/org/tron/plugins/Db.java

@@ -12,8 +12,13 @@
     mixinStandardHelpOptions = true,
     version = "db command 1.0",
     description = "An rich command set that provides high-level operations  for dbs.",
-    subcommands = {CommandLine.HelpCommand.class, DbMove.class, DbArchive.class},
-    commandListHeading = "%nCommands:%n%nThe most commonly used git commands are:%n"
+    subcommands = {CommandLine.HelpCommand.class,
+        DbMove.class,
+        DbArchive.class,
+        DbConvert.class,
+        DbLite.class
+    },
+    commandListHeading = "%nCommands:%n%nThe most commonly used db commands are:%n"
 )
 public class Db {
 

plugins/src/main/java/org/tron/plugins/DbArchive.java

@@ -3,25 +3,14 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.iq80.leveldb.impl.Iq80DBFactory.factory;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
-import java.util.Properties;
 import java.util.concurrent.Callable;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
@@ -30,11 +19,12 @@
 import org.iq80.leveldb.DB;
 import org.iq80.leveldb.Options;
 import org.iq80.leveldb.impl.Filename;
+import org.tron.plugins.utils.FileUtils;
 import picocli.CommandLine;
 import picocli.CommandLine.Option;
 
 @Slf4j(topic = "archive")
-@CommandLine.Command(name = "archive", description = "a helper to rewrite leveldb manifest.")
+@CommandLine.Command(name = "archive", description = "A helper to rewrite leveldb manifest.")
 public class DbArchive implements Callable<Integer> {
 
   @CommandLine.Spec
@@ -189,7 +179,7 @@ public void doArchive() {
       }
       try {
         if (checkManifest(levelDbFile.toString())) {
-          if (!checkEngine()) {
+          if (!FileUtils.isLevelDBEngine(srcDbPath)) {
             logger.info("Db {},not leveldb, ignored.", this.name);
             return;
           }
@@ -203,46 +193,6 @@ public void doArchive() {
         throw new RuntimeException("Db " + this.name + " archive failed.", e);
       }
     }
-
-    public boolean checkEngine() {
-      String dir = this.srcDbPath.toString();
-      String enginePath = dir + File.separator + "engine.properties";
-      if (!new File(enginePath).exists() && !writeProperty(enginePath, KEY_ENGINE, LEVELDB)) {
-        return false;
-      }
-      String engine = readProperty(enginePath, KEY_ENGINE);
-      return LEVELDB.equals(engine);
-    }
-
-    public static String readProperty(String file, String key) {
-      try (FileInputStream fileInputStream = new FileInputStream(file);
-           InputStream inputStream = new BufferedInputStream(fileInputStream)) {
-        Properties prop = new Properties();
-        prop.load(inputStream);
-        return new String(prop.getProperty(key, "").getBytes(StandardCharsets.ISO_8859_1),
-            UTF_8);
-      } catch (Exception e) {
-        logger.error("readProperty", e);
-        return "";
-      }
-    }
-
-    public static boolean writeProperty(String file, String key, String value) {
-      try (OutputStream out = new FileOutputStream(file);
-           FileInputStream fis = new FileInputStream(file);
-           BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, UTF_8))) {
-        BufferedReader bf = new BufferedReader(new InputStreamReader(fis, UTF_8));
-        Properties properties = new Properties();
-        properties.load(bf);
-        properties.setProperty(key, value);
-        properties.store(bw, "Generated by the application.  PLEASE DO NOT EDIT! ");
-      } catch (Exception e) {
-        logger.warn("writeProperty", e);
-        return false;
-      }
-      return true;
-    }
-
   }
 
 }