feature(db): optimize properties query (#5378)

chainbase/src/main/java/org/tron/core/db2/core/AbstractSnapshot.java

@@ -15,6 +15,8 @@ public abstract class AbstractSnapshot<K, V> implements Snapshot {
 
   protected WeakReference<Snapshot> next;
 
+  protected boolean isOptimized;
+
   @Override
   public Snapshot advance() {
     return new SnapshotImpl(this);
@@ -34,4 +36,9 @@ public void setNext(Snapshot next) {
   public String getDbName() {
     return db.getDbName();
   }
+
+  @Override
+  public boolean isOptimized(){
+    return isOptimized;
+  }
 }

chainbase/src/main/java/org/tron/core/db2/core/Snapshot.java

@@ -46,4 +46,8 @@ static boolean isImpl(Snapshot snapshot) {
   void updateSolidity();
 
   String getDbName();
+
+  boolean isOptimized();
+
+  void reloadToMem();
 }

chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java

@@ -30,6 +30,10 @@ public class SnapshotImpl extends AbstractSnapshot<Key, Value> {
     }
     previous = snapshot;
     snapshot.setNext(this);
+    isOptimized = snapshot.isOptimized();
+    if (isOptimized &&  root == previous) {
+      Streams.stream(root.iterator()).forEach( e -> put(e.getKey(),e.getValue()));
+    }
   }
 
   @Override
@@ -40,6 +44,7 @@ public byte[] get(byte[] key) {
   private byte[] get(Snapshot head, byte[] key) {
     Snapshot snapshot = head;
     Value value;
+
     while (Snapshot.isImpl(snapshot)) {
       if ((value = ((SnapshotImpl) snapshot).db.get(Key.of(key))) != null) {
         return value.getBytes();
@@ -83,6 +88,19 @@ public void merge(Snapshot from) {
     Streams.stream(fromImpl.db).forEach(e -> db.put(e.getKey(), e.getValue()));
   }
 
+  public void mergeAhead(Snapshot from) {
+    if (from instanceof SnapshotRoot) {
+      return ;
+    }
+    SnapshotImpl fromImpl = (SnapshotImpl) from;
+    Streams.stream(fromImpl.db).forEach(e -> {
+      if (db.get(e.getKey()) == null) {
+        db.put(e.getKey(), e.getValue());
+      }
+    }
+    );
+  }
+
   @Override
   public Snapshot retreat() {
     return previous;
@@ -177,4 +195,9 @@ public String getDbName() {
   public Snapshot newInstance() {
     return new SnapshotImpl(this);
   }
+
+  @Override
+  public void reloadToMem() {
+      mergeAhead(previous);
+  }
 }

chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java

@@ -221,6 +221,12 @@ public synchronized void commit() {
     }
 
     --activeSession;
+
+    dbs.forEach(db -> {
+      if (db.getHead().isOptimized()) {
+        db.getHead().reloadToMem();
+      }
+    });
   }
 
   public synchronized void pop() {

chainbase/src/main/java/org/tron/core/db2/core/SnapshotRoot.java

@@ -38,6 +38,7 @@ public SnapshotRoot(DB<byte[], byte[]> db) {
     if (CACHE_DBS.contains(this.db.getDbName())) {
       this.cache = CacheManager.allocate(CacheType.findByType(this.db.getDbName()));
     }
+    isOptimized = "properties".equalsIgnoreCase(db.getDbName());
   }
 
   private boolean needOptAsset() {
@@ -221,4 +222,7 @@ public String getDbName() {
   public Snapshot newInstance() {
     return new SnapshotRoot(db.newInstance());
   }
+
+  @Override
+  public void reloadToMem() { }
 }

framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java

@@ -0,0 +1,201 @@
+package org.tron.core.db2;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.File;
+import java.lang.reflect.Constructor;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.tron.common.application.Application;
+import org.tron.common.application.ApplicationFactory;
+import org.tron.common.application.TronApplicationContext;
+import org.tron.common.utils.FileUtil;
+import org.tron.core.Constant;
+import org.tron.core.config.DefaultConfig;
+import org.tron.core.config.args.Args;
+import org.tron.core.db2.core.Snapshot;
+import org.tron.core.db2.core.SnapshotImpl;
+import org.tron.core.db2.core.SnapshotManager;
+import org.tron.core.db2.core.SnapshotRoot;
+
+public class SnapshotImplTest {
+  private RevokingDbWithCacheNewValueTest.TestRevokingTronStore tronDatabase;
+  private TronApplicationContext context;
+  private Application appT;
+  private SnapshotManager revokingDatabase;
+
+  @Before
+  public void init() {
+    Args.setParam(new String[]{"-d", "output_revokingStore_test"}, Constant.TEST_CONF);
+    context = new TronApplicationContext(DefaultConfig.class);
+    appT = ApplicationFactory.create(context);
+
+    tronDatabase = new RevokingDbWithCacheNewValueTest.TestRevokingTronStore(
+        "testSnapshotRoot-testMerge");
+    revokingDatabase = context.getBean(SnapshotManager.class);
+    revokingDatabase.enable();
+    revokingDatabase.add(tronDatabase.getRevokingDB());
+  }
+
+  @After
+  public void removeDb() {
+    Args.clearParam();
+    context.destroy();
+    FileUtil.deleteDir(new File("output_revokingStore_test"));
+
+    tronDatabase.close();
+    revokingDatabase.shutdown();
+  }
+
+  /**
+   * linklist is: from -> root
+   * root:key1=>value1, key2=>value2
+   * from:key3=>value3, key4=>value4
+   * after construct, getSnapshotImplIns(root);
+   * from: key1=>value1, key2=>value2, key3=>value3, key4=>value4
+   * from: get key1 or key2, traverse 0 times
+   */
+  @Test
+  public void testMergeRoot() {
+    // linklist is: from -> root
+    SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb());
+    //root.setOptimized(true);
+
+    root.put("key1".getBytes(), "value1".getBytes());
+    root.put("key2".getBytes(), "value2".getBytes());
+    SnapshotImpl from = getSnapshotImplIns(root);
+    from.put("key3".getBytes(), "value3".getBytes());
+    from.put("key4".getBytes(), "value4".getBytes());
+
+    byte[] s1 = from.get("key1".getBytes());
+    assertEquals(new String("value1".getBytes()), new String(s1));
+    byte[] s2 = from.get("key2".getBytes());
+    assertEquals(new String("value2".getBytes()), new String(s2));
+  }
+
+  /**
+   * linklist is: from2 -> from -> root
+   * root:
+   * from:key1=>value1, key2=>value2
+   * from2:key3=>value3,key4=>value4
+   * before merge: from2.mergeAhead(from);
+   * from2: get key1 or key2, traverse 1 times
+   * after merge
+   * from2:key1=>value1, key2=>value2, value3=>value3,key4=>value4
+   * from2: get key1 or key2, traverse 0 times
+   *
+   */
+  @Test
+  public void testMergeAhead() {
+
+    // linklist is: from2 -> from -> root
+    SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb());
+    SnapshotImpl from = getSnapshotImplIns(root);
+    from.put("key1".getBytes(), "value1".getBytes());
+    from.put("key2".getBytes(), "value2".getBytes());
+
+    SnapshotImpl from2 = getSnapshotImplIns(from);
+    from2.put("key3".getBytes(), "value3".getBytes());
+    from2.put("key4".getBytes(), "value4".getBytes());
+
+    /*
+    // before merge  get data in from is success，traverse 0 times
+    byte[] s1 = from.get("key1".getBytes());
+    assertEquals(new String("value1".getBytes()), new String(s1));
+    byte[] s2 = from.get("key2".getBytes());
+    assertEquals(new String("value2".getBytes()), new String(s2));
+    // before merge  get data in from2 is success， traverse 0 times
+    byte[] s3 = from2.get("key3".getBytes());
+    assertEquals(new String("value3".getBytes()), new String(s3));
+    byte[] s4 = from2.get("key4".getBytes());
+    assertEquals(new String("value4".getBytes()), new String(s4));
+     */
+
+    // before merge from2 get data is success， traverse 1 times
+    byte[] s11 = from2.get("key1".getBytes());
+    assertEquals(new String("value1".getBytes()), new String(s11));
+    byte[] s12 = from2.get("key2".getBytes());
+    assertEquals(new String("value2".getBytes()), new String(s12));
+    // this can not get key3 and key4
+    assertNull(from.get("key3".getBytes()));
+    assertNull(from.get("key4".getBytes()));
+
+    // do mergeAhead
+    from2.mergeAhead(from);
+    /*
+    // after merge  get data in from is success， traverse 0 times
+    s1 = from.get("key1".getBytes());
+    assertEquals(new String("value1".getBytes()), new String(s1));
+    s2 = from.get("key2".getBytes());
+    assertEquals(new String("value2".getBytes()), new String(s2));
+
+    // after merge get data in from2 is success， traverse 0 times
+    s3 = from2.get("key3".getBytes());
+    assertEquals(new String("value3".getBytes()), new String(s3));
+    s4 = from2.get("key4".getBytes());
+    assertEquals(new String("value4".getBytes()), new String(s4));
+     */
+
+    // after merge from2 get data is success， traverse 0 times
+    byte[] s1 = from2.get("key1".getBytes());
+    assertEquals(new String("value1".getBytes()), new String(s1));
+    byte[] s2 = from2.get("key2".getBytes());
+    assertEquals(new String("value2".getBytes()), new String(s2));
+
+    // this can not get key3 and key4
+    assertNull(from.get("key3".getBytes()));
+    assertNull(from.get("key4".getBytes()));
+  }
+
+  /**
+   * from: key1=>value1, key2=>value2, key3=>value31
+   * from2: key3=>value32,key4=>value4
+   * after merge: from2.mergeAhead(from);
+   * from2: key1=>value1, key2=>value2, key3=>value32, key4=>value4
+   */
+  @Test
+  public void testMergeOverride() {
+    // linklist is: from2 -> from -> root
+    SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb());
+    SnapshotImpl from = getSnapshotImplIns(root);
+    from.put("key1".getBytes(), "value1".getBytes());
+    from.put("key2".getBytes(), "value2".getBytes());
+    from.put("key3".getBytes(), "value31".getBytes());
+
+    SnapshotImpl from2 = getSnapshotImplIns(from);
+    from2.put("key3".getBytes(), "value32".getBytes());
+    from2.put("key4".getBytes(), "value4".getBytes());
+    // do mergeAhead
+    from2.mergeAhead(from);
+
+    // after merge from2 get data is success， traverse 0 times
+    byte[] s1 = from2.get("key1".getBytes());
+    assertEquals(new String("value1".getBytes()), new String(s1));
+    byte[] s2 = from2.get("key2".getBytes());
+    assertEquals(new String("value2".getBytes()), new String(s2));
+    byte[] s3 = from2.get("key3".getBytes());
+    assertEquals(new String("value32".getBytes()), new String(s3));
+    byte[] s4 = from2.get("key4".getBytes());
+    assertEquals(new String("value4".getBytes()), new String(s4));
+  }
+
+  /**
+   * The constructor of SnapshotImpl is not public
+   * so reflection is used to construct the object here.
+   */
+  private SnapshotImpl getSnapshotImplIns(Snapshot snapshot) {
+    Class clazz = SnapshotImpl.class;
+    try {
+      Constructor constructor = clazz.getDeclaredConstructor(Snapshot.class);
+      constructor.setAccessible(true);
+      return (SnapshotImpl) constructor.newInstance(snapshot);
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+    return null;
+  }
+
+}