diff --git a/chainbase/src/main/java/org/tron/core/db2/common/DB.java b/chainbase/src/main/java/org/tron/core/db2/common/DB.java index fed5e52bb04..eae529c5ca9 100644 --- a/chainbase/src/main/java/org/tron/core/db2/common/DB.java +++ b/chainbase/src/main/java/org/tron/core/db2/common/DB.java @@ -15,7 +15,7 @@ public interface DB extends Iterable>, Instance> void remove(K k); - Iterator iterator(); + Iterator> iterator(); void close(); diff --git a/chainbase/src/main/java/org/tron/core/db2/core/AbstractSnapshot.java b/chainbase/src/main/java/org/tron/core/db2/core/AbstractSnapshot.java index ba6c77d43a2..496a0fd09ac 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/AbstractSnapshot.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/AbstractSnapshot.java @@ -15,8 +15,6 @@ public abstract class AbstractSnapshot implements Snapshot { protected WeakReference next; - protected boolean isOptimized; - @Override public Snapshot advance() { return new SnapshotImpl(this); @@ -36,9 +34,4 @@ public void setNext(Snapshot next) { public String getDbName() { return db.getDbName(); } - - @Override - public boolean isOptimized(){ - return isOptimized; - } } diff --git a/chainbase/src/main/java/org/tron/core/db2/core/Snapshot.java b/chainbase/src/main/java/org/tron/core/db2/core/Snapshot.java index db59713f34d..e1ca149b207 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/Snapshot.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/Snapshot.java @@ -46,6 +46,4 @@ static boolean isImpl(Snapshot snapshot) { void updateSolidity(); String getDbName(); - - boolean isOptimized(); } diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java index 83c55c238ea..ae8073f668b 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java @@ -30,16 +30,6 @@ public class SnapshotImpl extends AbstractSnapshot { } previous = snapshot; snapshot.setNext(this); - // inherit - isOptimized = snapshot.isOptimized(); - // merge for DynamicPropertiesStore，about 100 keys - if (isOptimized) { - if (root == previous ){ - Streams.stream(root.iterator()).forEach( e -> put(e.getKey(),e.getValue())); - }else { - merge(previous); - } - } } @Override @@ -50,10 +40,6 @@ public byte[] get(byte[] key) { private byte[] get(Snapshot head, byte[] key) { Snapshot snapshot = head; Value value; - if (isOptimized) { - value = db.get(Key.of(key)); - return value == null ? null: value.getBytes(); - } while (Snapshot.isImpl(snapshot)) { if ((value = ((SnapshotImpl) snapshot).db.get(Key.of(key))) != null) { return value.getBytes(); diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotRoot.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotRoot.java index 9afb5277b9c..709e2ae1b62 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotRoot.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotRoot.java @@ -1,14 +1,18 @@ package org.tron.core.db2.core; -import ch.qos.logback.core.encoder.ByteArrayUtil; import com.google.common.collect.Maps; import com.google.common.collect.Streams; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import lombok.Getter; +import org.tron.common.cache.CacheManager; +import org.tron.common.cache.CacheType; +import org.tron.common.cache.TronCache; +import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; import org.tron.core.ChainBaseManager; import org.tron.core.capsule.AccountCapsule; @@ -23,11 +27,17 @@ public class SnapshotRoot extends AbstractSnapshot { private Snapshot solidity; private boolean isAccountDB; + private TronCache cache; + private static final List CACHE_DBS = CommonParameter.getInstance() + .getStorage().getCacheDbs(); + public SnapshotRoot(DB db) { this.db = db; solidity = this; - isOptimized = "properties".equalsIgnoreCase(db.getDbName()); isAccountDB = "account".equalsIgnoreCase(db.getDbName()); + if (CACHE_DBS.contains(this.db.getDbName())) { + this.cache = CacheManager.allocate(CacheType.findByType(this.db.getDbName())); + } } private boolean needOptAsset() { @@ -37,11 +47,18 @@ private boolean needOptAsset() { @Override public byte[] get(byte[] key) { - return db.get(key); + WrappedByteArray cache = getCache(key); + if (cache != null) { + return cache.getBytes(); + } + byte[] value = db.get(key); + putCache(key, value); + return value; } @Override public void put(byte[] key, byte[] value) { + byte[] v = value; if (needOptAsset()) { if (ByteArray.isEmpty(value)) { remove(key); @@ -56,10 +73,10 @@ public void put(byte[] key, byte[] value) { } assetStore.putAccount(item.getInstance()); item.clearAsset(); - db.put(key, item.getData()); - } else { - db.put(key, value); + v = item.getData(); } + db.put(key, v); + putCache(key, v); } @Override @@ -68,6 +85,7 @@ public void remove(byte[] key) { ChainBaseManager.getInstance().getAccountAssetStore().deleteAccount(key); } db.remove(key); + putCache(key, null); } @Override @@ -81,6 +99,7 @@ public void merge(Snapshot from) { processAccount(batch); } else { ((Flusher) db).flush(batch); + putCache(batch); } } @@ -97,6 +116,7 @@ public void merge(List snapshots) { processAccount(batch); } else { ((Flusher) db).flush(batch); + putCache(batch); } } @@ -120,11 +140,37 @@ private void processAccount(Map batch) { } }); ((Flusher) db).flush(accounts); + putCache(accounts); if (assets.size() > 0) { assetStore.updateByBatch(AccountAssetStore.convert(assets)); } } + private boolean cached() { + return Objects.nonNull(this.cache); + } + + private void putCache(byte[] key, byte[] value) { + if (cached()) { + cache.put(WrappedByteArray.of(key), WrappedByteArray.of(value)); + } + } + + private void putCache(Map values) { + if (cached()) { + values.forEach(cache::put); + } + } + + private WrappedByteArray getCache(byte[] key) { + if (cached()) { + return cache.getIfPresent(WrappedByteArray.of(key)); + } + return null; + } + + // second cache + @Override public Snapshot retreat() { return this; @@ -142,11 +188,17 @@ public Iterator> iterator() { @Override public void close() { + if (cached()) { + CacheManager.release(cache); + } ((Flusher) db).close(); } @Override public void reset() { + if (cached()) { + CacheManager.release(cache); + } ((Flusher) db).reset(); } diff --git a/chainbase/src/main/java/org/tron/core/service/MortgageService.java b/chainbase/src/main/java/org/tron/core/service/MortgageService.java index 5dd99626930..e9b00a38201 100644 --- a/chainbase/src/main/java/org/tron/core/service/MortgageService.java +++ b/chainbase/src/main/java/org/tron/core/service/MortgageService.java @@ -1,14 +1,9 @@ package org.tron.core.service; import com.google.protobuf.ByteString; - import java.math.BigInteger; -import java.util.ArrayList; import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Map; - import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -18,7 +13,6 @@ import org.tron.common.utils.StringUtil; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.WitnessCapsule; -import org.tron.core.config.Parameter.ChainConstant; import org.tron.core.exception.BalanceInsufficientException; import org.tron.core.store.AccountStore; import org.tron.core.store.DelegationStore; @@ -52,31 +46,17 @@ public void initStore(WitnessStore witnessStore, DelegationStore delegationStore } public void payStandbyWitness() { - List witnessCapsules = witnessStore.getAllWitnesses(); - Map witnessCapsuleMap = new HashMap<>(); - List witnessAddressList = new ArrayList<>(); - for (WitnessCapsule witnessCapsule : witnessCapsules) { - witnessAddressList.add(witnessCapsule.getAddress()); - witnessCapsuleMap.put(witnessCapsule.getAddress(), witnessCapsule); - } - witnessAddressList.sort(Comparator.comparingLong((ByteString b) -> witnessCapsuleMap.get(b).getVoteCount()) - .reversed().thenComparing(Comparator.comparingInt(ByteString::hashCode).reversed())); - if (witnessAddressList.size() > ChainConstant.WITNESS_STANDBY_LENGTH) { - witnessAddressList = witnessAddressList.subList(0, ChainConstant.WITNESS_STANDBY_LENGTH); + List witnessStandbys = witnessStore.getWitnessStandby(); + long voteSum = witnessStandbys.stream().mapToLong(WitnessCapsule::getVoteCount).sum(); + if (voteSum < 1) { + return; } - long voteSum = 0; long totalPay = dynamicPropertiesStore.getWitness127PayPerBlock(); - for (ByteString b : witnessAddressList) { - voteSum += witnessCapsuleMap.get(b).getVoteCount(); - } - - if (voteSum > 0) { - for (ByteString b : witnessAddressList) { - double eachVotePay = (double) totalPay / voteSum; - long pay = (long) (witnessCapsuleMap.get(b).getVoteCount() * eachVotePay); - logger.debug("Pay {} stand reward {}.", Hex.toHexString(b.toByteArray()), pay); - payReward(b.toByteArray(), pay); - } + double eachVotePay = (double) totalPay / voteSum; + for (WitnessCapsule w : witnessStandbys) { + long pay = (long) (w.getVoteCount() * eachVotePay); + payReward(w.getAddress().toByteArray(), pay); + logger.debug("Pay {} stand reward {}.", Hex.toHexString(w.getAddress().toByteArray()), pay); } } diff --git a/chainbase/src/main/java/org/tron/core/store/WitnessStore.java b/chainbase/src/main/java/org/tron/core/store/WitnessStore.java index 22324bca108..48a0d207cfd 100644 --- a/chainbase/src/main/java/org/tron/core/store/WitnessStore.java +++ b/chainbase/src/main/java/org/tron/core/store/WitnessStore.java @@ -1,6 +1,8 @@ package org.tron.core.store; import com.google.common.collect.Streams; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map.Entry; import java.util.stream.Collectors; @@ -9,16 +11,25 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.tron.common.cache.CacheManager; +import org.tron.common.cache.CacheStrategies; +import org.tron.common.cache.CacheType; +import org.tron.common.cache.TronCache; import org.tron.core.capsule.WitnessCapsule; +import org.tron.core.config.Parameter; import org.tron.core.db.TronStoreWithRevoking; @Slf4j(topic = "DB") @Component public class WitnessStore extends TronStoreWithRevoking { + private final TronCache> witnessStandby; + @Autowired protected WitnessStore(@Value("witness") String dbName) { super(dbName); + String strategy = String.format(CacheStrategies.PATTERNS, 1, 1, "30s", 1); + witnessStandby = CacheManager.allocate(CacheType.witnessStandby, strategy); } /** @@ -35,4 +46,33 @@ public WitnessCapsule get(byte[] key) { byte[] value = revokingDB.getUnchecked(key); return ArrayUtils.isEmpty(value) ? null : new WitnessCapsule(value); } + + public List getWitnessStandby() { + List list = + witnessStandby.getIfPresent(Parameter.ChainConstant.WITNESS_STANDBY_LENGTH); + if (list != null) { + return list; + } + return updateWitnessStandby(null); + } + + public List updateWitnessStandby(List all) { + List ret; + if (all == null) { + all = getAllWitnesses(); + } + all.sort(Comparator.comparingLong(WitnessCapsule::getVoteCount) + .reversed().thenComparing(Comparator.comparingInt( + (WitnessCapsule w) -> w.getAddress().hashCode()).reversed())); + if (all.size() > Parameter.ChainConstant.WITNESS_STANDBY_LENGTH) { + ret = new ArrayList<>(all.subList(0, Parameter.ChainConstant.WITNESS_STANDBY_LENGTH)); + } else { + ret = new ArrayList<>(all); + } + // trim voteCount = 0 + ret.removeIf(w -> w.getVoteCount() < 1); + witnessStandby.put(Parameter.ChainConstant.WITNESS_STANDBY_LENGTH, ret); + return ret; + } + } diff --git a/common/src/main/java/org/tron/common/cache/CacheManager.java b/common/src/main/java/org/tron/common/cache/CacheManager.java new file mode 100644 index 00000000000..fa1fbff193b --- /dev/null +++ b/common/src/main/java/org/tron/common/cache/CacheManager.java @@ -0,0 +1,44 @@ +package org.tron.common.cache; + +import com.google.common.cache.CacheLoader; +import com.google.common.cache.CacheStats; +import com.google.common.collect.Maps; +import java.util.Map; +import java.util.stream.Collectors; +import org.tron.common.parameter.CommonParameter; + +public class CacheManager { + + private static final Map> CACHES = Maps.newConcurrentMap(); + + public static TronCache allocate(CacheType name) { + TronCache cache = new TronCache<>(name, CommonParameter.getInstance() + .getStorage().getCacheStrategy(name)); + CACHES.put(name, cache); + return cache; + } + + public static TronCache allocate(CacheType name, String strategy) { + TronCache cache = new TronCache<>(name, strategy); + CACHES.put(name, cache); + return cache; + } + + public static TronCache allocate(CacheType name, String strategy, + CacheLoader loader) { + TronCache cache = new TronCache<>(name, strategy, loader); + CACHES.put(name, cache); + return cache; + } + + + public static void release(TronCache cache) { + cache.invalidateAll(); + } + + public static Map stats() { + return CACHES.values().stream().collect(Collectors.toMap(c -> c.getName().toString(), + TronCache::stats)); + } + +} diff --git a/common/src/main/java/org/tron/common/cache/CacheStrategies.java b/common/src/main/java/org/tron/common/cache/CacheStrategies.java new file mode 100644 index 00000000000..b282ca1a687 --- /dev/null +++ b/common/src/main/java/org/tron/common/cache/CacheStrategies.java @@ -0,0 +1,70 @@ +package org.tron.common.cache; + +import static org.tron.common.cache.CacheType.abi; +import static org.tron.common.cache.CacheType.account; +import static org.tron.common.cache.CacheType.assetIssueV2; +import static org.tron.common.cache.CacheType.code; +import static org.tron.common.cache.CacheType.contract; +import static org.tron.common.cache.CacheType.delegatedResource; +import static org.tron.common.cache.CacheType.delegatedResourceAccountIndex; +import static org.tron.common.cache.CacheType.delegation; +import static org.tron.common.cache.CacheType.properties; +import static org.tron.common.cache.CacheType.recentBlock; +import static org.tron.common.cache.CacheType.storageRow; +import static org.tron.common.cache.CacheType.votes; +import static org.tron.common.cache.CacheType.witness; +import static org.tron.common.cache.CacheType.witnessSchedule; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class CacheStrategies { + + + public static final String PATTERNS = + "initialCapacity=%d,maximumSize=%d,expireAfterAccess=%s,concurrencyLevel=%d,recordStats"; + public static final List CACHE_BIG_DBS = Collections.singletonList(delegation); + private static final int CPUS = Runtime.getRuntime().availableProcessors(); + public static final String CACHE_STRATEGY_DEFAULT = + String.format(PATTERNS, 1000, 1000, "30s", CPUS); + private static final String CACHE_STRATEGY_SMALL_DEFAULT = + String.format(PATTERNS, 100, 100, "30s", CPUS); + private static final List CACHE_SMALL_DBS = Arrays.asList(recentBlock, witness, + witnessSchedule, delegatedResource, delegatedResourceAccountIndex, + votes, abi); + private static final String CACHE_STRATEGY_NORMAL_DEFAULT = + String.format(PATTERNS, 500, 500, "30s", CPUS); + private static final List CACHE_NORMAL_DBS = Arrays.asList(code, contract, + assetIssueV2, properties); + private static final String CACHE_STRATEGY_BIG_DEFAULT = + String.format(PATTERNS, 10000, 10000, "30s", CPUS); + private static final String CACHE_STRATEGY_HUGE_DEFAULT = + String.format(PATTERNS, 20000, 20000, "30s", CPUS); + private static final List CACHE_HUGE_DBS = Arrays.asList(storageRow, account); + + public static final List CACHE_DBS = Stream.of(CACHE_SMALL_DBS, CACHE_NORMAL_DBS, + CACHE_BIG_DBS, CACHE_HUGE_DBS).flatMap(Collection::stream).map(CacheType::toString) + .collect(Collectors.toList()); + + + public static String getCacheStrategy(CacheType dbName) { + String defaultStrategy = CACHE_STRATEGY_DEFAULT; + if (CACHE_SMALL_DBS.contains(dbName)) { + defaultStrategy = CACHE_STRATEGY_SMALL_DEFAULT; + } + if (CACHE_NORMAL_DBS.contains(dbName)) { + defaultStrategy = CACHE_STRATEGY_NORMAL_DEFAULT; + } + if (CACHE_BIG_DBS.contains(dbName)) { + defaultStrategy = CACHE_STRATEGY_BIG_DEFAULT; + } + if (CACHE_HUGE_DBS.contains(dbName)) { + defaultStrategy = CACHE_STRATEGY_HUGE_DEFAULT; + } + return defaultStrategy; + } +} diff --git a/common/src/main/java/org/tron/common/cache/CacheType.java b/common/src/main/java/org/tron/common/cache/CacheType.java new file mode 100644 index 00000000000..b9b28d0bd7c --- /dev/null +++ b/common/src/main/java/org/tron/common/cache/CacheType.java @@ -0,0 +1,46 @@ +package org.tron.common.cache; + +import java.util.Arrays; + +public enum CacheType { + // for 127 SR + witnessStandby("witnessStandby"), + // for leveldb or rocksdb cache + recentBlock("recent-block"), + witness("witness"), + witnessSchedule("witness_schedule"), + delegatedResource("DelegatedResource"), + delegatedResourceAccountIndex("DelegatedResourceAccountIndex"), + votes("votes"), + abi("abi"), + code("code"), + contract("contract"), + assetIssueV2("asset-issue-v2"), + properties("properties"), + delegation("delegation"), + storageRow("storage-row"), + account("account"); + // for leveldb or rocksdb cache + + public final String type; + + CacheType(String type) { + this.type = type; + } + + public static CacheType findByType(String type) { + return Arrays.stream(CacheType.values()).filter(c -> c.type.equals(type)).findFirst() + .orElseThrow(() -> new IllegalArgumentException(type)); + } + + @Override + public String toString() { + return type; + } + + + + + + +} diff --git a/common/src/main/java/org/tron/common/cache/TronCache.java b/common/src/main/java/org/tron/common/cache/TronCache.java new file mode 100644 index 00000000000..4faf73f864a --- /dev/null +++ b/common/src/main/java/org/tron/common/cache/TronCache.java @@ -0,0 +1,64 @@ +package org.tron.common.cache; + +import com.google.common.base.Objects; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.CacheStats; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import lombok.Getter; + +public class TronCache { + + @Getter + private final CacheType name; + private final Cache cache; + + TronCache(CacheType name, String strategy) { + this.name = name; + this.cache = CacheBuilder.from(strategy).build(); + } + + TronCache(CacheType name, String strategy, CacheLoader loader) { + this.name = name; + this.cache = CacheBuilder.from(strategy).build(loader); + } + + public void put(K k, V v) { + this.cache.put(k, v); + } + + public V getIfPresent(K k) { + return this.cache.getIfPresent(k); + } + + public V get(K k, Callable loader) throws ExecutionException { + return this.cache.get(k, loader); + } + + public CacheStats stats() { + return this.cache.stats(); + } + + public void invalidateAll() { + this.cache.invalidateAll(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TronCache tronCache = (TronCache) o; + return Objects.equal(name, tronCache.name); + } + + @Override + public int hashCode() { + return Objects.hashCode(name); + } +} diff --git a/common/src/main/java/org/tron/common/prometheus/GuavaCacheExports.java b/common/src/main/java/org/tron/common/prometheus/GuavaCacheExports.java new file mode 100644 index 00000000000..e8e39d5b3ad --- /dev/null +++ b/common/src/main/java/org/tron/common/prometheus/GuavaCacheExports.java @@ -0,0 +1,89 @@ +package org.tron.common.prometheus; + +import static io.prometheus.client.SampleNameFilter.ALLOW_ALL; + +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import io.prometheus.client.Predicate; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.tron.common.cache.CacheManager; + +/** + * Exports metrics about for guava cache. + *

+ * Example usage: + * 

+ * {@code
+ *   new GuavaCacheExports().register();
+ * }
+ *
+ * Example metrics being exported: + * 
+ *   tron:guava_cache_hit_rate{type="account"} 0.135679
+ *   tron:guava_cache_request{type="account"} 3000
+ *
+ */ +public class GuavaCacheExports extends Collector { + + private static final String TRON_GUAVA_CACHE_HIT_RATE = "tron:guava_cache_hit_rate"; + private static final String TRON_GUAVA_CACHE_REQUEST = "tron:guava_cache_request"; + private static final String TRON_GUAVA_CACHE_EVICTION_COUNT = "tron:guava_cache_eviction_count"; + + + public GuavaCacheExports() { + } + + + void addHitRateMetrics(List sampleFamilies, Predicate nameFilter) { + if (nameFilter.test(TRON_GUAVA_CACHE_HIT_RATE)) { + GaugeMetricFamily hitRate = new GaugeMetricFamily( + TRON_GUAVA_CACHE_HIT_RATE, + "Hit rate of a guava cache.", + Collections.singletonList("type")); + CacheManager.stats().forEach((k, v) -> hitRate + .addMetric(Collections.singletonList(k), v.hitRate())); + sampleFamilies.add(hitRate); + } + } + + void addRequestMetrics(List sampleFamilies, Predicate nameFilter) { + if (nameFilter.test(TRON_GUAVA_CACHE_REQUEST)) { + GaugeMetricFamily request = new GaugeMetricFamily( + TRON_GUAVA_CACHE_REQUEST, + "Request of a guava cache.", + Collections.singletonList("type")); + CacheManager.stats().forEach((k, v) -> request + .addMetric(Collections.singletonList(k), v.requestCount())); + sampleFamilies.add(request); + } + } + + void addEvictionCountMetrics(List sampleFamilies, + Predicate nameFilter) { + if (nameFilter.test(TRON_GUAVA_CACHE_EVICTION_COUNT)) { + GaugeMetricFamily request = new GaugeMetricFamily( + TRON_GUAVA_CACHE_EVICTION_COUNT, + "Eviction count of a guava cache.", + Collections.singletonList("type")); + CacheManager.stats().forEach((k, v) -> request + .addMetric(Collections.singletonList(k), v.evictionCount())); + sampleFamilies.add(request); + } + } + + @Override + public List collect() { + return collect(null); + } + + @Override + public List collect(Predicate nameFilter) { + List mfs = new ArrayList<>(); + addHitRateMetrics(mfs, nameFilter == null ? ALLOW_ALL : nameFilter); + addRequestMetrics(mfs, nameFilter == null ? ALLOW_ALL : nameFilter); + addEvictionCountMetrics(mfs, nameFilter == null ? ALLOW_ALL : nameFilter); + return mfs; + } +} diff --git a/common/src/main/java/org/tron/common/prometheus/Metrics.java b/common/src/main/java/org/tron/common/prometheus/Metrics.java index fd13fa3fa5a..3f37cf331aa 100644 --- a/common/src/main/java/org/tron/common/prometheus/Metrics.java +++ b/common/src/main/java/org/tron/common/prometheus/Metrics.java @@ -27,6 +27,7 @@ public static synchronized void init() { try { DefaultExports.initialize(); new OperatingSystemExports().register(CollectorRegistry.defaultRegistry); + new GuavaCacheExports().register(CollectorRegistry.defaultRegistry); int port = CommonParameter.getInstance().getMetricsPrometheusPort(); new HTTPServer.Builder().withPort(port).build(); logger.info("prometheus exposed on port : {}", port); diff --git a/common/src/main/java/org/tron/common/utils/DbOptionalsUtils.java b/common/src/main/java/org/tron/common/utils/DbOptionalsUtils.java index 69965bf39ab..ac743ebc7b7 100644 --- a/common/src/main/java/org/tron/common/utils/DbOptionalsUtils.java +++ b/common/src/main/java/org/tron/common/utils/DbOptionalsUtils.java @@ -12,7 +12,6 @@ public class DbOptionalsUtils { public static final int DEFAULT_BLOCK_SIZE = 4 * 1024; public static final int DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024 * 1024; public static final int DEFAULT_WRITE_BUFFER_SIZE_M = 64 * 1024 * 1024; - public static final int DEFAULT_WRITE_BUFFER_SIZE_L = 256 * 1024 * 1024; public static final long DEFAULT_CACHE_SIZE = 32 * 1024 * 1024L; public static final int DEFAULT_MAX_OPEN_FILES = 100; /** @@ -30,13 +29,10 @@ public class DbOptionalsUtils { */ public static final int DEFAULT_MAX_OPEN_FILES_L = 100; // Read a lot - public static final List DB_M = Arrays.asList( "code", "contract"); + public static final List DB_M = Arrays.asList("code", "contract"); // Read frequently public static final List DB_L = Arrays.asList("account", "delegation", "storage-row"); - // Write frequently - public static final List DB_WRITE_L = Arrays.asList("block", "account", - "transactionRetStore", "storage-row", "trans"); private DbOptionalsUtils() { throw new IllegalStateException("DbOptionalsUtils class"); @@ -80,10 +76,6 @@ public static Options newDefaultDbOptions(String name ,Options defaultOptions) { adjustDefaultDbOptionsForL(dbOptions); } - if (DB_WRITE_L.contains(name)) { - adjustDefaultDbOptionsForWriteL(dbOptions); - } - return dbOptions; } @@ -97,8 +89,4 @@ private static void adjustDefaultDbOptionsForL(Options defaultOptions) { defaultOptions.writeBufferSize(DEFAULT_WRITE_BUFFER_SIZE_M); } - private static void adjustDefaultDbOptionsForWriteL(Options defaultOptions) { - - defaultOptions.writeBufferSize(DEFAULT_WRITE_BUFFER_SIZE_L); - } } diff --git a/common/src/main/java/org/tron/core/config/args/Storage.java b/common/src/main/java/org/tron/core/config/args/Storage.java index 6d911f3b0ac..2394a108c6e 100644 --- a/common/src/main/java/org/tron/core/config/args/Storage.java +++ b/common/src/main/java/org/tron/core/config/args/Storage.java @@ -15,9 +15,11 @@ package org.tron.core.config.args; +import com.google.common.collect.Maps; import com.typesafe.config.Config; import com.typesafe.config.ConfigObject; import java.io.File; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; import lombok.Getter; @@ -25,6 +27,8 @@ import org.apache.commons.lang3.StringUtils; import org.iq80.leveldb.CompressionType; import org.iq80.leveldb.Options; +import org.tron.common.cache.CacheStrategies; +import org.tron.common.cache.CacheType; import org.tron.common.utils.DbOptionalsUtils; import org.tron.common.utils.FileUtil; import org.tron.common.utils.Property; @@ -72,6 +76,8 @@ public class Storage { private static final String CHECKPOINT_VERSION_KEY = "storage.checkpoint.version"; private static final String CHECKPOINT_SYNC_KEY = "storage.checkpoint.sync"; + private static final String CACHE_STRATEGIES = "storage.cache.strategies"; + /** * Default values of directory */ @@ -139,6 +145,13 @@ public class Storage { @Setter private int estimatedBlockTransactions; + // second cache + private final Map cacheStrategies = Maps.newConcurrentMap(); + + @Getter + private final List cacheDbs = CacheStrategies.CACHE_DBS; + // second cache + /** * Key: dbName, Value: Property object of that database */ @@ -213,6 +226,19 @@ public static int getEstimatedTransactionsFromConfig(final Config config) { return estimatedTransactions; } + + public void setCacheStrategies(Config config) { + if (config.hasPath(CACHE_STRATEGIES)) { + config.getConfig(CACHE_STRATEGIES).resolve().entrySet().forEach(c -> + this.cacheStrategies.put(CacheType.valueOf(c.getKey()), + c.getValue().unwrapped().toString())); + } + } + + public String getCacheStrategy(CacheType dbName) { + return this.cacheStrategies.getOrDefault(dbName, CacheStrategies.getCacheStrategy(dbName)); + } + private Property createProperty(final ConfigObject conf) { Property property = new Property(); diff --git a/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java b/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java index 767463a6a5b..4a98c933bd1 100644 --- a/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java +++ b/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java @@ -108,6 +108,10 @@ public List getAllWitnesses() { return witnessStore.getAllWitnesses(); } + public List updateWitnessStandby(List all) { + return witnessStore.updateWitnessStandby(all); + } + public void saveStateFlag(int flag) { dynamicPropertiesStore.saveStateFlag(flag); } diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index 012169bdb87..fc6cdd55c15 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -151,11 +151,13 @@ public void doMaintenance() { if (dynamicPropertiesStore.allowChangeDelegation()) { long nextCycle = dynamicPropertiesStore.getCurrentCycleNumber() + 1; dynamicPropertiesStore.saveCurrentCycleNumber(nextCycle); - consensusDelegate.getAllWitnesses().forEach(witness -> { + List all = consensusDelegate.getAllWitnesses(); + all.forEach(witness -> { delegationStore.setBrokerage(nextCycle, witness.createDbKey(), delegationStore.getBrokerage(witness.createDbKey())); delegationStore.setWitnessVote(nextCycle, witness.createDbKey(), witness.getVoteCount()); }); + consensusDelegate.updateWitnessStandby(all); } } diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 7311f94a80d..a0b428df59f 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -520,6 +520,7 @@ public static void setParam(final String[] args, final String confFileName) { PARAMETER.storage.setDefaultDbOptions(config); PARAMETER.storage.setPropertyMapFromConfig(config); + PARAMETER.storage.setCacheStrategies(config); PARAMETER.seedNode = new SeedNode(); PARAMETER.seedNode.setIpList(Optional.ofNullable(PARAMETER.seedNodes) diff --git a/framework/src/test/java/org/tron/core/config/args/StorageTest.java b/framework/src/test/java/org/tron/core/config/args/StorageTest.java index a6d9d9e7f73..eb349a2d146 100644 --- a/framework/src/test/java/org/tron/core/config/args/StorageTest.java +++ b/framework/src/test/java/org/tron/core/config/args/StorageTest.java @@ -93,7 +93,7 @@ public void getOptions() { Assert.assertEquals(1000, options.maxOpenFiles()); options = StorageUtils.getOptionsByDbName("trans"); - Assert.assertEquals(256 * 1024 * 1024, options.writeBufferSize()); + Assert.assertEquals(16 * 1024 * 1024, options.writeBufferSize()); Assert.assertEquals(50, options.maxOpenFiles()); }