From 371455b1f20b3f3ba3494386e510fee95d126a29 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Wed, 29 Nov 2023 12:45:40 -0700 Subject: [PATCH 1/2] Implement synthetic records for event recovery scenarios Signed-off-by: Matt Hess --- .../record/GenesisRecordsBuilder.java | 13 +- .../state/NoOpGenesisRecordsBuilder.java | 13 +- .../main/java/com/hedera/node/app/Hedera.java | 55 +- .../node/app/config/ConfigProviderImpl.java | 30 + .../record/GenesisRecordsConsensusHook.java | 81 +- .../app/config/ConfigProviderImplTest.java | 14 + .../GenesisRecordsConsensusHookTest.java | 131 ++- .../test/utils/TestFixturesKeyLookup.java | 5 + .../token/impl/ReadableAccountStoreImpl.java | 5 + .../service/token/impl/TokenServiceImpl.java | 47 +- .../impl/comparator/TokenComparators.java | 4 + .../schemas/SyntheticRecordsGenerator.java | 264 ++++++ .../token/impl/schemas/TokenSchema.java | 367 +++------ .../src/main/java/module-info.java | 3 + .../test/ReadableAccountStoreImplTest.java | 15 + .../impl/test/schemas/GenesisSchemaTest.java | 656 --------------- .../test/schemas/SyntheticAccountsData.java | 68 ++ .../SyntheticRecordsGeneratorTest.java | 251 ++++++ .../impl/test/schemas/TokenSchemaTest.java | 770 ++++++++++++++++++ .../service/token/ReadableAccountStore.java | 6 + 20 files changed, 1777 insertions(+), 1021 deletions(-) create mode 100644 hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/schemas/SyntheticRecordsGenerator.java delete mode 100644 hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/GenesisSchemaTest.java create mode 100644 hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/SyntheticAccountsData.java create mode 100644 hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/SyntheticRecordsGeneratorTest.java create mode 100644 hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/TokenSchemaTest.java diff --git a/hedera-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/workflows/record/GenesisRecordsBuilder.java b/hedera-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/workflows/record/GenesisRecordsBuilder.java index 95f6622fd928..9c6b5d503065 100644 --- a/hedera-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/workflows/record/GenesisRecordsBuilder.java +++ b/hedera-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/workflows/record/GenesisRecordsBuilder.java @@ -17,9 +17,8 @@ package com.hedera.node.app.spi.workflows.record; import com.hedera.hapi.node.state.token.Account; -import com.hedera.hapi.node.token.CryptoCreateTransactionBody; import edu.umd.cs.findbugs.annotations.NonNull; -import java.util.Map; +import java.util.SortedSet; /** * A class that stores entities created during node startup, for the purpose of creating synthetic @@ -29,25 +28,25 @@ public interface GenesisRecordsBuilder { /** * Tracks the system accounts created during node startup */ - void systemAccounts(@NonNull final Map accounts); + void systemAccounts(@NonNull final SortedSet accounts); /** * Tracks the staking accounts created during node startup */ - void stakingAccounts(@NonNull final Map accounts); + void stakingAccounts(@NonNull final SortedSet accounts); /** * Tracks miscellaneous accounts created during node startup. These accounts are typically used for testing */ - void miscAccounts(@NonNull final Map accounts); + void miscAccounts(@NonNull final SortedSet accounts); /** * Tracks the treasury clones created during node startup */ - void treasuryClones(@NonNull final Map accounts); + void treasuryClones(@NonNull final SortedSet accounts); /** * Tracks the blocklist accounts created during node startup */ - void blocklistAccounts(@NonNull final Map accounts); + void blocklistAccounts(@NonNull final SortedSet accounts); } diff --git a/hedera-node/hedera-app-spi/src/testFixtures/java/com/hedera/node/app/spi/fixtures/state/NoOpGenesisRecordsBuilder.java b/hedera-node/hedera-app-spi/src/testFixtures/java/com/hedera/node/app/spi/fixtures/state/NoOpGenesisRecordsBuilder.java index 97988d2949d7..9624c892f0f3 100644 --- a/hedera-node/hedera-app-spi/src/testFixtures/java/com/hedera/node/app/spi/fixtures/state/NoOpGenesisRecordsBuilder.java +++ b/hedera-node/hedera-app-spi/src/testFixtures/java/com/hedera/node/app/spi/fixtures/state/NoOpGenesisRecordsBuilder.java @@ -33,34 +33,33 @@ */ import com.hedera.hapi.node.state.token.Account; -import com.hedera.hapi.node.token.CryptoCreateTransactionBody; import com.hedera.node.app.spi.workflows.record.GenesisRecordsBuilder; import edu.umd.cs.findbugs.annotations.NonNull; -import java.util.Map; +import java.util.SortedSet; public class NoOpGenesisRecordsBuilder implements GenesisRecordsBuilder { @Override - public void systemAccounts(@NonNull final Map accounts) { + public void systemAccounts(@NonNull final SortedSet accounts) { // Intentional no-op } @Override - public void stakingAccounts(@NonNull final Map accounts) { + public void stakingAccounts(@NonNull final SortedSet accounts) { // Intentional no-op } @Override - public void miscAccounts(@NonNull final Map accounts) { + public void miscAccounts(@NonNull final SortedSet accounts) { // Intentional no-op } @Override - public void treasuryClones(@NonNull final Map accounts) { + public void treasuryClones(@NonNull final SortedSet accounts) { // Intentional no-op } @Override - public void blocklistAccounts(@NonNull Map accounts) { + public void blocklistAccounts(@NonNull SortedSet accounts) { // Intentional no-op } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java index d3c40eedca8f..b3c9a7c2a5bd 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java @@ -52,6 +52,7 @@ import com.hedera.node.app.service.networkadmin.impl.NetworkServiceImpl; import com.hedera.node.app.service.schedule.impl.ScheduleServiceImpl; import com.hedera.node.app.service.token.impl.TokenServiceImpl; +import com.hedera.node.app.service.token.impl.schemas.SyntheticRecordsGenerator; import com.hedera.node.app.service.util.impl.UtilServiceImpl; import com.hedera.node.app.services.ServicesRegistryImpl; import com.hedera.node.app.spi.HapiUtils; @@ -186,6 +187,7 @@ public final class Hedera implements SwirldMain { private ThrottleAccumulator backendThrottle; private ThrottleAccumulator frontendThrottle; private CongestionMultipliers congestionMultipliers; + private final SyntheticRecordsGenerator recordsGenerator; /** * The application name from the platform's perspective. This is currently locked in at the old main class name and @@ -241,7 +243,9 @@ public Hedera(@NonNull final ConstructableRegistry constructableRegistry) { () -> HapiUtils.toString(version.getServicesVersion()), () -> HapiUtils.toString(version.getHapiVersion())); - // Create a record builder for any genesis records that need to be created + // Create a records generator for any synthetic records that need to be CREATED + this.recordsGenerator = new SyntheticRecordsGenerator(); + // Create a records builder for any genesis records that need to be RECORDED this.genesisRecordsBuilder = new GenesisRecordsConsensusHook(); // Create all the service implementations @@ -257,7 +261,12 @@ public Hedera(@NonNull final ConstructableRegistry constructableRegistry) { new FreezeServiceImpl(), new NetworkServiceImpl(), new ScheduleServiceImpl(), - new TokenServiceImpl(), + new TokenServiceImpl( + recordsGenerator::sysAcctRecords, + recordsGenerator::stakingAcctRecords, + recordsGenerator::treasuryAcctRecords, + recordsGenerator::multiUseAcctRecords, + recordsGenerator::blocklistAcctRecords), new UtilServiceImpl(), new RecordCacheService(), new BlockRecordService(), @@ -354,8 +363,39 @@ private void onStateInitialized( @NonNull final SwirldDualState dualState, @NonNull final InitTrigger trigger, @Nullable final SoftwareVersion previousVersion) { + // Initialize the configuration from disk. We must do this BEFORE we run migration, because the various + // migration methods may depend on configuration to do their work. For example, the token service migration code + // needs to know the token treasury account, which has an account ID specified in config. The initial config + // file in state, created by the file service migration, will match what we have here, so we don't have to worry + // about re-loading config after migration. + logger.info("Initializing configuration with trigger {}", trigger); + switch (trigger) { + case EVENT_STREAM_RECOVERY -> configProvider = + new ConfigProviderImpl(platform.getContext().getConfiguration()); + case GENESIS -> configProvider = new ConfigProviderImpl(true); + case RESTART, RECONNECT -> configProvider = new ConfigProviderImpl(false); + } + logConfiguration(); + + // Determine if we need to create synthetic records for system entities + final var blockRecordState = state.createReadableStates(BlockRecordService.NAME); + boolean createSynthRecords = false; + if (!blockRecordState.isEmpty()) { + final var blockInfo = blockRecordState + .getSingleton(BlockRecordService.BLOCK_INFO_STATE_KEY) + .get(); + if (blockInfo == null || blockInfo.consTimeOfLastHandledTxn() == null) { + createSynthRecords = true; + } + } else { + createSynthRecords = true; + } + if (createSynthRecords) { + recordsGenerator.createRecords(configProvider.getConfiguration(), genesisRecordsBuilder); + } - // We do nothing for EVENT_STREAM_RECOVERY. This is a special case that is handled by the platform. + // We do nothing else for EVENT_STREAM_RECOVERY, which for now is broken. This is a special case that is handled + // by the platform, and we need to figure out how to make it work with the modular app. if (trigger == EVENT_STREAM_RECOVERY) { logger.debug("Skipping state initialization for trigger {}", trigger); return; @@ -706,15 +746,6 @@ public void shutdownGrpcServer() { private void genesis(@NonNull final MerkleHederaState state) { logger.debug("Genesis Initialization"); - // Initialize the configuration from disk (genesis case). We must do this BEFORE we run migration, because - // the various migration methods may depend on configuration to do their work. For example, the token service - // migration code needs to know the token treasury account, which has an account ID specified in config. - // The initial config file in state, created by the file service migration, will match what we have here, - // so we don't have to worry about re-loading config after migration. - logger.info("Initializing genesis configuration"); - this.configProvider = new ConfigProviderImpl(true); - logConfiguration(); - logger.info("Initializing ThrottleManager"); this.throttleManager = new ThrottleManager(); diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/config/ConfigProviderImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/config/ConfigProviderImpl.java index d537ed845e75..28b6f70fb1cb 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/config/ConfigProviderImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/config/ConfigProviderImpl.java @@ -65,11 +65,16 @@ import com.swirlds.common.threading.locks.Locks; import com.swirlds.config.api.Configuration; import com.swirlds.config.api.ConfigurationBuilder; +import com.swirlds.config.api.source.ConfigSource; import com.swirlds.config.extensions.sources.SystemEnvironmentConfigSource; import com.swirlds.config.extensions.sources.SystemPropertiesConfigSource; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.io.IOException; +import java.util.NoSuchElementException; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.inject.Singleton; import org.apache.logging.log4j.LogManager; @@ -110,6 +115,31 @@ public ConfigProviderImpl(final boolean useGenesisSource) { configuration = new AtomicReference<>(new VersionedConfigImpl(config, 0)); } + /** + * Creates a new config provider based on an existing configuration. Note that this constructor assumes + * this is NOT a genesis case. + * @param existing the existing configuration to embed in the provider + */ + public ConfigProviderImpl(@NonNull final Configuration existing) { + final var builder = createConfigurationBuilder(); + addFileSources(builder, false); + builder.withSources(new ConfigSource() { + @NonNull + @Override + public Set getPropertyNames() { + return existing.getPropertyNames().collect(Collectors.toSet()); + } + + @Nullable + @Override + public String getValue(@NonNull String propertyName) throws NoSuchElementException { + return existing.getValue(propertyName); + } + }); + final Configuration config = builder.build(); + configuration = new AtomicReference<>(new VersionedConfigImpl(config, 0)); + } + @Override @NonNull public VersionedConfiguration getConfiguration() { diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/workflows/handle/record/GenesisRecordsConsensusHook.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/workflows/handle/record/GenesisRecordsConsensusHook.java index 8f7931d881d3..a90306ac103b 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/workflows/handle/record/GenesisRecordsConsensusHook.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/workflows/handle/record/GenesisRecordsConsensusHook.java @@ -16,6 +16,7 @@ package com.hedera.node.app.workflows.handle.record; +import static com.hedera.node.app.spi.HapiUtils.ACCOUNT_ID_COMPARATOR; import static com.hedera.node.app.spi.HapiUtils.FUNDING_ACCOUNT_EXPIRY; import static java.util.Objects.requireNonNull; @@ -33,8 +34,8 @@ import edu.umd.cs.findbugs.annotations.Nullable; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; import javax.inject.Singleton; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -49,12 +50,14 @@ public class GenesisRecordsConsensusHook implements GenesisRecordsBuilder, Conse private static final String SYSTEM_ACCOUNT_CREATION_MEMO = "Synthetic system creation"; private static final String STAKING_MEMO = "Release 0.24.1 migration record"; private static final String TREASURY_CLONE_MEMO = "Synthetic zero-balance treasury clone"; + private static final Comparator ACCOUNT_COMPARATOR = + Comparator.comparing(Account::accountId, ACCOUNT_ID_COMPARATOR); - private Map systemAccounts = new HashMap<>(); - private Map stakingAccounts = new HashMap<>(); - private Map miscAccounts = new HashMap<>(); - private Map treasuryClones = new HashMap<>(); - private Map blocklistAccounts = new HashMap<>(); + private SortedSet systemAccounts = new TreeSet<>(ACCOUNT_COMPARATOR); + private SortedSet stakingAccounts = new TreeSet<>(ACCOUNT_COMPARATOR); + private SortedSet miscAccounts = new TreeSet<>(ACCOUNT_COMPARATOR); + private SortedSet treasuryClones = new TreeSet<>(ACCOUNT_COMPARATOR); + private SortedSet blocklistAccounts = new TreeSet<>(ACCOUNT_COMPARATOR); /** * ⚠️⚠️ Note: though this method will be called each time a new platform event is received, @@ -75,83 +78,84 @@ public void process(@NonNull final TokenContext context) { if (!systemAccounts.isEmpty()) { createAccountRecordBuilders(systemAccounts, context, SYSTEM_ACCOUNT_CREATION_MEMO); - systemAccounts = Collections.emptyMap(); + systemAccounts = Collections.emptySortedSet(); } + log.info("Queued {} system account records with consTime {}", systemAccounts.size(), consensusTime); if (!stakingAccounts.isEmpty()) { final var implicitAutoRenewPeriod = FUNDING_ACCOUNT_EXPIRY - consensusTime.getEpochSecond(); createAccountRecordBuilders(stakingAccounts, context, STAKING_MEMO, implicitAutoRenewPeriod); - stakingAccounts = Collections.emptyMap(); + stakingAccounts = Collections.emptySortedSet(); } + log.info("Queued {} staking account records with consTime {}", stakingAccounts.size(), consensusTime); if (!miscAccounts.isEmpty()) { createAccountRecordBuilders(miscAccounts, context, null); - miscAccounts = Collections.emptyMap(); + miscAccounts = Collections.emptySortedSet(); } + log.info("Queued {} misc account records with consTime {}", miscAccounts.size(), consensusTime); if (!treasuryClones.isEmpty()) { createAccountRecordBuilders(treasuryClones, context, TREASURY_CLONE_MEMO); - treasuryClones = Collections.emptyMap(); + treasuryClones = Collections.emptySortedSet(); } + log.info("Queued {} treasury clone account records with consTime {}", treasuryClones.size(), consensusTime); if (!blocklistAccounts.isEmpty()) { createAccountRecordBuilders(blocklistAccounts, context, null); - blocklistAccounts = Collections.emptyMap(); + blocklistAccounts = Collections.emptySortedSet(); } + log.info("Queued {} blocklist account records with consTime {}", blocklistAccounts.size(), consensusTime); } @Override - public void systemAccounts(@NonNull final Map accounts) { - systemAccounts.putAll(requireNonNull(accounts)); + public void systemAccounts(@NonNull final SortedSet accounts) { + systemAccounts.addAll(requireNonNull(accounts)); } @Override - public void stakingAccounts(@NonNull final Map accounts) { - stakingAccounts.putAll(requireNonNull(accounts)); + public void stakingAccounts(@NonNull final SortedSet accounts) { + stakingAccounts.addAll(requireNonNull(accounts)); } @Override - public void miscAccounts(@NonNull final Map accounts) { - miscAccounts.putAll(requireNonNull(accounts)); + public void miscAccounts(@NonNull final SortedSet accounts) { + miscAccounts.addAll(requireNonNull(accounts)); } @Override - public void treasuryClones(@NonNull final Map accounts) { - treasuryClones.putAll(requireNonNull(accounts)); + public void treasuryClones(@NonNull final SortedSet accounts) { + treasuryClones.addAll(requireNonNull(accounts)); } @Override - public void blocklistAccounts(@NonNull final Map accounts) { - blocklistAccounts.putAll(requireNonNull(accounts)); + public void blocklistAccounts(@NonNull final SortedSet accounts) { + blocklistAccounts.addAll(requireNonNull(accounts)); } private void createAccountRecordBuilders( - @NonNull final Map map, + @NonNull final SortedSet map, @NonNull final TokenContext context, @Nullable final String recordMemo) { createAccountRecordBuilders(map, context, recordMemo, null); } private void createAccountRecordBuilders( - @NonNull final Map map, + @NonNull final SortedSet accts, @NonNull final TokenContext context, @Nullable final String recordMemo, @Nullable final Long overrideAutoRenewPeriod) { - final var orderedAccts = map.keySet().stream() - .sorted(Comparator.comparingLong(acct -> acct.accountId().accountNum())) - .toList(); - for (final Account key : orderedAccts) { + for (final Account account : accts) { // we create preceding records on genesis for each system account created. // This is an exception and should not fail with MAX_CHILD_RECORDS_EXCEEDED final var recordBuilder = context.addUncheckedPrecedingChildRecordBuilder(GenesisAccountRecordBuilder.class); - final var accountId = requireNonNull(key.accountId()); - recordBuilder.accountID(accountId); + recordBuilder.accountID(account.accountId()); if (recordMemo != null) { recordBuilder.memo(recordMemo); } - var txnBody = map.get(key); + var txnBody = newCryptoCreate(account); if (overrideAutoRenewPeriod != null) { txnBody.autoRenewPeriod(Duration.newBuilder().seconds(overrideAutoRenewPeriod)); } @@ -159,7 +163,20 @@ private void createAccountRecordBuilders( Transaction.newBuilder().body(TransactionBody.newBuilder().cryptoCreateAccount(txnBody)); recordBuilder.transaction(txnBuilder.build()); - log.debug("Queued synthetic CryptoCreate for {} account {}", recordMemo, accountId); + log.debug("Queued synthetic CryptoCreate for {} account {}", recordMemo, account); } } + + private static CryptoCreateTransactionBody.Builder newCryptoCreate(@NonNull final Account account) { + return CryptoCreateTransactionBody.newBuilder() + .key(account.key()) + .memo(account.memo()) + .declineReward(account.declineReward()) + .receiverSigRequired(account.receiverSigRequired()) + .autoRenewPeriod(Duration.newBuilder() + .seconds(account.autoRenewSeconds()) + .build()) + .initialBalance(account.tinybarBalance()) + .alias(account.alias()); + } } diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/config/ConfigProviderImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/config/ConfigProviderImplTest.java index 1a94b2693e4c..765c9f0932e3 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/config/ConfigProviderImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/config/ConfigProviderImplTest.java @@ -24,6 +24,7 @@ import com.hedera.hapi.node.base.Setting; import com.hedera.node.config.VersionedConfiguration; import com.hedera.pbj.runtime.io.buffer.Bytes; +import com.swirlds.config.api.ConfigurationBuilder; import java.nio.charset.StandardCharsets; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -145,6 +146,19 @@ void testGenesisPropertiesFileIsOptional(final EnvironmentVariables environment) assertThat(bar).isEqualTo("456"); } + @Test + void providerGetsCorrectValueFromExistingConfig() { + // given + final var subject = new ConfigProviderImpl( + ConfigurationBuilder.create().withValue("key", "value").build()); + + // when + final var result = subject.getConfiguration(); + + // then + assertThat(result.getValue("key")).isEqualTo("value"); + } + @Test void testUpdateDoesUseApplicationProperties() { // given diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/workflows/handle/record/GenesisRecordsConsensusHookTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/workflows/handle/record/GenesisRecordsConsensusHookTest.java index 4c4d2ba7f79b..30340042dcfe 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/workflows/handle/record/GenesisRecordsConsensusHookTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/workflows/handle/record/GenesisRecordsConsensusHookTest.java @@ -16,25 +16,21 @@ package com.hedera.node.app.workflows.handle.record; -import static com.hedera.node.app.spi.HapiUtils.FUNDING_ACCOUNT_EXPIRY; import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.verify; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.verifyNoInteractions; import com.hedera.hapi.node.base.AccountID; -import com.hedera.hapi.node.base.Duration; import com.hedera.hapi.node.base.Timestamp; -import com.hedera.hapi.node.base.Transaction; import com.hedera.hapi.node.state.blockrecords.BlockInfo; import com.hedera.hapi.node.state.token.Account; -import com.hedera.hapi.node.token.CryptoCreateTransactionBody; -import com.hedera.hapi.node.transaction.TransactionBody; import com.hedera.node.app.records.ReadableBlockRecordStore; +import com.hedera.node.app.service.token.impl.comparator.TokenComparators; import com.hedera.node.app.service.token.records.GenesisAccountRecordBuilder; import com.hedera.node.app.service.token.records.TokenContext; import java.time.Instant; -import java.util.Map; +import java.util.TreeSet; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -53,10 +49,6 @@ class GenesisRecordsConsensusHookTest { Account.newBuilder().accountId(ACCOUNT_ID_1).build(); private static final Account ACCOUNT_2 = Account.newBuilder().accountId(ACCOUNT_ID_2).build(); - private static final CryptoCreateTransactionBody ACCT_1_CREATE = - CryptoCreateTransactionBody.newBuilder().memo("builder1").build(); - private static final CryptoCreateTransactionBody ACCT_2_CREATE = - CryptoCreateTransactionBody.newBuilder().memo("builder2").build(); private static final Instant CONSENSUS_NOW = Instant.parse("2023-08-10T00:00:00Z"); private static final String EXPECTED_SYSTEM_ACCOUNT_CREATION_MEMO = "Synthetic system creation"; @@ -88,102 +80,100 @@ void setup() { @Test void processCreatesSystemAccounts() { - subject.systemAccounts(Map.of(ACCOUNT_1, ACCT_1_CREATE.copyBuilder(), ACCOUNT_2, ACCT_2_CREATE.copyBuilder())); + final var accts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + accts.add(ACCOUNT_1); + accts.add(ACCOUNT_2); + subject.systemAccounts(accts); subject.process(context); - verifyBuilderInvoked(ACCOUNT_ID_1, ACCT_1_CREATE, EXPECTED_SYSTEM_ACCOUNT_CREATION_MEMO); - verifyBuilderInvoked(ACCOUNT_ID_2, ACCT_2_CREATE, EXPECTED_SYSTEM_ACCOUNT_CREATION_MEMO); + verifyBuilderInvoked(ACCOUNT_ID_1, EXPECTED_SYSTEM_ACCOUNT_CREATION_MEMO); + verifyBuilderInvoked(ACCOUNT_ID_2, EXPECTED_SYSTEM_ACCOUNT_CREATION_MEMO); } @Test - void processCreatesStakingAccountsWithImplicitExpiry() { - subject.stakingAccounts(Map.of(ACCOUNT_1, ACCT_1_CREATE.copyBuilder(), ACCOUNT_2, ACCT_2_CREATE.copyBuilder())); + void processCreatesStakingAccounts() { + final var accts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + accts.add(ACCOUNT_1); + accts.add(ACCOUNT_2); + subject.stakingAccounts(accts); subject.process(context); - final var expectedAutoRenew = FUNDING_ACCOUNT_EXPIRY - CONSENSUS_NOW.getEpochSecond(); - verifyBuilderInvoked( - ACCOUNT_ID_1, - ACCT_1_CREATE - .copyBuilder() - .autoRenewPeriod( - Duration.newBuilder().seconds(expectedAutoRenew).build()) - .build(), - EXPECTED_STAKING_MEMO); - verifyBuilderInvoked( - ACCOUNT_ID_2, - ACCT_2_CREATE - .copyBuilder() - .autoRenewPeriod( - Duration.newBuilder().seconds(expectedAutoRenew).build()) - .build(), - EXPECTED_STAKING_MEMO); + verifyBuilderInvoked(ACCOUNT_ID_1, EXPECTED_STAKING_MEMO); + verifyBuilderInvoked(ACCOUNT_ID_2, EXPECTED_STAKING_MEMO); } @Test void processCreatesMultipurposeAccounts() { - subject.miscAccounts(Map.of(ACCOUNT_1, ACCT_1_CREATE.copyBuilder(), ACCOUNT_2, ACCT_2_CREATE.copyBuilder())); + final var accts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + accts.add(ACCOUNT_1); + accts.add(ACCOUNT_2); + subject.miscAccounts(accts); subject.process(context); - verifyBuilderInvoked(ACCOUNT_ID_1, ACCT_1_CREATE, null); - verifyBuilderInvoked(ACCOUNT_ID_2, ACCT_2_CREATE, null); + verifyBuilderInvoked(ACCOUNT_ID_1, null); + verifyBuilderInvoked(ACCOUNT_ID_2, null); } @Test void processCreatesTreasuryClones() { - subject.treasuryClones(Map.of(ACCOUNT_1, ACCT_1_CREATE.copyBuilder(), ACCOUNT_2, ACCT_2_CREATE.copyBuilder())); + final var accts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + accts.add(ACCOUNT_1); + accts.add(ACCOUNT_2); + subject.treasuryClones(accts); subject.process(context); - verifyBuilderInvoked(ACCOUNT_ID_1, ACCT_1_CREATE, EXPECTED_TREASURY_CLONE_MEMO); - verifyBuilderInvoked(ACCOUNT_ID_2, ACCT_2_CREATE, EXPECTED_TREASURY_CLONE_MEMO); + verifyBuilderInvoked(ACCOUNT_ID_1, EXPECTED_TREASURY_CLONE_MEMO); + verifyBuilderInvoked(ACCOUNT_ID_2, EXPECTED_TREASURY_CLONE_MEMO); } @Test void processCreatesBlocklistAccounts() { - subject.blocklistAccounts( - Map.of(ACCOUNT_1, ACCT_1_CREATE.copyBuilder(), ACCOUNT_2, ACCT_2_CREATE.copyBuilder())); + final var accts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + accts.add(ACCOUNT_1); + accts.add(ACCOUNT_2); + subject.blocklistAccounts(accts); subject.process(context); - verifyBuilderInvoked(ACCOUNT_ID_1, ACCT_1_CREATE, null); - verifyBuilderInvoked(ACCOUNT_ID_2, ACCT_2_CREATE, null); + verifyBuilderInvoked(ACCOUNT_ID_1, null); + verifyBuilderInvoked(ACCOUNT_ID_2, null); } @Test void processCreatesAllRecords() { final var acctId3 = ACCOUNT_ID_1.copyBuilder().accountNum(3).build(); final var acct3 = ACCOUNT_1.copyBuilder().accountId(acctId3).build(); - final var acct3Create = ACCT_1_CREATE.copyBuilder().memo("builder3").build(); final var acctId4 = ACCOUNT_ID_1.copyBuilder().accountNum(4).build(); final var acct4 = ACCOUNT_1.copyBuilder().accountId(acctId4).build(); - final var acct4Create = ACCT_1_CREATE.copyBuilder().memo("builder4").build(); final var acctId5 = ACCOUNT_ID_1.copyBuilder().accountNum(5).build(); final var acct5 = ACCOUNT_1.copyBuilder().accountId(acctId5).build(); - final var acct5Create = ACCT_1_CREATE.copyBuilder().memo("builder5").build(); - subject.systemAccounts(Map.of(ACCOUNT_1, ACCT_1_CREATE.copyBuilder())); - subject.stakingAccounts(Map.of(ACCOUNT_2, ACCT_2_CREATE.copyBuilder())); - subject.miscAccounts(Map.of(acct3, acct3Create.copyBuilder())); - subject.treasuryClones(Map.of(acct4, acct4Create.copyBuilder())); - subject.blocklistAccounts(Map.of(acct5, acct5Create.copyBuilder())); + final var sysAccts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + sysAccts.add(ACCOUNT_1); + subject.systemAccounts(sysAccts); + final var stakingAccts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + stakingAccts.add(ACCOUNT_2); + subject.stakingAccounts(stakingAccts); + final var miscAccts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + miscAccts.add(acct3); + subject.miscAccounts(miscAccts); + final var treasuryAccts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + treasuryAccts.add(acct4); + subject.treasuryClones(treasuryAccts); + final var blocklistAccts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + blocklistAccts.add(acct5); + subject.blocklistAccounts(blocklistAccts); // Call the first time to make sure records are generated subject.process(context); - verifyBuilderInvoked(ACCOUNT_ID_1, ACCT_1_CREATE, EXPECTED_SYSTEM_ACCOUNT_CREATION_MEMO); - verifyBuilderInvoked( - ACCOUNT_ID_2, - ACCT_2_CREATE - .copyBuilder() - .autoRenewPeriod(Duration.newBuilder() - .seconds(FUNDING_ACCOUNT_EXPIRY - CONSENSUS_NOW.getEpochSecond()) - .build()) - .build(), - EXPECTED_STAKING_MEMO); - verifyBuilderInvoked(acctId3, acct3Create, null); - verifyBuilderInvoked(acctId4, acct4Create, EXPECTED_TREASURY_CLONE_MEMO); - verifyBuilderInvoked(acctId5, acct5Create, null); + verifyBuilderInvoked(ACCOUNT_ID_1, EXPECTED_SYSTEM_ACCOUNT_CREATION_MEMO); + verifyBuilderInvoked(ACCOUNT_ID_2, EXPECTED_STAKING_MEMO); + verifyBuilderInvoked(acctId3, null); + verifyBuilderInvoked(acctId4, EXPECTED_TREASURY_CLONE_MEMO); + verifyBuilderInvoked(acctId5, null); // Call process() a second time to make sure no other records are created Mockito.clearInvocations(genesisAccountRecordBuilder); @@ -207,8 +197,9 @@ void processCreatesNoRecordsAfterRunning() { .nanos(CONSENSUS_NOW.getNano())) .build()); // Add a single account, so we know the subject isn't skipping processing because there's no data - subject.stakingAccounts( - Map.of(Account.newBuilder().accountId(ACCOUNT_ID_1).build(), ACCT_1_CREATE.copyBuilder())); + final var accts = new TreeSet<>(TokenComparators.ACCOUNT_COMPARATOR); + accts.add(ACCOUNT_1); + subject.stakingAccounts(accts); subject.process(context); @@ -245,22 +236,14 @@ void blocklistAccountsNullParam() { Assertions.assertThatThrownBy(() -> subject.blocklistAccounts(null)).isInstanceOf(NullPointerException.class); } - private void verifyBuilderInvoked( - final AccountID acctId, final CryptoCreateTransactionBody acctCreateBody, final String expectedMemo) { + private void verifyBuilderInvoked(final AccountID acctId, final String expectedMemo) { verify(genesisAccountRecordBuilder).accountID(acctId); - verify(genesisAccountRecordBuilder).transaction(asCryptoCreateTxn(acctCreateBody)); if (expectedMemo != null) verify(genesisAccountRecordBuilder, atLeastOnce()).memo(expectedMemo); //noinspection DataFlowIssue verify(genesisAccountRecordBuilder, Mockito.never()).memo(null); } - private static Transaction asCryptoCreateTxn(CryptoCreateTransactionBody body) { - return Transaction.newBuilder() - .body(TransactionBody.newBuilder().cryptoCreateAccount(body)) - .build(); - } - private static BlockInfo defaultStartupBlockInfo() { return BlockInfo.newBuilder() .consTimeOfLastHandledTxn((Timestamp) null) diff --git a/hedera-node/hedera-mono-service/src/testFixtures/java/com/hedera/test/utils/TestFixturesKeyLookup.java b/hedera-node/hedera-mono-service/src/testFixtures/java/com/hedera/test/utils/TestFixturesKeyLookup.java index 5d8b5c36eee7..e47494dea120 100644 --- a/hedera-node/hedera-mono-service/src/testFixtures/java/com/hedera/test/utils/TestFixturesKeyLookup.java +++ b/hedera-node/hedera-mono-service/src/testFixtures/java/com/hedera/test/utils/TestFixturesKeyLookup.java @@ -79,6 +79,11 @@ public boolean containsAlias(@NonNull Bytes alias) { return aliases.contains(new ProtoBytes(alias)); } + @Override + public boolean contains(@NonNull AccountID accountID) { + return accounts.contains(accountID); + } + @Override @Nullable public AccountID getAccountIDByAlias(@NonNull final Bytes alias) { diff --git a/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/ReadableAccountStoreImpl.java b/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/ReadableAccountStoreImpl.java index 0315f466d1a3..1b6d05f81cd0 100644 --- a/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/ReadableAccountStoreImpl.java +++ b/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/ReadableAccountStoreImpl.java @@ -115,6 +115,11 @@ public boolean containsAlias(@NonNull Bytes alias) { return aliases.contains(new ProtoBytes(alias)); } + @Override + public boolean contains(@NonNull final AccountID accountID) { + return accountState().contains(accountID); + } + /* Helper methods */ /** diff --git a/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/TokenServiceImpl.java b/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/TokenServiceImpl.java index 6e435410511d..e84845c6192b 100644 --- a/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/TokenServiceImpl.java +++ b/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/TokenServiceImpl.java @@ -18,10 +18,15 @@ import static java.util.Objects.requireNonNull; +import com.hedera.hapi.node.state.token.Account; import com.hedera.node.app.service.token.TokenService; +import com.hedera.node.app.service.token.impl.schemas.SyntheticRecordsGenerator; import com.hedera.node.app.service.token.impl.schemas.TokenSchema; import com.hedera.node.app.spi.state.SchemaRegistry; import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.Collections; +import java.util.SortedSet; +import java.util.function.Supplier; /** An implementation of the {@link TokenService} interface. */ public class TokenServiceImpl implements TokenService { @@ -32,10 +37,50 @@ public class TokenServiceImpl implements TokenService { public static final String TOKEN_RELS_KEY = "TOKEN_RELS"; public static final String STAKING_INFO_KEY = "STAKING_INFOS"; public static final String STAKING_NETWORK_REWARDS_KEY = "STAKING_NETWORK_REWARDS"; + private final Supplier> sysAccts; + private final Supplier> stakingAccts; + private final Supplier> treasuryAccts; + private final Supplier> miscAccts; + private final Supplier> blocklistAccts; + + /** + * Constructor for the token service. Each of the given suppliers should produce a {@link SortedSet} + * of {@link Account} objects, where each account object represents a SYNTHETIC RECORD (see {@link + * SyntheticRecordsGenerator} for more details). Even though these sorted sets contain account objects, + * these account objects may or may not yet exist in state. They're needed for event recovery circumstances + * @param sysAccts + * @param stakingAccts + * @param treasuryAccts + * @param miscAccts + * @param blocklistAccts + */ + public TokenServiceImpl( + @NonNull final Supplier> sysAccts, + @NonNull final Supplier> stakingAccts, + @NonNull final Supplier> treasuryAccts, + @NonNull final Supplier> miscAccts, + @NonNull final Supplier> blocklistAccts) { + this.sysAccts = sysAccts; + this.stakingAccts = stakingAccts; + this.treasuryAccts = treasuryAccts; + this.miscAccts = miscAccts; + this.blocklistAccts = blocklistAccts; + } + + /** + * Necessary default constructor. See all params constructor for more details + */ + public TokenServiceImpl() { + this.sysAccts = Collections::emptySortedSet; + this.stakingAccts = Collections::emptySortedSet; + this.treasuryAccts = Collections::emptySortedSet; + this.miscAccts = Collections::emptySortedSet; + this.blocklistAccts = Collections::emptySortedSet; + } @Override public void registerSchemas(@NonNull SchemaRegistry registry) { requireNonNull(registry); - registry.register(new TokenSchema()); + registry.register(new TokenSchema(sysAccts, stakingAccts, treasuryAccts, miscAccts, blocklistAccts)); } } diff --git a/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/comparator/TokenComparators.java b/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/comparator/TokenComparators.java index 3f220088a430..352ed133adfd 100644 --- a/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/comparator/TokenComparators.java +++ b/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/comparator/TokenComparators.java @@ -21,6 +21,7 @@ import com.hedera.hapi.node.base.AccountAmount; import com.hedera.hapi.node.base.TokenID; import com.hedera.hapi.node.base.TokenTransferList; +import com.hedera.hapi.node.state.token.Account; import java.util.Comparator; import java.util.Objects; @@ -29,6 +30,9 @@ private TokenComparators() { throw new IllegalStateException("Utility Class"); } + public static final Comparator ACCOUNT_COMPARATOR = + Comparator.comparing(Account::accountId, ACCOUNT_ID_COMPARATOR); + public static final Comparator ACCOUNT_AMOUNT_COMPARATOR = Comparator.comparing(AccountAmount::accountID, ACCOUNT_ID_COMPARATOR); public static final Comparator TOKEN_ID_COMPARATOR = Comparator.comparingLong(TokenID::tokenNum); diff --git a/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/schemas/SyntheticRecordsGenerator.java b/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/schemas/SyntheticRecordsGenerator.java new file mode 100644 index 000000000000..79ca6548837f --- /dev/null +++ b/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/schemas/SyntheticRecordsGenerator.java @@ -0,0 +1,264 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.service.token.impl.schemas; + +import static com.hedera.node.app.service.token.impl.comparator.TokenComparators.ACCOUNT_COMPARATOR; +import static com.hedera.node.app.spi.HapiUtils.EMPTY_KEY_LIST; +import static com.hedera.node.app.spi.HapiUtils.FUNDING_ACCOUNT_EXPIRY; + +import com.google.common.annotations.VisibleForTesting; +import com.hedera.hapi.node.base.AccountID; +import com.hedera.hapi.node.base.Key; +import com.hedera.hapi.node.state.token.Account; +import com.hedera.node.app.service.token.impl.BlocklistParser; +import com.hedera.node.app.spi.workflows.record.GenesisRecordsBuilder; +import com.hedera.node.config.data.AccountsConfig; +import com.hedera.node.config.data.BootstrapConfig; +import com.hedera.node.config.data.HederaConfig; +import com.hedera.node.config.data.LedgerConfig; +import com.swirlds.config.api.Configuration; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.stream.LongStream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class SyntheticRecordsGenerator { + private static final Logger log = LogManager.getLogger(SyntheticRecordsGenerator.class); + private static final long FIRST_RESERVED_SYSTEM_CONTRACT = 350L; + private static final long LAST_RESERVED_SYSTEM_CONTRACT = 399L; + private static final long FIRST_POST_SYSTEM_FILE_ENTITY = 200L; + + private final BlocklistParser blocklistParser; + private final SortedSet systemAcctRcds = new TreeSet<>(ACCOUNT_COMPARATOR); + private final SortedSet stakingAcctRcds = new TreeSet<>(ACCOUNT_COMPARATOR); + private final SortedSet treasuryAcctRcds = new TreeSet<>(ACCOUNT_COMPARATOR); + private final SortedSet multiUseAcctRcds = new TreeSet<>(ACCOUNT_COMPARATOR); + private final SortedSet blocklistAcctRcds = new TreeSet<>(ACCOUNT_COMPARATOR); + + /** + * Create a new instance + */ + public SyntheticRecordsGenerator() { + blocklistParser = new BlocklistParser(); + } + + public SortedSet sysAcctRecords() { + return systemAcctRcds; + } + + public SortedSet stakingAcctRecords() { + return stakingAcctRcds; + } + + public SortedSet treasuryAcctRecords() { + return treasuryAcctRcds; + } + + public SortedSet multiUseAcctRecords() { + return multiUseAcctRcds; + } + + public SortedSet blocklistAcctRecords() { + return blocklistAcctRcds; + } + + /** + * Creates the synthetic records needed in various startup scenarios (e.g. genesis or event recovery). + * Actually, this method creates {@link Account} objects; since these objects will ultimately be + * written to the record stream later, we'll refer to them as "records" throughout this method. + * @param configuration the current configuration of the node + * @param recordsKeeper the record builder that tracks the synthetic records + */ + public void createRecords( + @NonNull final Configuration configuration, @NonNull final GenesisRecordsBuilder recordsKeeper) { + // We will use these various configs for creating accounts. It would be nice to consolidate them somehow + final var accountsConfig = configuration.getConfigData(AccountsConfig.class); + final var bootstrapConfig = configuration.getConfigData(BootstrapConfig.class); + final var ledgerConfig = configuration.getConfigData(LedgerConfig.class); + final var hederaConfig = configuration.getConfigData(HederaConfig.class); + + // This key is used for all system accounts + final var superUserKey = superUserKey(bootstrapConfig); + + // Create a synthetic record for every system account. Now, it turns out that while accounts 1-100 (inclusive) + // are system accounts, accounts 200-349 (inclusive) and 400-750 (inclusive) are "clones" of system accounts. So + // we can just create all of these up front. Basically, every account from 1 to 750 (inclusive) other than those + // set aside for files (101-199 inclusive) and contracts (350-399 inclusive) are the same, except for the + // treasury account, which has a balance + // ---------- Create system records ------------------------- + for (long num = 1; num <= ledgerConfig.numSystemAccounts(); num++) { + final var id = asAccountId(num, hederaConfig); + + final var accountTinyBars = num == accountsConfig.treasury() ? ledgerConfig.totalTinyBarFloat() : 0; + assert accountTinyBars >= 0L : "Negative account balance!"; + + final var account = createAccount(id, accountTinyBars, bootstrapConfig.systemEntityExpiry(), superUserKey); + systemAcctRcds.add(account); + } + recordsKeeper.systemAccounts(systemAcctRcds); + log.info("Created {} synthetic system records", sysAcctRecords().size()); + + // ---------- Create staking fund records ------------------------- + final var stakingRewardAccountId = asAccountId(accountsConfig.stakingRewardAccount(), hederaConfig); + final var nodeRewardAccountId = asAccountId(accountsConfig.nodeRewardAccount(), hederaConfig); + final var stakingFundAccounts = List.of(stakingRewardAccountId, nodeRewardAccountId); + for (final var id : stakingFundAccounts) { + final var stakingFundAccount = createAccount(id, 0, FUNDING_ACCOUNT_EXPIRY, EMPTY_KEY_LIST); + stakingAcctRcds.add(stakingFundAccount); + } + recordsKeeper.stakingAccounts(stakingAcctRcds); + log.info("Created {} synthetic staking records", stakingAcctRcds.size()); + + // ---------- Create multi-use records ------------------------- + for (long num = 900; num <= 1000; num++) { + final var id = asAccountId(num, hederaConfig); + final var account = createAccount(id, 0, bootstrapConfig.systemEntityExpiry(), superUserKey); + multiUseAcctRcds.add(account); + } + recordsKeeper.miscAccounts(multiUseAcctRcds); + log.info("Created {} synthetic multi-use records", multiUseAcctRcds.size()); + + // ---------- Create treasury clones ------------------------- + // Since version 0.28.6, all of these clone accounts should either all exist (on a restart) or all not exist + // (starting from genesis) + final var treasury = systemAcctRcds.stream() + .filter(a -> a.accountId().accountNum() == accountsConfig.treasury()) + .findFirst() + .orElseThrow(); + for (final var num : nonContractSystemNums(ledgerConfig.numReservedSystemEntities())) { + final var nextClone = createAccount( + asAccountId(num, hederaConfig), + 0, + treasury.expirationSecond(), + treasury.key(), + treasury.declineReward()); + treasuryAcctRcds.add(nextClone); + } + recordsKeeper.treasuryClones(treasuryAcctRcds); + log.info( + "Created {} zero-balance synthetic records cloning treasury properties in the {}-{} range", + treasuryAcctRcds.size(), + FIRST_POST_SYSTEM_FILE_ENTITY, + ledgerConfig.numReservedSystemEntities()); + + // ---------- Create blocklist records (if enabled) ------------------------- + if (accountsConfig.blocklistEnabled()) { + final var blocklistResourceName = accountsConfig.blocklistResource(); + final var blocklist = blocklistParser.parse(blocklistResourceName); + if (!blocklist.isEmpty()) { + int counter = 0; + for (final var blockedInfo : blocklist) { + final var acctBldr = blockedAccountWith(blockedInfo, bootstrapConfig) + // add a PLACEHOLDER account ID + .accountId(asAccountId(++counter, hederaConfig)); + blocklistAcctRcds.add(acctBldr.build()); + } + } else if (log.isDebugEnabled()) { + log.debug("No blocklist accounts found in {}", blocklistResourceName); + } + } + // Note: These account objects don't yet have CORRECT IDs! We will create or look up the real entity IDs when + // the EntityIDService becomes available + recordsKeeper.blocklistAccounts(blocklistAcctRcds); + log.info("Created {} PLACEHOLDER synthetic blocklist records", blocklistAcctRcds.size()); + } + + /** + * Creates a blocked Hedera account with the given memo and EVM address. + * A blocked account has receiverSigRequired flag set to true, key set to the genesis key, and balance set to 0. + * + * @param blockedInfo record containing EVM address and memo for the blocked account + * @return a Hedera account with the given memo and EVM address + */ + @NonNull + private Account.Builder blockedAccountWith( + @NonNull final BlocklistParser.BlockedInfo blockedInfo, @NonNull final BootstrapConfig bootstrapConfig) { + final var expiry = bootstrapConfig.systemEntityExpiry(); + final var acctBuilder = Account.newBuilder() + .receiverSigRequired(true) + .declineReward(true) + .deleted(false) + .expirationSecond(expiry) + .smartContract(false) + .key(superUserKey(bootstrapConfig)) + .autoRenewSeconds(expiry) + .alias(blockedInfo.evmAddress()); + + if (!blockedInfo.memo().isEmpty()) acctBuilder.memo(blockedInfo.memo()); + + return acctBuilder; + } + + /** + * Given an account number and a config, produce a correct AccountID. The config is needed to + * determine the correct shard and realm numbers. + */ + static AccountID asAccountId(final long acctNum, final HederaConfig hederaConfig) { + return AccountID.newBuilder() + .shardNum(hederaConfig.shard()) + .realmNum(hederaConfig.realm()) + .accountNum(acctNum) + .build(); + } + + @VisibleForTesting + public static long[] nonContractSystemNums(final long numReservedSystemEntities) { + return LongStream.rangeClosed(FIRST_POST_SYSTEM_FILE_ENTITY, numReservedSystemEntities) + .filter(i -> i < FIRST_RESERVED_SYSTEM_CONTRACT || i > LAST_RESERVED_SYSTEM_CONTRACT) + .toArray(); + } + + @NonNull + private Key superUserKey(@NonNull final BootstrapConfig bootstrapConfig) { + final var superUserKeyBytes = bootstrapConfig.genesisPublicKey(); + if (superUserKeyBytes.length() != 32) { + throw new IllegalStateException("'" + superUserKeyBytes + "' is not a possible Ed25519 public key"); + } + return Key.newBuilder().ed25519(superUserKeyBytes).build(); + } + + @NonNull + private Account createAccount( + @NonNull final AccountID id, final long balance, final long expiry, @NonNull final Key key) { + return createAccount(id, balance, expiry, key, true); + } + + @NonNull + private Account createAccount( + @NonNull final AccountID id, + final long balance, + final long expiry, + final Key key, + final boolean declineReward) { + return Account.newBuilder() + .accountId(id) + .receiverSigRequired(false) + .deleted(false) + .expirationSecond(expiry) + .memo("") + .smartContract(false) + .key(key) + .declineReward(declineReward) + .autoRenewSeconds(expiry) + .maxAutoAssociations(0) + .tinybarBalance(balance) + .build(); + } +} diff --git a/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/schemas/TokenSchema.java b/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/schemas/TokenSchema.java index dc7cba569a4d..58fbee15df47 100644 --- a/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/schemas/TokenSchema.java +++ b/hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/schemas/TokenSchema.java @@ -23,15 +23,13 @@ import static com.hedera.node.app.service.token.impl.TokenServiceImpl.STAKING_NETWORK_REWARDS_KEY; import static com.hedera.node.app.service.token.impl.TokenServiceImpl.TOKENS_KEY; import static com.hedera.node.app.service.token.impl.TokenServiceImpl.TOKEN_RELS_KEY; -import static com.hedera.node.app.service.token.impl.handlers.BaseCryptoHandler.asAccount; -import static com.hedera.node.app.spi.HapiUtils.EMPTY_KEY_LIST; -import static com.hedera.node.app.spi.HapiUtils.FUNDING_ACCOUNT_EXPIRY; +import static com.hedera.node.app.service.token.impl.comparator.TokenComparators.ACCOUNT_COMPARATOR; +import static com.hedera.node.app.service.token.impl.schemas.SyntheticRecordsGenerator.asAccountId; import static com.hedera.node.app.spi.Service.RELEASE_045_VERSION; +import static java.util.Objects.requireNonNull; import com.google.common.annotations.VisibleForTesting; import com.hedera.hapi.node.base.AccountID; -import com.hedera.hapi.node.base.Duration; -import com.hedera.hapi.node.base.Key; import com.hedera.hapi.node.base.NftID; import com.hedera.hapi.node.base.TokenID; import com.hedera.hapi.node.state.common.EntityIDPair; @@ -43,53 +41,66 @@ import com.hedera.hapi.node.state.token.StakingNodeInfo; import com.hedera.hapi.node.state.token.Token; import com.hedera.hapi.node.state.token.TokenRelation; -import com.hedera.hapi.node.token.CryptoCreateTransactionBody; -import com.hedera.node.app.service.token.impl.BlocklistParser; import com.hedera.node.app.service.token.impl.TokenServiceImpl; import com.hedera.node.app.spi.state.MigrationContext; import com.hedera.node.app.spi.state.Schema; import com.hedera.node.app.spi.state.StateDefinition; -import com.hedera.node.app.spi.state.WritableKVState; -import com.hedera.node.app.spi.workflows.record.GenesisRecordsBuilder; import com.hedera.node.config.data.AccountsConfig; -import com.hedera.node.config.data.BootstrapConfig; import com.hedera.node.config.data.HederaConfig; import com.hedera.node.config.data.LedgerConfig; import com.hedera.node.config.data.StakingConfig; import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.function.Supplier; import java.util.stream.LongStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; /** - * Genesis schema for the token service + * General schema for the token service */ public class TokenSchema extends Schema { private static final Logger log = LogManager.getLogger(TokenSchema.class); // These need to be big so databases are created at right scale. If they are too small then the on disk hash map // buckets will be too full which results in very poor performance. Have chosen 10 billion as should give us // plenty of runway. - private static final long MAX_ACCOUNTS = 10_000_000_000l; - private static final long MAX_TOKEN_RELS = 10_000_000_000l; - private static final long MAX_MINTABLE_NFTS = 10_000_000_000l; + private static final long MAX_ACCOUNTS = 10_000_000_000L; + private static final long MAX_TOKEN_RELS = 10_000_000_000L; + private static final long MAX_MINTABLE_NFTS = 10_000_000_000L; private static final long FIRST_RESERVED_SYSTEM_CONTRACT = 350L; private static final long LAST_RESERVED_SYSTEM_CONTRACT = 399L; private static final long FIRST_POST_SYSTEM_FILE_ENTITY = 200L; - private final BlocklistParser blocklistParser; + private final Supplier> sysAccts; + private final Supplier> stakingAccts; + private final Supplier> treasuryAccts; + private final Supplier> miscAccts; + private final Supplier> blocklistAccts; /** - * Create a new instance + * Constructor for this schema. Each of the supplier params should produce a {@link SortedSet} of + * {@link Account} objects, where each account object represents a _synthetic record_ (see {@link + * SyntheticRecordsGenerator} for more details). Even though these sorted sets contain account + * objects, these account objects may or may not yet exist in state. They're usually not needed, + * but are required for an event recovery situation. */ - public TokenSchema() { + public TokenSchema( + @NonNull final Supplier> sysAcctRcds, + @NonNull final Supplier> stakingAcctRcds, + @NonNull final Supplier> treasuryAcctRcds, + @NonNull final Supplier> miscAcctRcds, + @NonNull final Supplier> blocklistAcctRcds) { super(RELEASE_045_VERSION); - blocklistParser = new BlocklistParser(); + + this.sysAccts = sysAcctRcds; + this.stakingAccts = stakingAcctRcds; + this.treasuryAccts = treasuryAcctRcds; + this.miscAccts = miscAcctRcds; + this.blocklistAccts = blocklistAcctRcds; } @NonNull @@ -107,200 +118,142 @@ public Set statesToCreate() { @Override public void migrate(@NonNull MigrationContext ctx) { - // We will use these various configs for creating accounts. It would be nice to consolidate them somehow - final var accountsConfig = ctx.configuration().getConfigData(AccountsConfig.class); - final var bootstrapConfig = ctx.configuration().getConfigData(BootstrapConfig.class); - final var ledgerConfig = ctx.configuration().getConfigData(LedgerConfig.class); - final var hederaConfig = ctx.configuration().getConfigData(HederaConfig.class); - - // Get the record builder for creating any necessary (synthetic) records - final var recordsKeeper = ctx.genesisRecordsBuilder(); - - // Get the map for storing all the created accounts - final var accounts = ctx.newStates().get(ACCOUNTS_KEY); - final var isGenesis = ctx.previousStates().isEmpty(); if (isGenesis) { - createGenesisSchema( - ctx, accountsConfig, bootstrapConfig, ledgerConfig, hederaConfig, recordsKeeper, accounts); + createGenesisSchema(ctx); } } - private void createGenesisSchema( - final MigrationContext ctx, - final AccountsConfig accountsConfig, - final BootstrapConfig bootstrapConfig, - final LedgerConfig ledgerConfig, - final HederaConfig hederaConfig, - final GenesisRecordsBuilder recordsKeeper, - final WritableKVState accounts) { - // This key is used for all system accounts - final var superUserKey = superUserKey(bootstrapConfig); + private void createGenesisSchema(@NonNull MigrationContext ctx) { + // Create the network rewards state + initializeNetworkRewards(ctx); + initializeStakingNodeInfo(ctx); - // Create every system account, and add it to the accounts state. Now, it turns out that while accounts 1-100 - // (inclusive) are system accounts, accounts 200-349 (inclusive) and 400-750 (inclusive) are "clones" of system - // accounts. So we can just create all of these up front. Basically, every account from 1 to 750 (inclusive) - // other than those set aside for files (101-199 inclusive) and contracts (350-399 inclusive) are the same, - // except for the treasury account, which has a balance - // ---------- Create system accounts ------------------------- - final var systemAccts = new HashMap(); - for (long num = 1; num <= ledgerConfig.numSystemAccounts(); num++) { - final var id = asAccountId(num, hederaConfig); - if (accounts.contains(id)) { - continue; - } + // Get the map for storing all the created accounts + final var accounts = ctx.newStates().get(ACCOUNTS_KEY); - final var accountTinyBars = num == accountsConfig.treasury() ? ledgerConfig.totalTinyBarFloat() : 0; - assert accountTinyBars >= 0L : "Negative account balance!"; + // We will use these various configs for creating accounts. It would be nice to consolidate them somehow + final var ledgerConfig = ctx.configuration().getConfigData(LedgerConfig.class); + final var hederaConfig = ctx.configuration().getConfigData(HederaConfig.class); + final var accountsConfig = ctx.configuration().getConfigData(AccountsConfig.class); - final var account = createAccount(id, accountTinyBars, bootstrapConfig.systemEntityExpiry(), superUserKey); - systemAccts.put(account, newCryptoCreate(account)); - accounts.put(id, account); + // ---------- Create system accounts ------------------------- + int counter = 0; + for (final Account acct : sysAccts.get()) { + final var id = requireNonNull(acct.accountId()); + if (!accounts.contains(id)) { + accounts.put(id, acct); + counter++; + } } - recordsKeeper.systemAccounts(systemAccts); + log.info( + "Created {} system accounts (from {} total synthetic records)", + counter, + sysAccts.get().size()); // ---------- Create staking fund accounts ------------------------- - final var stakingAccts = new HashMap(); - final var stakingRewardAccountId = asAccountId(accountsConfig.stakingRewardAccount(), hederaConfig); - final var nodeRewardAccountId = asAccountId(accountsConfig.nodeRewardAccount(), hederaConfig); - final var stakingFundAccounts = List.of(stakingRewardAccountId, nodeRewardAccountId); - for (final var id : stakingFundAccounts) { - if (accounts.contains(id)) { - continue; + counter = 0; + for (final Account acct : stakingAccts.get()) { + final var id = requireNonNull(acct.accountId()); + if (!accounts.contains(id)) { + accounts.put(id, acct); + counter++; } - - final var stakingFundAccount = createAccount(id, 0, FUNDING_ACCOUNT_EXPIRY, EMPTY_KEY_LIST); - stakingAccts.put(stakingFundAccount, newCryptoCreate(stakingFundAccount)); - accounts.put(id, stakingFundAccount); } - recordsKeeper.stakingAccounts(stakingAccts); - - // ---------- Create multi-use accounts ------------------------- - final var multiAccts = new HashMap(); - for (long num = 900; num <= 1000; num++) { - final var id = asAccountId(num, hederaConfig); - if (accounts.contains(id)) { - continue; + log.info( + "Created {} staking accounts (from {} total synthetic records)", + counter, + stakingAccts.get().size()); + + // ---------- Create miscellaneous accounts ------------------------- + counter = 0; + for (final Account acct : treasuryAccts.get()) { + final var id = requireNonNull(acct.accountId()); + if (!accounts.contains(id)) { + accounts.put(id, acct); + counter++; } - - final var account = createAccount(id, 0, bootstrapConfig.systemEntityExpiry(), superUserKey); - multiAccts.put(account, newCryptoCreate(account)); - accounts.put(id, account); } - recordsKeeper.miscAccounts(multiAccts); + log.info( + "Created {} treasury clones (from {} total synthetic records)", + counter, + treasuryAccts.get().size()); // ---------- Create treasury clones ------------------------- - // Since version 0.28.6, all of these clone accounts should either all exist (on a restart) or all not exist - // (starting from genesis) - final Account treasury = accounts.get(asAccountId(accountsConfig.treasury(), hederaConfig)); - final var treasuryClones = new HashMap(); - for (final var num : nonContractSystemNums(ledgerConfig.numReservedSystemEntities())) { - final var nextCloneId = asAccountId(num, hederaConfig); - if (accounts.contains(nextCloneId)) { - continue; + counter = 0; + for (final Account acct : miscAccts.get()) { + final var id = requireNonNull(acct.accountId()); + if (!accounts.contains(id)) { + accounts.put(id, acct); + counter++; } - - final var nextClone = createAccount( - nextCloneId, 0, treasury.expirationSecond(), treasury.key(), treasury.declineReward()); - treasuryClones.put(nextClone, newCryptoCreate(nextClone)); - accounts.put(nextCloneId, nextClone); } - recordsKeeper.treasuryClones(treasuryClones); log.info( - "Created {} zero-balance accounts cloning treasury properties in the {}-{} range", - treasuryClones.size(), - FIRST_POST_SYSTEM_FILE_ENTITY, - ledgerConfig.numReservedSystemEntities()); + "Created {} miscellaneous accounts (from {} total synthetic records)", + counter, + miscAccts.get().size()); - // Create the network rewards state - initializeNetworkRewards(ctx); - initializeStakingNodeInfo(ctx); + // ---------- Create blocklist accounts ------------------------- + counter = 0; + final var newBlocklistAccts = new TreeSet<>(ACCOUNT_COMPARATOR); + if (accountsConfig.blocklistEnabled()) { + final var existingAliases = ctx.newStates().get(TokenServiceImpl.ALIASES_KEY); + + for (final Account acctWithoutId : blocklistAccts.get()) { + final var acctWithIdBldr = acctWithoutId.copyBuilder(); + final Account accountWithId; + if (!existingAliases.contains(acctWithoutId.alias())) { + // The account does not yet exist in state, so we create it with a new entity ID. This is where we + // replace the placeholder entity IDs assigned in the SyntheticRegordsGenerator with actual, real + // entity IDs + final var id = asAccountId(ctx.newEntityNum(), hederaConfig); + accountWithId = acctWithIdBldr.accountId(id).build(); + + // Put the account and its alias in state + accounts.put(accountWithId.accountIdOrThrow(), accountWithId); + existingAliases.put(accountWithId.alias(), accountWithId.accountIdOrThrow()); + counter++; + } else { + // The account already exists in state, so we look up its existing ID, but do NOT re-add it to state + final var existingAcctId = existingAliases.get(acctWithoutId.alias()); + accountWithId = acctWithIdBldr.accountId(existingAcctId).build(); + } + newBlocklistAccts.add(accountWithId); + } + } + // Since we may have replaced the placeholder entity IDs, we need to overwrite the builder's blocklist records. + // The overwritten "record" (here represented as an Account object) will simply be a copy of the record already + // there, but with a real entity ID instead of a placeholder entity ID + final var recordBuilder = ctx.genesisRecordsBuilder(); + if (!newBlocklistAccts.isEmpty()) { + recordBuilder.blocklistAccounts(newBlocklistAccts); + } + log.info( + "Overwrote {} blocklist records (from {} total synthetic records)", + newBlocklistAccts.size(), + blocklistAccts.get().size()); + log.info( + "Created {} blocklist accounts (from {} total synthetic records)", + counter, + blocklistAccts.get().size()); - // Safety check -- add up the balances of all accounts, they must match 50,000,000,000 HBARs (config) + // ---------- Balances Safety Check ------------------------- + // Aadd up the balances of all accounts, they must match 50,000,000,000 HBARs (config) var totalBalance = 0L; for (int i = 1; i < hederaConfig.firstUserEntity(); i++) { - final var account = accounts.get(AccountID.newBuilder() - .shardNum(hederaConfig.shard()) - .realmNum(hederaConfig.realm()) - .accountNum(i) - .build()); - + final var account = accounts.get(asAccountId(i, hederaConfig)); if (account != null) { totalBalance += account.tinybarBalance(); } } - if (totalBalance != ledgerConfig.totalTinyBarFloat()) { - throw new IllegalStateException("Total balance of all accounts does not match the total float"); + throw new IllegalStateException("Total balance of all accounts does not match the total float: actual: " + + totalBalance + " vs expected: " + ledgerConfig.totalTinyBarFloat()); } log.info( - "Ledger float is {} tinyBars in {} accounts.", + "Ledger float is {} tinyBars; {} modified accounts.", totalBalance, accounts.modifiedKeys().size()); - - // ---------- Create blocklist accounts (if enabled) ------------------------- - final Map blocklistAccts = new HashMap<>(); - if (accountsConfig.blocklistEnabled()) { - final var blocklistResourceName = accountsConfig.blocklistResource(); - final var blocklist = blocklistParser.parse(blocklistResourceName); - if (blocklist.isEmpty()) { - return; - } - - final var aliases = ctx.newStates().get(TokenServiceImpl.ALIASES_KEY); - - // We only want to create accounts that are not already in state, so we filter based on blocked account EVM - // addresses that don't yet exist in state - final var blockedToCreate = blocklist.stream() - .filter(blockedAccount -> aliases.get(blockedAccount.evmAddress()) == null) - .toList(); - - for (final var blockedInfo : blockedToCreate) { - final var newId = ctx.newEntityNum(); - final var account = blockedAccountWith(blockedInfo, bootstrapConfig) - .accountId(asAccount(newId)) - .build(); - blocklistAccts.put(account, newCryptoCreate(account)); - accounts.put(account.accountIdOrThrow(), account); - aliases.put(account.alias(), account.accountIdOrThrow()); - } - } - recordsKeeper.blocklistAccounts(blocklistAccts); - log.info("Created {} blocklist accounts", blocklistAccts.size()); - } - - /** - * Creates a blocked Hedera account with the given memo and EVM address. - * A blocked account has receiverSigRequired flag set to true, key set to the genesis key, and balance set to 0. - * - * @param blockedInfo record containing EVM address and memo for the blocked account - * @return a Hedera account with the given memo and EVM address - */ - @NonNull - private Account.Builder blockedAccountWith( - @NonNull final BlocklistParser.BlockedInfo blockedInfo, @NonNull final BootstrapConfig bootstrapConfig) { - final var expiry = bootstrapConfig.systemEntityExpiry(); - final var acctBuilder = Account.newBuilder() - .receiverSigRequired(true) - .declineReward(true) - .deleted(false) - .expirationSecond(expiry) - .smartContract(false) - .key(superUserKey(bootstrapConfig)) - .autoRenewSeconds(expiry) - .alias(blockedInfo.evmAddress()); - - if (!blockedInfo.memo().isEmpty()) acctBuilder.memo(blockedInfo.memo()); - - return acctBuilder; - } - - private static AccountID asAccountId(final long acctNum, final HederaConfig hederaConfig) { - return AccountID.newBuilder() - .shardNum(hederaConfig.shard()) - .realmNum(hederaConfig.realm()) - .accountNum(acctNum) - .build(); } @VisibleForTesting @@ -310,43 +263,6 @@ public static long[] nonContractSystemNums(final long numReservedSystemEntities) .toArray(); } - @NonNull - private Key superUserKey(@NonNull final BootstrapConfig bootstrapConfig) { - final var superUserKeyBytes = bootstrapConfig.genesisPublicKey(); - if (superUserKeyBytes.length() != 32) { - throw new IllegalStateException("'" + superUserKeyBytes + "' is not a possible Ed25519 public key"); - } - return Key.newBuilder().ed25519(superUserKeyBytes).build(); - } - - @NonNull - private Account createAccount( - @NonNull final AccountID id, final long balance, final long expiry, @NonNull final Key key) { - return createAccount(id, balance, expiry, key, true); - } - - @NonNull - private Account createAccount( - @NonNull final AccountID id, - final long balance, - final long expiry, - final Key key, - final boolean declineReward) { - return Account.newBuilder() - .accountId(id) - .receiverSigRequired(false) - .deleted(false) - .expirationSecond(expiry) - .memo("") - .smartContract(false) - .key(key) - .declineReward(declineReward) - .autoRenewSeconds(expiry) - .maxAutoAssociations(0) - .tinybarBalance(balance) - .build(); - } - private void initializeStakingNodeInfo(@NonNull final MigrationContext ctx) { // TODO: This need to go through address book and set all the nodes final var config = ctx.configuration(); @@ -383,17 +299,4 @@ private void initializeNetworkRewards(@NonNull final MigrationContext ctx) { .build(); networkRewardsState.put(networkRewards); } - - private static CryptoCreateTransactionBody.Builder newCryptoCreate(@NonNull final Account account) { - return CryptoCreateTransactionBody.newBuilder() - .key(account.key()) - .memo(account.memo()) - .declineReward(account.declineReward()) - .receiverSigRequired(account.receiverSigRequired()) - .autoRenewPeriod(Duration.newBuilder() - .seconds(account.autoRenewSeconds()) - .build()) - .initialBalance(account.tinybarBalance()) - .alias(account.alias()); - } } diff --git a/hedera-node/hedera-token-service-impl/src/main/java/module-info.java b/hedera-node/hedera-token-service-impl/src/main/java/module-info.java index 07403c511867..94977c2addda 100644 --- a/hedera-node/hedera-token-service-impl/src/main/java/module-info.java +++ b/hedera-node/hedera-token-service-impl/src/main/java/module-info.java @@ -39,4 +39,7 @@ com.hedera.node.app.service.token.impl.test; exports com.hedera.node.app.service.token.impl.handlers.transfer to com.hedera.node.app; + exports com.hedera.node.app.service.token.impl.schemas to + com.hedera.node.app, + com.hedera.node.app.service.token.impl.api.test; } diff --git a/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/ReadableAccountStoreImplTest.java b/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/ReadableAccountStoreImplTest.java index 6883c85a637b..e86fa737395c 100644 --- a/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/ReadableAccountStoreImplTest.java +++ b/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/ReadableAccountStoreImplTest.java @@ -30,6 +30,7 @@ import com.hedera.node.app.service.token.impl.ReadableAccountStoreImpl; import com.hedera.node.app.service.token.impl.test.handlers.util.CryptoHandlerTestBase; import com.hedera.pbj.runtime.io.buffer.Bytes; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -235,4 +236,18 @@ void getSizeOfState() { final var store = new ReadableAccountStoreImpl(readableStates); assertEquals(readableStates.get(ACCOUNTS).size(), store.sizeOfAccountState()); } + + @Test + void containsWorksAsExpected() { + // Subject is pre-populated with this ID + assertThat(subject.contains(id)).isTrue(); + + // Pass any account ID that isn't in the store + assertThat(subject.contains( + AccountID.newBuilder().accountNum(Long.MAX_VALUE).build())) + .isFalse(); + + //noinspection DataFlowIssue + Assertions.assertThatThrownBy(() -> subject.contains(null)).isInstanceOf(NullPointerException.class); + } } diff --git a/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/GenesisSchemaTest.java b/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/GenesisSchemaTest.java deleted file mode 100644 index dc9b9f8738c2..000000000000 --- a/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/GenesisSchemaTest.java +++ /dev/null @@ -1,656 +0,0 @@ -/* - * Copyright (C) 2023 Hedera Hashgraph, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.hedera.node.app.service.token.impl.test.schemas; - -import static com.hedera.node.app.service.token.impl.TokenServiceImpl.ACCOUNTS_KEY; -import static com.hedera.node.app.service.token.impl.TokenServiceImpl.ALIASES_KEY; -import static com.hedera.node.app.service.token.impl.handlers.BaseCryptoHandler.asAccount; -import static com.hedera.node.app.spi.HapiUtils.EMPTY_KEY_LIST; -import static java.util.Collections.emptyMap; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.verify; - -import com.hedera.hapi.node.base.AccountID; -import com.hedera.hapi.node.base.Duration; -import com.hedera.hapi.node.state.common.EntityNumber; -import com.hedera.hapi.node.state.token.Account; -import com.hedera.hapi.node.token.CryptoCreateTransactionBody; -import com.hedera.node.app.ids.EntityIdService; -import com.hedera.node.app.ids.WritableEntityIdStore; -import com.hedera.node.app.service.token.impl.TokenServiceImpl; -import com.hedera.node.app.service.token.impl.schemas.TokenSchema; -import com.hedera.node.app.spi.fixtures.info.FakeNetworkInfo; -import com.hedera.node.app.spi.fixtures.state.MapWritableKVState; -import com.hedera.node.app.spi.fixtures.state.MapWritableStates; -import com.hedera.node.app.spi.info.NetworkInfo; -import com.hedera.node.app.spi.state.EmptyReadableStates; -import com.hedera.node.app.spi.state.WritableSingletonState; -import com.hedera.node.app.spi.state.WritableSingletonStateBase; -import com.hedera.node.app.spi.state.WritableStates; -import com.hedera.node.app.spi.throttle.HandleThrottleParser; -import com.hedera.node.app.spi.workflows.record.GenesisRecordsBuilder; -import com.hedera.node.app.workflows.handle.record.MigrationContextImpl; -import com.hedera.node.config.testfixtures.HederaTestConfigBuilder; -import com.hedera.pbj.runtime.io.buffer.Bytes; -import com.swirlds.config.api.Configuration; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.IntStream; -import org.assertj.core.api.Assertions; -import org.bouncycastle.util.Arrays; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -final class GenesisSchemaTest { - - private static final String GENESIS_KEY = "0aa8e21064c61eab86e2a9c164565b4e7a9a4146106e0a6cd03a8c395a110e92"; - private static final long EXPECTED_TREASURY_TINYBARS_BALANCE = 5000000000000000000L; - private static final int NUM_SYSTEM_ACCOUNTS = 312; - private static final long EXPECTED_ENTITY_EXPIRY = 1812637686L; - private static final long TREASURY_ACCOUNT_NUM = 2L; - private static final long NUM_RESERVED_SYSTEM_ENTITIES = 750L; - private static final String EVM_ADDRESS_0 = "e261e26aecce52b3788fac9625896ffbc6bb4424"; - private static final String EVM_ADDRESS_1 = "ce16e8eb8f4bf2e65ba9536c07e305b912bafacf"; - private static final String EVM_ADDRESS_2 = "f39fd6e51aad88f6f4ce6ab8827279cfffb92266"; - private static final String EVM_ADDRESS_3 = "70997970c51812dc3a010c7d01b50e0d17dc79c8"; - private static final String EVM_ADDRESS_4 = "7e5f4552091a69125d5dfcb7b8c2659029395bdf"; - private static final String EVM_ADDRESS_5 = "a04a864273e77be6fe500ad2f5fad320d9168bb6"; - private static final String[] EVM_ADDRESSES = { - EVM_ADDRESS_0, EVM_ADDRESS_1, EVM_ADDRESS_2, EVM_ADDRESS_3, EVM_ADDRESS_4, EVM_ADDRESS_5 - }; - private static final long BEGINNING_ENTITY_ID = 3000; - - @Mock - private GenesisRecordsBuilder genesisRecordsBuilder; - - @Mock - private HandleThrottleParser handleThrottling; - - @Captor - private ArgumentCaptor> sysAcctMapCaptor; - - @Captor - private ArgumentCaptor> stakingAcctMapCaptor; - - @Captor - private ArgumentCaptor> multiuseAcctMapCaptor; - - @Captor - private ArgumentCaptor> treasuryCloneMapCaptor; - - @Captor - private ArgumentCaptor> blocklistMapCaptor; - - private MapWritableKVState accounts; - private MapWritableKVState aliases; - private WritableStates newStates; - private Configuration config; - private NetworkInfo networkInfo; - private WritableEntityIdStore entityIdStore; - - @BeforeEach - void setUp() { - accounts = MapWritableKVState.builder(TokenServiceImpl.ACCOUNTS_KEY) - .build(); - aliases = MapWritableKVState.builder(ALIASES_KEY).build(); - - newStates = newStatesInstance(accounts, aliases, newWritableEntityIdState()); - - entityIdStore = new WritableEntityIdStore(newStates); - - networkInfo = new FakeNetworkInfo(); - - config = buildConfig(NUM_SYSTEM_ACCOUNTS, true); - } - - @Test - void createsAllAccounts() { - final var schema = new TokenSchema(); - final var migrationContext = new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore); - - schema.migrate(migrationContext); - - // Verify created system accounts - verify(genesisRecordsBuilder).systemAccounts(sysAcctMapCaptor.capture()); - final var sysAcctsResult = sysAcctMapCaptor.getValue(); - Assertions.assertThat(sysAcctsResult) - .isNotNull() - .hasSize(NUM_SYSTEM_ACCOUNTS) - .allSatisfy((account, builder) -> { - verifySystemAccount(account); - verifyCryptoCreateBuilder(account, builder); - }); - Assertions.assertThat( - sysAcctsResult.keySet().stream().map(Account::accountId).map(AccountID::accountNum)) - .allMatch(acctNum -> 1 <= acctNum && acctNum <= NUM_SYSTEM_ACCOUNTS); - - // Verify created staking accounts - verify(genesisRecordsBuilder).stakingAccounts(stakingAcctMapCaptor.capture()); - final var stakingAcctsResult = stakingAcctMapCaptor.getValue(); - Assertions.assertThat(stakingAcctsResult).isNotNull().hasSize(2).allSatisfy((account, builder) -> { - verifyStakingAccount(account); - verifyCryptoCreateBuilder(account, builder); - }); - Assertions.assertThat(stakingAcctsResult.keySet().stream() - .map(Account::accountId) - .map(AccountID::accountNum) - .toArray()) - .containsExactlyInAnyOrder(800L, 801L); - - // Verify created multipurpose accounts - verify(genesisRecordsBuilder).miscAccounts(multiuseAcctMapCaptor.capture()); - final var multiuseAcctsResult = multiuseAcctMapCaptor.getValue(); - Assertions.assertThat(multiuseAcctsResult).isNotNull().hasSize(101).allSatisfy((account, builder) -> { - verifyMultiUseAccount(account); - verifyCryptoCreateBuilder(account, builder); - }); - Assertions.assertThat(multiuseAcctsResult.keySet().stream() - .map(Account::accountId) - .map(AccountID::accountNum)) - .allMatch(acctNum -> 900 <= acctNum && acctNum <= 1000); - - // Verify created treasury clones - verify(genesisRecordsBuilder).treasuryClones(treasuryCloneMapCaptor.capture()); - final var treasuryCloneAcctsResult = treasuryCloneMapCaptor.getValue(); - Assertions.assertThat(treasuryCloneAcctsResult).isNotNull().hasSize(388).allSatisfy((account, builder) -> { - verifyTreasuryCloneAccount(account); - verifyCryptoCreateBuilder(account, builder); - }); - Assertions.assertThat(treasuryCloneAcctsResult.keySet().stream() - .map(Account::accountId) - .map(AccountID::accountNum)) - .allMatch(acctNum -> - Arrays.contains(TokenSchema.nonContractSystemNums(NUM_RESERVED_SYSTEM_ENTITIES), acctNum)); - - // Verify created blocklist accounts - verify(genesisRecordsBuilder).blocklistAccounts(blocklistMapCaptor.capture()); - final var blocklistAcctsResult = blocklistMapCaptor.getValue(); - Assertions.assertThat(blocklistAcctsResult).isNotNull().hasSize(6).allSatisfy((account, builder) -> { - Assertions.assertThat(account).isNotNull(); - - Assertions.assertThat(account.accountId().accountNum()) - .isBetween(BEGINNING_ENTITY_ID, BEGINNING_ENTITY_ID + EVM_ADDRESSES.length); - Assertions.assertThat(account.receiverSigRequired()).isTrue(); - Assertions.assertThat(account.declineReward()).isTrue(); - Assertions.assertThat(account.deleted()).isFalse(); - Assertions.assertThat(account.expirationSecond()).isEqualTo(EXPECTED_ENTITY_EXPIRY); - Assertions.assertThat(account.autoRenewSeconds()).isEqualTo(EXPECTED_ENTITY_EXPIRY); - Assertions.assertThat(account.smartContract()).isFalse(); - Assertions.assertThat(account.key()).isNotNull(); - Assertions.assertThat(account.alias()).isNotNull(); - - verifyCryptoCreateBuilder(account, builder); - }); - } - - @Test - void someAccountsAlreadyExist() { - final var schema = new TokenSchema(); - - // We'll only configure 4 system accounts, half of which will already exist - config = buildConfig(4, true); - final var accts = new HashMap(); - IntStream.rangeClosed(1, 2).forEach(i -> putNewAccount(i, accts)); - // One of the two staking accounts will already exist - final var stakingAcctId = AccountID.newBuilder().accountNum(800L).build(); - accts.put(stakingAcctId, Account.newBuilder().accountId(stakingAcctId).build()); - // Half of the multipurpose accounts will already exist - IntStream.rangeClosed(900, 950).forEach(i -> putNewAccount(i, accts)); - // All but five of the treasury clones will already exist - IntStream.rangeClosed(200, 745).forEach(i -> { - if (isRegularAcctNum(i)) putNewAccount(i, accts); - }); - // Half of the blocklist accounts will already exist (simulated by the existence of alias mappings, not the - // account objects) - final var blocklistAccts = Map.of( - Bytes.fromHex(EVM_ADDRESS_0), asAccount(BEGINNING_ENTITY_ID), - Bytes.fromHex(EVM_ADDRESS_2), asAccount(BEGINNING_ENTITY_ID + 2), - Bytes.fromHex(EVM_ADDRESS_4), asAccount(BEGINNING_ENTITY_ID + 4)); - newStates = newStatesInstance( - new MapWritableKVState<>(ACCOUNTS_KEY, accts), - new MapWritableKVState<>(ALIASES_KEY, blocklistAccts), - newWritableEntityIdState()); - final var migrationContext = new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore); - - schema.migrate(migrationContext); - - verify(genesisRecordsBuilder).systemAccounts(sysAcctMapCaptor.capture()); - final var sysAcctsResult = sysAcctMapCaptor.getValue(); - // Only system accts with IDs 3 and 4 should have been created - Assertions.assertThat(sysAcctsResult).hasSize(2); - - verify(genesisRecordsBuilder).stakingAccounts(stakingAcctMapCaptor.capture()); - final var stakingAcctsResult = stakingAcctMapCaptor.getValue(); - // Only the staking acct with ID 801 should have been created - Assertions.assertThat(stakingAcctsResult).hasSize(1); - - verify(genesisRecordsBuilder).miscAccounts(multiuseAcctMapCaptor.capture()); - final var multiuseAcctsResult = multiuseAcctMapCaptor.getValue(); - // Only multi-use accts with IDs 951-1000 should have been created - Assertions.assertThat(multiuseAcctsResult).hasSize(50); - - verify(genesisRecordsBuilder).treasuryClones(treasuryCloneMapCaptor.capture()); - final var treasuryCloneAcctsResult = treasuryCloneMapCaptor.getValue(); - // Only treasury clones with IDs 746-750 should have been created - Assertions.assertThat(treasuryCloneAcctsResult).hasSize(5); - - verify(genesisRecordsBuilder).blocklistAccounts(blocklistMapCaptor.capture()); - final var blocklistAcctsResult = blocklistMapCaptor.getValue(); - // Only half of the blocklist accts should have been created - Assertions.assertThat(blocklistAcctsResult).hasSize(3); - } - - @Test - void allAccountsAlreadyExist() { - final var schema = new TokenSchema(); - - // All the system accounts will already exist - final var accts = new HashMap(); - IntStream.rangeClosed(1, NUM_SYSTEM_ACCOUNTS).forEach(i -> putNewAccount(i, accts)); - // Both of the two staking accounts will already exist - IntStream.rangeClosed(800, 801).forEach(i -> putNewAccount(i, accts)); - // All the multipurpose accounts will already exist - IntStream.rangeClosed(900, 1000).forEach(i -> putNewAccount(i, accts)); - // All the treasury clones will already exist - IntStream.rangeClosed(200, 750).forEach(i -> { - if (isRegularAcctNum(i)) putNewAccount(i, accts); - }); - // All the blocklist accounts will already exist - final var blocklistEvmAliasMappings = Map.of( - Bytes.fromHex(EVM_ADDRESS_0), asAccount(BEGINNING_ENTITY_ID), - Bytes.fromHex(EVM_ADDRESS_1), asAccount(BEGINNING_ENTITY_ID + 1), - Bytes.fromHex(EVM_ADDRESS_2), asAccount(BEGINNING_ENTITY_ID + 2), - Bytes.fromHex(EVM_ADDRESS_3), asAccount(BEGINNING_ENTITY_ID + 3), - Bytes.fromHex(EVM_ADDRESS_4), asAccount(BEGINNING_ENTITY_ID + 4), - Bytes.fromHex(EVM_ADDRESS_5), asAccount(BEGINNING_ENTITY_ID + 5)); - newStates = newStatesInstance( - new MapWritableKVState<>(ACCOUNTS_KEY, accts), - new MapWritableKVState<>(ALIASES_KEY, blocklistEvmAliasMappings), - newWritableEntityIdState()); - final var migrationContext = new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore); - - schema.migrate(migrationContext); - - verify(genesisRecordsBuilder).systemAccounts(emptyMap()); - verify(genesisRecordsBuilder).stakingAccounts(emptyMap()); - verify(genesisRecordsBuilder).miscAccounts(emptyMap()); - verify(genesisRecordsBuilder).treasuryClones(emptyMap()); - verify(genesisRecordsBuilder).blocklistAccounts(emptyMap()); - } - - @Test - void blocklistNotEnabled() { - final var schema = new TokenSchema(); - - // None of the blocklist accounts will exist, but they shouldn't be created since blocklists aren't enabled - config = buildConfig(NUM_SYSTEM_ACCOUNTS, false); - final var migrationContext = new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore); - - schema.migrate(migrationContext); - - verify(genesisRecordsBuilder).blocklistAccounts(emptyMap()); - } - - @Test - void systemAccountsCreated() { - final var schema = new TokenSchema(); - schema.migrate(new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore)); - - for (int i = 1; i <= 100; i++) { - final var balance = i == 2 ? EXPECTED_TREASURY_TINYBARS_BALANCE : 0L; - - final var account = accounts.get(accountID(i)); - verifySystemAccount(account); - assertThat(account.accountId()).isEqualTo(accountID(i)); - assertThat(account.keyOrThrow().ed25519OrThrow().toHex()).isEqualTo(GENESIS_KEY); - assertBasicAccount(account, balance, EXPECTED_ENTITY_EXPIRY); - assertThat(account.autoRenewSeconds()).isEqualTo(EXPECTED_ENTITY_EXPIRY); - } - } - - @Test - void accountsBetweenFilesAndContracts() { - final var schema = new TokenSchema(); - schema.migrate(new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore)); - - for (int i = 200; i < 350; i++) { - final var account = accounts.get(accountID(i)); - assertThat(account).isNotNull(); - assertThat(account.accountId()).isEqualTo(accountID(i)); - assertThat(account.keyOrThrow().ed25519OrThrow().toHex()).isEqualTo(GENESIS_KEY); - assertBasicAccount(account, 0, EXPECTED_ENTITY_EXPIRY); - } - } - - @Test - void contractEntityIdsNotUsed() { - final var schema = new TokenSchema(); - schema.migrate(new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore)); - - for (int i = 350; i < 400; i++) { - assertThat(accounts.contains(accountID(i))).isFalse(); - } - } - - @Test - void accountsAfterContracts() { - final var schema = new TokenSchema(); - schema.migrate(new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore)); - - for (int i = 400; i <= 750; i++) { - final var account = accounts.get(accountID(i)); - assertThat(account).isNotNull(); - assertThat(account.accountId()).isEqualTo(accountID(i)); - assertThat(account.keyOrThrow().ed25519OrThrow().toHex()).isEqualTo(GENESIS_KEY); - assertBasicAccount(account, 0, EXPECTED_ENTITY_EXPIRY); - } - } - - @Test - void entityIdsBetweenSystemAccountsAndRewardAccountsAreEmpty() { - final var schema = new TokenSchema(); - schema.migrate(new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore)); - - for (int i = 751; i < 800; i++) { - assertThat(accounts.contains(accountID(i))).isFalse(); - } - } - - @Test - void stakingRewardAccounts() { - final var schema = new TokenSchema(); - schema.migrate(new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore)); - - final var stakingRewardAccount = accounts.get(accountID(800)); - verifyStakingAccount(stakingRewardAccount); - - final var nodeRewardAccount = accounts.get(accountID(801)); - verifyStakingAccount(nodeRewardAccount); - } - - @Test - void entityIdsAfterRewardAccountsAreEmpty() { - final var schema = new TokenSchema(); - schema.migrate(new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore)); - - for (int i = 802; i < 900; i++) { - assertThat(accounts.contains(accountID(i))).isFalse(); - } - } - - @Test - void miscAccountsAfter900() { - final var schema = new TokenSchema(); - schema.migrate(new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore)); - - for (int i = 900; i <= 1000; i++) { - final var account = accounts.get(accountID(i)); - assertThat(account).isNotNull(); - assertThat(account.accountId()).isEqualTo(accountID(i)); - verifyMultiUseAccount(account); - } - } - - @Test - void blocklistAccountIdsMatchEntityIds() { - final var schema = new TokenSchema(); - schema.migrate(new MigrationContextImpl( - EmptyReadableStates.INSTANCE, - newStates, - config, - networkInfo, - genesisRecordsBuilder, - handleThrottling, - entityIdStore)); - - for (int i = 0; i < EVM_ADDRESSES.length; i++) { - final var acctId = aliases.get(Bytes.fromHex(EVM_ADDRESSES[i])); - assertThat(acctId).isEqualTo(accountID((int) BEGINNING_ENTITY_ID + i + 1)); - } - } - - private void verifySystemAccount(final Account account) { - assertThat(account).isNotNull(); - final long expectedBalance = - account.accountId().accountNum() == TREASURY_ACCOUNT_NUM ? EXPECTED_TREASURY_TINYBARS_BALANCE : 0; - assertBasicAccount(account, expectedBalance, EXPECTED_ENTITY_EXPIRY); - assertThat(account.keyOrThrow().ed25519OrThrow().toHex()).isEqualTo(GENESIS_KEY); - Assertions.assertThat(account.autoRenewSeconds()).isEqualTo(EXPECTED_ENTITY_EXPIRY); - } - - private void verifyStakingAccount(final Account account) { - assertBasicAccount(account, 0, 33197904000L); - Assertions.assertThat(account.key()).isEqualTo(EMPTY_KEY_LIST); - } - - private void verifyMultiUseAccount(final Account account) { - assertBasicAccount(account, 0, EXPECTED_ENTITY_EXPIRY); - assertThat(account.keyOrThrow().ed25519OrThrow().toHex()).isEqualTo(GENESIS_KEY); - } - - private void verifyTreasuryCloneAccount(final Account account) { - assertBasicAccount(account, 0, EXPECTED_ENTITY_EXPIRY); - assertThat(account.keyOrThrow().ed25519OrThrow().toHex()).isEqualTo(GENESIS_KEY); - Assertions.assertThat(account.autoRenewSeconds()).isEqualTo(EXPECTED_ENTITY_EXPIRY); - } - - private void assertBasicAccount(Account account, long balance, long expiry) { - assertThat(account).isNotNull(); - assertThat(account.tinybarBalance()).isEqualTo(balance); - assertThat(account.alias()).isEqualTo(Bytes.EMPTY); - assertThat(account.expirationSecond()).isEqualTo(expiry); - assertThat(account.memo()).isEmpty(); - assertThat(account.deleted()).isFalse(); - assertThat(account.stakedToMe()).isZero(); - assertThat(account.stakePeriodStart()).isZero(); - assertThat(account.stakedId().kind()).isEqualTo(Account.StakedIdOneOfType.UNSET); - assertThat(account.receiverSigRequired()).isFalse(); - assertThat(account.hasHeadNftId()).isFalse(); - assertThat(account.headNftSerialNumber()).isZero(); - assertThat(account.numberOwnedNfts()).isZero(); - assertThat(account.maxAutoAssociations()).isZero(); - assertThat(account.usedAutoAssociations()).isZero(); - assertThat(account.declineReward()).isTrue(); - assertThat(account.numberAssociations()).isZero(); - assertThat(account.smartContract()).isFalse(); - assertThat(account.numberPositiveBalances()).isZero(); - assertThat(account.ethereumNonce()).isZero(); - assertThat(account.stakeAtStartOfLastRewardedPeriod()).isZero(); - assertThat(account.hasAutoRenewAccountId()).isFalse(); - assertThat(account.contractKvPairsNumber()).isZero(); - assertThat(account.cryptoAllowances()).isEmpty(); - assertThat(account.approveForAllNftAllowances()).isEmpty(); - assertThat(account.tokenAllowances()).isEmpty(); - assertThat(account.numberTreasuryTitles()).isZero(); - assertThat(account.expiredAndPendingRemoval()).isFalse(); - assertThat(account.firstContractStorageKey()).isEqualTo(Bytes.EMPTY); - } - - private AccountID accountID(int num) { - return AccountID.newBuilder().accountNum(num).build(); - } - - /** - * Compares the given account (already assumed to be correct) to the given crypto create - * transaction body builder - */ - private void verifyCryptoCreateBuilder( - final Account acctResult, final CryptoCreateTransactionBody.Builder builderSubject) { - Assertions.assertThat(builderSubject).isNotNull(); - Assertions.assertThat(builderSubject.build()) - .isEqualTo(CryptoCreateTransactionBody.newBuilder() - .key(acctResult.key()) - .memo(acctResult.memo()) - .declineReward(acctResult.declineReward()) - .receiverSigRequired(acctResult.receiverSigRequired()) - .initialBalance(acctResult.tinybarBalance()) - .autoRenewPeriod(Duration.newBuilder() - .seconds(acctResult.autoRenewSeconds()) - .build()) - .alias(acctResult.alias()) - .build()); - } - - private void putNewAccount(final long num, final HashMap accts) { - final var acctId = AccountID.newBuilder().accountNum(num).build(); - final var balance = num == TREASURY_ACCOUNT_NUM ? EXPECTED_TREASURY_TINYBARS_BALANCE : 0L; - final var acct = - Account.newBuilder().accountId(acctId).tinybarBalance(balance).build(); - accts.put(acctId, acct); - } - - private Configuration buildConfig(final int numSystemAccounts, final boolean blocklistEnabled) { - return HederaTestConfigBuilder.create() - // Accounts Config - .withValue("accounts.treasury", TREASURY_ACCOUNT_NUM) - .withValue("accounts.stakingRewardAccount", 800L) - .withValue("accounts.nodeRewardAccount", 801L) - .withValue("accounts.blocklist.enabled", blocklistEnabled) - .withValue("accounts.blocklist.path", "blocklist-parsing/test-evm-addresses-blocklist.csv") - // Bootstrap Config - .withValue("bootstrap.genesisPublicKey", "0x" + GENESIS_KEY) - .withValue("bootstrap.system.entityExpiry", EXPECTED_ENTITY_EXPIRY) - // Hedera Config - .withValue("hedera.realm", 0L) - .withValue("hedera.shard", 0L) - // Ledger Config - .withValue("ledger.numSystemAccounts", numSystemAccounts) - .withValue("ledger.numReservedSystemEntities", NUM_RESERVED_SYSTEM_ENTITIES) - .withValue("ledger.totalTinyBarFloat", EXPECTED_TREASURY_TINYBARS_BALANCE) - .getOrCreateConfig(); - } - - private WritableSingletonState newWritableEntityIdState() { - return new WritableSingletonStateBase<>( - EntityIdService.ENTITY_ID_STATE_KEY, () -> new EntityNumber(BEGINNING_ENTITY_ID), c -> {}); - } - - private MapWritableStates newStatesInstance( - final MapWritableKVState accts, - final MapWritableKVState aliases, - final WritableSingletonState entityIdState) { - return MapWritableStates.builder() - .state(accts) - .state(aliases) - .state(MapWritableKVState.builder(TokenServiceImpl.STAKING_INFO_KEY) - .build()) - .state(new WritableSingletonStateBase<>( - TokenServiceImpl.STAKING_NETWORK_REWARDS_KEY, () -> null, c -> {})) - .state(entityIdState) - .build(); - } - - /** - * @return true if the given account number is NOT a staking account number or system contract - */ - private boolean isRegularAcctNum(final long i) { - // Skip the staking account nums - if (Arrays.contains(new long[] {800, 801}, i)) return false; - // Skip the system contract account nums - return i < 350 || i > 399; - } -} diff --git a/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/SyntheticAccountsData.java b/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/SyntheticAccountsData.java new file mode 100644 index 000000000000..3ed451a7becf --- /dev/null +++ b/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/SyntheticAccountsData.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.service.token.impl.test.schemas; + +import com.hedera.node.config.testfixtures.HederaTestConfigBuilder; +import com.swirlds.config.api.Configuration; +import com.swirlds.test.framework.config.TestConfigBuilder; + +final class SyntheticAccountsData { + + static final String GENESIS_KEY = "0aa8e21064c61eab86e2a9c164565b4e7a9a4146106e0a6cd03a8c395a110e92"; + static final long EXPECTED_TREASURY_TINYBARS_BALANCE = 5000000000000000000L; + static final int DEFAULT_NUM_SYSTEM_ACCOUNTS = 312; + static final long EXPECTED_ENTITY_EXPIRY = 1812637686L; + static final long TREASURY_ACCOUNT_NUM = 2L; + static final long NUM_RESERVED_SYSTEM_ENTITIES = 750L; + static final String EVM_ADDRESS_0 = "e261e26aecce52b3788fac9625896ffbc6bb4424"; + static final String EVM_ADDRESS_1 = "ce16e8eb8f4bf2e65ba9536c07e305b912bafacf"; + static final String EVM_ADDRESS_2 = "f39fd6e51aad88f6f4ce6ab8827279cfffb92266"; + static final String EVM_ADDRESS_3 = "70997970c51812dc3a010c7d01b50e0d17dc79c8"; + static final String EVM_ADDRESS_4 = "7e5f4552091a69125d5dfcb7b8c2659029395bdf"; + static final String EVM_ADDRESS_5 = "a04a864273e77be6fe500ad2f5fad320d9168bb6"; + static final String[] EVM_ADDRESSES = { + EVM_ADDRESS_0, EVM_ADDRESS_1, EVM_ADDRESS_2, EVM_ADDRESS_3, EVM_ADDRESS_4, EVM_ADDRESS_5 + }; + + private SyntheticAccountsData() { + throw new IllegalStateException("Data and utility class"); + } + + static Configuration buildConfig(final int numSystemAccounts, final boolean blocklistEnabled) { + return configBuilder(numSystemAccounts, blocklistEnabled).getOrCreateConfig(); + } + + static TestConfigBuilder configBuilder(final int numSystemAccounts, final boolean blocklistEnabled) { + return HederaTestConfigBuilder.create() + // Accounts Config + .withValue("accounts.treasury", TREASURY_ACCOUNT_NUM) + .withValue("accounts.stakingRewardAccount", 800L) + .withValue("accounts.nodeRewardAccount", 801L) + .withValue("accounts.blocklist.enabled", blocklistEnabled) + .withValue("accounts.blocklist.path", "blocklist-parsing/test-evm-addresses-blocklist.csv") + // Bootstrap Config + .withValue("bootstrap.genesisPublicKey", "0x" + GENESIS_KEY) + .withValue("bootstrap.system.entityExpiry", EXPECTED_ENTITY_EXPIRY) + // Hedera Config + .withValue("hedera.realm", 0L) + .withValue("hedera.shard", 0L) + // Ledger Config + .withValue("ledger.numSystemAccounts", numSystemAccounts) + .withValue("ledger.numReservedSystemEntities", NUM_RESERVED_SYSTEM_ENTITIES) + .withValue("ledger.totalTinyBarFloat", EXPECTED_TREASURY_TINYBARS_BALANCE); + } +} diff --git a/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/SyntheticRecordsGeneratorTest.java b/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/SyntheticRecordsGeneratorTest.java new file mode 100644 index 000000000000..9bfd6a15c36d --- /dev/null +++ b/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/SyntheticRecordsGeneratorTest.java @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.service.token.impl.test.schemas; + +import static com.hedera.node.app.service.token.impl.comparator.TokenComparators.ACCOUNT_COMPARATOR; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.DEFAULT_NUM_SYSTEM_ACCOUNTS; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EVM_ADDRESSES; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EXPECTED_ENTITY_EXPIRY; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EXPECTED_TREASURY_TINYBARS_BALANCE; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.GENESIS_KEY; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.NUM_RESERVED_SYSTEM_ENTITIES; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.TREASURY_ACCOUNT_NUM; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.buildConfig; +import static com.hedera.node.app.spi.HapiUtils.EMPTY_KEY_LIST; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; + +import com.hedera.hapi.node.base.AccountID; +import com.hedera.hapi.node.state.token.Account; +import com.hedera.node.app.service.token.impl.schemas.SyntheticRecordsGenerator; +import com.hedera.node.app.service.token.impl.schemas.TokenSchema; +import com.hedera.node.app.spi.workflows.record.GenesisRecordsBuilder; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import com.swirlds.config.api.Configuration; +import java.util.Collections; +import java.util.SortedSet; +import java.util.TreeSet; +import org.assertj.core.api.Assertions; +import org.bouncycastle.util.Arrays; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +final class SyntheticRecordsGeneratorTest { + + @Mock + private GenesisRecordsBuilder genesisRecordsBuilder; + + @Captor + private ArgumentCaptor> sysAcctRcdsCaptor; + + @Captor + private ArgumentCaptor> stakingAcctRcdsCaptor; + + @Captor + private ArgumentCaptor> multiuseAcctRcdsCaptor; + + @Captor + private ArgumentCaptor> treasuryCloneRcdsCaptor; + + @Captor + private ArgumentCaptor> blocklistAcctRcdsCaptor; + + private Configuration config; + + @BeforeEach + void setUp() { + config = buildConfig(DEFAULT_NUM_SYSTEM_ACCOUNTS, true); + } + + @Test + void createsAllSyntheticRecords() { + final var subject = new SyntheticRecordsGenerator(); + subject.createRecords(config, genesisRecordsBuilder); + + // Verify system records created + verify(genesisRecordsBuilder).systemAccounts(sysAcctRcdsCaptor.capture()); + final var sysAcctRcdsResult = sysAcctRcdsCaptor.getValue(); + Assertions.assertThat(sysAcctRcdsResult) + .isNotNull() + .hasSize(DEFAULT_NUM_SYSTEM_ACCOUNTS) + .allSatisfy(this::verifySystemSynthRecord); + Assertions.assertThat(sysAcctRcdsResult.stream().map(Account::accountId).map(AccountID::accountNum)) + .allMatch(acctNum -> 1 <= acctNum && acctNum <= DEFAULT_NUM_SYSTEM_ACCOUNTS); + final var aggregateBalance = sysAcctRcdsResult.stream() + .mapToLong(a -> a != null ? a.tinybarBalance() : 0) + .sum(); + Assertions.assertThat(aggregateBalance).isEqualTo(EXPECTED_TREASURY_TINYBARS_BALANCE); + + // Verify staking records created + verify(genesisRecordsBuilder).stakingAccounts(stakingAcctRcdsCaptor.capture()); + final var stakingAcctRcdsResult = stakingAcctRcdsCaptor.getValue(); + Assertions.assertThat(stakingAcctRcdsResult).isNotNull().hasSize(2).allSatisfy(this::verifyStakingSynthRecord); + Assertions.assertThat(stakingAcctRcdsResult.stream() + .map(Account::accountId) + .map(AccountID::accountNum) + .toArray()) + .containsExactly(800L, 801L); + + // Verify multipurpose records created + verify(genesisRecordsBuilder).miscAccounts(multiuseAcctRcdsCaptor.capture()); + final var multiuseAcctsResult = multiuseAcctRcdsCaptor.getValue(); + Assertions.assertThat(multiuseAcctsResult).isNotNull().hasSize(101).allSatisfy(this::verifyMultiUseSynthRecord); + Assertions.assertThat( + multiuseAcctsResult.stream().map(Account::accountId).map(AccountID::accountNum)) + .allMatch(acctNum -> 900 <= acctNum && acctNum <= 1000); + + // Verify treasury clone records created + verify(genesisRecordsBuilder).treasuryClones(treasuryCloneRcdsCaptor.capture()); + final var treasuryCloneAcctsResult = treasuryCloneRcdsCaptor.getValue(); + Assertions.assertThat(treasuryCloneAcctsResult) + .isNotNull() + .hasSize(501) + .allSatisfy(this::verifyTreasuryCloneSynthRecord); + Assertions.assertThat(treasuryCloneAcctsResult.stream() + .map(Account::accountId) + .map(AccountID::accountNum)) + .allMatch(acctNum -> + Arrays.contains(TokenSchema.nonContractSystemNums(NUM_RESERVED_SYSTEM_ENTITIES), acctNum)); + + // Verify blocklist records created + verify(genesisRecordsBuilder).blocklistAccounts(blocklistAcctRcdsCaptor.capture()); + final var expectedBlocklistAcctRcdsSize = 6; + final var blocklistAcctsResult = blocklistAcctRcdsCaptor.getValue(); + Assertions.assertThat(blocklistAcctsResult).isNotNull().hasSize(expectedBlocklistAcctRcdsSize); + for (final Account acctRecord : blocklistAcctsResult) { + Assertions.assertThat(acctRecord).isNotNull(); + + // These account ID numbers are placeholders until we can get a real entity ID assigned by the + // EntityIdService (which happens later) + final var placeholderAcctNum = acctRecord.accountId().accountNum().longValue(); + Assertions.assertThat(placeholderAcctNum).isBetween(1L, (long) expectedBlocklistAcctRcdsSize); + Assertions.assertThat(acctRecord.receiverSigRequired()).isTrue(); + Assertions.assertThat(acctRecord.declineReward()).isTrue(); + Assertions.assertThat(acctRecord.deleted()).isFalse(); + Assertions.assertThat(acctRecord.expirationSecond()).isEqualTo(EXPECTED_ENTITY_EXPIRY); + Assertions.assertThat(acctRecord.autoRenewSeconds()).isEqualTo(EXPECTED_ENTITY_EXPIRY); + Assertions.assertThat(acctRecord.smartContract()).isFalse(); + Assertions.assertThat(acctRecord.key()).isNotNull(); + Assertions.assertThat(acctRecord.alias()) + .isEqualTo(Bytes.fromHex(EVM_ADDRESSES[(int) placeholderAcctNum - 1])); + } + } + + @Test + void blocklistNotEnabled() { + final var subject = new SyntheticRecordsGenerator(); + + config = buildConfig(DEFAULT_NUM_SYSTEM_ACCOUNTS, false); + + subject.createRecords(config, genesisRecordsBuilder); + + // No synthetic records should be created when the blocklist isn't enabled + verify(genesisRecordsBuilder).blocklistAccounts(Collections.emptySortedSet()); + } + + @Test + void correctEntityIdsUsed() { + final var subject = new SyntheticRecordsGenerator(); + subject.createRecords(config, genesisRecordsBuilder); + + verify(genesisRecordsBuilder).systemAccounts(sysAcctRcdsCaptor.capture()); + verify(genesisRecordsBuilder).stakingAccounts(stakingAcctRcdsCaptor.capture()); + verify(genesisRecordsBuilder).treasuryClones(treasuryCloneRcdsCaptor.capture()); + verify(genesisRecordsBuilder).miscAccounts(multiuseAcctRcdsCaptor.capture()); + verify(genesisRecordsBuilder).blocklistAccounts(blocklistAcctRcdsCaptor.capture()); + final var allSynthRcds = new TreeSet<>(ACCOUNT_COMPARATOR); + allSynthRcds.addAll(sysAcctRcdsCaptor.getValue()); + allSynthRcds.addAll(stakingAcctRcdsCaptor.getValue()); + allSynthRcds.addAll(treasuryCloneRcdsCaptor.getValue()); + allSynthRcds.addAll(multiuseAcctRcdsCaptor.getValue()); + allSynthRcds.addAll(blocklistAcctRcdsCaptor.getValue()); + Assertions.assertThat(allSynthRcds.stream().map(Account::accountId).map(AccountID::accountNum)) + .allMatch(i -> + // Verify contract entity IDs aren't used + (i < 350 || i >= 400) + && + // Verify entity IDs between system account records and reward account records aren't + // used + (i < 751 || i >= 800) + && + // Verify entity IDs between staking reward records and misc account records aren't used + (i < 802 || i >= 900)); + } + + private void verifySystemSynthRecord(final Account account) { + assertThat(account).isNotNull(); + final long expectedBalance = + account.accountId().accountNum() == TREASURY_ACCOUNT_NUM ? EXPECTED_TREASURY_TINYBARS_BALANCE : 0; + assertBasicSynthRecord(account, expectedBalance, EXPECTED_ENTITY_EXPIRY); + assertThat(account.keyOrThrow().ed25519OrThrow().toHex()).isEqualTo(GENESIS_KEY); + Assertions.assertThat(account.autoRenewSeconds()).isEqualTo(EXPECTED_ENTITY_EXPIRY); + } + + private void verifyStakingSynthRecord(final Account account) { + assertBasicSynthRecord(account, 0, 33197904000L); + Assertions.assertThat(account.key()).isEqualTo(EMPTY_KEY_LIST); + } + + private void verifyMultiUseSynthRecord(final Account account) { + assertBasicSynthRecord(account, 0, EXPECTED_ENTITY_EXPIRY); + assertThat(account.keyOrThrow().ed25519OrThrow().toHex()).isEqualTo(GENESIS_KEY); + } + + private void verifyTreasuryCloneSynthRecord(final Account account) { + assertBasicSynthRecord(account, 0, EXPECTED_ENTITY_EXPIRY); + assertThat(account.keyOrThrow().ed25519OrThrow().toHex()).isEqualTo(GENESIS_KEY); + Assertions.assertThat(account.autoRenewSeconds()).isEqualTo(EXPECTED_ENTITY_EXPIRY); + } + + private void assertBasicSynthRecord(Account account, long balance, long expiry) { + assertThat(account).isNotNull(); + assertThat(account.tinybarBalance()).isEqualTo(balance); + assertThat(account.alias()).isEqualTo(Bytes.EMPTY); + assertThat(account.expirationSecond()).isEqualTo(expiry); + assertThat(account.memo()).isEmpty(); + assertThat(account.deleted()).isFalse(); + assertThat(account.stakedToMe()).isZero(); + assertThat(account.stakePeriodStart()).isZero(); + assertThat(account.stakedId().kind()).isEqualTo(Account.StakedIdOneOfType.UNSET); + assertThat(account.receiverSigRequired()).isFalse(); + assertThat(account.hasHeadNftId()).isFalse(); + assertThat(account.headNftSerialNumber()).isZero(); + assertThat(account.numberOwnedNfts()).isZero(); + assertThat(account.maxAutoAssociations()).isZero(); + assertThat(account.usedAutoAssociations()).isZero(); + assertThat(account.declineReward()).isTrue(); + assertThat(account.numberAssociations()).isZero(); + assertThat(account.smartContract()).isFalse(); + assertThat(account.numberPositiveBalances()).isZero(); + assertThat(account.ethereumNonce()).isZero(); + assertThat(account.stakeAtStartOfLastRewardedPeriod()).isZero(); + assertThat(account.hasAutoRenewAccountId()).isFalse(); + assertThat(account.contractKvPairsNumber()).isZero(); + assertThat(account.cryptoAllowances()).isEmpty(); + assertThat(account.approveForAllNftAllowances()).isEmpty(); + assertThat(account.tokenAllowances()).isEmpty(); + assertThat(account.numberTreasuryTitles()).isZero(); + assertThat(account.expiredAndPendingRemoval()).isFalse(); + assertThat(account.firstContractStorageKey()).isEqualTo(Bytes.EMPTY); + } +} diff --git a/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/TokenSchemaTest.java b/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/TokenSchemaTest.java new file mode 100644 index 000000000000..76a0b7a62a90 --- /dev/null +++ b/hedera-node/hedera-token-service-impl/src/test/java/com/hedera/node/app/service/token/impl/test/schemas/TokenSchemaTest.java @@ -0,0 +1,770 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.service.token.impl.test.schemas; + +import static com.hedera.node.app.service.token.impl.TokenServiceImpl.ACCOUNTS_KEY; +import static com.hedera.node.app.service.token.impl.TokenServiceImpl.ALIASES_KEY; +import static com.hedera.node.app.service.token.impl.TokenServiceImpl.STAKING_INFO_KEY; +import static com.hedera.node.app.service.token.impl.TokenServiceImpl.STAKING_NETWORK_REWARDS_KEY; +import static com.hedera.node.app.service.token.impl.comparator.TokenComparators.ACCOUNT_COMPARATOR; +import static com.hedera.node.app.service.token.impl.handlers.BaseCryptoHandler.asAccount; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.DEFAULT_NUM_SYSTEM_ACCOUNTS; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EVM_ADDRESSES; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EVM_ADDRESS_0; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EVM_ADDRESS_1; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EVM_ADDRESS_2; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EVM_ADDRESS_3; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EVM_ADDRESS_4; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EVM_ADDRESS_5; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.EXPECTED_TREASURY_TINYBARS_BALANCE; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.NUM_RESERVED_SYSTEM_ENTITIES; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.TREASURY_ACCOUNT_NUM; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.buildConfig; +import static com.hedera.node.app.service.token.impl.test.schemas.SyntheticAccountsData.configBuilder; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; + +import com.hedera.hapi.node.base.AccountID; +import com.hedera.hapi.node.state.common.EntityNumber; +import com.hedera.hapi.node.state.token.Account; +import com.hedera.hapi.node.state.token.StakingNodeInfo; +import com.hedera.node.app.ids.EntityIdService; +import com.hedera.node.app.ids.WritableEntityIdStore; +import com.hedera.node.app.service.token.impl.TokenServiceImpl; +import com.hedera.node.app.service.token.impl.schemas.TokenSchema; +import com.hedera.node.app.spi.fixtures.info.FakeNetworkInfo; +import com.hedera.node.app.spi.fixtures.state.MapWritableKVState; +import com.hedera.node.app.spi.fixtures.state.MapWritableStates; +import com.hedera.node.app.spi.info.NetworkInfo; +import com.hedera.node.app.spi.info.NodeInfo; +import com.hedera.node.app.spi.state.EmptyReadableStates; +import com.hedera.node.app.spi.state.WritableSingletonState; +import com.hedera.node.app.spi.state.WritableSingletonStateBase; +import com.hedera.node.app.spi.state.WritableStates; +import com.hedera.node.app.spi.throttle.HandleThrottleParser; +import com.hedera.node.app.spi.workflows.record.GenesisRecordsBuilder; +import com.hedera.node.app.workflows.handle.record.MigrationContextImpl; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import com.swirlds.config.api.Configuration; +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.stream.IntStream; +import org.assertj.core.api.Assertions; +import org.bouncycastle.util.Arrays; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +final class TokenSchemaTest { + + private static final long BEGINNING_ENTITY_ID = 3000; + + private static final AccountID[] ACCT_IDS = new AccountID[1001]; + + static { + IntStream.rangeClosed(1, 1000).forEach(i -> ACCT_IDS[i] = asAccount(i)); + } + + private static final long[] NON_CONTRACT_RESERVED_NUMS = + TokenSchema.nonContractSystemNums(NUM_RESERVED_SYSTEM_ENTITIES); + + static { + // Precondition check + Assertions.assertThat(NON_CONTRACT_RESERVED_NUMS).hasSize(501); + } + + @Mock + private GenesisRecordsBuilder genesisRecordsBuilder; + + @Mock + private HandleThrottleParser handleThrottling; + + @Captor + private ArgumentCaptor> blocklistMapCaptor; + + private MapWritableKVState accounts; + private WritableStates newStates; + private Configuration config; + private NetworkInfo networkInfo; + private WritableEntityIdStore entityIdStore; + + @BeforeEach + void setUp() { + accounts = MapWritableKVState.builder(TokenServiceImpl.ACCOUNTS_KEY) + .build(); + + newStates = newStatesInstance( + accounts, + MapWritableKVState.builder(ALIASES_KEY).build(), + newWritableEntityIdState()); + + entityIdStore = new WritableEntityIdStore(newStates); + + networkInfo = new FakeNetworkInfo(); + + config = buildConfig(DEFAULT_NUM_SYSTEM_ACCOUNTS, true); + } + + @Test + void nonGenesisDoesntCreate() { + // To simulate a non-genesis case, we'll add a single account object to the `previousStates` param + accounts = MapWritableKVState.builder(TokenServiceImpl.ACCOUNTS_KEY) + .value(ACCT_IDS[1], Account.DEFAULT) + .build(); + final var nonEmptyPrevStates = newStatesInstance( + accounts, + MapWritableKVState.builder(ALIASES_KEY).build(), + newWritableEntityIdState()); + final var schema = newSubjectWithAllExpected(); + final var migrationContext = new MigrationContextImpl( + nonEmptyPrevStates, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore); + + schema.migrate(migrationContext); + + final var acctsStateResult = newStates.get(ACCOUNTS_KEY); + assertThat(acctsStateResult.isModified()).isFalse(); + final var nodeRewardsStateResult = newStates.getSingleton(STAKING_NETWORK_REWARDS_KEY); + assertThat(nodeRewardsStateResult.isModified()).isFalse(); + final var nodeInfoStateResult = newStates.get(STAKING_INFO_KEY); + assertThat(nodeInfoStateResult.isModified()).isFalse(); + } + + @Test + void initializesStakingData() { + final var schema = newSubjectWithAllExpected(); + final var migrationContext = new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore); + + schema.migrate(migrationContext); + + final var nodeRewardsStateResult = newStates.getSingleton(STAKING_NETWORK_REWARDS_KEY); + assertThat(nodeRewardsStateResult.isModified()).isTrue(); + final var nodeInfoStateResult = newStates.get(STAKING_INFO_KEY); + assertThat(nodeInfoStateResult.isModified()).isTrue(); + } + + @Test + void createsAllAccounts() { + final var schema = newSubjectWithAllExpected(); + final var migrationContext = new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore); + + schema.migrate(migrationContext); + + final var acctsStateResult = newStates.get(ACCOUNTS_KEY); + Assertions.assertThat(acctsStateResult).isNotNull(); + + // Verify created system accounts + for (int i = 1; i < DEFAULT_NUM_SYSTEM_ACCOUNTS; i++) { + assertThat(acctsStateResult.get(ACCT_IDS[i])).isNotNull(); + } + + // Verify created staking accounts + assertThat(acctsStateResult.get(ACCT_IDS[800])).isNotNull(); + assertThat(acctsStateResult.get(ACCT_IDS[801])).isNotNull(); + + // Verify created multipurpose accounts + for (int i = 900; i < 1001; i++) { + assertThat(acctsStateResult.get(ACCT_IDS[i])).isNotNull(); + } + + // Verify created treasury clones + for (final long nonContractSysNum : NON_CONTRACT_RESERVED_NUMS) { + if (nonContractSysNum < DEFAULT_NUM_SYSTEM_ACCOUNTS) { + // we've already checked that this account is not null (there's some overlap with system accounts), so + // skip checking this account again + continue; + } + assertThat(acctsStateResult.get(ACCT_IDS[(int) nonContractSysNum])).isNotNull(); + } + + // Verify overwritten blocklist account RECORDS + verify(genesisRecordsBuilder).blocklistAccounts(blocklistMapCaptor.capture()); + final var blocklistAcctsResult = blocklistMapCaptor.getValue(); + Assertions.assertThat(blocklistAcctsResult).isNotNull().hasSize(6).allSatisfy(account -> { + Assertions.assertThat(account).isNotNull(); + Assertions.assertThat(account.accountId().accountNum()) + .isBetween(BEGINNING_ENTITY_ID, BEGINNING_ENTITY_ID + EVM_ADDRESSES.length); + }); + + // Verify created blocklist account OBJECTS + final long expectedBlocklistIndex = BEGINNING_ENTITY_ID + EVM_ADDRESSES.length; + for (int i = 3001; i <= expectedBlocklistIndex; i++) { + final var acct = + acctsStateResult.get(AccountID.newBuilder().accountNum(i).build()); + assertThat(acct).isNotNull(); + assertThat(acct.alias()).isEqualTo(Bytes.fromHex(EVM_ADDRESSES[i - (int) BEGINNING_ENTITY_ID - 1])); + } + + // Finally, verify that the size is exactly as expected + assertThat(acctsStateResult.size()) + .isEqualTo( + // All the system accounts + DEFAULT_NUM_SYSTEM_ACCOUNTS + + + // Both of the two staking accounts + 2 + + + // All the misc accounts + 101 + + + // All treasury clones (which is 501 - allSysAccts(4), + this::allStakingAccts, + this::allMiscAccts, + this::allTreasuryClones, + this::allBlocklistAccts); + + // We'll only configure 4 system accounts, half of which will already exist + config = buildConfig(4, true); + final var accts = new HashMap(); + IntStream.rangeClosed(1, 2).forEach(i -> putNewAccount(i, accts, testMemo())); + // One of the two staking accounts will already exist + final var stakingAcctId = ACCT_IDS[800]; + accts.put( + stakingAcctId, + Account.newBuilder().accountId(stakingAcctId).memo(testMemo()).build()); + // Half of the multipurpose accounts will already exist + IntStream.rangeClosed(900, 950).forEach(i -> putNewAccount(i, accts, testMemo())); + // All but five of the treasury clones will already exist + IntStream.rangeClosed(200, 745).forEach(i -> { + if (isRegularAcctNum(i)) { + putNewAccount(i, accts, testMemo()); + } + }); + // Half of the blocklist accounts will already exist (simulated by the existence of alias + // mappings only, not the account objects) + final var blocklistAccts = Map.of( + Bytes.fromHex(EVM_ADDRESS_0), asAccount(BEGINNING_ENTITY_ID - 2), + Bytes.fromHex(EVM_ADDRESS_1), asAccount(BEGINNING_ENTITY_ID - 1), + Bytes.fromHex(EVM_ADDRESS_2), asAccount(BEGINNING_ENTITY_ID)); + newStates = newStatesInstance( + new MapWritableKVState<>(ACCOUNTS_KEY, accts), + new MapWritableKVState<>(ALIASES_KEY, blocklistAccts), + newWritableEntityIdState()); + final var migrationContext = new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore); + + schema.migrate(migrationContext); + + final var acctsStateResult = newStates.get(ACCOUNTS_KEY); + // Verify the pre-inserted didn't change + IntStream.rangeClosed(1, 2) + .forEach(i -> assertThat(acctsStateResult.get(ACCT_IDS[i])).isEqualTo(accts.get(ACCT_IDS[i]))); + // Only system accts with IDs 3 and 4 should have been created + IntStream.rangeClosed(3, 4).forEach(i -> assertThat(acctsStateResult.get(ACCT_IDS[i])) + .isNotNull() + .extracting("memo") + .asString() + .doesNotStartWith("test-original")); + + // Verify the pre-inserted didn't change + assertThat(acctsStateResult.get(ACCT_IDS[800])).isEqualTo(accts.get(ACCT_IDS[800])); + // Only the staking acct with ID 801 should have been created + assertThat(acctsStateResult.get(ACCT_IDS[801])) + .isNotNull() + .extracting("memo") + .asString() + .doesNotStartWith("test-original"); + + // Verify the pre-inserted didn't change + IntStream.rangeClosed(900, 950) + .forEach(i -> assertThat(acctsStateResult.get(ACCT_IDS[i])).isEqualTo(accts.get(ACCT_IDS[i]))); + // Only multi-use accts with IDs 951-1000 should have been created + IntStream.rangeClosed(951, 1000).forEach(i -> assertThat(acctsStateResult.get(ACCT_IDS[i])) + .isNotNull() + .extracting("memo") + .asString() + .doesNotStartWith("test-original")); + + // Verify the pre-inserted didn't change + IntStream.rangeClosed(250, 745) + .forEach(i -> assertThat(acctsStateResult.get(ACCT_IDS[i])).isEqualTo(accts.get(ACCT_IDS[i]))); + // Only treasury clones with IDs 746-750 should have been created + IntStream.rangeClosed(746, 750).forEach(i -> assertThat(acctsStateResult.get(ACCT_IDS[i])) + .isNotNull() + .extracting("memo") + .asString() + .doesNotStartWith("test-original")); + + final var aliasStateResult = newStates.get(ALIASES_KEY); + // Verify the pre-inserted didn't change + assertThat(aliasStateResult.get(Bytes.fromHex(EVM_ADDRESSES[0]))) + .isEqualTo(AccountID.newBuilder() + .accountNum(BEGINNING_ENTITY_ID - 2) + .build()); + assertThat(aliasStateResult.get(Bytes.fromHex(EVM_ADDRESSES[1]))) + .isEqualTo(AccountID.newBuilder() + .accountNum(BEGINNING_ENTITY_ID - 1) + .build()); + assertThat(aliasStateResult.get(Bytes.fromHex(EVM_ADDRESSES[2]))) + .isEqualTo( + AccountID.newBuilder().accountNum(BEGINNING_ENTITY_ID).build()); + // Only half of the blocklist accts should have been "created" (i.e. added to the alias state) + assertThat(aliasStateResult.get(Bytes.fromHex(EVM_ADDRESSES[3]))) + .isEqualTo(AccountID.newBuilder() + .accountNum(BEGINNING_ENTITY_ID + 1) + .build()); + assertThat(aliasStateResult.get(Bytes.fromHex(EVM_ADDRESSES[4]))) + .isEqualTo(AccountID.newBuilder() + .accountNum(BEGINNING_ENTITY_ID + 2) + .build()); + assertThat(aliasStateResult.get(Bytes.fromHex(EVM_ADDRESSES[5]))) + .isEqualTo(AccountID.newBuilder() + .accountNum(BEGINNING_ENTITY_ID + 3) + .build()); + + final int numModifiedAccts = acctsStateResult.modifiedKeys().size(); + assertThat(numModifiedAccts) + .isEqualTo( + // Half of the four configured system accounts + 2 + + + // One of the two staking accounts + 1 + + + // Half of the misc accounts + 50 + + + // The five remaining treasury accounts + 5 + + + // Half of the blocklist accounts + 3); + } + + @Test + void allAccountsAlreadyExist() { + // Initializing the schema will happen with _all_ the expected account records, but none of those accounts + // should be created in the migration + final var schema = newSubjectWithAllExpected(); + + // All the system accounts will already exist + final var accts = new HashMap(); + IntStream.rangeClosed(1, DEFAULT_NUM_SYSTEM_ACCOUNTS).forEach(i -> putNewAccount(i, accts, testMemo())); + // Both of the two staking accounts will already exist + IntStream.rangeClosed(800, 801).forEach(i -> putNewAccount(i, accts, testMemo())); + // All the multipurpose accounts will already exist + IntStream.rangeClosed(900, 1000).forEach(i -> putNewAccount(i, accts, testMemo())); + // All the treasury clones will already exist + IntStream.rangeClosed(200, 750).forEach(i -> { + if (isRegularAcctNum(i)) putNewAccount(i, accts, testMemo()); + }); + // All the blocklist accounts will already exist + final var blocklistEvmAliasMappings = Map.of( + Bytes.fromHex(EVM_ADDRESS_0), asAccount(BEGINNING_ENTITY_ID), + Bytes.fromHex(EVM_ADDRESS_1), asAccount(BEGINNING_ENTITY_ID + 1), + Bytes.fromHex(EVM_ADDRESS_2), asAccount(BEGINNING_ENTITY_ID + 2), + Bytes.fromHex(EVM_ADDRESS_3), asAccount(BEGINNING_ENTITY_ID + 3), + Bytes.fromHex(EVM_ADDRESS_4), asAccount(BEGINNING_ENTITY_ID + 4), + Bytes.fromHex(EVM_ADDRESS_5), asAccount(BEGINNING_ENTITY_ID + 5)); + newStates = newStatesInstance( + new MapWritableKVState<>(ACCOUNTS_KEY, accts), + new MapWritableKVState<>(ALIASES_KEY, blocklistEvmAliasMappings), + newWritableEntityIdState()); + final var migrationContext = new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore); + + schema.migrate(migrationContext); + + // Verify that none of the accounts changed + final var acctsStateResult = newStates.get(ACCOUNTS_KEY); + IntStream.rangeClosed(1, DEFAULT_NUM_SYSTEM_ACCOUNTS) + .forEach(i -> assertThat(acctsStateResult.get(ACCT_IDS[i])).isEqualTo(accts.get(ACCT_IDS[i]))); + IntStream.rangeClosed(800, 801) + .forEach(i -> assertThat(acctsStateResult.get(ACCT_IDS[i])).isEqualTo(accts.get(ACCT_IDS[i]))); + // All the multipurpose accounts will already exist + IntStream.rangeClosed(900, 1000) + .forEach(i -> assertThat(acctsStateResult.get(ACCT_IDS[i])).isEqualTo(accts.get(ACCT_IDS[i]))); + // All the treasury clones will already exist + IntStream.rangeClosed(200, (int) NUM_RESERVED_SYSTEM_ENTITIES).forEach(i -> { + if (isRegularAcctNum(i)) + assertThat(acctsStateResult.get(ACCT_IDS[i])).isEqualTo(accts.get(ACCT_IDS[i])); + else assertThat(acctsStateResult.get(ACCT_IDS[i])).isNull(); + }); + // All the blocklist accounts will already exist + IntStream.rangeClosed((int) BEGINNING_ENTITY_ID + 1, (int) BEGINNING_ENTITY_ID + EVM_ADDRESSES.length) + .forEach(i -> { + final var acctId = AccountID.newBuilder().accountNum(i).build(); + assertThat(acctsStateResult.get(acctId)).isEqualTo(accts.get(acctId)); + }); + + // Since all accounts were already created, no state should not have been modified + assertThat(acctsStateResult.isModified()).isFalse(); + final var aliasStateResult = newStates.get(ALIASES_KEY); + assertThat(aliasStateResult.isModified()).isFalse(); + } + + @Test + void blocklistNotEnabled() { + final var schema = newSubjectWithAllExpected(); + + // None of the blocklist accounts will exist, but they shouldn't be created since blocklists aren't enabled + config = buildConfig(DEFAULT_NUM_SYSTEM_ACCOUNTS, false); + final var migrationContext = new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore); + + schema.migrate(migrationContext); + + final var acctsStateResult = newStates.get(ACCOUNTS_KEY); + final var aliasStateResult = newStates.get(ALIASES_KEY); + for (int i = 0; i < EVM_ADDRESSES.length; i++) { + assertThat(acctsStateResult.get(AccountID.newBuilder() + .accountNum(BEGINNING_ENTITY_ID + 1 + i) + .build())) + .isNull(); + assertThat(aliasStateResult.get(Bytes.fromHex(EVM_ADDRESSES[i]))).isNull(); + } + } + + @Test + void createsSystemAccountsOnly() { + final var schema = new TokenSchema( + this::allDefaultSysAccts, + Collections::emptySortedSet, + Collections::emptySortedSet, + Collections::emptySortedSet, + Collections::emptySortedSet); + schema.migrate(new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore)); + + final var acctsStateResult = newStates.get(ACCOUNTS_KEY); + for (int i = 1; i < DEFAULT_NUM_SYSTEM_ACCOUNTS; i++) { + assertThat(acctsStateResult.get(ACCT_IDS[i])).isNotNull(); + } + + assertThat(acctsStateResult.size()).isEqualTo(DEFAULT_NUM_SYSTEM_ACCOUNTS); + } + + @Test + void createsStakingRewardAccountsOnly() { + // Since we aren't creating all accounts, we overwrite the config so as not to expect the total ledger balance + // to be present in the resulting objects + config = overridingLedgerBalanceWithZero(); + + final var schema = new TokenSchema( + Collections::emptySortedSet, + this::allStakingAccts, + Collections::emptySortedSet, + Collections::emptySortedSet, + Collections::emptySortedSet); + schema.migrate(new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore)); + + final var acctsStateResult = newStates.get(ACCOUNTS_KEY); + final var stakingRewardAccount = acctsStateResult.get(ACCT_IDS[800]); + assertThat(stakingRewardAccount).isNotNull(); + final var nodeRewardAccount = accounts.get(ACCT_IDS[801]); + assertThat(nodeRewardAccount).isNotNull(); + + // Finally, verify that the size is exactly as expected + assertThat(acctsStateResult.modifiedKeys()).hasSize(2); + } + + @Test + void createsTreasuryAccountsOnly() { + // Since we aren't creating all accounts, we overwrite the config so as not to expect the total ledger balance + // to be present in the resulting objects + config = overridingLedgerBalanceWithZero(); + + final var schema = new TokenSchema( + Collections::emptySortedSet, + Collections::emptySortedSet, + this::allTreasuryClones, + Collections::emptySortedSet, + Collections::emptySortedSet); + schema.migrate(new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore)); + + final var acctsStateResult = newStates.get(ACCOUNTS_KEY); + for (final long reservedNum : NON_CONTRACT_RESERVED_NUMS) { + assertThat(acctsStateResult.get(ACCT_IDS[(int) reservedNum])).isNotNull(); + } + + assertThat(acctsStateResult.modifiedKeys()).hasSize(NON_CONTRACT_RESERVED_NUMS.length); + } + + @Test + void createsMiscAccountsOnly() { + // Since we aren't creating all accounts, we overwrite the config so as not to expect the total ledger balance + // to be present in the resulting objects + config = overridingLedgerBalanceWithZero(); + + final var schema = new TokenSchema( + Collections::emptySortedSet, + Collections::emptySortedSet, + Collections::emptySortedSet, + this::allMiscAccts, + Collections::emptySortedSet); + schema.migrate(new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore)); + + final var acctsStateResult = newStates.get(ACCOUNTS_KEY); + + for (int i = 900; i <= 1000; i++) { + assertThat(acctsStateResult.get(ACCT_IDS[i])).isNotNull(); + } + } + + @Test + void createsBlocklistAccountsOnly() { + // Since we aren't creating all accounts, we overwrite the config so as not to expect the total ledger balance + // to be present in the resulting objects + config = overridingLedgerBalanceWithZero(); + + final var schema = new TokenSchema( + Collections::emptySortedSet, + Collections::emptySortedSet, + Collections::emptySortedSet, + Collections::emptySortedSet, + this::allBlocklistAccts); + schema.migrate(new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore)); + + // Verify that the assigned account ID matches the expected entity IDs + for (int i = 0; i < EVM_ADDRESSES.length; i++) { + final var acctId = newStates.get(ALIASES_KEY).get(Bytes.fromHex(EVM_ADDRESSES[i])); + assertThat(acctId).isEqualTo(asAccount((int) BEGINNING_ENTITY_ID + i + 1)); + } + } + + @Test + void onlyExpectedIdsUsed() { + final var schema = newSubjectWithAllExpected(); + schema.migrate(new MigrationContextImpl( + EmptyReadableStates.INSTANCE, + newStates, + config, + networkInfo, + genesisRecordsBuilder, + handleThrottling, + entityIdStore)); + + // Verify contract entity IDs aren't used + for (int i = 350; i < 400; i++) { + assertThat(accounts.contains(ACCT_IDS[i])).isFalse(); + } + + // Verify entity IDs between system and staking reward accounts aren't used + for (int i = 751; i < 800; i++) { + assertThat(accounts.contains(asAccount(i))).isFalse(); + } + + // Verify entity IDs between staking rewards and misc accounts are empty + for (int i = 802; i < 900; i++) { + assertThat(accounts.contains(asAccount(i))).isFalse(); + } + } + + private Configuration overridingLedgerBalanceWithZero() { + return configBuilder(DEFAULT_NUM_SYSTEM_ACCOUNTS, true) + .withValue("ledger.totalTinyBarFloat", "0") + .getOrCreateConfig(); + } + + private void putNewAccount(final int num, final HashMap accts, final String memo) { + final var balance = num == TREASURY_ACCOUNT_NUM ? EXPECTED_TREASURY_TINYBARS_BALANCE : 0L; + final var acct = Account.newBuilder() + .accountId(ACCT_IDS[num]) + .tinybarBalance(balance) + .memo(memo) + .build(); + accts.put(ACCT_IDS[num], acct); + } + + private WritableSingletonState newWritableEntityIdState() { + return new WritableSingletonStateBase<>( + EntityIdService.ENTITY_ID_STATE_KEY, () -> new EntityNumber(BEGINNING_ENTITY_ID), c -> {}); + } + + private MapWritableStates newStatesInstance( + final MapWritableKVState accts, + final MapWritableKVState aliases, + final WritableSingletonState entityIdState) { + //noinspection ReturnOfNull + return MapWritableStates.builder() + .state(accts) + .state(aliases) + .state(MapWritableKVState.builder(TokenServiceImpl.STAKING_INFO_KEY) + .build()) + .state(new WritableSingletonStateBase<>(STAKING_NETWORK_REWARDS_KEY, () -> null, c -> {})) + .state(entityIdState) + .build(); + } + + private SortedSet allDefaultSysAccts() { + return allSysAccts(DEFAULT_NUM_SYSTEM_ACCOUNTS); + } + + private SortedSet allSysAccts(final int numSysAccts) { + return newAccounts(1, numSysAccts); + } + + private SortedSet allStakingAccts() { + return newAccounts(800, 801); + } + + private SortedSet allMiscAccts() { + return newAccounts(900, 1000); + } + + private SortedSet allTreasuryClones() { + final var accts = new TreeSet<>(ACCOUNT_COMPARATOR); + for (final long nonContractSysNum : NON_CONTRACT_RESERVED_NUMS) { + accts.add(newAcctWithId((int) nonContractSysNum)); + } + + return accts; + } + + private SortedSet allBlocklistAccts() { + final var accts = new TreeSet<>(ACCOUNT_COMPARATOR); + for (int i = 0; i < EVM_ADDRESSES.length; i++) { + accts.add(newAcctWithId((int) BEGINNING_ENTITY_ID + i) + .copyBuilder() + .alias(Bytes.fromHex(EVM_ADDRESSES[i])) + .build()); + } + + return accts; + } + + private TokenSchema newSubjectWithAllExpected() { + return new TokenSchema( + this::allDefaultSysAccts, + this::allStakingAccts, + this::allMiscAccts, + this::allTreasuryClones, + this::allBlocklistAccts); + } + + /** + * @return true if the given account number is NOT a staking account number or system contract + */ + private static boolean isRegularAcctNum(final long i) { + // Skip the staking account nums + if (Arrays.contains(new long[] {800, 801}, i)) return false; + // Skip the system contract account nums + return i < 350 || i > 399; + } + + private static String testMemo() { + return "test-original" + Instant.now(); + } + + private static SortedSet newAccounts(final int lower, final int higher) { + final var accts = new TreeSet<>(ACCOUNT_COMPARATOR); + for (int i = lower; i <= higher; i++) { + accts.add(newAcctWithId(i)); + } + + return accts; + } + + private static Account newAcctWithId(final int id) { + final var acctId = id <= ACCT_IDS.length ? ACCT_IDS[id] : asAccount(id); + final var balance = id == TREASURY_ACCOUNT_NUM ? EXPECTED_TREASURY_TINYBARS_BALANCE : 0; + return Account.newBuilder().accountId(acctId).tinybarBalance(balance).build(); + } +} diff --git a/hedera-node/hedera-token-service/src/main/java/com/hedera/node/app/service/token/ReadableAccountStore.java b/hedera-node/hedera-token-service/src/main/java/com/hedera/node/app/service/token/ReadableAccountStore.java index 3c89e2535c3c..e5394f57e2ab 100644 --- a/hedera-node/hedera-token-service/src/main/java/com/hedera/node/app/service/token/ReadableAccountStore.java +++ b/hedera-node/hedera-token-service/src/main/java/com/hedera/node/app/service/token/ReadableAccountStore.java @@ -58,6 +58,12 @@ public interface ReadableAccountStore { */ boolean containsAlias(@NonNull final Bytes alias); + /** + * Returns true if the given account ID exists in state. + * @param accountID the ID to check + */ + boolean contains(@NonNull final AccountID accountID); + /** * Returns the number of accounts in state. * From 4fb7847d6dc7f630e071f31eef7fda9d80228ebd Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Thu, 30 Nov 2023 14:43:32 -0700 Subject: [PATCH 2/2] Revert new config provider constructor Signed-off-by: Matt Hess --- .../main/java/com/hedera/node/app/Hedera.java | 7 +---- .../node/app/config/ConfigProviderImpl.java | 30 ------------------- .../app/config/ConfigProviderImplTest.java | 14 --------- 3 files changed, 1 insertion(+), 50 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java index b3c9a7c2a5bd..663746a04a1a 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java @@ -369,12 +369,7 @@ private void onStateInitialized( // file in state, created by the file service migration, will match what we have here, so we don't have to worry // about re-loading config after migration. logger.info("Initializing configuration with trigger {}", trigger); - switch (trigger) { - case EVENT_STREAM_RECOVERY -> configProvider = - new ConfigProviderImpl(platform.getContext().getConfiguration()); - case GENESIS -> configProvider = new ConfigProviderImpl(true); - case RESTART, RECONNECT -> configProvider = new ConfigProviderImpl(false); - } + configProvider = new ConfigProviderImpl(trigger == GENESIS); logConfiguration(); // Determine if we need to create synthetic records for system entities diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/config/ConfigProviderImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/config/ConfigProviderImpl.java index 28b6f70fb1cb..d537ed845e75 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/config/ConfigProviderImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/config/ConfigProviderImpl.java @@ -65,16 +65,11 @@ import com.swirlds.common.threading.locks.Locks; import com.swirlds.config.api.Configuration; import com.swirlds.config.api.ConfigurationBuilder; -import com.swirlds.config.api.source.ConfigSource; import com.swirlds.config.extensions.sources.SystemEnvironmentConfigSource; import com.swirlds.config.extensions.sources.SystemPropertiesConfigSource; import edu.umd.cs.findbugs.annotations.NonNull; -import edu.umd.cs.findbugs.annotations.Nullable; import java.io.IOException; -import java.util.NoSuchElementException; -import java.util.Set; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import javax.inject.Inject; import javax.inject.Singleton; import org.apache.logging.log4j.LogManager; @@ -115,31 +110,6 @@ public ConfigProviderImpl(final boolean useGenesisSource) { configuration = new AtomicReference<>(new VersionedConfigImpl(config, 0)); } - /** - * Creates a new config provider based on an existing configuration. Note that this constructor assumes - * this is NOT a genesis case. - * @param existing the existing configuration to embed in the provider - */ - public ConfigProviderImpl(@NonNull final Configuration existing) { - final var builder = createConfigurationBuilder(); - addFileSources(builder, false); - builder.withSources(new ConfigSource() { - @NonNull - @Override - public Set getPropertyNames() { - return existing.getPropertyNames().collect(Collectors.toSet()); - } - - @Nullable - @Override - public String getValue(@NonNull String propertyName) throws NoSuchElementException { - return existing.getValue(propertyName); - } - }); - final Configuration config = builder.build(); - configuration = new AtomicReference<>(new VersionedConfigImpl(config, 0)); - } - @Override @NonNull public VersionedConfiguration getConfiguration() { diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/config/ConfigProviderImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/config/ConfigProviderImplTest.java index 765c9f0932e3..1a94b2693e4c 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/config/ConfigProviderImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/config/ConfigProviderImplTest.java @@ -24,7 +24,6 @@ import com.hedera.hapi.node.base.Setting; import com.hedera.node.config.VersionedConfiguration; import com.hedera.pbj.runtime.io.buffer.Bytes; -import com.swirlds.config.api.ConfigurationBuilder; import java.nio.charset.StandardCharsets; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -146,19 +145,6 @@ void testGenesisPropertiesFileIsOptional(final EnvironmentVariables environment) assertThat(bar).isEqualTo("456"); } - @Test - void providerGetsCorrectValueFromExistingConfig() { - // given - final var subject = new ConfigProviderImpl( - ConfigurationBuilder.create().withValue("key", "value").build()); - - // when - final var result = subject.getConfiguration(); - - // then - assertThat(result.getValue("key")).isEqualTo("value"); - } - @Test void testUpdateDoesUseApplicationProperties() { // given