Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Implement synthetic records for immediate genesis reconnect scenario #10176

Merged
merged 3 commits into from
Dec 7, 2023
Merged

feat: Implement synthetic records for immediate genesis reconnect scenario #10176

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
371455b
Implement synthetic records for event recovery scenarios
mhess-swl Nov 29, 2023
4fb7847
Revert new config provider constructor
mhess-swl Nov 30, 2023
5f8a227
Merge branch 'develop' into 09757_modular-synthetic-records
mhess-swl Dec 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions ...app-spi/src/main/java/com/hedera/node/app/spi/workflows/record/GenesisRecordsBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@
package com.hedera.node.app.spi.workflows.record;

import com.hedera.hapi.node.state.token.Account;
import com.hedera.hapi.node.token.CryptoCreateTransactionBody;
import edu.umd.cs.findbugs.annotations.NonNull;
import java.util.Map;
import java.util.SortedSet;

/**
* A class that stores entities created during node startup, for the purpose of creating synthetic
Expand All @@ -29,25 +28,25 @@ public interface GenesisRecordsBuilder {
/**
* Tracks the system accounts created during node startup
*/
void systemAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
void systemAccounts(@NonNull final SortedSet<Account> accounts);

/**
* Tracks the staking accounts created during node startup
*/
void stakingAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
void stakingAccounts(@NonNull final SortedSet<Account> accounts);

/**
* Tracks miscellaneous accounts created during node startup. These accounts are typically used for testing
*/
void miscAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
void miscAccounts(@NonNull final SortedSet<Account> accounts);

/**
* Tracks the treasury clones created during node startup
*/
void treasuryClones(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
void treasuryClones(@NonNull final SortedSet<Account> accounts);

/**
* Tracks the blocklist accounts created during node startup
*/
void blocklistAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
void blocklistAccounts(@NonNull final SortedSet<Account> accounts);
}
13 changes: 6 additions & 7 deletions ...c/testFixtures/java/com/hedera/node/app/spi/fixtures/state/NoOpGenesisRecordsBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,34 +33,33 @@
*/

import com.hedera.hapi.node.state.token.Account;
import com.hedera.hapi.node.token.CryptoCreateTransactionBody;
import com.hedera.node.app.spi.workflows.record.GenesisRecordsBuilder;
import edu.umd.cs.findbugs.annotations.NonNull;
import java.util.Map;
import java.util.SortedSet;

public class NoOpGenesisRecordsBuilder implements GenesisRecordsBuilder {
@Override
public void systemAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
public void systemAccounts(@NonNull final SortedSet<Account> accounts) {
// Intentional no-op
}

@Override
public void stakingAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
public void stakingAccounts(@NonNull final SortedSet<Account> accounts) {
// Intentional no-op
}

@Override
public void miscAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
public void miscAccounts(@NonNull final SortedSet<Account> accounts) {
// Intentional no-op
}

@Override
public void treasuryClones(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
public void treasuryClones(@NonNull final SortedSet<Account> accounts) {
// Intentional no-op
}

@Override
public void blocklistAccounts(@NonNull Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
public void blocklistAccounts(@NonNull SortedSet<Account> accounts) {
// Intentional no-op
}
}
50 changes: 38 additions & 12 deletions hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import com.hedera.node.app.service.networkadmin.impl.NetworkServiceImpl;
import com.hedera.node.app.service.schedule.impl.ScheduleServiceImpl;
import com.hedera.node.app.service.token.impl.TokenServiceImpl;
import com.hedera.node.app.service.token.impl.schemas.SyntheticRecordsGenerator;
import com.hedera.node.app.service.util.impl.UtilServiceImpl;
import com.hedera.node.app.services.ServicesRegistryImpl;
import com.hedera.node.app.spi.HapiUtils;
Expand Down Expand Up @@ -187,6 +188,7 @@ public final class Hedera implements SwirldMain {
private ThrottleAccumulator backendThrottle;
private ThrottleAccumulator frontendThrottle;
private CongestionMultipliers congestionMultipliers;
private final SyntheticRecordsGenerator recordsGenerator;

/**
* The application name from the platform's perspective. This is currently locked in at the old main class name and
Expand Down Expand Up @@ -242,7 +244,9 @@ public Hedera(@NonNull final ConstructableRegistry constructableRegistry) {
() -> HapiUtils.toString(version.getServicesVersion()),
() -> HapiUtils.toString(version.getHapiVersion()));

// Create a record builder for any genesis records that need to be created
// Create a records generator for any synthetic records that need to be CREATED
this.recordsGenerator = new SyntheticRecordsGenerator();
// Create a records builder for any genesis records that need to be RECORDED
this.genesisRecordsBuilder = new GenesisRecordsConsensusHook();
tinker-michaelj marked this conversation as resolved.
Show resolved Hide resolved

// Create all the service implementations
Expand All @@ -258,7 +262,12 @@ public Hedera(@NonNull final ConstructableRegistry constructableRegistry) {
new FreezeServiceImpl(),
new NetworkServiceImpl(),
new ScheduleServiceImpl(),
new TokenServiceImpl(),
new TokenServiceImpl(
recordsGenerator::sysAcctRecords,
recordsGenerator::stakingAcctRecords,
recordsGenerator::treasuryAcctRecords,
recordsGenerator::multiUseAcctRecords,
recordsGenerator::blocklistAcctRecords),
new UtilServiceImpl(),
new RecordCacheService(),
new BlockRecordService(),
Expand Down Expand Up @@ -355,8 +364,34 @@ private void onStateInitialized(
@NonNull final SwirldDualState dualState,
@NonNull final InitTrigger trigger,
@Nullable final SoftwareVersion previousVersion) {
// Initialize the configuration from disk. We must do this BEFORE we run migration, because the various
// migration methods may depend on configuration to do their work. For example, the token service migration code
// needs to know the token treasury account, which has an account ID specified in config. The initial config
// file in state, created by the file service migration, will match what we have here, so we don't have to worry
// about re-loading config after migration.
logger.info("Initializing configuration with trigger {}", trigger);
configProvider = new ConfigProviderImpl(trigger == GENESIS);
logConfiguration();

// We do nothing for EVENT_STREAM_RECOVERY. This is a special case that is handled by the platform.
// Determine if we need to create synthetic records for system entities
final var blockRecordState = state.createReadableStates(BlockRecordService.NAME);
boolean createSynthRecords = false;
if (!blockRecordState.isEmpty()) {
final var blockInfo = blockRecordState
.<BlockInfo>getSingleton(BlockRecordService.BLOCK_INFO_STATE_KEY)
.get();
if (blockInfo == null || blockInfo.consTimeOfLastHandledTxn() == null) {
createSynthRecords = true;
}
} else {
createSynthRecords = true;
}
if (createSynthRecords) {
recordsGenerator.createRecords(configProvider.getConfiguration(), genesisRecordsBuilder);
}

// We do nothing else for EVENT_STREAM_RECOVERY, which for now is broken. This is a special case that is handled
// by the platform, and we need to figure out how to make it work with the modular app.
if (trigger == EVENT_STREAM_RECOVERY) {
logger.debug("Skipping state initialization for trigger {}", trigger);
return;
Expand Down Expand Up @@ -718,15 +753,6 @@ public void shutdownGrpcServer() {
private void genesis(@NonNull final MerkleHederaState state) {
logger.debug("Genesis Initialization");

// Initialize the configuration from disk (genesis case). We must do this BEFORE we run migration, because
// the various migration methods may depend on configuration to do their work. For example, the token service
// migration code needs to know the token treasury account, which has an account ID specified in config.
// The initial config file in state, created by the file service migration, will match what we have here,
// so we don't have to worry about re-loading config after migration.
logger.info("Initializing genesis configuration");
this.configProvider = new ConfigProviderImpl(true);
logConfiguration();

logger.info("Initializing ThrottleManager");
this.throttleManager = new ThrottleManager();

Expand Down
81 changes: 49 additions & 32 deletions ...rc/main/java/com/hedera/node/app/workflows/handle/record/GenesisRecordsConsensusHook.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package com.hedera.node.app.workflows.handle.record;

import static com.hedera.node.app.spi.HapiUtils.ACCOUNT_ID_COMPARATOR;
import static com.hedera.node.app.spi.HapiUtils.FUNDING_ACCOUNT_EXPIRY;
import static java.util.Objects.requireNonNull;

Expand All @@ -33,8 +34,8 @@
import edu.umd.cs.findbugs.annotations.Nullable;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;
import javax.inject.Singleton;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -49,12 +50,14 @@ public class GenesisRecordsConsensusHook implements GenesisRecordsBuilder, Conse
private static final String SYSTEM_ACCOUNT_CREATION_MEMO = "Synthetic system creation";
private static final String STAKING_MEMO = "Release 0.24.1 migration record";
private static final String TREASURY_CLONE_MEMO = "Synthetic zero-balance treasury clone";
private static final Comparator<Account> ACCOUNT_COMPARATOR =
Comparator.comparing(Account::accountId, ACCOUNT_ID_COMPARATOR);

private Map<Account, CryptoCreateTransactionBody.Builder> systemAccounts = new HashMap<>();
private Map<Account, CryptoCreateTransactionBody.Builder> stakingAccounts = new HashMap<>();
private Map<Account, CryptoCreateTransactionBody.Builder> miscAccounts = new HashMap<>();
private Map<Account, CryptoCreateTransactionBody.Builder> treasuryClones = new HashMap<>();
private Map<Account, CryptoCreateTransactionBody.Builder> blocklistAccounts = new HashMap<>();
private SortedSet<Account> systemAccounts = new TreeSet<>(ACCOUNT_COMPARATOR);
private SortedSet<Account> stakingAccounts = new TreeSet<>(ACCOUNT_COMPARATOR);
private SortedSet<Account> miscAccounts = new TreeSet<>(ACCOUNT_COMPARATOR);
private SortedSet<Account> treasuryClones = new TreeSet<>(ACCOUNT_COMPARATOR);
private SortedSet<Account> blocklistAccounts = new TreeSet<>(ACCOUNT_COMPARATOR);

/**
* <b> ⚠️⚠️ Note: though this method will be called each time a new platform event is received,
Expand All @@ -75,91 +78,105 @@ public void process(@NonNull final TokenContext context) {

if (!systemAccounts.isEmpty()) {
createAccountRecordBuilders(systemAccounts, context, SYSTEM_ACCOUNT_CREATION_MEMO);
systemAccounts = Collections.emptyMap();
systemAccounts = Collections.emptySortedSet();
}
log.info("Queued {} system account records with consTime {}", systemAccounts.size(), consensusTime);

if (!stakingAccounts.isEmpty()) {
final var implicitAutoRenewPeriod = FUNDING_ACCOUNT_EXPIRY - consensusTime.getEpochSecond();
createAccountRecordBuilders(stakingAccounts, context, STAKING_MEMO, implicitAutoRenewPeriod);
stakingAccounts = Collections.emptyMap();
stakingAccounts = Collections.emptySortedSet();
}
log.info("Queued {} staking account records with consTime {}", stakingAccounts.size(), consensusTime);

if (!miscAccounts.isEmpty()) {
createAccountRecordBuilders(miscAccounts, context, null);
miscAccounts = Collections.emptyMap();
miscAccounts = Collections.emptySortedSet();
}
log.info("Queued {} misc account records with consTime {}", miscAccounts.size(), consensusTime);

if (!treasuryClones.isEmpty()) {
createAccountRecordBuilders(treasuryClones, context, TREASURY_CLONE_MEMO);
treasuryClones = Collections.emptyMap();
treasuryClones = Collections.emptySortedSet();
}
log.info("Queued {} treasury clone account records with consTime {}", treasuryClones.size(), consensusTime);

if (!blocklistAccounts.isEmpty()) {
createAccountRecordBuilders(blocklistAccounts, context, null);
blocklistAccounts = Collections.emptyMap();
blocklistAccounts = Collections.emptySortedSet();
}
log.info("Queued {} blocklist account records with consTime {}", blocklistAccounts.size(), consensusTime);
}

@Override
public void systemAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
systemAccounts.putAll(requireNonNull(accounts));
public void systemAccounts(@NonNull final SortedSet<Account> accounts) {
systemAccounts.addAll(requireNonNull(accounts));
}

@Override
public void stakingAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
stakingAccounts.putAll(requireNonNull(accounts));
public void stakingAccounts(@NonNull final SortedSet<Account> accounts) {
stakingAccounts.addAll(requireNonNull(accounts));
}

@Override
public void miscAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
miscAccounts.putAll(requireNonNull(accounts));
public void miscAccounts(@NonNull final SortedSet<Account> accounts) {
miscAccounts.addAll(requireNonNull(accounts));
}

@Override
public void treasuryClones(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
treasuryClones.putAll(requireNonNull(accounts));
public void treasuryClones(@NonNull final SortedSet<Account> accounts) {
treasuryClones.addAll(requireNonNull(accounts));
}

@Override
public void blocklistAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
blocklistAccounts.putAll(requireNonNull(accounts));
public void blocklistAccounts(@NonNull final SortedSet<Account> accounts) {
blocklistAccounts.addAll(requireNonNull(accounts));
}

private void createAccountRecordBuilders(
@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> map,
@NonNull final SortedSet<Account> map,
@NonNull final TokenContext context,
@Nullable final String recordMemo) {
createAccountRecordBuilders(map, context, recordMemo, null);
}

private void createAccountRecordBuilders(
@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> map,
@NonNull final SortedSet<Account> accts,
@NonNull final TokenContext context,
@Nullable final String recordMemo,
@Nullable final Long overrideAutoRenewPeriod) {
final var orderedAccts = map.keySet().stream()
.sorted(Comparator.comparingLong(acct -> acct.accountId().accountNum()))
.toList();
for (final Account key : orderedAccts) {
for (final Account account : accts) {
// we create preceding records on genesis for each system account created.
// This is an exception and should not fail with MAX_CHILD_RECORDS_EXCEEDED
final var recordBuilder =
context.addUncheckedPrecedingChildRecordBuilder(GenesisAccountRecordBuilder.class);
final var accountId = requireNonNull(key.accountId());
recordBuilder.accountID(accountId);
recordBuilder.accountID(account.accountId());
if (recordMemo != null) {
recordBuilder.memo(recordMemo);
}

var txnBody = map.get(key);
var txnBody = newCryptoCreate(account);
if (overrideAutoRenewPeriod != null) {
txnBody.autoRenewPeriod(Duration.newBuilder().seconds(overrideAutoRenewPeriod));
}
var txnBuilder =
Transaction.newBuilder().body(TransactionBody.newBuilder().cryptoCreateAccount(txnBody));
recordBuilder.transaction(txnBuilder.build());

log.debug("Queued synthetic CryptoCreate for {} account {}", recordMemo, accountId);
log.debug("Queued synthetic CryptoCreate for {} account {}", recordMemo, account);
}
}

private static CryptoCreateTransactionBody.Builder newCryptoCreate(@NonNull final Account account) {
return CryptoCreateTransactionBody.newBuilder()
.key(account.key())
.memo(account.memo())
.declineReward(account.declineReward())
.receiverSigRequired(account.receiverSigRequired())
.autoRenewPeriod(Duration.newBuilder()
.seconds(account.autoRenewSeconds())
.build())
.initialBalance(account.tinybarBalance())
.alias(account.alias());
}
}
Loading
Loading