From e9db09068b8d57c7fe7a90b6b682b81879988ef7 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Mon, 4 Dec 2023 15:35:13 +0100 Subject: [PATCH 01/72] Add storage limit check exception for EVM address --- fvm/transactionInvoker.go | 1 + fvm/transactionStorageLimiter.go | 23 ++++++++- fvm/transactionStorageLimiter_test.go | 71 ++++++++++++++++++--------- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/fvm/transactionInvoker.go b/fvm/transactionInvoker.go index acd4088c4bd..57c0c449cbf 100644 --- a/fvm/transactionInvoker.go +++ b/fvm/transactionInvoker.go @@ -406,6 +406,7 @@ func (executor *transactionExecutor) normalExecution() ( // actual balance, for the purpose of calculating storage capacity, because the payer will have to pay for this tx. executor.txnState.RunWithAllLimitsDisabled(func() { err = executor.CheckStorageLimits( + executor.ctx, executor.env, bodySnapshot, executor.proc.Transaction.Payer, diff --git a/fvm/transactionStorageLimiter.go b/fvm/transactionStorageLimiter.go index 9d504adf7bf..a5ed69395b8 100644 --- a/fvm/transactionStorageLimiter.go +++ b/fvm/transactionStorageLimiter.go @@ -11,6 +11,7 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" ) @@ -33,6 +34,7 @@ type TransactionStorageLimiter struct{} // The payers balance is considered to be maxTxFees lower that its actual balance, due to the fact that // the fee deduction step happens after the storage limit check. func (limiter TransactionStorageLimiter) CheckStorageLimits( + ctx Context, env environment.Environment, snapshot *snapshot.ExecutionSnapshot, payer flow.Address, @@ -44,7 +46,7 @@ func (limiter TransactionStorageLimiter) CheckStorageLimits( defer env.StartChildSpan(trace.FVMTransactionStorageUsedCheck).End() - err := limiter.checkStorageLimits(env, snapshot, payer, maxTxFees) + err := limiter.checkStorageLimits(ctx, env, snapshot, payer, maxTxFees) if err != nil { return fmt.Errorf("storage limit check failed: %w", err) } @@ -55,6 +57,7 @@ func (limiter TransactionStorageLimiter) CheckStorageLimits( // storage limit is exceeded. The returned list include addresses of updated // registers (and the payer's address). func (limiter TransactionStorageLimiter) getStorageCheckAddresses( + ctx Context, snapshot *snapshot.ExecutionSnapshot, payer flow.Address, maxTxFees uint64, @@ -77,6 +80,10 @@ func (limiter TransactionStorageLimiter) getStorageCheckAddresses( continue } + if limiter.shouldSkipSpecialAddress(ctx, address) { + continue + } + _, ok = dedup[address] if ok { continue @@ -99,12 +106,13 @@ func (limiter TransactionStorageLimiter) getStorageCheckAddresses( // checkStorageLimits checks if the transaction changed the storage of any // address and exceeded the storage limit. func (limiter TransactionStorageLimiter) checkStorageLimits( + ctx Context, env environment.Environment, snapshot *snapshot.ExecutionSnapshot, payer flow.Address, maxTxFees uint64, ) error { - addresses := limiter.getStorageCheckAddresses(snapshot, payer, maxTxFees) + addresses := limiter.getStorageCheckAddresses(ctx, snapshot, payer, maxTxFees) usages := make([]uint64, len(addresses)) @@ -155,3 +163,14 @@ func (limiter TransactionStorageLimiter) checkStorageLimits( return nil } + +// shouldSkipSpecialAddress returns true if the address is a special address where storage +// limits are not enforced. +// This is currently only the EVM storage address. This is a temporary solution. +func (limiter TransactionStorageLimiter) shouldSkipSpecialAddress( + ctx Context, + address flow.Address, +) bool { + sc := systemcontracts.SystemContractsForChain(ctx.Chain.ChainID()) + return sc.EVM.Address == address +} diff --git a/fvm/transactionStorageLimiter_test.go b/fvm/transactionStorageLimiter_test.go index b9b2a87ec3a..2b62066b7e9 100644 --- a/fvm/transactionStorageLimiter_test.go +++ b/fvm/transactionStorageLimiter_test.go @@ -1,6 +1,8 @@ package fvm_test import ( + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/fvm/systemcontracts" "testing" "github.com/onflow/cadence" @@ -24,10 +26,14 @@ func TestTransactionStorageLimiter(t *testing.T) { }, } + ctx := fvm.Context{ + EnvironmentParams: environment.EnvironmentParams{ + Chain: flow.Emulator.Chain(), + }, + } + t.Run("capacity > storage -> OK", func(t *testing.T) { - chain := flow.Mainnet.Chain() env := &fvmmock.Environment{} - env.On("Chain").Return(chain) env.On("LimitAccountStorage").Return(true) env.On("StartChildSpan", mock.Anything).Return( tracing.NewMockTracerSpan()) @@ -40,13 +46,11 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, executionSnapshot, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(ctx, env, executionSnapshot, flow.EmptyAddress, 0) require.NoError(t, err, "Transaction with higher capacity than storage used should work") }) t.Run("capacity = storage -> OK", func(t *testing.T) { - chain := flow.Mainnet.Chain() env := &fvmmock.Environment{} - env.On("Chain").Return(chain) env.On("LimitAccountStorage").Return(true) env.On("StartChildSpan", mock.Anything).Return( tracing.NewMockTracerSpan()) @@ -59,13 +63,11 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, executionSnapshot, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(ctx, env, executionSnapshot, flow.EmptyAddress, 0) require.NoError(t, err, "Transaction with equal capacity than storage used should work") }) t.Run("capacity = storage -> OK (dedup payer)", func(t *testing.T) { - chain := flow.Mainnet.Chain() env := &fvmmock.Environment{} - env.On("Chain").Return(chain) env.On("LimitAccountStorage").Return(true) env.On("StartChildSpan", mock.Anything).Return( tracing.NewMockTracerSpan()) @@ -78,13 +80,11 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, executionSnapshot, owner, 0) + err := d.CheckStorageLimits(ctx, env, executionSnapshot, owner, 0) require.NoError(t, err, "Transaction with equal capacity than storage used should work") }) t.Run("capacity < storage -> Not OK", func(t *testing.T) { - chain := flow.Mainnet.Chain() env := &fvmmock.Environment{} - env.On("Chain").Return(chain) env.On("LimitAccountStorage").Return(true) env.On("StartChildSpan", mock.Anything).Return( tracing.NewMockTracerSpan()) @@ -97,13 +97,11 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, executionSnapshot, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(ctx, env, executionSnapshot, flow.EmptyAddress, 0) require.Error(t, err, "Transaction with lower capacity than storage used should fail") }) t.Run("capacity > storage -> OK (payer not updated)", func(t *testing.T) { - chain := flow.Mainnet.Chain() env := &fvmmock.Environment{} - env.On("Chain").Return(chain) env.On("LimitAccountStorage").Return(true) env.On("StartChildSpan", mock.Anything).Return( tracing.NewMockTracerSpan()) @@ -118,13 +116,11 @@ func TestTransactionStorageLimiter(t *testing.T) { executionSnapshot = &snapshot.ExecutionSnapshot{} d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, executionSnapshot, owner, 1) + err := d.CheckStorageLimits(ctx, env, executionSnapshot, owner, 1) require.NoError(t, err, "Transaction with higher capacity than storage used should work") }) t.Run("capacity < storage -> Not OK (payer not updated)", func(t *testing.T) { - chain := flow.Mainnet.Chain() env := &fvmmock.Environment{} - env.On("Chain").Return(chain) env.On("LimitAccountStorage").Return(true) env.On("StartChildSpan", mock.Anything).Return( tracing.NewMockTracerSpan()) @@ -139,13 +135,11 @@ func TestTransactionStorageLimiter(t *testing.T) { executionSnapshot = &snapshot.ExecutionSnapshot{} d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, executionSnapshot, owner, 1000) + err := d.CheckStorageLimits(ctx, env, executionSnapshot, owner, 1000) require.Error(t, err, "Transaction with lower capacity than storage used should fail") }) t.Run("if ctx LimitAccountStorage false-> OK", func(t *testing.T) { - chain := flow.Mainnet.Chain() env := &fvmmock.Environment{} - env.On("Chain").Return(chain) env.On("LimitAccountStorage").Return(false) env.On("StartChildSpan", mock.Anything).Return( tracing.NewMockTracerSpan()) @@ -159,13 +153,11 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, executionSnapshot, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(ctx, env, executionSnapshot, flow.EmptyAddress, 0) require.NoError(t, err, "Transaction with higher capacity than storage used should work") }) t.Run("non existing accounts or any other errors on fetching storage used -> Not OK", func(t *testing.T) { - chain := flow.Mainnet.Chain() env := &fvmmock.Environment{} - env.On("Chain").Return(chain) env.On("LimitAccountStorage").Return(true) env.On("StartChildSpan", mock.Anything).Return( tracing.NewMockTracerSpan()) @@ -178,9 +170,40 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, executionSnapshot, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(ctx, env, executionSnapshot, flow.EmptyAddress, 0) require.Error(t, err, "check storage used on non existing account (not general registers) should fail") }) + t.Run("special account is skipped", func(t *testing.T) { + + sc := systemcontracts.SystemContractsForChain(ctx.Chain.ChainID()) + evm := sc.EVM.Address + + executionSnapshot := &snapshot.ExecutionSnapshot{ + WriteSet: map[flow.RegisterID]flow.RegisterValue{ + flow.NewRegisterID(string(evm.Bytes()), "a"): flow.RegisterValue("foo"), + }, + } + + env := &fvmmock.Environment{} + env.On("LimitAccountStorage").Return(true) + env.On("StartChildSpan", mock.Anything).Return( + tracing.NewMockTracerSpan()) + env.On("GetStorageUsed", mock.Anything).Return(uint64(0), errors.NewAccountNotFoundError(owner)) + env.On("AccountsStorageCapacity", mock.Anything, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + require.Len(t, args.Get(0).([]flow.Address), 0) + }). + Return( + // since the special account is skipped, the resulting array from AccountsStorageCapacity should be empty + cadence.NewArray([]cadence.Value{}), + nil, + ) + + d := &fvm.TransactionStorageLimiter{} + err := d.CheckStorageLimits(ctx, env, executionSnapshot, flow.EmptyAddress, 0) + require.NoError(t, err) + }) + } func bytesToUFix64(b uint64) cadence.Value { From a88c912ef22d0943e3e0ca07d10c032b717b5637 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 5 Dec 2023 14:57:30 +0100 Subject: [PATCH 02/72] lint fix --- fvm/transactionStorageLimiter_test.go | 49 +++++++++++++++------------ 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/fvm/transactionStorageLimiter_test.go b/fvm/transactionStorageLimiter_test.go index 2b62066b7e9..d15fbf52187 100644 --- a/fvm/transactionStorageLimiter_test.go +++ b/fvm/transactionStorageLimiter_test.go @@ -1,8 +1,6 @@ package fvm_test import ( - "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/systemcontracts" "testing" "github.com/onflow/cadence" @@ -10,9 +8,11 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/fvm" + "github.com/onflow/flow-go/fvm/environment" fvmmock "github.com/onflow/flow-go/fvm/environment/mock" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" ) @@ -156,25 +156,32 @@ func TestTransactionStorageLimiter(t *testing.T) { err := d.CheckStorageLimits(ctx, env, executionSnapshot, flow.EmptyAddress, 0) require.NoError(t, err, "Transaction with higher capacity than storage used should work") }) - t.Run("non existing accounts or any other errors on fetching storage used -> Not OK", func(t *testing.T) { - env := &fvmmock.Environment{} - env.On("LimitAccountStorage").Return(true) - env.On("StartChildSpan", mock.Anything).Return( - tracing.NewMockTracerSpan()) - env.On("GetStorageUsed", mock.Anything).Return(uint64(0), errors.NewAccountNotFoundError(owner)) - env.On("AccountsStorageCapacity", mock.Anything, mock.Anything, mock.Anything).Return( - cadence.NewArray([]cadence.Value{ - bytesToUFix64(100), - }), - nil, - ) + t.Run( + "non existing accounts or any other errors on fetching storage used -> Not OK", + func(t *testing.T) { + env := &fvmmock.Environment{} + env.On("LimitAccountStorage").Return(true) + env.On("StartChildSpan", mock.Anything).Return( + tracing.NewMockTracerSpan()) + env.On("GetStorageUsed", mock.Anything). + Return(uint64(0), errors.NewAccountNotFoundError(owner)) + env.On("AccountsStorageCapacity", mock.Anything, mock.Anything, mock.Anything).Return( + cadence.NewArray([]cadence.Value{ + bytesToUFix64(100), + }), + nil, + ) - d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(ctx, env, executionSnapshot, flow.EmptyAddress, 0) - require.Error(t, err, "check storage used on non existing account (not general registers) should fail") - }) + d := &fvm.TransactionStorageLimiter{} + err := d.CheckStorageLimits(ctx, env, executionSnapshot, flow.EmptyAddress, 0) + require.Error( + t, + err, + "check storage used on non existing account (not general registers) should fail", + ) + }, + ) t.Run("special account is skipped", func(t *testing.T) { - sc := systemcontracts.SystemContractsForChain(ctx.Chain.ChainID()) evm := sc.EVM.Address @@ -188,7 +195,8 @@ func TestTransactionStorageLimiter(t *testing.T) { env.On("LimitAccountStorage").Return(true) env.On("StartChildSpan", mock.Anything).Return( tracing.NewMockTracerSpan()) - env.On("GetStorageUsed", mock.Anything).Return(uint64(0), errors.NewAccountNotFoundError(owner)) + env.On("GetStorageUsed", mock.Anything). + Return(uint64(0), errors.NewAccountNotFoundError(owner)) env.On("AccountsStorageCapacity", mock.Anything, mock.Anything, mock.Anything). Run(func(args mock.Arguments) { require.Len(t, args.Get(0).([]flow.Address), 0) @@ -203,7 +211,6 @@ func TestTransactionStorageLimiter(t *testing.T) { err := d.CheckStorageLimits(ctx, env, executionSnapshot, flow.EmptyAddress, 0) require.NoError(t, err) }) - } func bytesToUFix64(b uint64) cadence.Value { From 4a149e9b9e1e15394da0a37532cf2e1b344f9213 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Mon, 11 Dec 2023 18:11:18 +0100 Subject: [PATCH 03/72] Add migration infrastructure --- .../migrations/account_based_migration.go | 446 ++++++++++++------ .../ledger/migrations/account_migration.go | 100 ---- .../migrations/storage_fees_migration.go | 63 --- .../migrations/storage_used_migration.go | 147 ++++++ cmd/util/ledger/migrations/utils.go | 61 --- .../reporters/fungible_token_tracker.go | 4 +- .../util/migration_runtime_interface.go | 295 ++++++++++++ cmd/util/ledger/util/nop_meter.go | 49 ++ cmd/util/ledger/util/payload_grouping.go | 252 ++++++++++ cmd/util/ledger/util/payload_grouping_test.go | 135 ++++++ cmd/util/ledger/util/util.go | 145 ++++++ ledger/trie.go | 10 + 12 files changed, 1346 insertions(+), 361 deletions(-) delete mode 100644 cmd/util/ledger/migrations/account_migration.go delete mode 100644 cmd/util/ledger/migrations/storage_fees_migration.go create mode 100644 cmd/util/ledger/migrations/storage_used_migration.go delete mode 100644 cmd/util/ledger/migrations/utils.go create mode 100644 cmd/util/ledger/util/migration_runtime_interface.go create mode 100644 cmd/util/ledger/util/nop_meter.go create mode 100644 cmd/util/ledger/util/payload_grouping.go create mode 100644 cmd/util/ledger/util/payload_grouping_test.go create mode 100644 cmd/util/ledger/util/util.go diff --git a/cmd/util/ledger/migrations/account_based_migration.go b/cmd/util/ledger/migrations/account_based_migration.go index b7ca65a662c..d408b279a75 100644 --- a/cmd/util/ledger/migrations/account_based_migration.go +++ b/cmd/util/ledger/migrations/account_based_migration.go @@ -1,190 +1,366 @@ package migrations import ( + "container/heap" + "context" "fmt" + "io" + "sync" + "time" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/cmd/util/ledger/util" "github.com/onflow/flow-go/ledger" - "github.com/onflow/flow-go/ledger/common/convert" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/util" + moduleUtil "github.com/onflow/flow-go/module/util" ) -// PayloadToAccount takes a payload and return: -// - (address, true, nil) if the payload is for an account, the account address is returned -// - ("", false, nil) if the payload is not for an account -// - ("", false, err) if running into any exception -func PayloadToAccount(p ledger.Payload) (string, bool, error) { - k, err := p.Key() - if err != nil { - return "", false, fmt.Errorf("could not find key for payload: %w", err) - } - id, err := convert.LedgerKeyToRegisterID(k) - if err != nil { - return "", false, fmt.Errorf("error converting key to register ID") - } - if len([]byte(id.Owner)) != flow.AddressLength { - return "", false, nil - } - return id.Owner, true, nil +// logTopNDurations is the number of longest migrations to log at the end of the migration +const logTopNDurations = 20 + +// AccountBasedMigration is an interface for migrations that migrate account by account +// concurrently getting all the payloads for each account at a time. +type AccountBasedMigration interface { + InitMigration( + log zerolog.Logger, + allPayloads []*ledger.Payload, + nWorkers int, + ) error + MigrateAccount( + ctx context.Context, + address common.Address, + payloads []*ledger.Payload, + ) ([]*ledger.Payload, error) } -// PayloadGroup groups payloads by account. -// For global payloads, it's stored under NonAccountPayloads field -type PayloadGroup struct { - NonAccountPayloads []ledger.Payload - Accounts map[string][]ledger.Payload +// CreateAccountBasedMigration creates a migration function that migrates the payloads +// account by account using the given migrations +// accounts are processed concurrently using the given number of workers +// but each account is processed sequentially by the given migrations in order. +// The migrations InitMigration function is called once before the migration starts +// And the Close function is called once after the migration finishes if the migration +// is a finisher. +func CreateAccountBasedMigration( + log zerolog.Logger, + nWorker int, + migrations []AccountBasedMigration, +) func(payloads []*ledger.Payload) ([]*ledger.Payload, error) { + return func(payloads []*ledger.Payload) ([]*ledger.Payload, error) { + return MigrateByAccount( + log, + nWorker, + payloads, + migrations, + ) + } } -// PayloadGrouping is a reducer function that adds the given payload to the corresponding -// group under its account -func PayloadGrouping(groups *PayloadGroup, payload ledger.Payload) (*PayloadGroup, error) { - address, isAccount, err := PayloadToAccount(payload) - if err != nil { - return nil, err +// MigrateByAccount takes migrations and all the Payloads, +// and returns the migrated Payloads. +func MigrateByAccount( + log zerolog.Logger, + nWorker int, + allPayloads []*ledger.Payload, + migrations []AccountBasedMigration, +) ( + []*ledger.Payload, + error, +) { + if len(allPayloads) == 0 { + return allPayloads, nil } - if isAccount { - groups.Accounts[address] = append(groups.Accounts[address], payload) - } else { - groups.NonAccountPayloads = append(groups.NonAccountPayloads, payload) + for i, migrator := range migrations { + if err := migrator.InitMigration( + log.With(). + Int("migration_index", i). + Logger(), + allPayloads, + nWorker, + ); err != nil { + return nil, fmt.Errorf("could not init migration: %w", err) + } } - return groups, nil -} + log.Info(). + Int("inner_migrations", len(migrations)). + Int("nWorker", nWorker). + Msgf("created account migrations") + + defer func() { + for i, migrator := range migrations { + // close the migrator if it's a Closer + if migrator, ok := migrator.(io.Closer); ok { + log.Info(). + Int("migration_index", i). + Type("migration", migrator). + Msg("closing migration") + if err := migrator.Close(); err != nil { + log.Error().Err(err).Msg("error closing migration") + } + } + } + }() -// AccountMigrator takes all the payloads that belong to the given account -// and return the migrated payloads -type AccountMigrator interface { - MigratePayloads(account string, payloads []ledger.Payload) ([]ledger.Payload, error) -} + // group the Payloads by account + accountGroups := util.GroupPayloadsByAccount(log, allPayloads, nWorker) + + // migrate the Payloads under accounts + migrated, err := MigrateGroupConcurrently(log, migrations, accountGroups, nWorker) -// MigrateByAccount teaks a migrator function and all the payloads, and return the migrated payloads -func MigrateByAccount(migrator AccountMigrator, allPayloads []ledger.Payload, nWorker int) ( - []ledger.Payload, error) { - groups := &PayloadGroup{ - NonAccountPayloads: make([]ledger.Payload, 0), - Accounts: make(map[string][]ledger.Payload), + if err != nil { + return nil, fmt.Errorf("could not migrate accounts: %w", err) } - log.Info().Msgf("start grouping for a total of %v payloads", len(allPayloads)) + log.Info(). + Int("account_count", accountGroups.Len()). + Int("payload_count", len(allPayloads)). + Msgf("finished migrating Payloads") - var err error - logGrouping := util.LogProgress("grouping payload", len(allPayloads), log.Logger) - for i, payload := range allPayloads { - groups, err = PayloadGrouping(groups, payload) - if err != nil { - return nil, err - } - logGrouping(i) - } + return migrated, nil +} - log.Info().Msgf("finish grouping for payloads by account: %v groups in total, %v NonAccountPayloads", - len(groups.Accounts), len(groups.NonAccountPayloads)) +// MigrateGroupConcurrently migrate the Payloads in the given account groups. +// It uses nWorker to process the Payloads concurrently. The Payloads in each account +// are processed sequentially by the given migrations in order. +func MigrateGroupConcurrently( + log zerolog.Logger, + migrations []AccountBasedMigration, + accountGroups *util.PayloadAccountGrouping, + nWorker int, +) ([]*ledger.Payload, error) { - // migrate the payloads under accounts - migrated, err := MigrateGroupConcurrently(migrator, groups.Accounts, nWorker) + ctx := context.Background() + ctx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) - if err != nil { - return nil, fmt.Errorf("could not migrate group: %w", err) + jobs := make(chan jobMigrateAccountGroup, accountGroups.Len()) + + wg := sync.WaitGroup{} + wg.Add(nWorker) + resultCh := make(chan *migrationResult, accountGroups.Len()) + for i := 0; i < nWorker; i++ { + go func() { + defer wg.Done() + + for { + select { + case <-ctx.Done(): + return + case job, ok := <-jobs: + if !ok { + return + } + start := time.Now() + + // This is not an account, but common values for all accounts. + if job.Address == common.ZeroAddress { + resultCh <- &migrationResult{ + migrationDuration: migrationDuration{ + Address: job.Address, + Duration: time.Since(start), + }, + Migrated: job.Payloads, + } + continue + } + + if _, ok := knownProblematicAccounts[job.Address]; ok { + log.Info(). + Hex("address", job.Address[:]). + Msg("skipping problematic account") + resultCh <- &migrationResult{ + migrationDuration: migrationDuration{ + Address: job.Address, + Duration: time.Since(start), + }, + Migrated: job.Payloads, + } + continue + } + + var err error + accountMigrated := job.Payloads + for m, migrator := range migrations { + + select { + case <-ctx.Done(): + return + default: + } + + accountMigrated, err = migrator.MigrateAccount(ctx, job.Address, accountMigrated) + if err != nil { + log.Error(). + Err(err). + Int("migration_index", m). + Type("migration", migrator). + Hex("address", job.Address[:]). + Msg("could not migrate account") + cancel(fmt.Errorf("could not migrate account: %w", err)) + return + } + } + + resultCh <- &migrationResult{ + migrationDuration: migrationDuration{ + Address: job.Address, + Duration: time.Since(start), + }, + Migrated: accountMigrated, + } + } + } + }() } - log.Info().Msgf("finished migrating payloads for %v account", len(groups.Accounts)) + go func() { + for { + g, err := accountGroups.Next() + if err != nil { + cancel(fmt.Errorf("could not get next account group: %w", err)) + return + } - // add the non accounts which don't need to be migrated - migrated = append(migrated, groups.NonAccountPayloads...) + if g == nil { + break + } - log.Info().Msgf("finished migrating all account based payloads, total migrated payloads: %v", len(migrated)) + job := jobMigrateAccountGroup{ + Address: g.Address, + Payloads: g.Payloads, + } - return migrated, nil -} + select { + case <-ctx.Done(): + return + case jobs <- job: + } + } + }() -// MigrateGroupSequentially migrate the payloads in the given payloadsByAccount map which -// using the migrator -func MigrateGroupSequentially( - migrator AccountMigrator, - payloadsByAccount map[string][]ledger.Payload, -) ( - []ledger.Payload, error) { + // read job results + logAccount := moduleUtil.LogProgress("processing account group", accountGroups.Len(), log) + + migrated := make([]*ledger.Payload, 0) + durations := newMigrationDurations(logTopNDurations) + for i := 0; i < accountGroups.Len(); i++ { + select { + case <-ctx.Done(): + break + case result := <-resultCh: + durations.Add(result) + + accountMigrated := result.Migrated + migrated = append(migrated, accountMigrated...) + logAccount(1) + } + } + close(jobs) - logAccount := util.LogProgress("processing account group", len(payloadsByAccount), log.Logger) + // make sure to exit all workers before returning from this function + // so that the migrator can be closed properly + log.Info().Msg("waiting for migration workers to finish") + wg.Wait() - i := 0 - migrated := make([]ledger.Payload, 0) - for address, payloads := range payloadsByAccount { - accountMigrated, err := migrator.MigratePayloads(address, payloads) - if err != nil { - return nil, fmt.Errorf("could not migrate for account address %v: %w", address, err) - } + log.Info(). + Array("top_longest_migrations", durations.Array()). + Msgf("Top longest migrations") - migrated = append(migrated, accountMigrated...) - logAccount(i) - i++ + if ctx.Err() != nil { + return nil, fmt.Errorf("fail to migrate payload: %w", ctx.Err()) } return migrated, nil } +var knownProblematicAccounts = map[common.Address]string{ + // Testnet accounts with broken contracts + mustHexToAddress("434a1f199a7ae3ba"): "Broken contract FanTopPermission", + mustHexToAddress("454c9991c2b8d947"): "Broken contract Test", + mustHexToAddress("48602d8056ff9d93"): "Broken contract FanTopPermission", + mustHexToAddress("5d63c34d7f05e5a4"): "Broken contract FanTopPermission", + mustHexToAddress("5e3448b3cffb97f2"): "Broken contract FanTopPermission", + mustHexToAddress("7d8c7e050c694eaa"): "Broken contract Test", + mustHexToAddress("ba53f16ede01972d"): "Broken contract FanTopPermission", + mustHexToAddress("c843c1f5a4805c3a"): "Broken contract FanTopPermission", + mustHexToAddress("48d3be92e6e4a973"): "Broken contract FanTopPermission", + // Mainnet account +} + type jobMigrateAccountGroup struct { - Account string - Payloads []ledger.Payload + Address common.Address + Payloads []*ledger.Payload } type migrationResult struct { - Migrated []ledger.Payload - Err error + migrationDuration + + Migrated []*ledger.Payload } -// MigrateGroupConcurrently migrate the payloads in the given payloadsByAccount map which -// using the migrator -// It's similar to MigrateGroupSequentially, except it will migrate different groups concurrently -func MigrateGroupConcurrently( - migrator AccountMigrator, - payloadsByAccount map[string][]ledger.Payload, - nWorker int, -) ( - []ledger.Payload, error) { +type migrationDuration struct { + Address common.Address + Duration time.Duration +} - jobs := make(chan jobMigrateAccountGroup, len(payloadsByAccount)) - go func() { - for account, payloads := range payloadsByAccount { - jobs <- jobMigrateAccountGroup{ - Account: account, - Payloads: payloads, - } - } - close(jobs) - }() +// migrationDurations implements heap methods for the timer results +type migrationDurations struct { + v []migrationDuration - resultCh := make(chan *migrationResult) - for i := 0; i < int(nWorker); i++ { - go func() { - for job := range jobs { - accountMigrated, err := migrator.MigratePayloads(job.Account, job.Payloads) - resultCh <- &migrationResult{ - Migrated: accountMigrated, - Err: err, - } - } - }() + KeepTopN int +} + +// newMigrationDurations creates a new migrationDurations which are used to track the +// accounts that took the longest time to migrate. +func newMigrationDurations(keepTopN int) *migrationDurations { + return &migrationDurations{ + v: make([]migrationDuration, 0, keepTopN), + KeepTopN: keepTopN, } +} - // read job results - logAccount := util.LogProgress("processing account group", len(payloadsByAccount), log.Logger) +func (h *migrationDurations) Len() int { return len(h.v) } +func (h *migrationDurations) Less(i, j int) bool { + return h.v[i].Duration < h.v[j].Duration +} +func (h *migrationDurations) Swap(i, j int) { + h.v[i], h.v[j] = h.v[j], h.v[i] +} +func (h *migrationDurations) Push(x interface{}) { + h.v = append(h.v, x.(migrationDuration)) +} +func (h *migrationDurations) Pop() interface{} { + old := h.v + n := len(old) + x := old[n-1] + h.v = old[0 : n-1] + return x +} - migrated := make([]ledger.Payload, 0) +func (h *migrationDurations) Array() zerolog.LogArrayMarshaler { + array := zerolog.Arr() + for _, result := range h.v { + array = array.Str(fmt.Sprintf("%s: %s", result.Address.Hex(), result.Duration.String())) + } + return array +} - for i := 0; i < len(payloadsByAccount); i++ { - result := <-resultCh - if result.Err != nil { - return nil, fmt.Errorf("fail to migrate payload: %w", result.Err) +func (h *migrationDurations) Add(result *migrationResult) { + if h.Len() < h.KeepTopN || result.Duration > h.v[0].Duration { + if h.Len() == h.KeepTopN { + heap.Pop(h) // remove the element with the smallest duration } - - accountMigrated := result.Migrated - migrated = append(migrated, accountMigrated...) - logAccount(i) + heap.Push(h, result.migrationDuration) } +} - return migrated, nil +func mustHexToAddress(hex string) common.Address { + address, err := common.HexToAddress(hex) + if err != nil { + panic(err) + } + return address } diff --git a/cmd/util/ledger/migrations/account_migration.go b/cmd/util/ledger/migrations/account_migration.go deleted file mode 100644 index e0bd7c029dd..00000000000 --- a/cmd/util/ledger/migrations/account_migration.go +++ /dev/null @@ -1,100 +0,0 @@ -package migrations - -import ( - "fmt" - - "github.com/rs/zerolog/log" - - "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/ledger" - "github.com/onflow/flow-go/ledger/common/convert" - "github.com/onflow/flow-go/model/flow" -) - -func MigrateAccountUsage(payloads []ledger.Payload, nWorker int) ([]ledger.Payload, error) { - return MigrateByAccount(AccountUsageMigrator{}, payloads, nWorker) -} - -func payloadSize(key ledger.Key, payload ledger.Payload) (uint64, error) { - id, err := convert.LedgerKeyToRegisterID(key) - if err != nil { - return 0, err - } - - return uint64(registerSize(id, payload)), nil -} - -func isAccountKey(key ledger.Key) bool { - return string(key.KeyParts[1].Value) == flow.AccountStatusKey -} - -type AccountUsageMigrator struct{} - -// AccountUsageMigrator iterate through each payload, and calculate the storage usage -// and update the accoutns status with the updated storage usage -func (m AccountUsageMigrator) MigratePayloads(account string, payloads []ledger.Payload) ([]ledger.Payload, error) { - var status *environment.AccountStatus - var statusIndex int - totalSize := uint64(0) - for i, payload := range payloads { - key, err := payload.Key() - if err != nil { - return nil, err - } - if isAccountKey(key) { - statusIndex = i - status, err = environment.AccountStatusFromBytes(payload.Value()) - if err != nil { - return nil, fmt.Errorf("could not parse account status: %w", err) - } - - } - - size, err := payloadSize(key, payload) - if err != nil { - return nil, err - } - totalSize += size - } - - err := compareUsage(status, totalSize) - if err != nil { - log.Error().Msgf("%v", err) - } - - if status == nil { - return nil, fmt.Errorf("could not find account status for account %v", account) - } - - // update storage used - status.SetStorageUsed(totalSize) - - newValue := status.ToBytes() - newPayload, err := newPayloadWithValue(payloads[statusIndex], newValue) - if err != nil { - return nil, fmt.Errorf("cannot create new payload with value: %w", err) - } - - payloads[statusIndex] = newPayload - - return payloads, nil -} - -func compareUsage(status *environment.AccountStatus, totalSize uint64) error { - oldSize := status.StorageUsed() - if oldSize != totalSize { - return fmt.Errorf("old size: %v, new size: %v", oldSize, totalSize) - } - return nil -} - -// newPayloadWithValue returns a new payload with the key from the given payload, and -// the value from the argument -func newPayloadWithValue(payload ledger.Payload, value ledger.Value) (ledger.Payload, error) { - key, err := payload.Key() - if err != nil { - return ledger.Payload{}, err - } - newPayload := ledger.NewPayload(key, payload.Value()) - return *newPayload, nil -} diff --git a/cmd/util/ledger/migrations/storage_fees_migration.go b/cmd/util/ledger/migrations/storage_fees_migration.go deleted file mode 100644 index 6babd9ed279..00000000000 --- a/cmd/util/ledger/migrations/storage_fees_migration.go +++ /dev/null @@ -1,63 +0,0 @@ -package migrations - -import ( - fvm "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/ledger" - "github.com/onflow/flow-go/ledger/common/convert" - "github.com/onflow/flow-go/ledger/common/utils" - "github.com/onflow/flow-go/model/flow" -) - -// iterates through registers keeping a map of register sizes -// after it has reached the end it add storage used and storage capacity for each address -func StorageFeesMigration(payload []ledger.Payload) ([]ledger.Payload, error) { - storageUsed := make(map[string]uint64) - newPayload := make([]ledger.Payload, len(payload)) - - for i, p := range payload { - err := incrementStorageUsed(p, storageUsed) - if err != nil { - return nil, err - } - newPayload[i] = p - } - - for s, u := range storageUsed { - // this is the storage used by the storage_used register we are about to add - id := flow.NewRegisterID( - string(flow.BytesToAddress([]byte(s)).Bytes()), - "storage_used") - storageUsedByStorageUsed := fvm.RegisterSize(id, make([]byte, 8)) - u = u + uint64(storageUsedByStorageUsed) - - newPayload = append(newPayload, *ledger.NewPayload( - convert.RegisterIDToLedgerKey(id), - utils.Uint64ToBinary(u), - )) - } - return newPayload, nil -} - -func incrementStorageUsed(p ledger.Payload, used map[string]uint64) error { - k, err := p.Key() - if err != nil { - return err - } - id, err := convert.LedgerKeyToRegisterID(k) - if err != nil { - return err - } - if len([]byte(id.Owner)) != flow.AddressLength { - // not an address - return nil - } - if _, ok := used[id.Owner]; !ok { - used[id.Owner] = 0 - } - used[id.Owner] = used[id.Owner] + uint64(registerSize(id, p)) - return nil -} - -func registerSize(id flow.RegisterID, p ledger.Payload) int { - return fvm.RegisterSize(id, p.Value()) -} diff --git a/cmd/util/ledger/migrations/storage_used_migration.go b/cmd/util/ledger/migrations/storage_used_migration.go new file mode 100644 index 00000000000..4b6ddd5140c --- /dev/null +++ b/cmd/util/ledger/migrations/storage_used_migration.go @@ -0,0 +1,147 @@ +package migrations + +import ( + "context" + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/fvm/environment" + fvm "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" +) + +// AccountUsageMigrator iterates through each payload, and calculate the storage usage +// and update the accounts status with the updated storage usage +type AccountUsageMigrator struct { + log zerolog.Logger +} + +var _ AccountBasedMigration = &AccountUsageMigrator{} + +func (m *AccountUsageMigrator) InitMigration( + log zerolog.Logger, + _ []*ledger.Payload, + _ int, +) error { + m.log = log.With().Str("component", "AccountUsageMigrator").Logger() + return nil +} + +const oldAccountStatusSize = 25 + +func (m *AccountUsageMigrator) MigrateAccount( + _ context.Context, + address common.Address, + payloads []*ledger.Payload, +) ([]*ledger.Payload, error) { + + var status *environment.AccountStatus + var statusIndex int + actualSize := uint64(0) + for i, payload := range payloads { + key, err := payload.Key() + if err != nil { + return nil, err + } + if isAccountKey(key) { + statusIndex = i + status, err = environment.AccountStatusFromBytes(payload.Value()) + if err != nil { + return nil, fmt.Errorf("could not parse account status: %w", err) + } + + } + + size, err := payloadSize(key, payload) + if err != nil { + return nil, err + } + actualSize += size + } + + if status == nil { + return nil, fmt.Errorf("could not find account status for account %v", address.Hex()) + } + + isOldVersionOfStatusRegister := len(payloads[statusIndex].Value()) == oldAccountStatusSize + + same := m.compareUsage(isOldVersionOfStatusRegister, status, actualSize) + if same { + // there is no problem with the usage, return + return payloads, nil + } + + if isOldVersionOfStatusRegister { + // size will grow by 8 bytes because of the on-the-fly + // migration of account status in AccountStatusFromBytes + actualSize += 8 + } + + // update storage used + status.SetStorageUsed(actualSize) + + newValue := status.ToBytes() + newPayload, err := newPayloadWithValue(payloads[statusIndex], newValue) + if err != nil { + return nil, fmt.Errorf("cannot create new payload with value: %w", err) + } + + payloads[statusIndex] = &newPayload + + return payloads, nil +} + +func (m *AccountUsageMigrator) compareUsage( + isOldVersionOfStatusRegister bool, + status *environment.AccountStatus, + actualSize uint64, +) bool { + oldSize := status.StorageUsed() + if isOldVersionOfStatusRegister { + // size will be reported as 8 bytes larger than the actual size due to on-the-fly + // migration of account status in AccountStatusFromBytes + oldSize -= 8 + } + + if oldSize != actualSize { + m.log.Info(). + Uint64("old_size", oldSize). + Uint64("new_size", actualSize). + Msg("account storage used usage mismatch") + return false + } + return true +} + +// newPayloadWithValue returns a new payload with the key from the given payload, and +// the value from the argument +func newPayloadWithValue(payload *ledger.Payload, value ledger.Value) (ledger.Payload, error) { + key, err := payload.Key() + if err != nil { + return ledger.Payload{}, err + } + newPayload := ledger.NewPayload(key, value) + return *newPayload, nil +} + +func registerSize(id flow.RegisterID, p *ledger.Payload) int { + return fvm.RegisterSize(id, p.Value()) +} + +func payloadSize(key ledger.Key, payload *ledger.Payload) (uint64, error) { + id, err := convert.LedgerKeyToRegisterID(key) + if err != nil { + return 0, err + } + + return uint64(registerSize(id, payload)), nil +} + +func isAccountKey(key ledger.Key) bool { + return string(key.KeyParts[1].Value) == flow.AccountStatusKey +} diff --git a/cmd/util/ledger/migrations/utils.go b/cmd/util/ledger/migrations/utils.go deleted file mode 100644 index 11510008f74..00000000000 --- a/cmd/util/ledger/migrations/utils.go +++ /dev/null @@ -1,61 +0,0 @@ -package migrations - -import ( - "fmt" - - "github.com/onflow/atree" - - "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/model/flow" -) - -type AccountsAtreeLedger struct { - Accounts environment.Accounts -} - -func NewAccountsAtreeLedger(accounts environment.Accounts) *AccountsAtreeLedger { - return &AccountsAtreeLedger{Accounts: accounts} -} - -var _ atree.Ledger = &AccountsAtreeLedger{} - -func (a *AccountsAtreeLedger) GetValue(owner, key []byte) ([]byte, error) { - v, err := a.Accounts.GetValue( - flow.NewRegisterID( - string(flow.BytesToAddress(owner).Bytes()), - string(key))) - if err != nil { - return nil, fmt.Errorf("getting value failed: %w", err) - } - return v, nil -} - -func (a *AccountsAtreeLedger) SetValue(owner, key, value []byte) error { - err := a.Accounts.SetValue( - flow.NewRegisterID( - string(flow.BytesToAddress(owner).Bytes()), - string(key)), - value) - if err != nil { - return fmt.Errorf("setting value failed: %w", err) - } - return nil -} - -func (a *AccountsAtreeLedger) ValueExists(owner, key []byte) (exists bool, err error) { - v, err := a.GetValue(owner, key) - if err != nil { - return false, fmt.Errorf("checking value existence failed: %w", err) - } - - return len(v) > 0, nil -} - -// AllocateStorageIndex allocates new storage index under the owner accounts to store a new register -func (a *AccountsAtreeLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { - v, err := a.Accounts.AllocateStorageIndex(flow.BytesToAddress(owner)) - if err != nil { - return atree.StorageIndex{}, fmt.Errorf("storage address allocation failed: %w", err) - } - return v, nil -} diff --git a/cmd/util/ledger/reporters/fungible_token_tracker.go b/cmd/util/ledger/reporters/fungible_token_tracker.go index 24a2c09ac56..a84d8282b7a 100644 --- a/cmd/util/ledger/reporters/fungible_token_tracker.go +++ b/cmd/util/ledger/reporters/fungible_token_tracker.go @@ -13,7 +13,7 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" - "github.com/onflow/flow-go/cmd/util/ledger/migrations" + "github.com/onflow/flow-go/cmd/util/ledger/util" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/systemcontracts" @@ -147,7 +147,7 @@ func (r *FungibleTokenTracker) worker( state.DefaultParameters()) accounts := environment.NewAccounts(txnState) storage := cadenceRuntime.NewStorage( - &migrations.AccountsAtreeLedger{Accounts: accounts}, + &util.AccountsAtreeLedger{Accounts: accounts}, nil, ) diff --git a/cmd/util/ledger/util/migration_runtime_interface.go b/cmd/util/ledger/util/migration_runtime_interface.go new file mode 100644 index 00000000000..4726e3ef06c --- /dev/null +++ b/cmd/util/ledger/util/migration_runtime_interface.go @@ -0,0 +1,295 @@ +package util + +import ( + "fmt" + "time" + + "go.opentelemetry.io/otel/attribute" + + "github.com/onflow/atree" + "github.com/onflow/cadence" + "github.com/onflow/cadence/runtime" + "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/runtime/interpreter" + + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/model/flow" +) + +// MigrationRuntimeInterface is a runtime interface that can be used in migrations. +type MigrationRuntimeInterface struct { + Accounts environment.Accounts + Programs *environment.Programs + + // GetOrLoadProgramFunc allows for injecting extra logic + GetOrLoadProgramFunc func(location runtime.Location, load func() (*interpreter.Program, error)) (*interpreter.Program, error) +} + +func (m MigrationRuntimeInterface) ResolveLocation( + identifiers []runtime.Identifier, + location runtime.Location, +) ([]runtime.ResolvedLocation, error) { + + addressLocation, isAddress := location.(common.AddressLocation) + + // if the location is not an address location, e.g. an identifier location (`import Crypto`), + // then return a single resolved location which declares all identifiers. + if !isAddress { + return []runtime.ResolvedLocation{ + { + Location: location, + Identifiers: identifiers, + }, + }, nil + } + + // if the location is an address, + // and no specific identifiers where requested in the import statement, + // then fetch all identifiers at this address + if len(identifiers) == 0 { + address := flow.Address(addressLocation.Address) + + contractNames, err := m.Accounts.GetContractNames(address) + if err != nil { + return nil, fmt.Errorf("ResolveLocation failed: %w", err) + } + + // if there are no contractNames deployed, + // then return no resolved locations + if len(contractNames) == 0 { + return nil, nil + } + + identifiers = make([]runtime.Identifier, len(contractNames)) + + for i := range identifiers { + identifiers[i] = runtime.Identifier{ + Identifier: contractNames[i], + } + } + } + + // return one resolved location per identifier. + // each resolved location is an address contract location + resolvedLocations := make([]runtime.ResolvedLocation, len(identifiers)) + for i := range resolvedLocations { + identifier := identifiers[i] + resolvedLocations[i] = runtime.ResolvedLocation{ + Location: common.AddressLocation{ + Address: addressLocation.Address, + Name: identifier.Identifier, + }, + Identifiers: []runtime.Identifier{identifier}, + } + } + + return resolvedLocations, nil +} + +func (m MigrationRuntimeInterface) GetCode(location runtime.Location) ([]byte, error) { + contractLocation, ok := location.(common.AddressLocation) + if !ok { + return nil, fmt.Errorf("GetCode failed: expected AddressLocation") + } + + add, err := m.Accounts.GetContract(contractLocation.Name, flow.Address(contractLocation.Address)) + if err != nil { + return nil, fmt.Errorf("GetCode failed: %w", err) + } + + return add, nil +} + +func (m MigrationRuntimeInterface) GetAccountContractCode( + l common.AddressLocation, +) (code []byte, err error) { + return m.Accounts.GetContract(l.Name, flow.Address(l.Address)) +} + +func (m MigrationRuntimeInterface) GetOrLoadProgram(location runtime.Location, load func() (*interpreter.Program, error)) (*interpreter.Program, error) { + if m.GetOrLoadProgramFunc != nil { + return m.GetOrLoadProgramFunc(location, load) + } + + return m.Programs.GetOrLoadProgram(location, load) +} + +func (m MigrationRuntimeInterface) MeterMemory(_ common.MemoryUsage) error { + return nil +} + +func (m MigrationRuntimeInterface) MeterComputation(_ common.ComputationKind, _ uint) error { + return nil +} + +func (m MigrationRuntimeInterface) GetValue(_, _ []byte) (value []byte, err error) { + panic("unexpected GetValue call") +} + +func (m MigrationRuntimeInterface) SetValue(_, _, _ []byte) (err error) { + panic("unexpected SetValue call") +} + +func (m MigrationRuntimeInterface) CreateAccount(_ runtime.Address) (address runtime.Address, err error) { + panic("unexpected CreateAccount call") +} + +func (m MigrationRuntimeInterface) AddEncodedAccountKey(_ runtime.Address, _ []byte) error { + panic("unexpected AddEncodedAccountKey call") +} + +func (m MigrationRuntimeInterface) RevokeEncodedAccountKey(_ runtime.Address, _ int) (publicKey []byte, err error) { + panic("unexpected RevokeEncodedAccountKey call") +} + +func (m MigrationRuntimeInterface) AddAccountKey(_ runtime.Address, _ *runtime.PublicKey, _ runtime.HashAlgorithm, _ int) (*runtime.AccountKey, error) { + panic("unexpected AddAccountKey call") +} + +func (m MigrationRuntimeInterface) GetAccountKey(_ runtime.Address, _ int) (*runtime.AccountKey, error) { + panic("unexpected GetAccountKey call") +} + +func (m MigrationRuntimeInterface) RevokeAccountKey(_ runtime.Address, _ int) (*runtime.AccountKey, error) { + panic("unexpected RevokeAccountKey call") +} + +func (m MigrationRuntimeInterface) UpdateAccountContractCode(_ common.AddressLocation, _ []byte) (err error) { + panic("unexpected UpdateAccountContractCode call") +} + +func (m MigrationRuntimeInterface) RemoveAccountContractCode(common.AddressLocation) (err error) { + panic("unexpected RemoveAccountContractCode call") +} + +func (m MigrationRuntimeInterface) GetSigningAccounts() ([]runtime.Address, error) { + panic("unexpected GetSigningAccounts call") +} + +func (m MigrationRuntimeInterface) ProgramLog(_ string) error { + panic("unexpected ProgramLog call") +} + +func (m MigrationRuntimeInterface) EmitEvent(_ cadence.Event) error { + panic("unexpected EmitEvent call") +} + +func (m MigrationRuntimeInterface) ValueExists(_, _ []byte) (exists bool, err error) { + panic("unexpected ValueExists call") +} + +func (m MigrationRuntimeInterface) GenerateUUID() (uint64, error) { + panic("unexpected GenerateUUID call") +} + +func (m MigrationRuntimeInterface) GetComputationLimit() uint64 { + panic("unexpected GetComputationLimit call") +} + +func (m MigrationRuntimeInterface) SetComputationUsed(_ uint64) error { + panic("unexpected SetComputationUsed call") +} + +func (m MigrationRuntimeInterface) DecodeArgument(_ []byte, _ cadence.Type) (cadence.Value, error) { + panic("unexpected DecodeArgument call") +} + +func (m MigrationRuntimeInterface) GetCurrentBlockHeight() (uint64, error) { + panic("unexpected GetCurrentBlockHeight call") +} + +func (m MigrationRuntimeInterface) GetBlockAtHeight(_ uint64) (block runtime.Block, exists bool, err error) { + panic("unexpected GetBlockAtHeight call") +} + +func (m MigrationRuntimeInterface) ReadRandom([]byte) error { + panic("unexpected UnsafeRandom call") +} + +func (m MigrationRuntimeInterface) VerifySignature(_ []byte, _ string, _ []byte, _ []byte, _ runtime.SignatureAlgorithm, _ runtime.HashAlgorithm) (bool, error) { + panic("unexpected VerifySignature call") +} + +func (m MigrationRuntimeInterface) Hash(_ []byte, _ string, _ runtime.HashAlgorithm) ([]byte, error) { + panic("unexpected Hash call") +} + +func (m MigrationRuntimeInterface) GetAccountBalance(_ common.Address) (value uint64, err error) { + panic("unexpected GetAccountBalance call") +} + +func (m MigrationRuntimeInterface) GetAccountAvailableBalance(_ common.Address) (value uint64, err error) { + panic("unexpected GetAccountAvailableBalance call") +} + +func (m MigrationRuntimeInterface) GetStorageUsed(_ runtime.Address) (value uint64, err error) { + panic("unexpected GetStorageUsed call") +} + +func (m MigrationRuntimeInterface) GetStorageCapacity(_ runtime.Address) (value uint64, err error) { + panic("unexpected GetStorageCapacity call") +} + +func (m MigrationRuntimeInterface) ImplementationDebugLog(_ string) error { + panic("unexpected ImplementationDebugLog call") +} + +func (m MigrationRuntimeInterface) ValidatePublicKey(_ *runtime.PublicKey) error { + panic("unexpected ValidatePublicKey call") +} + +func (m MigrationRuntimeInterface) GetAccountContractNames(_ runtime.Address) ([]string, error) { + panic("unexpected GetAccountContractNames call") +} + +func (m MigrationRuntimeInterface) AllocateStorageIndex(_ []byte) (atree.StorageIndex, error) { + panic("unexpected AllocateStorageIndex call") +} + +func (m MigrationRuntimeInterface) ComputationUsed() (uint64, error) { + panic("unexpected ComputationUsed call") +} + +func (m MigrationRuntimeInterface) MemoryUsed() (uint64, error) { + panic("unexpected MemoryUsed call") +} + +func (m MigrationRuntimeInterface) InteractionUsed() (uint64, error) { + panic("unexpected InteractionUsed call") +} + +func (m MigrationRuntimeInterface) SetInterpreterSharedState(_ *interpreter.SharedState) { + panic("unexpected SetInterpreterSharedState call") +} + +func (m MigrationRuntimeInterface) GetInterpreterSharedState() *interpreter.SharedState { + panic("unexpected GetInterpreterSharedState call") +} + +func (m MigrationRuntimeInterface) AccountKeysCount(_ runtime.Address) (uint64, error) { + panic("unexpected AccountKeysCount call") +} + +func (m MigrationRuntimeInterface) BLSVerifyPOP(_ *runtime.PublicKey, _ []byte) (bool, error) { + panic("unexpected BLSVerifyPOP call") +} + +func (m MigrationRuntimeInterface) BLSAggregateSignatures(_ [][]byte) ([]byte, error) { + panic("unexpected BLSAggregateSignatures call") +} + +func (m MigrationRuntimeInterface) BLSAggregatePublicKeys(_ []*runtime.PublicKey) (*runtime.PublicKey, error) { + panic("unexpected BLSAggregatePublicKeys call") +} + +func (m MigrationRuntimeInterface) ResourceOwnerChanged(_ *interpreter.Interpreter, _ *interpreter.CompositeValue, _ common.Address, _ common.Address) { + panic("unexpected ResourceOwnerChanged call") +} + +func (m MigrationRuntimeInterface) GenerateAccountID(_ common.Address) (uint64, error) { + panic("unexpected GenerateAccountID call") +} + +func (m MigrationRuntimeInterface) RecordTrace(_ string, _ runtime.Location, _ time.Duration, _ []attribute.KeyValue) { + panic("unexpected RecordTrace call") +} diff --git a/cmd/util/ledger/util/nop_meter.go b/cmd/util/ledger/util/nop_meter.go new file mode 100644 index 00000000000..dc8a9c2ac6b --- /dev/null +++ b/cmd/util/ledger/util/nop_meter.go @@ -0,0 +1,49 @@ +package util + +import ( + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/fvm/meter" +) + +// NopMeter is a meter that does nothing. It can be used in migrations. +type NopMeter struct{} + +func (n NopMeter) ComputationAvailable(_ common.ComputationKind, _ uint) bool { + return false +} + +func (n NopMeter) MeterComputation(_ common.ComputationKind, _ uint) error { + return nil +} + +func (n NopMeter) ComputationUsed() (uint64, error) { + return 0, nil +} + +func (n NopMeter) ComputationIntensities() meter.MeteredComputationIntensities { + return meter.MeteredComputationIntensities{} +} + +func (n NopMeter) MeterMemory(_ common.MemoryUsage) error { + return nil +} + +func (n NopMeter) MemoryUsed() (uint64, error) { + return 0, nil +} + +func (n NopMeter) MeterEmittedEvent(_ uint64) error { + return nil +} + +func (n NopMeter) TotalEmittedEventBytes() uint64 { + return 0 +} + +func (n NopMeter) InteractionUsed() (uint64, error) { + return 0, nil +} + +var _ environment.Meter = NopMeter{} diff --git a/cmd/util/ledger/util/payload_grouping.go b/cmd/util/ledger/util/payload_grouping.go new file mode 100644 index 00000000000..185863d6ade --- /dev/null +++ b/cmd/util/ledger/util/payload_grouping.go @@ -0,0 +1,252 @@ +package util + +import ( + "bytes" + "fmt" + "sort" + "sync" + "time" + + "github.com/rs/zerolog" + + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" +) + +// encodedKeyAddressPrefixLength is the length of the address prefix in the encoded key +// 2 for uint16 of number of key parts +// 4 for uint32 of the length of the first key part +// 2 for uint32 of the key part type +// 8 for the address which is the actual length of the first key part +const encodedKeyAddressPrefixLength = 2 + 4 + 2 + flow.AddressLength + +// minSizeForSplitSortingIntoGoroutines below this size, no need to split +// the sorting into goroutines +const minSizeForSplitSortingIntoGoroutines = 100_000 + +// PayloadAccountGroup is a grouping of payloads by account +type PayloadAccountGroup struct { + Address common.Address + Payloads []*ledger.Payload +} + +// PayloadAccountGrouping is a grouping of payloads by account. +type PayloadAccountGrouping struct { + payloads sortablePayloads + indexes []int + + current int +} + +// Next returns the next account group. If there is no more account group, it returns nil. +// The zero address is used for global Payloads and is not an actual account. +func (g *PayloadAccountGrouping) Next() (*PayloadAccountGroup, error) { + if g.current == len(g.indexes) { + // reached the end + return nil, nil + } + + accountStartIndex := g.indexes[g.current] + accountEndIndex := len(g.payloads) + if g.current != len(g.indexes)-1 { + accountEndIndex = g.indexes[g.current+1] + } + g.current++ + + address, err := payloadToAddress(g.payloads[accountStartIndex]) + if err != nil { + return nil, fmt.Errorf("failed to get address from payload: %w", err) + } + + return &PayloadAccountGroup{ + Address: address, + Payloads: g.payloads[accountStartIndex:accountEndIndex], + }, nil +} + +// Len returns the number of accounts +func (g *PayloadAccountGrouping) Len() int { + return len(g.indexes) +} + +// GroupPayloadsByAccount takes a list of payloads and groups them by account. +// it uses nWorkers to sort the payloads by address and find the start and end indexes of +// each account. +func GroupPayloadsByAccount( + log zerolog.Logger, + payloads []*ledger.Payload, + nWorkers int, +) *PayloadAccountGrouping { + if len(payloads) == 0 { + return &PayloadAccountGrouping{} + } + p := sortablePayloads(payloads) + + start := time.Now() + log.Info(). + Int("payloads", len(payloads)). + Int("workers", nWorkers). + Msg("Sorting payloads by address") + + // sort the payloads by address + sortPayloads(0, len(p), p, make(sortablePayloads, len(p)), nWorkers) + end := time.Now() + + log.Info(). + Int("payloads", len(payloads)). + Str("duration", end.Sub(start).Round(1*time.Second).String()). + Msg("Sorted. Finding account boundaries in sorted payloads") + + start = time.Now() + // find the indexes of the payloads that start a new account + indexes := make([]int, 0, len(p)) + for i := 0; i < len(p); { + indexes = append(indexes, i) + i = p.FindLastOfTheSameKey(i) + 1 + } + end = time.Now() + + log.Info(). + Int("accounts", len(indexes)). + Str("duration", end.Sub(start).Round(1*time.Second).String()). + Msg("Done grouping payloads by account") + + return &PayloadAccountGrouping{ + payloads: p, + indexes: indexes, + } +} + +// payloadToAddress takes a payload and return: +// - (address, true, nil) if the payload is for an account, the account address is returned +// - ("", false, nil) if the payload is not for an account +// - ("", false, err) if running into any exception +// The zero address is used for global Payloads and is not an actual account +func payloadToAddress(p *ledger.Payload) (common.Address, error) { + k, err := p.Key() + if err != nil { + return common.ZeroAddress, fmt.Errorf("could not find key for payload: %w", err) + } + + id, err := convert.LedgerKeyToRegisterID(k) + if err != nil { + return common.ZeroAddress, fmt.Errorf("error converting key to register ID") + } + + if len([]byte(id.Owner)) != flow.AddressLength { + return common.ZeroAddress, nil + } + + address, err := common.BytesToAddress([]byte(id.Owner)) + if err != nil { + return common.ZeroAddress, fmt.Errorf("invalid account address: %w", err) + } + + return address, nil +} + +type sortablePayloads []*ledger.Payload + +func (s sortablePayloads) Len() int { + return len(s) +} + +func (s sortablePayloads) Less(i, j int) bool { + return s.Compare(i, j) < 0 +} + +func (s sortablePayloads) Compare(i, j int) int { + // sort descending to force one of the big accounts to be more at the beginning + return bytes.Compare( + s[j].EncodedKey()[:encodedKeyAddressPrefixLength], + s[i].EncodedKey()[:encodedKeyAddressPrefixLength], + ) +} + +func (s sortablePayloads) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s sortablePayloads) FindLastOfTheSameKey(i int) int { + low := i + step := 1 + for low+step < len(s) && s.Compare(low+step, i) == 0 { + low += step + step *= 2 + } + + high := low + step + if high > len(s) { + high = len(s) + } + + for low < high { + mid := (low + high) / 2 + if s.Compare(mid, i) == 0 { + low = mid + 1 + } else { + high = mid + } + } + + return low - 1 +} + +// sortPayloads sorts the payloads in the range [i, j) using goroutines and merges +// the results using the intermediate buffer. The goroutine allowance is the number +// of goroutines that can be used for sorting. If the allowance is less than 2, +// the payloads are sorted using the built-in sort. +// The buffer must be of the same length as the source and can be disposed after. +func sortPayloads(i, j int, source, buffer sortablePayloads, goroutineAllowance int) { + // if the length is less than 2, no need to sort + if j-i <= 1 { + return + } + + // if we are out of goroutine allowance, sort with built-in sortß + // if the length is less than minSizeForSplit, sort with built-in sort + if goroutineAllowance < 2 || j-i < minSizeForSplitSortingIntoGoroutines { + sort.Sort(source[i:j]) + return + } + + goroutineAllowance -= 2 + allowance1 := goroutineAllowance / 2 + allowance2 := goroutineAllowance - allowance1 + mid := (i + j) / 2 + + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + sortPayloads(i, mid, source, buffer, allowance1) + wg.Done() + }() + go func() { + sortPayloads(mid, j, source, buffer, allowance2) + wg.Done() + }() + wg.Wait() + + mergeInto(source, buffer, i, mid, j) +} + +func mergeInto(source, buffer sortablePayloads, i int, mid int, j int) { + left := i + right := mid + for k := i; k < j; k++ { + if left < mid && (right >= j || source.Compare(left, right) <= 0) { + buffer[k] = source[left] + left++ + } else { + buffer[k] = source[right] + right++ + } + } + + for k := i; k < j; k++ { + source[k] = buffer[k] + } +} diff --git a/cmd/util/ledger/util/payload_grouping_test.go b/cmd/util/ledger/util/payload_grouping_test.go new file mode 100644 index 00000000000..e2efe0654ef --- /dev/null +++ b/cmd/util/ledger/util/payload_grouping_test.go @@ -0,0 +1,135 @@ +package util_test + +import ( + "crypto/rand" + "encoding/hex" + rand2 "math/rand" + "runtime" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" +) + +func TestGroupPayloadsByAccount(t *testing.T) { + log := zerolog.New(zerolog.NewTestWriter(t)) + payloads := generateRandomPayloads(1000000) + tmp := make([]*ledger.Payload, len(payloads)) + copy(tmp, payloads) + + groups := util.GroupPayloadsByAccount(log, payloads, 0) + + require.Greater(t, groups.Len(), 1) +} + +func TestGroupPayloadsByAccountCompareResults(t *testing.T) { + log := zerolog.Nop() + payloads := generateRandomPayloads(1000000) + tmp1 := make([]*ledger.Payload, len(payloads)) + tmp2 := make([]*ledger.Payload, len(payloads)) + copy(tmp1, payloads) + copy(tmp2, payloads) + + groups1 := util.GroupPayloadsByAccount(log, tmp1, 0) + groups2 := util.GroupPayloadsByAccount(log, tmp2, runtime.NumCPU()) + + require.Equal(t, groups1.Len(), groups2.Len()) + for { + group1, err1 := groups1.Next() + group2, err2 := groups2.Next() + + require.NoError(t, err1) + require.NoError(t, err2) + + if group1 == nil { + require.Nil(t, group2) + break + } + + require.Equal(t, group1.Address, group2.Address) + require.Equal(t, len(group1.Payloads), len(group2.Payloads)) + } +} + +func BenchmarkGroupPayloadsByAccount(b *testing.B) { + log := zerolog.Nop() + payloads := generateRandomPayloads(10000000) + tmp := make([]*ledger.Payload, len(payloads)) + + bench := func(b *testing.B, nWorker int) func(b *testing.B) { + return func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + copy(tmp, payloads) + b.StartTimer() + util.GroupPayloadsByAccount(log, tmp, nWorker) + } + } + } + + b.Run("1 worker", bench(b, 1)) + b.Run("2 worker", bench(b, 2)) + b.Run("4 worker", bench(b, 4)) + b.Run("8 worker", bench(b, 8)) + b.Run("max worker", bench(b, runtime.NumCPU())) +} + +// GeneratePayloads generates n random payloads +// with a random number of payloads per account (exponentially distributed) +func generateRandomPayloads(n int) []*ledger.Payload { + const meanPayloadsPerAccount = 100 + const minPayloadsPerAccount = 1 + + payloads := make([]*ledger.Payload, 0, n) + + for i := 0; i < n; { + + registersForAccount := minPayloadsPerAccount + int(rand2.ExpFloat64()*(meanPayloadsPerAccount-minPayloadsPerAccount)) + if registersForAccount > n-i { + registersForAccount = n - i + } + i += registersForAccount + + accountKey := generateRandomAccountKey() + for j := 0; j < registersForAccount; j++ { + payloads = append(payloads, + ledger.NewPayload( + accountKey, + []byte(generateRandomString(10)), + )) + } + } + + return payloads +} + +func generateRandomAccountKey() ledger.Key { + return convert.RegisterIDToLedgerKey(flow.RegisterID{ + Owner: generateRandomAddress(), + Key: generateRandomString(10), + }) +} + +func generateRandomString(i int) string { + buf := make([]byte, i) + _, err := rand.Read(buf) + if err != nil { + panic(err) + } + return hex.EncodeToString(buf) +} + +func generateRandomAddress() string { + buf := make([]byte, flow.AddressLength) + _, err := rand.Read(buf) + if err != nil { + panic(err) + } + return string(buf) +} diff --git a/cmd/util/ledger/util/util.go b/cmd/util/ledger/util/util.go new file mode 100644 index 00000000000..6845b613286 --- /dev/null +++ b/cmd/util/ledger/util/util.go @@ -0,0 +1,145 @@ +package util + +import ( + "fmt" + + "github.com/onflow/atree" + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" +) + +type AccountsAtreeLedger struct { + Accounts environment.Accounts +} + +func NewAccountsAtreeLedger(accounts environment.Accounts) *AccountsAtreeLedger { + return &AccountsAtreeLedger{Accounts: accounts} +} + +var _ atree.Ledger = &AccountsAtreeLedger{} + +func (a *AccountsAtreeLedger) GetValue(owner, key []byte) ([]byte, error) { + v, err := a.Accounts.GetValue( + flow.NewRegisterID( + string(flow.BytesToAddress(owner).Bytes()), + string(key))) + if err != nil { + return nil, fmt.Errorf("getting value failed: %w", err) + } + return v, nil +} + +func (a *AccountsAtreeLedger) SetValue(owner, key, value []byte) error { + err := a.Accounts.SetValue( + flow.NewRegisterID( + string(flow.BytesToAddress(owner).Bytes()), + string(key)), + value) + if err != nil { + return fmt.Errorf("setting value failed: %w", err) + } + return nil +} + +func (a *AccountsAtreeLedger) ValueExists(owner, key []byte) (exists bool, err error) { + v, err := a.GetValue(owner, key) + if err != nil { + return false, fmt.Errorf("checking value existence failed: %w", err) + } + + return len(v) > 0, nil +} + +// AllocateStorageIndex allocates new storage index under the owner accounts to store a new register +func (a *AccountsAtreeLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { + v, err := a.Accounts.AllocateStorageIndex(flow.BytesToAddress(owner)) + if err != nil { + return atree.StorageIndex{}, fmt.Errorf("storage address allocation failed: %w", err) + } + return v, nil +} + +type PayloadSnapshot struct { + Payloads map[flow.RegisterID]*ledger.Payload +} + +var _ snapshot.StorageSnapshot = (*PayloadSnapshot)(nil) + +func NewPayloadSnapshot(payloads []*ledger.Payload) (*PayloadSnapshot, error) { + l := &PayloadSnapshot{ + Payloads: make(map[flow.RegisterID]*ledger.Payload, len(payloads)), + } + for _, payload := range payloads { + key, err := payload.Key() + if err != nil { + return nil, err + } + id, err := convert.LedgerKeyToRegisterID(key) + if err != nil { + return nil, err + } + l.Payloads[id] = payload + } + return l, nil +} + +func (p PayloadSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { + value := p.Payloads[id] + return value.Value(), nil +} + +// NopMemoryGauge is a no-op implementation of the MemoryGauge interface +type NopMemoryGauge struct{} + +func (n NopMemoryGauge) MeterMemory(common.MemoryUsage) error { + return nil +} + +var _ common.MemoryGauge = (*NopMemoryGauge)(nil) + +type PayloadsReadonlyLedger struct { + Snapshot *PayloadSnapshot + + AllocateStorageIndexFunc func(owner []byte) (atree.StorageIndex, error) + SetValueFunc func(owner, key, value []byte) (err error) +} + +func (p *PayloadsReadonlyLedger) GetValue(owner, key []byte) (value []byte, err error) { + v, err := p.Snapshot.Get(flow.NewRegisterID(string(owner), string(key))) + if err != nil { + return nil, fmt.Errorf("getting value failed: %w", err) + } + return v, nil +} + +func (p *PayloadsReadonlyLedger) SetValue(owner, key, value []byte) (err error) { + if p.SetValueFunc != nil { + return p.SetValueFunc(owner, key, value) + } + + panic("SetValue not expected to be called") +} + +func (p *PayloadsReadonlyLedger) ValueExists(owner, key []byte) (exists bool, err error) { + _, ok := p.Snapshot.Payloads[flow.NewRegisterID(string(owner), string(key))] + return ok, nil +} + +func (p *PayloadsReadonlyLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { + if p.AllocateStorageIndexFunc != nil { + return p.AllocateStorageIndexFunc(owner) + } + + panic("AllocateStorageIndex not expected to be called") +} + +func NewPayloadsReadonlyLedger(snapshot *PayloadSnapshot) *PayloadsReadonlyLedger { + return &PayloadsReadonlyLedger{Snapshot: snapshot} +} + +var _ atree.Ledger = &PayloadsReadonlyLedger{} diff --git a/ledger/trie.go b/ledger/trie.go index 17f2ba1a232..a46b90780b2 100644 --- a/ledger/trie.go +++ b/ledger/trie.go @@ -319,6 +319,16 @@ func (p *Payload) Key() (Key, error) { return *k, nil } +// EncodedKey returns payload key. +// CAUTION: do not modify returned encoded key +// because it shares underlying data with payload key. +func (p *Payload) EncodedKey() []byte { + if p == nil { + return nil + } + return p.encKey +} + // Value returns payload value. // CAUTION: do not modify returned value because it shares underlying data with payload value. func (p *Payload) Value() Value { From cacba0dc90235ed8d4a11ea8cb10ebf88a846365 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 12 Dec 2023 16:26:10 +0100 Subject: [PATCH 04/72] address review comments --- .../migrations/account_based_migration.go | 15 ++++- .../migrations/storage_used_migration.go | 8 +-- .../util/migration_runtime_interface.go | 2 +- cmd/util/ledger/util/payload_grouping.go | 63 +++++++++++++------ cmd/util/ledger/util/util.go | 7 ++- 5 files changed, 67 insertions(+), 28 deletions(-) diff --git a/cmd/util/ledger/migrations/account_based_migration.go b/cmd/util/ledger/migrations/account_based_migration.go index d408b279a75..d0b5e7b1256 100644 --- a/cmd/util/ledger/migrations/account_based_migration.go +++ b/cmd/util/ledger/migrations/account_based_migration.go @@ -239,16 +239,23 @@ func MigrateGroupConcurrently( case jobs <- job: } } + close(jobs) }() // read job results - logAccount := moduleUtil.LogProgress("processing account group", accountGroups.Len(), log) + logAccount := moduleUtil.LogProgress( + "processing account group", + accountGroups.Len(), + log, + ) - migrated := make([]*ledger.Payload, 0) + migrated := make([]*ledger.Payload, accountGroups.AllPayloadsCount()) durations := newMigrationDurations(logTopNDurations) + contextDone := false for i := 0; i < accountGroups.Len(); i++ { select { case <-ctx.Done(): + contextDone = true break case result := <-resultCh: durations.Add(result) @@ -257,8 +264,10 @@ func MigrateGroupConcurrently( migrated = append(migrated, accountMigrated...) logAccount(1) } + if contextDone { + break + } } - close(jobs) // make sure to exit all workers before returning from this function // so that the migrator can be closed properly diff --git a/cmd/util/ledger/migrations/storage_used_migration.go b/cmd/util/ledger/migrations/storage_used_migration.go index 4b6ddd5140c..28d35570dec 100644 --- a/cmd/util/ledger/migrations/storage_used_migration.go +++ b/cmd/util/ledger/migrations/storage_used_migration.go @@ -91,7 +91,7 @@ func (m *AccountUsageMigrator) MigrateAccount( return nil, fmt.Errorf("cannot create new payload with value: %w", err) } - payloads[statusIndex] = &newPayload + payloads[statusIndex] = newPayload return payloads, nil } @@ -120,13 +120,13 @@ func (m *AccountUsageMigrator) compareUsage( // newPayloadWithValue returns a new payload with the key from the given payload, and // the value from the argument -func newPayloadWithValue(payload *ledger.Payload, value ledger.Value) (ledger.Payload, error) { +func newPayloadWithValue(payload *ledger.Payload, value ledger.Value) (*ledger.Payload, error) { key, err := payload.Key() if err != nil { - return ledger.Payload{}, err + return &ledger.Payload{}, err } newPayload := ledger.NewPayload(key, value) - return *newPayload, nil + return newPayload, nil } func registerSize(id flow.RegisterID, p *ledger.Payload) int { diff --git a/cmd/util/ledger/util/migration_runtime_interface.go b/cmd/util/ledger/util/migration_runtime_interface.go index 4726e3ef06c..c72d8493095 100644 --- a/cmd/util/ledger/util/migration_runtime_interface.go +++ b/cmd/util/ledger/util/migration_runtime_interface.go @@ -203,7 +203,7 @@ func (m MigrationRuntimeInterface) GetBlockAtHeight(_ uint64) (block runtime.Blo } func (m MigrationRuntimeInterface) ReadRandom([]byte) error { - panic("unexpected UnsafeRandom call") + panic("unexpected ReadRandom call") } func (m MigrationRuntimeInterface) VerifySignature(_ []byte, _ string, _ []byte, _ []byte, _ runtime.SignatureAlgorithm, _ runtime.HashAlgorithm) (bool, error) { diff --git a/cmd/util/ledger/util/payload_grouping.go b/cmd/util/ledger/util/payload_grouping.go index 185863d6ade..f8dd5599401 100644 --- a/cmd/util/ledger/util/payload_grouping.go +++ b/cmd/util/ledger/util/payload_grouping.go @@ -19,7 +19,7 @@ import ( // encodedKeyAddressPrefixLength is the length of the address prefix in the encoded key // 2 for uint16 of number of key parts // 4 for uint32 of the length of the first key part -// 2 for uint32 of the key part type +// 2 for uint16 of the key part type // 8 for the address which is the actual length of the first key part const encodedKeyAddressPrefixLength = 2 + 4 + 2 + flow.AddressLength @@ -27,6 +27,8 @@ const encodedKeyAddressPrefixLength = 2 + 4 + 2 + flow.AddressLength // the sorting into goroutines const minSizeForSplitSortingIntoGoroutines = 100_000 +const estimatedNumOfAccount = 30_000_000 + // PayloadAccountGroup is a grouping of payloads by account type PayloadAccountGroup struct { Address common.Address @@ -38,6 +40,8 @@ type PayloadAccountGrouping struct { payloads sortablePayloads indexes []int + payloadsCount int + current int } @@ -72,6 +76,11 @@ func (g *PayloadAccountGrouping) Len() int { return len(g.indexes) } +// AllPayloadsCount the number of accounts +func (g *PayloadAccountGrouping) AllPayloadsCount() int { + return g.payloadsCount +} + // GroupPayloadsByAccount takes a list of payloads and groups them by account. // it uses nWorkers to sort the payloads by address and find the start and end indexes of // each account. @@ -102,10 +111,10 @@ func GroupPayloadsByAccount( start = time.Now() // find the indexes of the payloads that start a new account - indexes := make([]int, 0, len(p)) + indexes := make([]int, 0, estimatedNumOfAccount) for i := 0; i < len(p); { indexes = append(indexes, i) - i = p.FindLastOfTheSameKey(i) + 1 + i = p.FindNextKeyIndex(i) } end = time.Now() @@ -117,13 +126,15 @@ func GroupPayloadsByAccount( return &PayloadAccountGrouping{ payloads: p, indexes: indexes, + + payloadsCount: len(payloads), } } // payloadToAddress takes a payload and return: -// - (address, true, nil) if the payload is for an account, the account address is returned -// - ("", false, nil) if the payload is not for an account -// - ("", false, err) if running into any exception +// - (address, nil) if the payload is for an account, the account address is returned +// - (common.ZeroAddress, nil) if the payload is not for an account +// - (common.ZeroAddress, err) if running into any exception // The zero address is used for global Payloads and is not an actual account func payloadToAddress(p *ledger.Payload) (common.Address, error) { k, err := p.Key() @@ -170,7 +181,7 @@ func (s sortablePayloads) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s sortablePayloads) FindLastOfTheSameKey(i int) int { +func (s sortablePayloads) FindNextKeyIndex(i int) int { low := i step := 1 for low+step < len(s) && s.Compare(low+step, i) == 0 { @@ -192,7 +203,7 @@ func (s sortablePayloads) FindLastOfTheSameKey(i int) int { } } - return low - 1 + return low } // sortPayloads sorts the payloads in the range [i, j) using goroutines and merges @@ -206,7 +217,7 @@ func sortPayloads(i, j int, source, buffer sortablePayloads, goroutineAllowance return } - // if we are out of goroutine allowance, sort with built-in sortß + // if we are out of goroutine allowance, sort with built-in sort // if the length is less than minSizeForSplit, sort with built-in sort if goroutineAllowance < 2 || j-i < minSizeForSplitSortingIntoGoroutines { sort.Sort(source[i:j]) @@ -236,17 +247,33 @@ func sortPayloads(i, j int, source, buffer sortablePayloads, goroutineAllowance func mergeInto(source, buffer sortablePayloads, i int, mid int, j int) { left := i right := mid - for k := i; k < j; k++ { - if left < mid && (right >= j || source.Compare(left, right) <= 0) { - buffer[k] = source[left] - left++ + k := i + for left < mid && right < j { + // More elements in the both partitions to process. + if source.Compare(left, right) <= 0 { + // Move left partition elements with the same address to buffer. + nextLeft := source.FindNextKeyIndex(left) + n := copy(buffer[k:], source[left:nextLeft]) + left = nextLeft + k += n } else { - buffer[k] = source[right] - right++ + // Move right partition elements with the same address to buffer. + nextRight := source.FindNextKeyIndex(right) + n := copy(buffer[k:], source[right:nextRight]) + right = nextRight + k += n } } - - for k := i; k < j; k++ { - source[k] = buffer[k] + // At this point: + // - one partition is exhausted. + // - remaining elements in the other partition (already sorted) can be copied over. + if left < mid { + // Copy remaining elements in the left partition. + copy(buffer[k:], source[left:mid]) + } else { + // Copy remaining elements in the right partition. + copy(buffer[k:], source[right:j]) } + // Copy merged buffer back to source. + copy(source[i:j], buffer[i:j]) } diff --git a/cmd/util/ledger/util/util.go b/cmd/util/ledger/util/util.go index 6845b613286..b2015b11e0f 100644 --- a/cmd/util/ledger/util/util.go +++ b/cmd/util/ledger/util/util.go @@ -59,7 +59,7 @@ func (a *AccountsAtreeLedger) ValueExists(owner, key []byte) (exists bool, err e func (a *AccountsAtreeLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { v, err := a.Accounts.AllocateStorageIndex(flow.BytesToAddress(owner)) if err != nil { - return atree.StorageIndex{}, fmt.Errorf("storage address allocation failed: %w", err) + return atree.StorageIndex{}, fmt.Errorf("storage index allocation failed: %w", err) } return v, nil } @@ -89,7 +89,10 @@ func NewPayloadSnapshot(payloads []*ledger.Payload) (*PayloadSnapshot, error) { } func (p PayloadSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { - value := p.Payloads[id] + value, exists := p.Payloads[id] + if !exists { + return nil, nil + } return value.Value(), nil } From d488ae3b4c39f4e99c8ae94607d6eb2dbe876405 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Wed, 13 Dec 2023 17:01:39 +0100 Subject: [PATCH 05/72] review comments --- cmd/util/ledger/migrations/account_based_migration.go | 4 ++-- cmd/util/ledger/migrations/storage_used_migration.go | 2 +- cmd/util/ledger/util/payload_grouping.go | 6 +----- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cmd/util/ledger/migrations/account_based_migration.go b/cmd/util/ledger/migrations/account_based_migration.go index 9764f340c4f..2376021a36e 100644 --- a/cmd/util/ledger/migrations/account_based_migration.go +++ b/cmd/util/ledger/migrations/account_based_migration.go @@ -217,6 +217,7 @@ func MigrateGroupConcurrently( } go func() { + defer close(jobs) for { g, err := accountGroups.Next() if err != nil { @@ -239,7 +240,6 @@ func MigrateGroupConcurrently( case jobs <- job: } } - close(jobs) }() // read job results @@ -251,7 +251,7 @@ func MigrateGroupConcurrently( ), ) - migrated := make([]*ledger.Payload, accountGroups.AllPayloadsCount()) + migrated := make([]*ledger.Payload, 0, accountGroups.AllPayloadsCount()) durations := newMigrationDurations(logTopNDurations) contextDone := false for i := 0; i < accountGroups.Len(); i++ { diff --git a/cmd/util/ledger/migrations/storage_used_migration.go b/cmd/util/ledger/migrations/storage_used_migration.go index 28d35570dec..806d91ce45d 100644 --- a/cmd/util/ledger/migrations/storage_used_migration.go +++ b/cmd/util/ledger/migrations/storage_used_migration.go @@ -123,7 +123,7 @@ func (m *AccountUsageMigrator) compareUsage( func newPayloadWithValue(payload *ledger.Payload, value ledger.Value) (*ledger.Payload, error) { key, err := payload.Key() if err != nil { - return &ledger.Payload{}, err + return nil, err } newPayload := ledger.NewPayload(key, value) return newPayload, nil diff --git a/cmd/util/ledger/util/payload_grouping.go b/cmd/util/ledger/util/payload_grouping.go index f8dd5599401..9874016744f 100644 --- a/cmd/util/ledger/util/payload_grouping.go +++ b/cmd/util/ledger/util/payload_grouping.go @@ -40,8 +40,6 @@ type PayloadAccountGrouping struct { payloads sortablePayloads indexes []int - payloadsCount int - current int } @@ -78,7 +76,7 @@ func (g *PayloadAccountGrouping) Len() int { // AllPayloadsCount the number of accounts func (g *PayloadAccountGrouping) AllPayloadsCount() int { - return g.payloadsCount + return len(g.payloads) } // GroupPayloadsByAccount takes a list of payloads and groups them by account. @@ -126,8 +124,6 @@ func GroupPayloadsByAccount( return &PayloadAccountGrouping{ payloads: p, indexes: indexes, - - payloadsCount: len(payloads), } } From 8679ef282f052599c28459534bcd80674b97dbaf Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Wed, 13 Dec 2023 17:54:43 +0100 Subject: [PATCH 06/72] test fix --- fvm/fvm_blockcontext_test.go | 6 ++++++ fvm/transactionStorageLimiter.go | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fvm/fvm_blockcontext_test.go b/fvm/fvm_blockcontext_test.go index dcf73b9280b..f905ba06517 100644 --- a/fvm/fvm_blockcontext_test.go +++ b/fvm/fvm_blockcontext_test.go @@ -948,6 +948,9 @@ func TestBlockContext_ExecuteTransaction_StorageLimit(t *testing.T) { fvm.WithAccountCreationFee(fvm.DefaultAccountCreationFee), fvm.WithMinimumStorageReservation(fvm.DefaultMinimumStorageReservation), fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW), + // The evm account has a storage exception, and if we don't bootstrap with evm, + // the first created account will have that address. + fvm.WithSetupEVMEnabled(true), } t.Run("Storing too much data fails", newVMTest().withBootstrapProcedureOptions(bootstrapOptions...). @@ -1859,6 +1862,9 @@ func TestBlockContext_ExecuteTransaction_FailingTransactions(t *testing.T) { fvm.WithAccountCreationFee(fvm.DefaultAccountCreationFee), fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW), fvm.WithExecutionMemoryLimit(math.MaxUint64), + // The evm account has a storage exception, and if we don't bootstrap with evm, + // the first created account will have that address. + fvm.WithSetupEVMEnabled(true), ).run( func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { ctx.LimitAccountStorage = true // this test requires storage limits to be enforced diff --git a/fvm/transactionStorageLimiter.go b/fvm/transactionStorageLimiter.go index a5ed69395b8..c03a84ce832 100644 --- a/fvm/transactionStorageLimiter.go +++ b/fvm/transactionStorageLimiter.go @@ -74,13 +74,14 @@ func (limiter TransactionStorageLimiter) getStorageCheckAddresses( addresses = append(addresses, payer) } + sc := systemcontracts.SystemContractsForChain(ctx.Chain.ChainID()) for id := range snapshot.WriteSet { address, ok := addressFromRegisterId(id) if !ok { continue } - if limiter.shouldSkipSpecialAddress(ctx, address) { + if limiter.shouldSkipSpecialAddress(ctx, address, sc) { continue } @@ -170,7 +171,7 @@ func (limiter TransactionStorageLimiter) checkStorageLimits( func (limiter TransactionStorageLimiter) shouldSkipSpecialAddress( ctx Context, address flow.Address, + sc *systemcontracts.SystemContracts, ) bool { - sc := systemcontracts.SystemContractsForChain(ctx.Chain.ChainID()) return sc.EVM.Address == address } From 8a2a096adfd0d4d78070f0182fb2d03874a67a86 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Wed, 13 Dec 2023 17:59:34 +0100 Subject: [PATCH 07/72] fix coment --- cmd/util/ledger/util/payload_grouping.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/util/ledger/util/payload_grouping.go b/cmd/util/ledger/util/payload_grouping.go index 9874016744f..9aec5d76efa 100644 --- a/cmd/util/ledger/util/payload_grouping.go +++ b/cmd/util/ledger/util/payload_grouping.go @@ -74,7 +74,7 @@ func (g *PayloadAccountGrouping) Len() int { return len(g.indexes) } -// AllPayloadsCount the number of accounts +// AllPayloadsCount the number of payloads func (g *PayloadAccountGrouping) AllPayloadsCount() int { return len(g.payloads) } From 3b6372484f6930257ca16a569d6fb1e952ac3d16 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Wed, 13 Dec 2023 21:28:46 +0100 Subject: [PATCH 08/72] address review comments --- .../migrations/account_based_migration.go | 63 ++++++++++--------- .../migrations/storage_used_migration.go | 6 +- cmd/util/ledger/util/util.go | 6 ++ 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/cmd/util/ledger/migrations/account_based_migration.go b/cmd/util/ledger/migrations/account_based_migration.go index 2376021a36e..8a78728b7b6 100644 --- a/cmd/util/ledger/migrations/account_based_migration.go +++ b/cmd/util/ledger/migrations/account_based_migration.go @@ -33,6 +33,7 @@ type AccountBasedMigration interface { address common.Address, payloads []*ledger.Payload, ) ([]*ledger.Payload, error) + io.Closer } // CreateAccountBasedMigration creates a migration function that migrates the payloads @@ -91,15 +92,12 @@ func MigrateByAccount( defer func() { for i, migrator := range migrations { - // close the migrator if it's a Closer - if migrator, ok := migrator.(io.Closer); ok { - log.Info(). - Int("migration_index", i). - Type("migration", migrator). - Msg("closing migration") - if err := migrator.Close(); err != nil { - log.Error().Err(err).Msg("error closing migration") - } + log.Info(). + Int("migration_index", i). + Type("migration", migrator). + Msg("closing migration") + if err := migrator.Close(); err != nil { + log.Error().Err(err).Msg("error closing migration") } } }() @@ -155,12 +153,13 @@ func MigrateGroupConcurrently( } start := time.Now() - // This is not an account, but common values for all accounts. - if job.Address == common.ZeroAddress { + // This is not an account, but service level keys. + if util.IsServiceLevelAddress(job.Address) { resultCh <- &migrationResult{ migrationDuration: migrationDuration{ - Address: job.Address, - Duration: time.Since(start), + Address: job.Address, + Duration: time.Since(start), + PayloadCount: len(job.Payloads), }, Migrated: job.Payloads, } @@ -170,11 +169,13 @@ func MigrateGroupConcurrently( if _, ok := knownProblematicAccounts[job.Address]; ok { log.Info(). Hex("address", job.Address[:]). + Int("payload_count", len(job.Payloads)). Msg("skipping problematic account") resultCh <- &migrationResult{ migrationDuration: migrationDuration{ - Address: job.Address, - Duration: time.Since(start), + Address: job.Address, + Duration: time.Since(start), + PayloadCount: len(job.Payloads), }, Migrated: job.Payloads, } @@ -206,8 +207,9 @@ func MigrateGroupConcurrently( resultCh <- &migrationResult{ migrationDuration: migrationDuration{ - Address: job.Address, - Duration: time.Since(start), + Address: job.Address, + Duration: time.Since(start), + PayloadCount: len(job.Payloads), }, Migrated: accountMigrated, } @@ -301,6 +303,14 @@ var knownProblematicAccounts = map[common.Address]string{ // Mainnet account } +func mustHexToAddress(hex string) common.Address { + address, err := common.HexToAddress(hex) + if err != nil { + panic(err) + } + return address +} + type jobMigrateAccountGroup struct { Address common.Address Payloads []*ledger.Payload @@ -313,8 +323,9 @@ type migrationResult struct { } type migrationDuration struct { - Address common.Address - Duration time.Duration + Address common.Address + Duration time.Duration + PayloadCount int } // migrationDurations implements heap methods for the timer results @@ -354,7 +365,11 @@ func (h *migrationDurations) Pop() interface{} { func (h *migrationDurations) Array() zerolog.LogArrayMarshaler { array := zerolog.Arr() for _, result := range h.v { - array = array.Str(fmt.Sprintf("%s: %s", result.Address.Hex(), result.Duration.String())) + array = array.Str(fmt.Sprintf("%s [payloads: %d]: %s", + result.Address.Hex(), + result.PayloadCount, + result.Duration.String(), + )) } return array } @@ -367,11 +382,3 @@ func (h *migrationDurations) Add(result *migrationResult) { heap.Push(h, result.migrationDuration) } } - -func mustHexToAddress(hex string) common.Address { - address, err := common.HexToAddress(hex) - if err != nil { - panic(err) - } - return address -} diff --git a/cmd/util/ledger/migrations/storage_used_migration.go b/cmd/util/ledger/migrations/storage_used_migration.go index 806d91ce45d..3e79d67fb22 100644 --- a/cmd/util/ledger/migrations/storage_used_migration.go +++ b/cmd/util/ledger/migrations/storage_used_migration.go @@ -34,6 +34,10 @@ func (m *AccountUsageMigrator) InitMigration( const oldAccountStatusSize = 25 +func (m *AccountUsageMigrator) Close() error { + return nil +} + func (m *AccountUsageMigrator) MigrateAccount( _ context.Context, address common.Address, @@ -109,7 +113,7 @@ func (m *AccountUsageMigrator) compareUsage( } if oldSize != actualSize { - m.log.Info(). + m.log.Warn(). Uint64("old_size", oldSize). Uint64("new_size", actualSize). Msg("account storage used usage mismatch") diff --git a/cmd/util/ledger/util/util.go b/cmd/util/ledger/util/util.go index b2015b11e0f..f7e127a0daa 100644 --- a/cmd/util/ledger/util/util.go +++ b/cmd/util/ledger/util/util.go @@ -145,4 +145,10 @@ func NewPayloadsReadonlyLedger(snapshot *PayloadSnapshot) *PayloadsReadonlyLedge return &PayloadsReadonlyLedger{Snapshot: snapshot} } +// IsServiceLevelAddress returns true if the given address is the service level address. +// Which means it's not an actual account but instead holds service lever registers. +func IsServiceLevelAddress(address common.Address) bool { + return address == common.ZeroAddress +} + var _ atree.Ledger = &PayloadsReadonlyLedger{} From 12a8462440d09312548934539520567b8a33ef87 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 13:21:47 -0800 Subject: [PATCH 09/72] add delta view impl --- fvm/evm/emulator/state/delta.go | 414 +++++++++++++++ fvm/evm/emulator/state/delta_test.go | 739 +++++++++++++++++++++++++++ 2 files changed, 1153 insertions(+) create mode 100644 fvm/evm/emulator/state/delta.go create mode 100644 fvm/evm/emulator/state/delta_test.go diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go new file mode 100644 index 00000000000..8080c6c9f42 --- /dev/null +++ b/fvm/evm/emulator/state/delta.go @@ -0,0 +1,414 @@ +package state + +import ( + "fmt" + "math/big" + + gethCommon "github.com/ethereum/go-ethereum/common" + gethTypes "github.com/ethereum/go-ethereum/core/types" + gethCrypto "github.com/ethereum/go-ethereum/crypto" + + "github.com/onflow/flow-go/fvm/evm/types" +) + +// DeltaView captures the changes to the state during the execution +// +// for most of the read calls it checks its change logs and if no record is +// found it would redirect the call to the parent view. +type DeltaView struct { + parent types.ReadOnlyView + + // account changes + dirtyAddresses map[gethCommon.Address]interface{} + created map[gethCommon.Address]interface{} + suicided map[gethCommon.Address]interface{} + balances map[gethCommon.Address]*big.Int + nonces map[gethCommon.Address]uint64 + codes map[gethCommon.Address][]byte + codeHashes map[gethCommon.Address]gethCommon.Hash + + // states changes + dirtySlots map[types.SlotAddress]interface{} + states map[types.SlotAddress]gethCommon.Hash + + // transient storage + transient map[types.SlotAddress]gethCommon.Hash + + // access lists + accessListAddresses map[gethCommon.Address]interface{} + accessListSlots map[types.SlotAddress]interface{} + + // logs + logs []*gethTypes.Log + + // preimages + preimages map[gethCommon.Hash][]byte + + // refund + refund uint64 +} + +var _ types.HotView = &DeltaView{} + +// NewDeltaView constructs a new delta view +func NewDeltaView(parent types.ReadOnlyView) *DeltaView { + return &DeltaView{ + parent: parent, + + dirtyAddresses: make(map[gethCommon.Address]interface{}, 0), + created: make(map[gethCommon.Address]interface{}, 0), + suicided: make(map[gethCommon.Address]interface{}, 0), + balances: make(map[gethCommon.Address]*big.Int, 0), + nonces: make(map[gethCommon.Address]uint64, 0), + codes: make(map[gethCommon.Address][]byte, 0), + codeHashes: make(map[gethCommon.Address]gethCommon.Hash, 0), + dirtySlots: make(map[types.SlotAddress]interface{}, 0), + states: make(map[types.SlotAddress]gethCommon.Hash, 0), + transient: make(map[types.SlotAddress]gethCommon.Hash, 0), + accessListAddresses: make(map[gethCommon.Address]interface{}, 0), + accessListSlots: make(map[types.SlotAddress]interface{}, 0), + logs: make([]*gethTypes.Log, 0), + preimages: make(map[gethCommon.Hash][]byte, 0), + + // for refund we just copy the data + refund: parent.GetRefund(), + } +} + +// NewChildView constructs a new delta view having the current view as parent +func (d *DeltaView) NewChildView() *DeltaView { + return NewDeltaView(d) +} + +// Exist returns true if address exists +// +// it also returns true for both newly created accounts or accounts that has been flagged for deletion +func (d *DeltaView) Exist(addr gethCommon.Address) (bool, error) { + _, found := d.created[addr] + if found { + return true, nil + } + _, found = d.suicided[addr] + if found { + return true, nil + } + return d.parent.Exist(addr) +} + +// CreateAccount creates a new account for the given address +func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { + d.created[addr] = struct{}{} + // flag the address as dirty + d.dirtyAddresses[addr] = struct{}{} + return nil +} + +// IsCreated returns true if address has been created in this tx +func (d *DeltaView) IsCreated(addr gethCommon.Address) bool { + _, found := d.created[addr] + if found { + return true + } + return d.parent.IsCreated(addr) +} + +// HasSuicided returns true if address has been flagged for deletion +func (d *DeltaView) HasSuicided(addr gethCommon.Address) bool { + _, found := d.suicided[addr] + if found { + return true + } + return d.parent.HasSuicided(addr) +} + +// Suicide sets a flag to delete the account at the end of transaction +func (d *DeltaView) Suicide(addr gethCommon.Address) (bool, error) { + // if it doesn't exist, return false + exists, err := d.Exist(addr) + if err != nil { + return false, err + } + if !exists { + return false, nil + } + // flag the account for deletion + d.suicided[addr] = struct{}{} + + // flag the address as dirty + d.dirtyAddresses[addr] = struct{}{} + return true, nil +} + +// GetBalance returns the balance of the given address +func (d *DeltaView) GetBalance(addr gethCommon.Address) (*big.Int, error) { + if d.HasSuicided(addr) { + return big.NewInt(0), nil + } + val, found := d.balances[addr] + if found { + return val, nil + } + // if newly created and no balance is set yet + _, newlyCreated := d.created[addr] + if newlyCreated { + return big.NewInt(0), nil + } + return d.parent.GetBalance(addr) +} + +// AddBalance adds the amount to the current balance of the given address +func (d *DeltaView) AddBalance(addr gethCommon.Address, amount *big.Int) error { + // if amount is 0 skip + if amount.Sign() == 0 { + return nil + } + // get the latest balance + orgBalance, err := d.GetBalance(addr) + if err != nil { + return err + } + // update the balance + newBalance := new(big.Int).Add(orgBalance, amount) + d.balances[addr] = newBalance + + // flag the address as dirty + d.dirtyAddresses[addr] = struct{}{} + return nil +} + +// SubBalance subtracts the amount from the current balance of the given address +func (d *DeltaView) SubBalance(addr gethCommon.Address, amount *big.Int) error { + // if amount is 0 skip + if amount.Sign() == 0 { + return nil + } + + // get the latest balance + orgBalance, err := d.GetBalance(addr) + if err != nil { + return err + } + + // update the new balance + newBalance := new(big.Int).Sub(orgBalance, amount) + + // if new balance is negative error + if newBalance.Sign() < 0 { + return fmt.Errorf("account balance is negative %d", newBalance) + } + + // update the balance + d.balances[addr] = newBalance + + // flag the address as dirty + d.dirtyAddresses[addr] = struct{}{} + return nil +} + +// GetNonce returns the nonce of the given address +func (d *DeltaView) GetNonce(addr gethCommon.Address) (uint64, error) { + val, found := d.nonces[addr] + if found { + return val, nil + } + // if newly created + _, newlyCreated := d.created[addr] + if newlyCreated { + return 0, nil + } + return d.parent.GetNonce(addr) +} + +// SetNonce sets the nonce for the given address +func (d *DeltaView) SetNonce(addr gethCommon.Address, nonce uint64) error { + // update the nonce + d.nonces[addr] = nonce + + // flag the address as dirty + d.dirtyAddresses[addr] = struct{}{} + return nil +} + +// GetCode returns the code of the given address +func (d *DeltaView) GetCode(addr gethCommon.Address) ([]byte, error) { + code, found := d.codes[addr] + if found { + return code, nil + } + // if newly created + _, newlyCreated := d.created[addr] + if newlyCreated { + return nil, nil + } + return d.parent.GetCode(addr) +} + +// GetCodeSize returns the code size of the given address +func (d *DeltaView) GetCodeSize(addr gethCommon.Address) (int, error) { + code, err := d.GetCode(addr) + return len(code), err +} + +// GetCodeHash returns the code hash of the given address +func (d *DeltaView) GetCodeHash(addr gethCommon.Address) (gethCommon.Hash, error) { + codeHash, found := d.codeHashes[addr] + if found { + return codeHash, nil + } + // if newly created + _, newlyCreated := d.created[addr] + if newlyCreated { + return gethTypes.EmptyCodeHash, nil + } + return d.parent.GetCodeHash(addr) +} + +// SetCode sets the code for the given address +func (d *DeltaView) SetCode(addr gethCommon.Address, code []byte) error { + // update code + d.codes[addr] = code + + // update code hash + codeHash := gethTypes.EmptyCodeHash + if len(code) > 0 { + codeHash = gethCrypto.Keccak256Hash(code) + } + d.codeHashes[addr] = codeHash + + // flag the address as dirty + d.dirtyAddresses[addr] = struct{}{} + return nil +} + +// GetTransientState returns the value of the slot of the main state +func (d *DeltaView) GetState(sk types.SlotAddress) (gethCommon.Hash, error) { + val, found := d.states[sk] + if found { + return val, nil + } + return d.parent.GetState(sk) +} + +// SetState adds sets a value for the given slot of the main storage +func (d *DeltaView) SetState(sk types.SlotAddress, value gethCommon.Hash) error { + lastValue, err := d.GetState(sk) + if err != nil { + return err + } + // we skip the value is the same + // this step might look not helping with performance but we kept it to + // act similar to the Geth StateDB behaviour + if value == lastValue { + return nil + } + d.states[sk] = value + d.dirtySlots[sk] = struct{}{} + return nil +} + +// GetTransientState returns the value of the slot of the transient state +func (d *DeltaView) GetTransientState(sk types.SlotAddress) gethCommon.Hash { + val, found := d.transient[sk] + if found { + return val + } + return d.parent.GetTransientState(sk) +} + +// SetState adds sets a value for the given slot of the transient storage +func (d *DeltaView) SetTransientState(sk types.SlotAddress, value gethCommon.Hash) { + d.transient[sk] = value +} + +// GetRefund returns the total (gas) refund +func (d *DeltaView) GetRefund() uint64 { + return d.refund +} + +// AddRefund adds the amount to the total (gas) refund +func (d *DeltaView) AddRefund(amount uint64) error { + d.refund += amount + return nil +} + +// SubRefund subtracts the amount from the total (gas) refund +func (d *DeltaView) SubRefund(amount uint64) error { + if amount > d.refund { + return fmt.Errorf("refund counter below zero (gas: %d > refund: %d)", amount, d.refund) + } + d.refund -= amount + return nil +} + +// AddressInAccessList checks if the address is in the access list +func (d *DeltaView) AddressInAccessList(addr gethCommon.Address) bool { + _, addressFound := d.accessListAddresses[addr] + if !addressFound { + addressFound = d.parent.AddressInAccessList(addr) + } + return addressFound +} + +// AddAddressToAccessList adds an address to the access list +func (d *DeltaView) AddAddressToAccessList(addr gethCommon.Address) bool { + addrPresent := d.AddressInAccessList(addr) + d.accessListAddresses[addr] = struct{}{} + return !addrPresent +} + +// SlotInAccessList checks if the slot is in the access list +func (d *DeltaView) SlotInAccessList(sk types.SlotAddress) (addressOk bool, slotOk bool) { + addressFound := d.AddressInAccessList(sk.Address) + _, slotFound := d.accessListSlots[sk] + if !slotFound { + _, slotFound = d.parent.SlotInAccessList(sk) + } + return addressFound, slotFound +} + +// AddSlotToAccessList adds a slot to the access list +// it also adds the address to the address list +func (d *DeltaView) AddSlotToAccessList(sk types.SlotAddress) (addrAdded bool, slotAdded bool) { + addrPresent, slotPresent := d.SlotInAccessList(sk) + d.accessListAddresses[sk.Address] = struct{}{} + d.accessListSlots[sk] = struct{}{} + return !addrPresent, !slotPresent +} + +// AddLog appends a log to the log collection +func (d *DeltaView) AddLog(log *gethTypes.Log) { + d.logs = append(d.logs, log) +} + +// Logs returns the logs that has been captured in this view +func (d *DeltaView) Logs() []*gethTypes.Log { + return d.logs +} + +// AddPreimage adds a preimage +func (d *DeltaView) AddPreimage(hash gethCommon.Hash, preimage []byte) { + // make a copy (legacy behaviour) + pi := make([]byte, len(preimage)) + copy(pi, preimage) + d.preimages[hash] = pi +} + +// Preimages returns a map of preimages +func (d *DeltaView) Preimages() map[gethCommon.Hash][]byte { + return d.preimages +} + +// Commit for deltaview is a no-op +func (d *DeltaView) Commit() error { + return nil +} + +// DirtyAddresses returns a set of addresses that has been updated in this view +func (d *DeltaView) DirtyAddresses() map[gethCommon.Address]interface{} { + return d.dirtyAddresses +} + +// DirtySlots returns a set of slots that has been updated in this view +func (d *DeltaView) DirtySlots() map[types.SlotAddress]interface{} { + return d.dirtySlots +} diff --git a/fvm/evm/emulator/state/delta_test.go b/fvm/evm/emulator/state/delta_test.go new file mode 100644 index 00000000000..b6c81ab660a --- /dev/null +++ b/fvm/evm/emulator/state/delta_test.go @@ -0,0 +1,739 @@ +package state_test + +import ( + "fmt" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + gethCommon "github.com/ethereum/go-ethereum/common" + gethTypes "github.com/ethereum/go-ethereum/core/types" + gethCrypto "github.com/ethereum/go-ethereum/crypto" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/fvm/evm/emulator/state" + "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/fvm/evm/types" +) + +var emptyRefund = func() uint64 { + return 0 +} + +func TestDeltaView(t *testing.T) { + t.Parallel() + + t.Run("test account exist/creation/suicide functionality", func(t *testing.T) { + addr1 := testutils.RandomCommonAddress(t) + addr2 := testutils.RandomCommonAddress(t) + addr3 := testutils.RandomCommonAddress(t) + + view := state.NewDeltaView( + &MockedReadOnlyView{ + // we need get refund for parent + GetRefundFunc: emptyRefund, + ExistFunc: func(addr gethCommon.Address) (bool, error) { + switch addr { + case addr1: + return true, nil + case addr2: + return false, nil + default: + return false, fmt.Errorf("some error") + } + }, + HasSuicidedFunc: func(gethCommon.Address) bool { + return false + }, + }) + + // check existing account on the parent + found, err := view.Exist(addr1) + require.NoError(t, err) + require.True(t, found) + + // account doesn't exist on parent + found, err = view.Exist(addr2) + require.NoError(t, err) + require.False(t, found) + + // handling error on the parent + _, err = view.Exist(addr3) + require.Error(t, err) + + // create a account at address 2 + err = view.CreateAccount(addr2) + require.NoError(t, err) + require.True(t, view.IsCreated(addr2)) + + // now it should be found + found, err = view.Exist(addr2) + require.NoError(t, err) + require.True(t, found) + + // test HasSuicided first + success := view.HasSuicided(addr1) + require.False(t, success) + + // set addr1 for deletion + success, err = view.Suicide(addr1) + require.NoError(t, err) + require.True(t, success) + + // check HasSuicided now + success = view.HasSuicided(addr1) + require.True(t, success) + + // addr1 should still exist after suicide call + found, err = view.Exist(addr1) + require.NoError(t, err) + require.True(t, found) + }) + + t.Run("test account balance functionality", func(t *testing.T) { + addr1 := testutils.RandomCommonAddress(t) + addr1InitBal := big.NewInt(10) + addr2 := testutils.RandomCommonAddress(t) + addr2InitBal := big.NewInt(5) + addr3 := testutils.RandomCommonAddress(t) + + view := state.NewDeltaView( + &MockedReadOnlyView{ + // we need get refund for parent + GetRefundFunc: emptyRefund, + ExistFunc: func(addr gethCommon.Address) (bool, error) { + switch addr { + case addr1, addr2: + return true, nil + default: + return false, nil + } + }, + HasSuicidedFunc: func(gethCommon.Address) bool { + return false + }, + GetBalanceFunc: func(addr gethCommon.Address) (*big.Int, error) { + switch addr { + case addr1: + return addr1InitBal, nil + case addr2: + return addr2InitBal, nil + default: + return nil, fmt.Errorf("some error") + } + }, + }) + + // get balance through parent + bal, err := view.GetBalance(addr1) + require.NoError(t, err) + require.Equal(t, addr1InitBal, bal) + + // call suicide on addr + success, err := view.Suicide(addr1) + require.NoError(t, err) + require.True(t, success) + + // now it should return balance of zero + bal, err = view.GetBalance(addr1) + require.NoError(t, err) + require.Equal(t, big.NewInt(0), bal) + + // add balance to addr2 + amount := big.NewInt(7) + expected := new(big.Int).Add(addr2InitBal, amount) + err = view.AddBalance(addr2, amount) + require.NoError(t, err) + newBal, err := view.GetBalance(addr2) + require.NoError(t, err) + require.Equal(t, expected, newBal) + + // sub balance from addr2 + amount = big.NewInt(9) + expected = new(big.Int).Sub(newBal, amount) + err = view.SubBalance(addr2, amount) + require.NoError(t, err) + bal, err = view.GetBalance(addr2) + require.NoError(t, err) + require.Equal(t, expected, bal) + + // negative balance error + err = view.SubBalance(addr2, big.NewInt(100)) + require.Error(t, err) + + // handling error on the parent + _, err = view.GetBalance(addr3) + require.Error(t, err) + + // create a new account at addr3 + err = view.CreateAccount(addr3) + require.NoError(t, err) + + // now the balance should return 0 + bal, err = view.GetBalance(addr3) + require.NoError(t, err) + require.Equal(t, big.NewInt(0), bal) + }) + + t.Run("test nonce functionality", func(t *testing.T) { + addr1 := testutils.RandomCommonAddress(t) + addr1InitNonce := uint64(1) + addr2 := testutils.RandomCommonAddress(t) + + view := state.NewDeltaView( + &MockedReadOnlyView{ + // we need get refund for parent + GetRefundFunc: emptyRefund, + ExistFunc: func(addr gethCommon.Address) (bool, error) { + switch addr { + case addr1: + return true, nil + default: + return false, nil + } + }, + + GetNonceFunc: func(addr gethCommon.Address) (uint64, error) { + switch addr { + case addr1: + return addr1InitNonce, nil + default: + return 0, fmt.Errorf("some error") + } + }, + }) + + // get nonce through parent + nonce, err := view.GetNonce(addr1) + require.NoError(t, err) + require.Equal(t, addr1InitNonce, nonce) + + // set nonce + new := uint64(100) + err = view.SetNonce(addr1, new) + require.NoError(t, err) + nonce, err = view.GetNonce(addr1) + require.NoError(t, err) + require.Equal(t, new, nonce) + + // handling error on the parent + _, err = view.GetNonce(addr2) + require.Error(t, err) + + // create a new account at addr2 + err = view.CreateAccount(addr2) + require.NoError(t, err) + + // now the nonce should return 0 + nonce, err = view.GetNonce(addr2) + require.NoError(t, err) + require.Equal(t, uint64(0), nonce) + }) + + t.Run("test code functionality", func(t *testing.T) { + addr1 := testutils.RandomCommonAddress(t) + addr1InitCode := []byte("code1") + addr1IntiCodeHash := gethCommon.BytesToHash([]byte{1, 2}) + addr2 := testutils.RandomCommonAddress(t) + + view := state.NewDeltaView( + &MockedReadOnlyView{ + // we need get refund for parent + GetRefundFunc: emptyRefund, + ExistFunc: func(addr gethCommon.Address) (bool, error) { + switch addr { + case addr1: + return true, nil + default: + return false, nil + } + }, + GetCodeFunc: func(addr gethCommon.Address) ([]byte, error) { + switch addr { + case addr1: + return addr1InitCode, nil + default: + return nil, fmt.Errorf("some error") + } + }, + GetCodeSizeFunc: func(addr gethCommon.Address) (int, error) { + switch addr { + case addr1: + return len(addr1InitCode), nil + default: + return 0, fmt.Errorf("some error") + } + }, + GetCodeHashFunc: func(addr gethCommon.Address) (common.Hash, error) { + switch addr { + case addr1: + return addr1IntiCodeHash, nil + default: + return gethCommon.Hash{}, fmt.Errorf("some error") + } + }, + }) + + // get code through parent + code, err := view.GetCode(addr1) + require.NoError(t, err) + require.Equal(t, addr1InitCode, code) + + // get code size through parent + codeSize, err := view.GetCodeSize(addr1) + require.NoError(t, err) + require.Equal(t, len(addr1InitCode), codeSize) + + // get code hash through parent + codeHash, err := view.GetCodeHash(addr1) + require.NoError(t, err) + require.Equal(t, addr1IntiCodeHash, codeHash) + + // set code for addr1 + newCode := []byte("new code") + err = view.SetCode(addr1, newCode) + require.NoError(t, err) + + code, err = view.GetCode(addr1) + require.NoError(t, err) + require.Equal(t, newCode, code) + + codeSize, err = view.GetCodeSize(addr1) + require.NoError(t, err) + require.Equal(t, len(newCode), codeSize) + + codeHash, err = view.GetCodeHash(addr1) + require.NoError(t, err) + require.Equal(t, gethCrypto.Keccak256Hash(code), codeHash) + + // handling error on the parent + _, err = view.GetCode(addr2) + require.Error(t, err) + + // create a new account at addr2 + err = view.CreateAccount(addr2) + require.NoError(t, err) + + // now the code should return empty code + code, err = view.GetCode(addr2) + require.NoError(t, err) + require.Len(t, code, 0) + + codeHash, err = view.GetCodeHash(addr2) + require.NoError(t, err) + require.Equal(t, gethTypes.EmptyCodeHash, codeHash) + }) + + t.Run("test state access functionality", func(t *testing.T) { + slot1 := types.SlotAddress{ + Address: testutils.RandomCommonAddress(t), + Key: gethCommon.BytesToHash([]byte{1, 2}), + } + + slot1InitValue := gethCommon.BytesToHash([]byte{3, 4}) + + slot2 := types.SlotAddress{ + Address: testutils.RandomCommonAddress(t), + Key: gethCommon.BytesToHash([]byte{5, 6}), + } + + view := state.NewDeltaView( + &MockedReadOnlyView{ + // we need get refund for parent + GetRefundFunc: emptyRefund, + + GetStateFunc: func(slot types.SlotAddress) (gethCommon.Hash, error) { + switch slot { + case slot1: + return slot1InitValue, nil + default: + return gethCommon.Hash{}, fmt.Errorf("some error") + } + }, + }) + + // get state through parent + value, err := view.GetState(slot1) + require.NoError(t, err) + require.Equal(t, slot1InitValue, value) + + // handle error from parent + _, err = view.GetState(slot2) + require.Error(t, err) + + // check dirty slots + dirtySlots := view.DirtySlots() + require.Empty(t, dirtySlots) + + // set slot1 with some new value + newValue := gethCommon.BytesToHash([]byte{9, 8}) + err = view.SetState(slot1, newValue) + require.NoError(t, err) + + value, err = view.GetState(slot1) + require.NoError(t, err) + require.Equal(t, newValue, value) + + // check dirty slots + dirtySlots = view.DirtySlots() + require.Len(t, dirtySlots, 1) + + _, found := dirtySlots[slot1] + require.True(t, found) + }) + + t.Run("test transient state access functionality", func(t *testing.T) { + slot1 := types.SlotAddress{ + Address: testutils.RandomCommonAddress(t), + Key: gethCommon.BytesToHash([]byte{1, 2}), + } + + slot1InitValue := gethCommon.BytesToHash([]byte{3, 4}) + + view := state.NewDeltaView( + &MockedReadOnlyView{ + // we need get refund for parent + GetRefundFunc: emptyRefund, + GetTransientStateFunc: func(slot types.SlotAddress) gethCommon.Hash { + switch slot { + case slot1: + return slot1InitValue + default: + return gethCommon.Hash{} + } + }, + }) + + // get state through parent + value := view.GetTransientState(slot1) + require.Equal(t, slot1InitValue, value) + + // set slot1 with some new value + newValue := gethCommon.BytesToHash([]byte{9, 8}) + view.SetTransientState(slot1, newValue) + + value = view.GetTransientState(slot1) + require.Equal(t, newValue, value) + }) + + t.Run("test refund functionality", func(t *testing.T) { + initRefund := uint64(10) + view := state.NewDeltaView( + &MockedReadOnlyView{ + GetRefundFunc: func() uint64 { + return initRefund + }, + }) + + // get refund through parent + value := view.GetRefund() + require.Equal(t, initRefund, value) + + // add refund + addition := uint64(7) + err := view.AddRefund(addition) + require.NoError(t, err) + require.Equal(t, initRefund+addition, view.GetRefund()) + + // sub refund + subtract := uint64(2) + err = view.SubRefund(subtract) + require.NoError(t, err) + require.Equal(t, initRefund+addition-subtract, view.GetRefund()) + + // refund goes negative + err = view.SubRefund(1000) + require.Error(t, err) + }) + + t.Run("test access list functionality", func(t *testing.T) { + addr1 := testutils.RandomCommonAddress(t) + addr2 := testutils.RandomCommonAddress(t) + slot1 := types.SlotAddress{ + Address: testutils.RandomCommonAddress(t), + Key: gethCommon.BytesToHash([]byte{1, 2}), + } + + slot2 := types.SlotAddress{ + Address: testutils.RandomCommonAddress(t), + Key: gethCommon.BytesToHash([]byte{3, 4}), + } + + view := state.NewDeltaView( + &MockedReadOnlyView{ + GetRefundFunc: emptyRefund, + AddressInAccessListFunc: func(addr gethCommon.Address) bool { + switch addr { + case addr1: + return true + default: + return false + } + }, + SlotInAccessListFunc: func(slot types.SlotAddress) (addressOk bool, slotOk bool) { + switch slot { + case slot1: + return false, true + default: + return false, false + } + }, + }) + + // check address through parent + require.True(t, view.AddressInAccessList(addr1)) + + // add addr 2 to the list + require.False(t, view.AddressInAccessList(addr2)) + added := view.AddAddressToAccessList(addr2) + require.True(t, added) + require.True(t, view.AddressInAccessList(addr2)) + + // adding again + added = view.AddAddressToAccessList(addr2) + require.False(t, added) + + // check slot through parent + addrFound, slotFound := view.SlotInAccessList(slot1) + require.False(t, addrFound) + require.True(t, slotFound) + + // add slot 2 to the list + addrFound, slotFound = view.SlotInAccessList(slot2) + require.False(t, addrFound) + require.False(t, slotFound) + + addressAdded, slotAdded := view.AddSlotToAccessList(slot2) + require.True(t, addressAdded) + require.True(t, slotAdded) + + addrFound, slotFound = view.SlotInAccessList(slot2) + require.True(t, addrFound) + require.True(t, slotFound) + + // adding again + addressAdded, slotAdded = view.AddSlotToAccessList(slot2) + require.False(t, addressAdded) + require.False(t, slotAdded) + }) + + t.Run("test log functionality", func(t *testing.T) { + view := state.NewDeltaView( + &MockedReadOnlyView{ + GetRefundFunc: emptyRefund, + }) + + logs := view.Logs() + require.Empty(t, logs) + + log1 := &gethTypes.Log{ + Address: testutils.RandomCommonAddress(t), + } + view.AddLog(log1) + + log2 := &gethTypes.Log{ + Address: testutils.RandomCommonAddress(t), + } + view.AddLog(log2) + + logs = view.Logs() + require.Equal(t, []*gethTypes.Log{log1, log2}, logs) + }) + + t.Run("test preimage functionality", func(t *testing.T) { + view := state.NewDeltaView( + &MockedReadOnlyView{ + GetRefundFunc: emptyRefund, + }) + + preimages := view.Preimages() + require.Empty(t, preimages) + + preimage1 := []byte{1, 2} + hash1 := gethCommon.BytesToHash([]byte{2, 3}) + view.AddPreimage(hash1, preimage1) + + preimage2 := []byte{4, 5} + hash2 := gethCommon.BytesToHash([]byte{6, 7}) + view.AddPreimage(hash2, preimage2) + + expected := make(map[gethCommon.Hash][]byte) + expected[hash1] = preimage1 + expected[hash2] = preimage2 + + preimages = view.Preimages() + require.Equal(t, expected, preimages) + }) + + t.Run("test dirty addresses functionality", func(t *testing.T) { + addrCount := 6 + addresses := make([]gethCommon.Address, addrCount) + for i := 0; i < addrCount; i++ { + addresses[i] = testutils.RandomCommonAddress(t) + } + + view := state.NewDeltaView( + &MockedReadOnlyView{ + // we need get refund for parent + GetRefundFunc: emptyRefund, + ExistFunc: func(addr gethCommon.Address) (bool, error) { + return true, nil + }, + GetBalanceFunc: func(addr gethCommon.Address) (*big.Int, error) { + return big.NewInt(10), nil + }, + GetNonceFunc: func(addr gethCommon.Address) (uint64, error) { + return 0, nil + }, + HasSuicidedFunc: func(gethCommon.Address) bool { + return false + }, + }) + + // check dirty addresses + dirtyAddresses := view.DirtyAddresses() + require.Empty(t, dirtyAddresses) + + // create a account at address 1 + err := view.CreateAccount(addresses[0]) + require.NoError(t, err) + + // Suicide address 2 + _, err = view.Suicide(addresses[1]) + require.NoError(t, err) + + // add balance for address 3 + err = view.AddBalance(addresses[2], big.NewInt(5)) + require.NoError(t, err) + + // sub balance for address 4 + err = view.AddBalance(addresses[3], big.NewInt(5)) + require.NoError(t, err) + + // set nonce for address 5 + err = view.SetNonce(addresses[4], 5) + require.NoError(t, err) + + // set code for address 6 + err = view.SetCode(addresses[5], []byte{1, 2}) + require.NoError(t, err) + + // now check dirty addresses + dirtyAddresses = view.DirtyAddresses() + require.Len(t, dirtyAddresses, addrCount) + for _, addr := range addresses { + _, found := dirtyAddresses[addr] + require.True(t, found) + } + }) + +} + +type MockedReadOnlyView struct { + ExistFunc func(gethCommon.Address) (bool, error) + HasSuicidedFunc func(gethCommon.Address) bool + IsCreatedFunc func(gethCommon.Address) bool + GetBalanceFunc func(gethCommon.Address) (*big.Int, error) + GetNonceFunc func(gethCommon.Address) (uint64, error) + GetCodeFunc func(gethCommon.Address) ([]byte, error) + GetCodeHashFunc func(gethCommon.Address) (gethCommon.Hash, error) + GetCodeSizeFunc func(gethCommon.Address) (int, error) + GetStateFunc func(types.SlotAddress) (gethCommon.Hash, error) + GetTransientStateFunc func(types.SlotAddress) gethCommon.Hash + GetRefundFunc func() uint64 + AddressInAccessListFunc func(gethCommon.Address) bool + SlotInAccessListFunc func(types.SlotAddress) (addressOk bool, slotOk bool) +} + +var _ types.ReadOnlyView = &MockedReadOnlyView{} + +func (v *MockedReadOnlyView) Exist(addr gethCommon.Address) (bool, error) { + if v.ExistFunc == nil { + panic("Exist is not set in this mocked view") + } + return v.ExistFunc(addr) +} + +func (v *MockedReadOnlyView) IsCreated(addr gethCommon.Address) bool { + if v.IsCreatedFunc == nil { + panic("IsCreated is not set in this mocked view") + } + return v.IsCreatedFunc(addr) +} + +func (v *MockedReadOnlyView) HasSuicided(addr gethCommon.Address) bool { + if v.HasSuicidedFunc == nil { + panic("HasSuicided is not set in this mocked view") + } + return v.HasSuicidedFunc(addr) +} + +func (v *MockedReadOnlyView) GetBalance(addr gethCommon.Address) (*big.Int, error) { + if v.GetBalanceFunc == nil { + panic("GetBalance is not set in this mocked view") + } + return v.GetBalanceFunc(addr) +} + +func (v *MockedReadOnlyView) GetNonce(addr gethCommon.Address) (uint64, error) { + if v.GetNonceFunc == nil { + panic("GetNonce is not set in this mocked view") + } + return v.GetNonceFunc(addr) +} + +func (v *MockedReadOnlyView) GetCode(addr gethCommon.Address) ([]byte, error) { + if v.GetCodeFunc == nil { + panic("GetCode is not set in this mocked view") + } + return v.GetCodeFunc(addr) +} + +func (v *MockedReadOnlyView) GetCodeHash(addr gethCommon.Address) (gethCommon.Hash, error) { + if v.GetCodeHashFunc == nil { + panic("GetCodeHash is not set in this mocked view") + } + return v.GetCodeHashFunc(addr) +} + +func (v *MockedReadOnlyView) GetCodeSize(addr gethCommon.Address) (int, error) { + if v.GetCodeSizeFunc == nil { + panic("GetCodeSize is not set in this mocked view") + } + return v.GetCodeSizeFunc(addr) +} + +func (v *MockedReadOnlyView) GetState(slot types.SlotAddress) (gethCommon.Hash, error) { + if v.GetStateFunc == nil { + panic("GetState is not set in this mocked view") + } + return v.GetStateFunc(slot) +} + +func (v *MockedReadOnlyView) GetTransientState(slot types.SlotAddress) gethCommon.Hash { + if v.GetTransientStateFunc == nil { + panic("GetTransientState is not set in this mocked view") + } + return v.GetTransientStateFunc(slot) +} + +func (v *MockedReadOnlyView) GetRefund() uint64 { + if v.GetRefundFunc == nil { + panic("GetRefund is not set in this mocked view") + } + return v.GetRefundFunc() +} + +func (v *MockedReadOnlyView) AddressInAccessList(addr gethCommon.Address) bool { + if v.AddressInAccessListFunc == nil { + panic("AddressInAccessList is not set in this mocked view") + } + return v.AddressInAccessListFunc(addr) +} + +func (v *MockedReadOnlyView) SlotInAccessList(slot types.SlotAddress) (addressOk bool, slotOk bool) { + if v.SlotInAccessListFunc == nil { + panic("SlotInAccessList is not set in this mocked view") + } + return v.SlotInAccessListFunc(slot) +} From 09ea16d5c669d571581e2e2d3d0e82501c985d51 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 13:23:36 -0800 Subject: [PATCH 10/72] add stateDB impl --- fvm/evm/emulator/state/stateDB.go | 397 +++++++++++++++++++++++++ fvm/evm/emulator/state/stateDB_test.go | 114 +++++++ 2 files changed, 511 insertions(+) create mode 100644 fvm/evm/emulator/state/stateDB.go create mode 100644 fvm/evm/emulator/state/stateDB_test.go diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go new file mode 100644 index 00000000000..4581dda99ef --- /dev/null +++ b/fvm/evm/emulator/state/stateDB.go @@ -0,0 +1,397 @@ +package state + +import ( + "bytes" + stdErrors "errors" + "fmt" + "math/big" + "sort" + + gethCommon "github.com/ethereum/go-ethereum/common" + gethTypes "github.com/ethereum/go-ethereum/core/types" + gethParams "github.com/ethereum/go-ethereum/params" + + "github.com/onflow/atree" + "github.com/onflow/flow-go/fvm/errors" + "github.com/onflow/flow-go/fvm/evm/types" + "github.com/onflow/flow-go/model/flow" +) + +// TODO: question, does stateDB has to be thread safe ? +// error handling +// +// StateDB should not be used for running multiple transactions +type StateDB struct { + ledger atree.Ledger + root flow.Address + baseView types.BaseView + views []*DeltaView + dbErr error +} + +var _ types.StateDB = &StateDB{} + +func NewStateDB(ledger atree.Ledger, root flow.Address) (*StateDB, error) { + bv, err := NewBaseView(ledger, root) + if err != nil { + return nil, err + } + return &StateDB{ + ledger: ledger, + root: root, + baseView: bv, + views: []*DeltaView{NewDeltaView(bv)}, + dbErr: nil, + }, nil +} + +// Exist returns true if the given address exists in state. +// +// this should also return true for suicided accounts during the transaction execution. +func (db *StateDB) Exist(addr gethCommon.Address) bool { + exist, err := db.lastestView().Exist(addr) + db.handleError(err) + return exist +} + +// Empty returns whether the given account is empty. +// +// Empty is defined according to EIP161 (balance = nonce = code = 0). +func (db *StateDB) Empty(addr gethCommon.Address) bool { + if !db.Exist(addr) { + return true + } + return db.GetNonce(addr) == 0 && + db.GetBalance(addr).Sign() == 0 && + bytes.Equal(db.GetCodeHash(addr).Bytes(), gethTypes.EmptyCodeHash.Bytes()) +} + +// CreateAccount creates a new account for the given address +// it sets the nonce to zero +func (db *StateDB) CreateAccount(addr gethCommon.Address) { + // TODO: If a state object with the address already exists the balance is carried over to the new account. + // Carrying over the balance ensures that Ether doesn't disappear. + db.lastestView().CreateAccount(addr) +} + +// IsCreated returns true if address is recently created (context of a transaction) +func (db *StateDB) IsCreated(addr gethCommon.Address) bool { + return db.lastestView().IsCreated(addr) +} + +// Suicide flags the address for deletion. +// +// while this address exists for the rest of transaction, +// the balance of this account is return zero after the Suicide call. +func (db *StateDB) Suicide(addr gethCommon.Address) bool { + success, err := db.lastestView().Suicide(addr) + db.handleError(err) + return success +} + +// HasSuicided returns true if address is flaged with suicide. +func (db *StateDB) HasSuicided(addr gethCommon.Address) bool { + return db.lastestView().HasSuicided(addr) +} + +// SubBalance substitutes the amount from the balance of the given address +func (db *StateDB) SubBalance(addr gethCommon.Address, amount *big.Int) { + err := db.lastestView().SubBalance(addr, amount) + db.handleError(err) +} + +// SubBalance adds the amount from the balance of the given address +func (db *StateDB) AddBalance(addr gethCommon.Address, amount *big.Int) { + err := db.lastestView().AddBalance(addr, amount) + db.handleError(err) +} + +// GetBalance returns the balance of the given address +func (db *StateDB) GetBalance(addr gethCommon.Address) *big.Int { + bal, err := db.lastestView().GetBalance(addr) + db.handleError(err) + return bal +} + +// GetNonce returns the nonce of the given address +func (db *StateDB) GetNonce(addr gethCommon.Address) uint64 { + nonce, err := db.lastestView().GetNonce(addr) + db.handleError(err) + return nonce +} + +// SetNonce sets the nonce value for the given address +func (db *StateDB) SetNonce(addr gethCommon.Address, nonce uint64) { + db.lastestView().SetNonce(addr, nonce) +} + +func (db *StateDB) GetCodeHash(addr gethCommon.Address) gethCommon.Hash { + hash, err := db.lastestView().GetCodeHash(addr) + db.handleError(err) + return hash +} + +func (db *StateDB) GetCode(addr gethCommon.Address) []byte { + code, err := db.lastestView().GetCode(addr) + db.handleError(err) + return code +} + +func (db *StateDB) SetCode(addr gethCommon.Address, code []byte) { + db.lastestView().SetCode(addr, code) +} + +func (db *StateDB) GetCodeSize(addr gethCommon.Address) int { + codeSize, err := db.lastestView().GetCodeSize(addr) + db.handleError(err) + return codeSize +} + +func (db *StateDB) AddRefund(amount uint64) { + db.lastestView().AddRefund(amount) +} + +func (db *StateDB) SubRefund(amount uint64) { + db.lastestView().SubRefund(amount) +} + +func (db *StateDB) GetRefund() uint64 { + return db.lastestView().GetRefund() +} + +func (db *StateDB) GetCommittedState(addr gethCommon.Address, key gethCommon.Hash) gethCommon.Hash { + value, err := db.baseView.GetState(types.SlotAddress{Address: addr, Key: key}) + db.handleError(err) + return value +} + +func (db *StateDB) GetState(addr gethCommon.Address, key gethCommon.Hash) gethCommon.Hash { + state, err := db.lastestView().GetState(types.SlotAddress{Address: addr, Key: key}) + db.handleError(err) + return state +} + +func (db *StateDB) SetState(addr gethCommon.Address, key gethCommon.Hash, value gethCommon.Hash) { + db.lastestView().SetState(types.SlotAddress{Address: addr, Key: key}, value) +} + +func (db *StateDB) GetTransientState(addr gethCommon.Address, key gethCommon.Hash) gethCommon.Hash { + return db.lastestView().GetTransientState(types.SlotAddress{Address: addr, Key: key}) +} + +func (db *StateDB) SetTransientState(addr gethCommon.Address, key, value gethCommon.Hash) { + db.lastestView().SetTransientState(types.SlotAddress{Address: addr, Key: key}, value) +} + +func (db *StateDB) AddressInAccessList(addr gethCommon.Address) bool { + return db.lastestView().AddressInAccessList(addr) +} + +func (db *StateDB) SlotInAccessList(addr gethCommon.Address, key gethCommon.Hash) (addressOk bool, slotOk bool) { + return db.lastestView().SlotInAccessList(types.SlotAddress{Address: addr, Key: key}) +} + +// AddAddressToAccessList adds the given address to the access list. +func (db *StateDB) AddAddressToAccessList(addr gethCommon.Address) { + db.lastestView().AddAddressToAccessList(addr) +} + +// AddSlotToAccessList adds the given (address,slot) to the access list. +func (db *StateDB) AddSlotToAccessList(addr gethCommon.Address, key gethCommon.Hash) { + db.lastestView().AddSlotToAccessList(types.SlotAddress{Address: addr, Key: key}) +} + +func (db *StateDB) AddLog(log *gethTypes.Log) { + db.lastestView().AddLog(log) +} + +func (db *StateDB) AddPreimage(hash gethCommon.Hash, data []byte) { + db.lastestView().AddPreimage(hash, data) +} + +func (db *StateDB) RevertToSnapshot(index int) { + if index > len(db.views) { + db.dbErr = fmt.Errorf("invalid revert") + return + } + db.views = db.views[:index] +} + +func (db *StateDB) Snapshot() int { + newView := db.lastestView().NewChildView() + db.views = append(db.views, newView) + return len(db.views) - 1 +} + +func (db *StateDB) lastestView() *DeltaView { + return db.views[len(db.views)-1] +} + +func (db *StateDB) Logs( + blockHash gethCommon.Hash, + blockNumber uint64, + txHash gethCommon.Hash, + txIndex uint, +) []*gethTypes.Log { + allLogs := make([]*gethTypes.Log, 0) + for _, view := range db.views { + for _, log := range view.Logs() { + log.BlockNumber = blockNumber + log.BlockHash = blockHash + log.TxHash = txHash + log.TxIndex = txIndex + allLogs = append(allLogs, log) + } + } + return allLogs +} + +func (db *StateDB) Preimages() map[gethCommon.Hash][]byte { + preImages := make(map[gethCommon.Hash][]byte, 0) + for _, view := range db.views { + for k, v := range view.Preimages() { + preImages[k] = v + } + } + return preImages +} + +func (db *StateDB) Commit() error { + // return error if any has been acumulated + if db.dbErr != nil { + return db.dbErr + } + + var err error + + // iterate views and collect dirty addresses and slots + addresses := make(map[gethCommon.Address]interface{}) + slots := make(map[types.SlotAddress]interface{}) + for _, view := range db.views { + for key := range view.DirtyAddresses() { + addresses[key] = struct{}{} + } + for key := range view.DirtySlots() { + slots[key] = struct{}{} + } + } + + // sort addresses + sortedAddresses := make([]gethCommon.Address, 0, len(addresses)) + for addr := range addresses { + sortedAddresses = append(sortedAddresses, addr) + } + sort.Slice(sortedAddresses, + func(i, j int) bool { + return bytes.Compare(sortedAddresses[i][:], sortedAddresses[j][:]) < 0 + }) + + // update accounts + for _, addr := range sortedAddresses { + // TODO check if address is + if db.HasSuicided(addr) { + err = db.baseView.DeleteAccount(addr) + if err != nil { + return err + } + continue + } + if db.IsCreated(addr) { + err = db.baseView.CreateAccount( + addr, + db.GetBalance(addr), + db.GetNonce(addr), + db.GetCode(addr), + db.GetCodeHash(addr), + ) + if err != nil { + return err + } + continue + } + err = db.baseView.UpdateAccount( + addr, + db.GetBalance(addr), + db.GetNonce(addr), + db.GetCode(addr), + db.GetCodeHash(addr), + ) + if err != nil { + return err + } + } + + // sort slots + sortedSlots := make([]types.SlotAddress, 0, len(slots)) + for slot := range slots { + sortedSlots = append(sortedSlots, slot) + } + sort.Slice(sortedSlots, func(i, j int) bool { + comp := bytes.Compare(sortedSlots[i].Address[:], sortedSlots[j].Address[:]) + if comp == 0 { + return bytes.Compare(sortedSlots[i].Key[:], sortedSlots[j].Key[:]) < 0 + } + return comp < 0 + }) + + // update slots + for _, sk := range sortedSlots { + err = db.baseView.UpdateSlot( + sk, + db.GetState(sk.Address, sk.Key), + ) + if err != nil { + return err + } + } + + // don't purge views yet, people might call the logs etc + return db.baseView.Commit() +} + +// Error returns the memorized database failure occurred earlier. +func (s *StateDB) Error() error { + return s.dbErr +} + +// handleError capture the first non-nil error it is called with. +func (s *StateDB) handleError(err error) { + if err == nil { + return + } + + var atreeFatalError *atree.FatalError + // if is a atree fatal error or fvm fatal error (the second one captures external errors) + if stdErrors.As(err, &atreeFatalError) || errors.IsFailure(err) { + panic(types.NewFatalError(err)) + } + + // already no error is captured + if s.dbErr == nil { + s.dbErr = types.NewDatabaseError(err) + } +} + +func (db *StateDB) Prepare(rules gethParams.Rules, sender, coinbase gethCommon.Address, dest *gethCommon.Address, precompiles []gethCommon.Address, txAccesses gethTypes.AccessList) { + if rules.IsBerlin { + // no need for mutation + db.AddAddressToAccessList(sender) + + if dest != nil { + db.AddAddressToAccessList(*dest) + // If it's a create-tx, the destination will be added inside egethVM.create + } + for _, addr := range precompiles { + db.AddAddressToAccessList(addr) + } + for _, el := range txAccesses { + db.AddAddressToAccessList(el.Address) + for _, key := range el.StorageKeys { + db.AddSlotToAccessList(el.Address, key) + } + } + if rules.IsShanghai { // EIP-3651: warm coinbase + db.AddAddressToAccessList(coinbase) + } + } +} diff --git a/fvm/evm/emulator/state/stateDB_test.go b/fvm/evm/emulator/state/stateDB_test.go new file mode 100644 index 00000000000..c78aa054856 --- /dev/null +++ b/fvm/evm/emulator/state/stateDB_test.go @@ -0,0 +1,114 @@ +package state_test + +import ( + "math/big" + "testing" + + "github.com/onflow/flow-go/fvm/evm/emulator/state" + "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/model/flow" + "github.com/stretchr/testify/require" +) + +// Test revert +// TODO: add test for error handling +// Test logs + +func TestStateDB(t *testing.T) { + t.Parallel() + + t.Run("test commit functionality", func(t *testing.T) { + ledger := testutils.GetSimpleValueStore() + rootAddr := flow.Address{1, 2, 3, 4, 5, 6, 7, 8} + db, err := state.NewStateDB(ledger, rootAddr) + require.NoError(t, err) + + addr1 := testutils.RandomCommonAddress(t) + key1 := testutils.RandomCommonHash(t) + value1 := testutils.RandomCommonHash(t) + + db.CreateAccount(addr1) + require.NoError(t, db.Error()) + + db.AddBalance(addr1, big.NewInt(5)) + require.NoError(t, db.Error()) + + // should have code to be able to set state + db.SetCode(addr1, []byte{1, 2, 3}) + require.NoError(t, db.Error()) + + db.SetState(addr1, key1, value1) + + err = db.Commit() + require.NoError(t, err) + + // create a new db + db, err = state.NewStateDB(ledger, rootAddr) + require.NoError(t, err) + + bal := db.GetBalance(addr1) + require.NoError(t, db.Error()) + require.Equal(t, big.NewInt(5), bal) + + val := db.GetState(addr1, key1) + require.NoError(t, db.Error()) + require.Equal(t, value1, val) + }) + + t.Run("test snapshot and revert functionality", func(t *testing.T) { + ledger := testutils.GetSimpleValueStore() + rootAddr := flow.Address{1, 2, 3, 4, 5, 6, 7, 8} + db, err := state.NewStateDB(ledger, rootAddr) + require.NoError(t, err) + + addr1 := testutils.RandomCommonAddress(t) + require.False(t, db.Exist(addr1)) + require.NoError(t, db.Error()) + + snapshot1 := db.Snapshot() + require.Equal(t, 1, snapshot1) + + db.CreateAccount(addr1) + require.NoError(t, db.Error()) + + require.True(t, db.Exist(addr1)) + require.NoError(t, db.Error()) + + db.AddBalance(addr1, big.NewInt(5)) + require.NoError(t, db.Error()) + + bal := db.GetBalance(addr1) + require.NoError(t, db.Error()) + require.Equal(t, big.NewInt(5), bal) + + snapshot2 := db.Snapshot() + require.Equal(t, 2, snapshot2) + + db.AddBalance(addr1, big.NewInt(5)) + require.NoError(t, db.Error()) + + bal = db.GetBalance(addr1) + require.NoError(t, db.Error()) + require.Equal(t, big.NewInt(10), bal) + + // revert to snapshot 2 + db.RevertToSnapshot(snapshot2) + require.NoError(t, db.Error()) + + bal = db.GetBalance(addr1) + require.NoError(t, db.Error()) + require.Equal(t, big.NewInt(5), bal) + + // revert to snapshot 1 + db.RevertToSnapshot(snapshot1) + require.NoError(t, db.Error()) + + bal = db.GetBalance(addr1) + require.NoError(t, db.Error()) + require.Equal(t, big.NewInt(0), bal) + + // revert to an invalid snapshot + db.RevertToSnapshot(10) + require.Error(t, db.Error()) + }) +} From 6ec0fff5e3f756eb1b34979a8bdfc8799bc13879 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 13:26:04 -0800 Subject: [PATCH 11/72] use new statedb impl --- fvm/evm/emulator/emulator.go | 116 ++--- fvm/evm/emulator/emulator_test.go | 549 +++++++++++----------- fvm/evm/evm.go | 8 +- fvm/evm/handler/handler_benchmark_test.go | 18 +- fvm/evm/handler/handler_test.go | 18 +- fvm/evm/stdlib/contract.go | 94 +--- fvm/evm/stdlib/contract_test.go | 436 +---------------- fvm/evm/testutils/accounts.go | 7 +- fvm/evm/testutils/backend.go | 54 ++- fvm/evm/testutils/contract.go | 6 +- 10 files changed, 381 insertions(+), 925 deletions(-) diff --git a/fvm/evm/emulator/emulator.go b/fvm/evm/emulator/emulator.go index eef720912f7..1576a5531ad 100644 --- a/fvm/evm/emulator/emulator.go +++ b/fvm/evm/emulator/emulator.go @@ -5,28 +5,32 @@ import ( gethCommon "github.com/ethereum/go-ethereum/common" gethCore "github.com/ethereum/go-ethereum/core" - gethRawDB "github.com/ethereum/go-ethereum/core/rawdb" - gethState "github.com/ethereum/go-ethereum/core/state" gethTypes "github.com/ethereum/go-ethereum/core/types" gethVM "github.com/ethereum/go-ethereum/core/vm" gethCrypto "github.com/ethereum/go-ethereum/crypto" + "github.com/onflow/atree" + "github.com/onflow/flow-go/fvm/evm/emulator/state" "github.com/onflow/flow-go/fvm/evm/types" + "github.com/onflow/flow-go/model/flow" ) // Emulator handles operations against evm runtime type Emulator struct { - Database types.Database + rootAddr flow.Address + ledger atree.Ledger } var _ types.Emulator = &Emulator{} // NewEmulator constructs a new EVM Emulator func NewEmulator( - db types.Database, + ledger atree.Ledger, + rootAddr flow.Address, ) *Emulator { return &Emulator{ - Database: db, + rootAddr: rootAddr, + ledger: ledger, } } @@ -40,7 +44,7 @@ func newConfig(ctx types.BlockContext) *Config { // NewReadOnlyBlockView constructs a new readonly block view func (em *Emulator) NewReadOnlyBlockView(ctx types.BlockContext) (types.ReadOnlyBlockView, error) { - execState, err := newState(em.Database) + execState, err := state.NewStateDB(em.ledger, em.rootAddr) return &ReadOnlyBlockView{ state: execState, }, err @@ -51,14 +55,15 @@ func (em *Emulator) NewBlockView(ctx types.BlockContext) (types.BlockView, error cfg := newConfig(ctx) return &BlockView{ config: cfg, - database: em.Database, + rootAddr: em.rootAddr, + ledger: em.ledger, }, nil } // ReadOnlyBlockView provides a read only view of a block // could be used multiple times for queries type ReadOnlyBlockView struct { - state *gethState.StateDB + state types.StateDB } // BalanceOf returns the balance of the given address @@ -82,7 +87,8 @@ func (bv *ReadOnlyBlockView) NonceOf(address types.Address) (uint64, error) { // TODO: add block level commit (separation of trie commit to storage) type BlockView struct { config *Config - database types.Database + rootAddr flow.Address + ledger atree.Ledger } // DirectCall executes a direct call @@ -100,10 +106,7 @@ func (bl *BlockView) DirectCall(call *types.DirectCall) (*types.Result, error) { default: res, err = proc.run(call.Message(), types.DirectCallTxType) } - if err != nil { - return res, err - } - return res, bl.commit(res.StateRootHash) + return res, err } // RunTransaction runs an evm transaction @@ -126,15 +129,11 @@ func (bl *BlockView) RunTransaction( // update tx context origin proc.evm.TxContext.Origin = msg.From res, err := proc.run(msg, tx.Type()) - if err != nil { - return res, err - } - - return res, bl.commit(res.StateRootHash) + return res, err } func (bl *BlockView) newProcedure() (*procedure, error) { - execState, err := newState(bl.database) + execState, err := state.NewStateDB(bl.ledger, bl.rootAddr) if err != nil { return nil, err } @@ -148,51 +147,19 @@ func (bl *BlockView) newProcedure() (*procedure, error) { cfg.ChainConfig, cfg.EVMConfig, ), - state: execState, - database: bl.database, + state: execState, }, nil } -func (bl *BlockView) commit(rootHash gethCommon.Hash) error { - // commit atree changes back to the backend - err := bl.database.Commit(rootHash) - return handleCommitError(err) -} - type procedure struct { - config *Config - evm *gethVM.EVM - state *gethState.StateDB - database types.Database + config *Config + evm *gethVM.EVM + state types.StateDB } // commit commits the changes to the state. -func (proc *procedure) commit() (gethCommon.Hash, error) { - // commits the changes from the journal into the in memory trie. - // in the future if we want to move this to the block level we could use finalize - // to get the root hash - newRoot, err := proc.state.Commit(true) - if err != nil { - return gethTypes.EmptyRootHash, handleCommitError(err) - } - - // flush the trie to the lower level db - // the reason we have to do this, is the original database - // is designed to keep changes in memory until the state.Commit - // is called then the changes moves into the trie, but the trie - // would stay in memory for faster transaction execution. you - // have to explicitly ask the trie to commit to the underlying storage - err = proc.state.Database().TrieDB().Commit(newRoot, false) - if err != nil { - return gethTypes.EmptyRootHash, handleCommitError(err) - } - - // // remove the read registers (no history tracking) - // err = proc.database.DeleteAndCleanReadKey() - // if err != nil { - // return gethTypes.EmptyRootHash, types.NewFatalError(err) - // } - return newRoot, nil +func (proc *procedure) commit() error { + return proc.state.Commit() } func handleCommitError(err error) error { @@ -209,7 +176,6 @@ func handleCommitError(err error) error { } func (proc *procedure) mintTo(address types.Address, amount *big.Int) (*types.Result, error) { - var err error addr := address.ToCommon() res := &types.Result{ GasConsumed: proc.config.DirectCallBaseGasUsage, @@ -225,13 +191,10 @@ func (proc *procedure) mintTo(address types.Address, amount *big.Int) (*types.Re proc.state.AddBalance(addr, amount) // we don't need to increment any nonce, given the origin doesn't exist - res.StateRootHash, err = proc.commit() - - return res, err + return res, proc.commit() } func (proc *procedure) withdrawFrom(address types.Address, amount *big.Int) (*types.Result, error) { - var err error addr := address.ToCommon() res := &types.Result{ @@ -264,8 +227,7 @@ func (proc *procedure) withdrawFrom(address types.Address, amount *big.Int) (*ty nonce := proc.state.GetNonce(addr) proc.state.SetNonce(addr, nonce+1) - res.StateRootHash, err = proc.commit() - return res, err + return res, proc.commit() } func (proc *procedure) run(msg *gethCore.Message, txType uint8) (*types.Result, error) { @@ -299,31 +261,21 @@ func (proc *procedure) run(msg *gethCore.Message, txType uint8) (*types.Result, if msg.To == nil { res.DeployedContractAddress = types.NewAddress(gethCrypto.CreateAddress(msg.From, msg.Nonce)) } - res.Logs = proc.state.Logs() + res.Logs = proc.state.Logs( + // TODO pass proper hash values + gethCommon.Hash{}, + proc.config.BlockContext.BlockNumber.Uint64(), + gethCommon.Hash{}, + 0, + ) } else { res.Failed = true err = types.NewEVMExecutionError(execResult.Err) } } - var commitErr error - res.StateRootHash, commitErr = proc.commit() + commitErr := proc.commit() if commitErr != nil { return &res, commitErr } return &res, err } - -// Ramtin: this is the part of the code that we have to update if we hit performance problems -// the NewDatabase from the RawDB might have to change. -func newState(database types.Database) (*gethState.StateDB, error) { - root, err := database.GetRootHash() - if err != nil { - return nil, err - } - - return gethState.New(root, - gethState.NewDatabase( - gethRawDB.NewDatabase(database), - ), - nil) -} diff --git a/fvm/evm/emulator/emulator_test.go b/fvm/evm/emulator/emulator_test.go index caaf5853cac..fe6553a6cd0 100644 --- a/fvm/evm/emulator/emulator_test.go +++ b/fvm/evm/emulator/emulator_test.go @@ -1,7 +1,6 @@ package emulator_test import ( - "fmt" "math" "math/big" "testing" @@ -12,7 +11,6 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/fvm/evm/emulator" - "github.com/onflow/flow-go/fvm/evm/emulator/database" "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" @@ -21,18 +19,8 @@ import ( var blockNumber = big.NewInt(10) var defaultCtx = types.NewDefaultBlockContext(blockNumber.Uint64()) -func RunWithTestDB(t testing.TB, f func(types.Database)) { - testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { - testutils.RunWithTestFlowEVMRootAddress(t, backend, func(flowEVMRoot flow.Address) { - db, err := database.NewDatabase(backend, flowEVMRoot) - require.NoError(t, err) - f(db) - }) - }) -} - -func RunWithNewEmulator(t testing.TB, db types.Database, f func(*emulator.Emulator)) { - env := emulator.NewEmulator(db) +func RunWithNewEmulator(t testing.TB, backend *testutils.TestBackend, rootAddr flow.Address, f func(*emulator.Emulator)) { + env := emulator.NewEmulator(backend, rootAddr) f(env) } @@ -49,336 +37,341 @@ func RunWithNewReadOnlyBlockView(t testing.TB, em *emulator.Emulator, f func(blk } func TestNativeTokenBridging(t *testing.T) { - RunWithTestDB(t, func(db types.Database) { - originalBalance := big.NewInt(10000) - testAccount := types.NewAddressFromString("test") + testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { + testutils.RunWithTestFlowEVMRootAddress(t, backend, func(rootAddr flow.Address) { + originalBalance := big.NewInt(10000) + testAccount := types.NewAddressFromString("test") - t.Run("mint tokens to the first account", func(t *testing.T) { - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - RunWithNewBlockView(t, env, func(blk types.BlockView) { - res, err := blk.DirectCall(types.NewDepositCall(testAccount, originalBalance)) - require.NoError(t, err) - require.Equal(t, defaultCtx.DirectCallBaseGasUsage, res.GasConsumed) + t.Run("mint tokens to the first account", func(t *testing.T) { + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + RunWithNewBlockView(t, env, func(blk types.BlockView) { + res, err := blk.DirectCall(types.NewDepositCall(testAccount, originalBalance)) + require.NoError(t, err) + require.Equal(t, defaultCtx.DirectCallBaseGasUsage, res.GasConsumed) + }) }) }) - }) - t.Run("tokens withdraw", func(t *testing.T) { - amount := big.NewInt(1000) - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - RunWithNewReadOnlyBlockView(t, env, func(blk types.ReadOnlyBlockView) { - retBalance, err := blk.BalanceOf(testAccount) - require.NoError(t, err) - require.Equal(t, originalBalance, retBalance) + t.Run("tokens withdraw", func(t *testing.T) { + amount := big.NewInt(1000) + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + RunWithNewReadOnlyBlockView(t, env, func(blk types.ReadOnlyBlockView) { + retBalance, err := blk.BalanceOf(testAccount) + require.NoError(t, err) + require.Equal(t, originalBalance, retBalance) + }) }) - }) - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - RunWithNewBlockView(t, env, func(blk types.BlockView) { - res, err := blk.DirectCall(types.NewWithdrawCall(testAccount, amount)) - require.NoError(t, err) - require.Equal(t, defaultCtx.DirectCallBaseGasUsage, res.GasConsumed) + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + RunWithNewBlockView(t, env, func(blk types.BlockView) { + res, err := blk.DirectCall(types.NewWithdrawCall(testAccount, amount)) + require.NoError(t, err) + require.Equal(t, defaultCtx.DirectCallBaseGasUsage, res.GasConsumed) + }) }) - }) - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - RunWithNewReadOnlyBlockView(t, env, func(blk types.ReadOnlyBlockView) { - retBalance, err := blk.BalanceOf(testAccount) - require.NoError(t, err) - require.Equal(t, amount.Sub(originalBalance, amount), retBalance) + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + RunWithNewReadOnlyBlockView(t, env, func(blk types.ReadOnlyBlockView) { + retBalance, err := blk.BalanceOf(testAccount) + require.NoError(t, err) + require.Equal(t, amount.Sub(originalBalance, amount), retBalance) + }) }) }) }) }) - } func TestContractInteraction(t *testing.T) { - RunWithTestDB(t, func(db types.Database) { - - testContract := testutils.GetStorageTestContract(t) + testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { + testutils.RunWithTestFlowEVMRootAddress(t, backend, func(rootAddr flow.Address) { - testAccount := types.NewAddressFromString("test") - amount := big.NewInt(0).Mul(big.NewInt(1337), big.NewInt(gethParams.Ether)) - amountToBeTransfered := big.NewInt(0).Mul(big.NewInt(100), big.NewInt(gethParams.Ether)) + testContract := testutils.GetStorageTestContract(t) - // fund test account - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - RunWithNewBlockView(t, env, func(blk types.BlockView) { - _, err := blk.DirectCall(types.NewDepositCall(testAccount, amount)) - require.NoError(t, err) - }) - }) - - var contractAddr types.Address + testAccount := types.NewAddressFromString("test") + amount := big.NewInt(0).Mul(big.NewInt(1337), big.NewInt(gethParams.Ether)) + amountToBeTransfered := big.NewInt(0).Mul(big.NewInt(100), big.NewInt(gethParams.Ether)) - t.Run("deploy contract", func(t *testing.T) { - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { + // fund test account + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { RunWithNewBlockView(t, env, func(blk types.BlockView) { - res, err := blk.DirectCall( - types.NewDeployCall( - testAccount, - testContract.ByteCode, - math.MaxUint64, - amountToBeTransfered), - ) + _, err := blk.DirectCall(types.NewDepositCall(testAccount, amount)) require.NoError(t, err) - contractAddr = res.DeployedContractAddress - }) - RunWithNewReadOnlyBlockView(t, env, func(blk types.ReadOnlyBlockView) { - require.NotNil(t, contractAddr) - retCode, err := blk.CodeOf(contractAddr) - require.NoError(t, err) - require.NotEmpty(t, retCode) - - retBalance, err := blk.BalanceOf(contractAddr) - require.NoError(t, err) - require.Equal(t, amountToBeTransfered, retBalance) - - retBalance, err = blk.BalanceOf(testAccount) - require.NoError(t, err) - require.Equal(t, amount.Sub(amount, amountToBeTransfered), retBalance) }) }) - }) - - t.Run("call contract", func(t *testing.T) { - num := big.NewInt(10) - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - RunWithNewBlockView(t, env, func(blk types.BlockView) { - res, err := blk.DirectCall( - types.NewContractCall( - testAccount, - contractAddr, - testContract.MakeCallData(t, "store", num), - 1_000_000, - big.NewInt(0), // this should be zero because the contract doesn't have receiver - ), - ) - require.NoError(t, err) - require.GreaterOrEqual(t, res.GasConsumed, uint64(40_000)) + var contractAddr types.Address + + t.Run("deploy contract", func(t *testing.T) { + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + RunWithNewBlockView(t, env, func(blk types.BlockView) { + res, err := blk.DirectCall( + types.NewDeployCall( + testAccount, + testContract.ByteCode, + math.MaxUint64, + amountToBeTransfered), + ) + require.NoError(t, err) + contractAddr = res.DeployedContractAddress + }) + RunWithNewReadOnlyBlockView(t, env, func(blk types.ReadOnlyBlockView) { + require.NotNil(t, contractAddr) + retCode, err := blk.CodeOf(contractAddr) + require.NoError(t, err) + require.NotEmpty(t, retCode) + + retBalance, err := blk.BalanceOf(contractAddr) + require.NoError(t, err) + require.Equal(t, amountToBeTransfered, retBalance) + + retBalance, err = blk.BalanceOf(testAccount) + require.NoError(t, err) + require.Equal(t, amount.Sub(amount, amountToBeTransfered), retBalance) + }) }) }) - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - RunWithNewBlockView(t, env, func(blk types.BlockView) { - res, err := blk.DirectCall( - types.NewContractCall( - testAccount, - contractAddr, - testContract.MakeCallData(t, "retrieve"), - 1_000_000, - big.NewInt(0), // this should be zero because the contract doesn't have receiver - ), - ) - require.NoError(t, err) - - ret := new(big.Int).SetBytes(res.ReturnedValue) - require.Equal(t, num, ret) - require.GreaterOrEqual(t, res.GasConsumed, uint64(23_000)) + t.Run("call contract", func(t *testing.T) { + num := big.NewInt(10) + + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + RunWithNewBlockView(t, env, func(blk types.BlockView) { + res, err := blk.DirectCall( + types.NewContractCall( + testAccount, + contractAddr, + testContract.MakeCallData(t, "store", num), + 1_000_000, + big.NewInt(0), // this should be zero because the contract doesn't have receiver + ), + ) + require.NoError(t, err) + require.GreaterOrEqual(t, res.GasConsumed, uint64(40_000)) + }) }) - }) - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - RunWithNewBlockView(t, env, func(blk types.BlockView) { - res, err := blk.DirectCall( - types.NewContractCall( - testAccount, - contractAddr, - testContract.MakeCallData(t, "blockNumber"), - 1_000_000, - big.NewInt(0), // this should be zero because the contract doesn't have receiver - ), - ) - require.NoError(t, err) - - ret := new(big.Int).SetBytes(res.ReturnedValue) - require.Equal(t, blockNumber, ret) + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + RunWithNewBlockView(t, env, func(blk types.BlockView) { + res, err := blk.DirectCall( + types.NewContractCall( + testAccount, + contractAddr, + testContract.MakeCallData(t, "retrieve"), + 1_000_000, + big.NewInt(0), // this should be zero because the contract doesn't have receiver + ), + ) + require.NoError(t, err) + + ret := new(big.Int).SetBytes(res.ReturnedValue) + require.Equal(t, num, ret) + require.GreaterOrEqual(t, res.GasConsumed, uint64(23_000)) + }) }) - }) - - }) - t.Run("test sending transactions (happy case)", func(t *testing.T) { - account := testutils.GetTestEOAAccount(t, testutils.EOATestAccount1KeyHex) - fAddr := account.Address() - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - RunWithNewBlockView(t, env, func(blk types.BlockView) { - _, err := blk.DirectCall(types.NewDepositCall(fAddr, amount)) - require.NoError(t, err) + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + RunWithNewBlockView(t, env, func(blk types.BlockView) { + res, err := blk.DirectCall( + types.NewContractCall( + testAccount, + contractAddr, + testContract.MakeCallData(t, "blockNumber"), + 1_000_000, + big.NewInt(0), // this should be zero because the contract doesn't have receiver + ), + ) + require.NoError(t, err) + + ret := new(big.Int).SetBytes(res.ReturnedValue) + require.Equal(t, blockNumber, ret) + }) }) + }) - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - ctx := types.NewDefaultBlockContext(blockNumber.Uint64()) - ctx.GasFeeCollector = types.NewAddressFromString("coinbase") - coinbaseOrgBalance := gethCommon.Big1 - // small amount of money to create account - RunWithNewBlockView(t, env, func(blk types.BlockView) { - _, err := blk.DirectCall(types.NewDepositCall(ctx.GasFeeCollector, coinbaseOrgBalance)) - require.NoError(t, err) + t.Run("test sending transactions (happy case)", func(t *testing.T) { + account := testutils.GetTestEOAAccount(t, testutils.EOATestAccount1KeyHex) + fAddr := account.Address() + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + RunWithNewBlockView(t, env, func(blk types.BlockView) { + _, err := blk.DirectCall(types.NewDepositCall(fAddr, amount)) + require.NoError(t, err) + }) }) - blk, err := env.NewBlockView(ctx) - require.NoError(t, err) - tx := account.PrepareAndSignTx( - t, - testAccount.ToCommon(), // to - nil, // data - big.NewInt(1000), // amount - gethParams.TxGas, // gas limit - gethCommon.Big1, // gas fee - - ) - _, err = blk.RunTransaction(tx) - require.NoError(t, err) - - // check the balance of coinbase - RunWithNewReadOnlyBlockView(t, env, func(blk2 types.ReadOnlyBlockView) { - bal, err := blk2.BalanceOf(ctx.GasFeeCollector) + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + ctx := types.NewDefaultBlockContext(blockNumber.Uint64()) + ctx.GasFeeCollector = types.NewAddressFromString("coinbase") + coinbaseOrgBalance := gethCommon.Big1 + // small amount of money to create account + RunWithNewBlockView(t, env, func(blk types.BlockView) { + _, err := blk.DirectCall(types.NewDepositCall(ctx.GasFeeCollector, coinbaseOrgBalance)) + require.NoError(t, err) + }) + + blk, err := env.NewBlockView(ctx) require.NoError(t, err) - expected := gethParams.TxGas*gethCommon.Big1.Uint64() + gethCommon.Big1.Uint64() - require.Equal(t, expected, bal.Uint64()) + tx := account.PrepareAndSignTx( + t, + testAccount.ToCommon(), // to + nil, // data + big.NewInt(1000), // amount + gethParams.TxGas, // gas limit + gethCommon.Big1, // gas fee - nonce, err := blk2.NonceOf(fAddr) + ) + _, err = blk.RunTransaction(tx) require.NoError(t, err) - require.Equal(t, 1, int(nonce)) + + // check the balance of coinbase + RunWithNewReadOnlyBlockView(t, env, func(blk2 types.ReadOnlyBlockView) { + bal, err := blk2.BalanceOf(ctx.GasFeeCollector) + require.NoError(t, err) + expected := gethParams.TxGas*gethCommon.Big1.Uint64() + gethCommon.Big1.Uint64() + require.Equal(t, expected, bal.Uint64()) + + nonce, err := blk2.NonceOf(fAddr) + require.NoError(t, err) + require.Equal(t, 1, int(nonce)) + }) }) }) - }) - t.Run("test sending transactions (invalid nonce)", func(t *testing.T) { - account := testutils.GetTestEOAAccount(t, testutils.EOATestAccount1KeyHex) - fAddr := account.Address() - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - RunWithNewBlockView(t, env, func(blk types.BlockView) { - _, err := blk.DirectCall(types.NewDepositCall(fAddr, amount)) - require.NoError(t, err) + t.Run("test sending transactions (invalid nonce)", func(t *testing.T) { + account := testutils.GetTestEOAAccount(t, testutils.EOATestAccount1KeyHex) + fAddr := account.Address() + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + RunWithNewBlockView(t, env, func(blk types.BlockView) { + _, err := blk.DirectCall(types.NewDepositCall(fAddr, amount)) + require.NoError(t, err) + }) }) - }) - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - ctx := types.NewDefaultBlockContext(blockNumber.Uint64()) - blk, err := env.NewBlockView(ctx) - require.NoError(t, err) - tx := account.SignTx(t, - gethTypes.NewTransaction( - 100, // nonce - testAccount.ToCommon(), // to - big.NewInt(1000), // amount - gethParams.TxGas, // gas limit - gethCommon.Big1, // gas fee - nil, // data - ), - ) - _, err = blk.RunTransaction(tx) - require.Error(t, err) - require.True(t, types.IsEVMValidationError(err)) + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + ctx := types.NewDefaultBlockContext(blockNumber.Uint64()) + blk, err := env.NewBlockView(ctx) + require.NoError(t, err) + tx := account.SignTx(t, + gethTypes.NewTransaction( + 100, // nonce + testAccount.ToCommon(), // to + big.NewInt(1000), // amount + gethParams.TxGas, // gas limit + gethCommon.Big1, // gas fee + nil, // data + ), + ) + _, err = blk.RunTransaction(tx) + require.Error(t, err) + require.True(t, types.IsEVMValidationError(err)) + }) }) - }) - t.Run("test sending transactions (bad signature)", func(t *testing.T) { - RunWithNewEmulator(t, db, func(env *emulator.Emulator) { - ctx := types.NewDefaultBlockContext(blockNumber.Uint64()) - blk, err := env.NewBlockView(ctx) - require.NoError(t, err) - tx := gethTypes.NewTx(&gethTypes.LegacyTx{ - Nonce: 0, - GasPrice: gethCommon.Big1, - Gas: gethParams.TxGas, // gas limit - To: nil, // to - Value: big.NewInt(1000), // amount - Data: nil, // data - V: big.NewInt(1), - R: big.NewInt(2), - S: big.NewInt(3), + t.Run("test sending transactions (bad signature)", func(t *testing.T) { + RunWithNewEmulator(t, backend, rootAddr, func(env *emulator.Emulator) { + ctx := types.NewDefaultBlockContext(blockNumber.Uint64()) + blk, err := env.NewBlockView(ctx) + require.NoError(t, err) + tx := gethTypes.NewTx(&gethTypes.LegacyTx{ + Nonce: 0, + GasPrice: gethCommon.Big1, + Gas: gethParams.TxGas, // gas limit + To: nil, // to + Value: big.NewInt(1000), // amount + Data: nil, // data + V: big.NewInt(1), + R: big.NewInt(2), + S: big.NewInt(3), + }) + _, err = blk.RunTransaction(tx) + require.Error(t, err) + require.True(t, types.IsEVMValidationError(err)) }) - _, err = blk.RunTransaction(tx) - require.Error(t, err) - require.True(t, types.IsEVMValidationError(err)) }) - }) + }) }) } func TestTransfers(t *testing.T) { - RunWithTestDB(t, func(db types.Database) { - testAccount1 := types.NewAddressFromString("test1") - testAccount2 := types.NewAddressFromString("test2") - - amount := big.NewInt(0).Mul(big.NewInt(1337), big.NewInt(gethParams.Ether)) - amountToBeTransfered := big.NewInt(0).Mul(big.NewInt(100), big.NewInt(gethParams.Ether)) - - RunWithNewEmulator(t, db, func(em *emulator.Emulator) { - RunWithNewBlockView(t, em, func(blk types.BlockView) { - _, err := blk.DirectCall(types.NewDepositCall(testAccount1, amount)) - require.NoError(t, err) - }) - }) + testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { + testutils.RunWithTestFlowEVMRootAddress(t, backend, func(rootAddr flow.Address) { - RunWithNewEmulator(t, db, func(em *emulator.Emulator) { - RunWithNewBlockView(t, em, func(blk types.BlockView) { - _, err := blk.DirectCall(types.NewTransferCall(testAccount1, testAccount2, amountToBeTransfered)) - require.NoError(t, err) - }) - }) + testAccount1 := types.NewAddressFromString("test1") + testAccount2 := types.NewAddressFromString("test2") - RunWithNewEmulator(t, db, func(em *emulator.Emulator) { - RunWithNewReadOnlyBlockView(t, em, func(blk types.ReadOnlyBlockView) { - bal, err := blk.BalanceOf(testAccount2) - require.NoError(t, err) - require.Equal(t, amountToBeTransfered.Uint64(), bal.Uint64()) + amount := big.NewInt(0).Mul(big.NewInt(1337), big.NewInt(gethParams.Ether)) + amountToBeTransfered := big.NewInt(0).Mul(big.NewInt(100), big.NewInt(gethParams.Ether)) - bal, err = blk.BalanceOf(testAccount1) - require.NoError(t, err) - require.Equal(t, new(big.Int).Sub(amount, amountToBeTransfered).Uint64(), bal.Uint64()) + RunWithNewEmulator(t, backend, rootAddr, func(em *emulator.Emulator) { + RunWithNewBlockView(t, em, func(blk types.BlockView) { + _, err := blk.DirectCall(types.NewDepositCall(testAccount1, amount)) + require.NoError(t, err) + }) }) - }) - }) -} - -func TestDatabaseErrorHandling(t *testing.T) { - t.Run("test non-fatal db error handling", func(t *testing.T) { - db := &testutils.TestDatabase{ - GetRootHashFunc: func() (gethCommon.Hash, error) { - return gethTypes.EmptyRootHash, types.NewDatabaseError(fmt.Errorf("some non-fatal error")) - }, - } - - RunWithNewEmulator(t, db, func(em *emulator.Emulator) { - RunWithNewBlockView(t, em, func(blk types.BlockView) { - _, err := blk.DirectCall(types.NewDepositCall(types.EmptyAddress, big.NewInt(1))) - require.Error(t, err) - require.True(t, types.IsADatabaseError(err)) + RunWithNewEmulator(t, backend, rootAddr, func(em *emulator.Emulator) { + RunWithNewBlockView(t, em, func(blk types.BlockView) { + _, err := blk.DirectCall(types.NewTransferCall(testAccount1, testAccount2, amountToBeTransfered)) + require.NoError(t, err) + }) }) - }) - }) - t.Run("test fatal db error handling", func(t *testing.T) { - db := &testutils.TestDatabase{ - GetRootHashFunc: func() (gethCommon.Hash, error) { - return gethTypes.EmptyRootHash, types.NewFatalError(fmt.Errorf("some non-fatal error")) - }, - } + RunWithNewEmulator(t, backend, rootAddr, func(em *emulator.Emulator) { + RunWithNewReadOnlyBlockView(t, em, func(blk types.ReadOnlyBlockView) { + bal, err := blk.BalanceOf(testAccount2) + require.NoError(t, err) + require.Equal(t, amountToBeTransfered.Uint64(), bal.Uint64()) - RunWithNewEmulator(t, db, func(em *emulator.Emulator) { - RunWithNewBlockView(t, em, func(blk types.BlockView) { - _, err := blk.DirectCall(types.NewDepositCall(types.EmptyAddress, big.NewInt(1))) - require.Error(t, err) - require.True(t, types.IsAFatalError(err)) + bal, err = blk.BalanceOf(testAccount1) + require.NoError(t, err) + require.Equal(t, new(big.Int).Sub(amount, amountToBeTransfered).Uint64(), bal.Uint64()) + }) }) }) }) } +// TODO: fix me +// func TestDatabaseErrorHandling(t *testing.T) { + +// t.Run("test non-fatal db error handling", func(t *testing.T) { +// db := &testutils.TestDatabase{ +// GetRootHashFunc: func() (gethCommon.Hash, error) { +// return gethTypes.EmptyRootHash, types.NewDatabaseError(fmt.Errorf("some non-fatal error")) +// }, +// } + +// RunWithNewEmulator(t, backend, rootAddr, func(em *emulator.Emulator) { +// RunWithNewBlockView(t, em, func(blk types.BlockView) { +// _, err := blk.DirectCall(types.NewDepositCall(types.EmptyAddress, big.NewInt(1))) +// require.Error(t, err) +// require.True(t, types.IsADatabaseError(err)) +// }) +// }) +// }) + +// t.Run("test fatal db error handling", func(t *testing.T) { +// db := &testutils.TestDatabase{ +// GetRootHashFunc: func() (gethCommon.Hash, error) { +// return gethTypes.EmptyRootHash, types.NewFatalError(fmt.Errorf("some non-fatal error")) +// }, +// } + +// RunWithNewEmulator(t, db, func(em *emulator.Emulator) { +// RunWithNewBlockView(t, em, func(blk types.BlockView) { +// _, err := blk.DirectCall(types.NewDepositCall(types.EmptyAddress, big.NewInt(1))) +// require.Error(t, err) +// require.True(t, types.IsAFatalError(err)) +// }) +// }) +// }) +// } + func TestStorageNoSideEffect(t *testing.T) { t.Skip("we need to fix this issue ") testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { testutils.RunWithTestFlowEVMRootAddress(t, backend, func(flowEVMRoot flow.Address) { - db, err := database.NewDatabase(backend, flowEVMRoot) - require.NoError(t, err) - - em := emulator.NewEmulator(db) + var err error + em := emulator.NewEmulator(backend, flowEVMRoot) testAccount := types.NewAddressFromString("test") amount := big.NewInt(100) diff --git a/fvm/evm/evm.go b/fvm/evm/evm.go index a44b8be4552..800130bc86e 100644 --- a/fvm/evm/evm.go +++ b/fvm/evm/evm.go @@ -5,7 +5,6 @@ import ( "github.com/onflow/cadence/runtime/common" evm "github.com/onflow/flow-go/fvm/evm/emulator" - "github.com/onflow/flow-go/fvm/evm/emulator/database" "github.com/onflow/flow-go/fvm/evm/handler" "github.com/onflow/flow-go/fvm/evm/stdlib" "github.com/onflow/flow-go/fvm/evm/types" @@ -31,12 +30,7 @@ func SetupEnvironment( return err } - db, err := database.NewDatabase(backend, evmRootAddress) - if err != nil { - return err - } - - em := evm.NewEmulator(db) + em := evm.NewEmulator(backend, evmRootAddress) bs, err := handler.NewBlockStore(backend, evmRootAddress) if err != nil { diff --git a/fvm/evm/handler/handler_benchmark_test.go b/fvm/evm/handler/handler_benchmark_test.go index 73f0f0ed59d..6614819ef22 100644 --- a/fvm/evm/handler/handler_benchmark_test.go +++ b/fvm/evm/handler/handler_benchmark_test.go @@ -10,7 +10,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -func BenchmarkStorage(b *testing.B) { benchmarkStorageGrowth(b, 100, 100) } +func BenchmarkStorage(b *testing.B) { benchmarkStorageGrowth(b, 100, 1_000_000) } // benchmark func benchmarkStorageGrowth(b *testing.B, accountCount, setupKittyCount int) { @@ -21,12 +21,11 @@ func benchmarkStorageGrowth(b *testing.B, accountCount, setupKittyCount int) { backend, rootAddr, func(tc *testutils.TestContract) { - db, handler := SetupHandler(b, backend, rootAddr) - numOfAccounts := 100000 - accounts := make([]types.Account, numOfAccounts) + handler := SetupHandler(b, backend, rootAddr) + accounts := make([]types.Account, accountCount) // setup several of accounts // note that trie growth is the function of number of accounts - for i := 0; i < numOfAccounts; i++ { + for i := 0; i < accountCount; i++ { account := handler.AccountByAddress(handler.AllocateAddress(), true) account.Deposit(types.NewFlowTokenVault(types.Balance(100))) accounts[i] = account @@ -54,12 +53,9 @@ func benchmarkStorageGrowth(b *testing.B, accountCount, setupKittyCount int) { ) require.Equal(b, 2, len(backend.Events())) backend.DropEvents() // this would make things lighter + backend.ResetStats() // reset stats } - // measure the impact of mint after the setup phase - db.ResetReporter() - db.DropCache() - accounts[0].Call( tc.DeployedAt, tc.MakeCallData(b, @@ -73,8 +69,8 @@ func benchmarkStorageGrowth(b *testing.B, accountCount, setupKittyCount int) { types.Balance(0), ) - b.ReportMetric(float64(db.BytesRetrieved()), "bytes_read") - b.ReportMetric(float64(db.BytesStored()), "bytes_written") + b.ReportMetric(float64(backend.TotalBytesRead()), "bytes_read") + b.ReportMetric(float64(backend.TotalBytesWritten()), "bytes_written") b.ReportMetric(float64(backend.TotalStorageSize()), "total_storage_size") }) }) diff --git a/fvm/evm/handler/handler_test.go b/fvm/evm/handler/handler_test.go index db78cf2d827..5f87664202e 100644 --- a/fvm/evm/handler/handler_test.go +++ b/fvm/evm/handler/handler_test.go @@ -23,7 +23,6 @@ import ( "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/evm/emulator" - "github.com/onflow/flow-go/fvm/evm/emulator/database" "github.com/onflow/flow-go/fvm/evm/handler" "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/fvm/evm/types" @@ -194,7 +193,7 @@ func TestHandler_TransactionRun(t *testing.T) { testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { testutils.RunWithTestFlowEVMRootAddress(t, backend, func(rootAddr flow.Address) { - _, handler := SetupHandler(t, backend, rootAddr) + handler := SetupHandler(t, backend, rootAddr) eoa := testutils.GetTestEOAAccount(t, testutils.EOATestAccount1KeyHex) @@ -250,7 +249,7 @@ func TestHandler_OpsWithoutEmulator(t *testing.T) { testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { testutils.RunWithTestFlowEVMRootAddress(t, backend, func(rootAddr flow.Address) { - _, handler := SetupHandler(t, backend, rootAddr) + handler := SetupHandler(t, backend, rootAddr) // test call last executed block without initialization b := handler.LastExecutedBlock() @@ -300,7 +299,7 @@ func TestHandler_BridgedAccount(t *testing.T) { testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { testutils.RunWithTestFlowEVMRootAddress(t, backend, func(rootAddr flow.Address) { - _, handler := SetupHandler(t, backend, rootAddr) + handler := SetupHandler(t, backend, rootAddr) foa := handler.AccountByAddress(handler.AllocateAddress(), true) require.NotNil(t, foa) @@ -474,7 +473,7 @@ func TestHandler_BridgedAccount(t *testing.T) { // TODO update this test with events, gas metering, etc testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { testutils.RunWithTestFlowEVMRootAddress(t, backend, func(rootAddr flow.Address) { - _, handler := SetupHandler(t, backend, rootAddr) + handler := SetupHandler(t, backend, rootAddr) foa := handler.AccountByAddress(handler.AllocateAddress(), true) require.NotNil(t, foa) @@ -533,18 +532,15 @@ func assertPanic(t *testing.T, check checkError, f func()) { f() } -func SetupHandler(t testing.TB, backend types.Backend, rootAddr flow.Address) (*database.Database, *handler.ContractHandler) { +func SetupHandler(t testing.TB, backend types.Backend, rootAddr flow.Address) *handler.ContractHandler { bs, err := handler.NewBlockStore(backend, rootAddr) require.NoError(t, err) aa, err := handler.NewAddressAllocator(backend, rootAddr) require.NoError(t, err) - db, err := database.NewDatabase(backend, rootAddr) - require.NoError(t, err) - - emulator := emulator.NewEmulator(db) + emulator := emulator.NewEmulator(backend, rootAddr) handler := handler.NewContractHandler(flowTokenAddress, bs, aa, backend, emulator) - return db, handler + return handler } diff --git a/fvm/evm/stdlib/contract.go b/fvm/evm/stdlib/contract.go index d7fe8401d59..f39487982f4 100644 --- a/fvm/evm/stdlib/contract.go +++ b/fvm/evm/stdlib/contract.go @@ -3,7 +3,6 @@ package stdlib import ( _ "embed" "fmt" - "math" "math/big" "reflect" "regexp" @@ -40,8 +39,6 @@ const ContractName = "EVM" const evmAddressTypeBytesFieldName = "bytes" const evmAddressTypeQualifiedIdentifier = "EVM.EVMAddress" -const abiEncodingByteSize = 32 - var EVMTransactionBytesCadenceType = cadence.NewVariableSizedArrayType(cadence.TheUInt8Type) var evmTransactionBytesType = sema.NewVariableSizedType(nil, sema.UInt8Type) @@ -100,77 +97,6 @@ func (e abiDecodingError) Error() string { return b.String() } -func reportABIEncodingComputation( - inter *interpreter.Interpreter, - values *interpreter.ArrayValue, - evmAddressTypeID common.TypeID, - reportComputation func(intensity uint), -) { - values.Iterate(inter, func(element interpreter.Value) (resume bool) { - switch value := element.(type) { - case *interpreter.StringValue: - // Dynamic variables, such as strings, are encoded - // in 2+ chunks of 32 bytes. The first chunk contains - // the index where information for the string begin, - // the second chunk contains the number of bytes the - // string occupies, and the third chunk contains the - // value of the string itself. - computation := uint(2 * abiEncodingByteSize) - stringLength := len(value.Str) - chunks := math.Ceil(float64(stringLength) / float64(abiEncodingByteSize)) - computation += uint(chunks * abiEncodingByteSize) - reportComputation(computation) - - case interpreter.BoolValue, - interpreter.UInt8Value, - interpreter.UInt16Value, - interpreter.UInt32Value, - interpreter.UInt64Value, - interpreter.UInt128Value, - interpreter.UInt256Value, - interpreter.Int8Value, - interpreter.Int16Value, - interpreter.Int32Value, - interpreter.Int64Value, - interpreter.Int128Value, - interpreter.Int256Value: - - // Numeric and bool variables are also static variables - // with a fixed size of 32 bytes. - reportComputation(abiEncodingByteSize) - - case *interpreter.CompositeValue: - if value.TypeID() == evmAddressTypeID { - // EVM addresses are static variables with a fixed - // size of 32 bytes. - reportComputation(abiEncodingByteSize) - } else { - panic(abiEncodingError{ - Type: value.StaticType(inter), - }) - } - case *interpreter.ArrayValue: - // Dynamic variables, such as arrays & slices, are encoded - // in 2+ chunks of 32 bytes. The first chunk contains - // the index where information for the array begin, - // the second chunk contains the number of bytes the - // array occupies, and the third chunk contains the - // values of the array itself. - computation := uint(2 * abiEncodingByteSize) - reportComputation(computation) - reportABIEncodingComputation(inter, value, evmAddressTypeID, reportComputation) - - default: - panic(abiEncodingError{ - Type: element.StaticType(inter), - }) - } - - // continue iteration - return true - }) -} - // EVM.encodeABI const internalEVMTypeEncodeABIFunctionName = "encodeABI" @@ -209,15 +135,6 @@ func newInternalEVMTypeEncodeABIFunction( panic(errors.NewUnreachableError()) } - reportABIEncodingComputation( - inter, - valuesArray, - evmAddressTypeID, - func(intensity uint) { - inter.ReportComputation(environment.ComputationKindEVMEncodeABI, intensity) - }, - ) - size := valuesArray.Count() values := make([]any, 0, size) @@ -242,12 +159,19 @@ func newInternalEVMTypeEncodeABIFunction( return true }) - encodedValues, err := arguments.Pack(values...) + hexData, err := arguments.Pack(values...) if err != nil { panic(abiEncodingError{}) } - return interpreter.ByteSliceToByteArrayValue(inter, encodedValues) + encodedValues := interpreter.ByteSliceToByteArrayValue(inter, hexData) + + invocation.Interpreter.ReportComputation( + environment.ComputationKindEVMEncodeABI, + uint(encodedValues.Count()), + ) + + return encodedValues }, ) } diff --git a/fvm/evm/stdlib/contract_test.go b/fvm/evm/stdlib/contract_test.go index a10f90913de..8dbe73ac6b9 100644 --- a/fvm/evm/stdlib/contract_test.go +++ b/fvm/evm/stdlib/contract_test.go @@ -14,7 +14,6 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/fvm/blueprints" - "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/evm/stdlib" . "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/fvm/evm/types" @@ -263,7 +262,6 @@ func TestEVMEncodeABI(t *testing.T) { accountCodes := map[common.Location][]byte{} var events []cadence.Event - computation := uint(0) runtimeInterface := &TestRuntimeInterface{ Storage: NewTestLedger(nil, nil), OnGetSigningAccounts: func() ([]runtime.Address, error) { @@ -285,12 +283,6 @@ func TestEVMEncodeABI(t *testing.T) { OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { return json.Decode(nil, b) }, - OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { - if compKind == environment.ComputationKindEVMEncodeABI { - computation += intensity - } - return nil - }, } nextTransactionLocation := NewTransactionLocationGenerator() @@ -348,311 +340,6 @@ func TestEVMEncodeABI(t *testing.T) { encodedABI, result, ) - assert.Equal(t, computation, uint(len(cdcBytes))) -} - -func TestEVMEncodeABIComputation(t *testing.T) { - - t.Parallel() - - handler := &testContractHandler{} - - contractsAddress := flow.BytesToAddress([]byte{0x1}) - - transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) - scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) - - rt := runtime.NewInterpreterRuntime(runtime.Config{}) - - script := []byte(` - import EVM from 0x1 - - access(all) - fun main(): [UInt8] { - let address = EVM.EVMAddress( - bytes: [ - 122, 88, 192, 190, 114, 190, 33, 139, 65, 198, - 8, 183, 254, 124, 91, 182, 48, 115, 108, 113 - ] - ) - let arr: [UInt8] = [1, 2, 3, 4, 5] - - return EVM.encodeABI([ - "John Doe", - UInt64(33), - false, - address, - [arr], - ["one", "two", "three"] - ]) - } - `) - - accountCodes := map[common.Location][]byte{} - var events []cadence.Event - - computation := uint(0) - runtimeInterface := &TestRuntimeInterface{ - Storage: NewTestLedger(nil, nil), - OnGetSigningAccounts: func() ([]runtime.Address, error) { - return []runtime.Address{runtime.Address(contractsAddress)}, nil - }, - OnResolveLocation: SingleIdentifierLocationResolver(t), - OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { - accountCodes[location] = code - return nil - }, - OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { - code = accountCodes[location] - return code, nil - }, - OnEmitEvent: func(event cadence.Event) error { - events = append(events, event) - return nil - }, - OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { - return json.Decode(nil, b) - }, - OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { - if compKind == environment.ComputationKindEVMEncodeABI { - computation += intensity - } - return nil - }, - } - - nextTransactionLocation := NewTransactionLocationGenerator() - nextScriptLocation := NewScriptLocationGenerator() - - // Deploy contracts - - deployContracts( - t, - rt, - contractsAddress, - runtimeInterface, - transactionEnvironment, - nextTransactionLocation, - ) - - // Run script - - result, err := rt.ExecuteScript( - runtime.Script{ - Source: script, - Arguments: [][]byte{}, - }, - runtime.Context{ - Interface: runtimeInterface, - Environment: scriptEnvironment, - Location: nextScriptLocation(), - }, - ) - require.NoError(t, err) - - cdcBytes, ok := result.(cadence.Array) - require.True(t, ok) - // computation & len(cdcBytes.Values) is equal to 832 - assert.Equal(t, computation, uint(len(cdcBytes.Values))) -} - -func TestEVMEncodeABIComputationEmptyDynamicVariables(t *testing.T) { - - t.Parallel() - - handler := &testContractHandler{} - - contractsAddress := flow.BytesToAddress([]byte{0x1}) - - transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) - scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) - - rt := runtime.NewInterpreterRuntime(runtime.Config{}) - - script := []byte(` - import EVM from 0x1 - - access(all) - fun main(): [UInt8] { - return EVM.encodeABI([ - "", - [[""], [] as [String]], - [] as [UInt8], - ["", "", ""] - ]) - } - `) - - accountCodes := map[common.Location][]byte{} - var events []cadence.Event - - computation := uint(0) - runtimeInterface := &TestRuntimeInterface{ - Storage: NewTestLedger(nil, nil), - OnGetSigningAccounts: func() ([]runtime.Address, error) { - return []runtime.Address{runtime.Address(contractsAddress)}, nil - }, - OnResolveLocation: SingleIdentifierLocationResolver(t), - OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { - accountCodes[location] = code - return nil - }, - OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { - code = accountCodes[location] - return code, nil - }, - OnEmitEvent: func(event cadence.Event) error { - events = append(events, event) - return nil - }, - OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { - return json.Decode(nil, b) - }, - OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { - if compKind == environment.ComputationKindEVMEncodeABI { - computation += intensity - } - return nil - }, - } - - nextTransactionLocation := NewTransactionLocationGenerator() - nextScriptLocation := NewScriptLocationGenerator() - - // Deploy contracts - - deployContracts( - t, - rt, - contractsAddress, - runtimeInterface, - transactionEnvironment, - nextTransactionLocation, - ) - - // Run script - - result, err := rt.ExecuteScript( - runtime.Script{ - Source: script, - Arguments: [][]byte{}, - }, - runtime.Context{ - Interface: runtimeInterface, - Environment: scriptEnvironment, - Location: nextScriptLocation(), - }, - ) - require.NoError(t, err) - - cdcBytes, ok := result.(cadence.Array) - require.True(t, ok) - // computation & len(cdcBytes.Values) is equal to 832 - assert.Equal(t, computation, uint(len(cdcBytes.Values))) -} - -func TestEVMEncodeABIComputationDynamicVariablesAboveChunkSize(t *testing.T) { - - t.Parallel() - - handler := &testContractHandler{} - - contractsAddress := flow.BytesToAddress([]byte{0x1}) - - transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) - scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) - - rt := runtime.NewInterpreterRuntime(runtime.Config{}) - - script := []byte(` - import EVM from 0x1 - - access(all) - fun main(): [UInt8] { - let str = "abcdefghijklmnopqrstuvwxyz" - let arr: [UInt64] = [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, - 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53 - ] - - return EVM.encodeABI([ - str, - str.concat(str).concat(str), - [[str]], - arr, - [arr], - arr.concat(arr).concat(arr) - ]) - } - `) - - accountCodes := map[common.Location][]byte{} - var events []cadence.Event - - computation := uint(0) - runtimeInterface := &TestRuntimeInterface{ - Storage: NewTestLedger(nil, nil), - OnGetSigningAccounts: func() ([]runtime.Address, error) { - return []runtime.Address{runtime.Address(contractsAddress)}, nil - }, - OnResolveLocation: SingleIdentifierLocationResolver(t), - OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { - accountCodes[location] = code - return nil - }, - OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { - code = accountCodes[location] - return code, nil - }, - OnEmitEvent: func(event cadence.Event) error { - events = append(events, event) - return nil - }, - OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { - return json.Decode(nil, b) - }, - OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { - if compKind == environment.ComputationKindEVMEncodeABI { - computation += intensity - } - return nil - }, - } - - nextTransactionLocation := NewTransactionLocationGenerator() - nextScriptLocation := NewScriptLocationGenerator() - - // Deploy contracts - - deployContracts( - t, - rt, - contractsAddress, - runtimeInterface, - transactionEnvironment, - nextTransactionLocation, - ) - - // Run script - - result, err := rt.ExecuteScript( - runtime.Script{ - Source: script, - Arguments: [][]byte{}, - }, - runtime.Context{ - Interface: runtimeInterface, - Environment: scriptEnvironment, - Location: nextScriptLocation(), - }, - ) - require.NoError(t, err) - - cdcBytes, ok := result.(cadence.Array) - require.True(t, ok) - // computation & len(cdcBytes.Values) is equal to 832 - assert.Equal(t, computation, uint(len(cdcBytes.Values))) } func TestEVMDecodeABI(t *testing.T) { @@ -688,7 +375,6 @@ func TestEVMDecodeABI(t *testing.T) { accountCodes := map[common.Location][]byte{} var events []cadence.Event - computation := uint(0) runtimeInterface := &TestRuntimeInterface{ Storage: NewTestLedger(nil, nil), OnGetSigningAccounts: func() ([]runtime.Address, error) { @@ -710,12 +396,6 @@ func TestEVMDecodeABI(t *testing.T) { OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { return json.Decode(nil, b) }, - OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { - if compKind == environment.ComputationKindEVMDecodeABI { - computation += intensity - } - return nil - }, } nextTransactionLocation := NewTransactionLocationGenerator() @@ -771,120 +451,6 @@ func TestEVMDecodeABI(t *testing.T) { require.NoError(t, err) assert.Equal(t, cadence.NewBool(true), result) - assert.Equal(t, computation, uint(len(cdcBytes))) -} - -func TestEVMDecodeABIComputation(t *testing.T) { - - t.Parallel() - - handler := &testContractHandler{} - - contractsAddress := flow.BytesToAddress([]byte{0x1}) - - transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) - scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) - - rt := runtime.NewInterpreterRuntime(runtime.Config{}) - - script := []byte(` - import EVM from 0x1 - - access(all) - fun main(): [UInt8] { - let address = EVM.EVMAddress( - bytes: [ - 122, 88, 192, 190, 114, 190, 33, 139, 65, 198, - 8, 183, 254, 124, 91, 182, 48, 115, 108, 113 - ] - ) - let arr: [UInt8] = [1, 2, 3, 4, 5] - - let data = EVM.encodeABI([ - "John Doe", - UInt64(33), - true, - address, - [arr], - ["one", "two", "three"] - ]) - - let types = [ - Type(), Type(), Type(), Type(), - Type<[[UInt8]]>(), Type<[String]>() - ] - let values = EVM.decodeABI(types: types, data: data) - - return data - } - `) - - accountCodes := map[common.Location][]byte{} - var events []cadence.Event - - computation := uint(0) - runtimeInterface := &TestRuntimeInterface{ - Storage: NewTestLedger(nil, nil), - OnGetSigningAccounts: func() ([]runtime.Address, error) { - return []runtime.Address{runtime.Address(contractsAddress)}, nil - }, - OnResolveLocation: SingleIdentifierLocationResolver(t), - OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { - accountCodes[location] = code - return nil - }, - OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { - code = accountCodes[location] - return code, nil - }, - OnEmitEvent: func(event cadence.Event) error { - events = append(events, event) - return nil - }, - OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { - return json.Decode(nil, b) - }, - OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { - if compKind == environment.ComputationKindEVMDecodeABI { - computation += intensity - } - return nil - }, - } - - nextTransactionLocation := NewTransactionLocationGenerator() - nextScriptLocation := NewScriptLocationGenerator() - - // Deploy contracts - - deployContracts( - t, - rt, - contractsAddress, - runtimeInterface, - transactionEnvironment, - nextTransactionLocation, - ) - - // Run script - - result, err := rt.ExecuteScript( - runtime.Script{ - Source: script, - Arguments: [][]byte{}, - }, - runtime.Context{ - Interface: runtimeInterface, - Environment: scriptEnvironment, - Location: nextScriptLocation(), - }, - ) - require.NoError(t, err) - - cdcBytes, ok := result.(cadence.Array) - require.True(t, ok) - // computation & len(cdcBytes.Values) is equal to 832 - assert.Equal(t, computation, uint(len(cdcBytes.Values))) } func TestEVMEncodeDecodeABIRoundtrip(t *testing.T) { @@ -1503,7 +1069,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { assert.ErrorContains( t, err, - "failed to ABI encode value of type Character", + "failed to ABI encode value of type [Character]", ) }) diff --git a/fvm/evm/testutils/accounts.go b/fvm/evm/testutils/accounts.go index 34487b8e91f..8df4b712508 100644 --- a/fvm/evm/testutils/accounts.go +++ b/fvm/evm/testutils/accounts.go @@ -16,7 +16,6 @@ import ( "github.com/onflow/atree" "github.com/onflow/flow-go/fvm/evm/emulator" - "github.com/onflow/flow-go/fvm/evm/emulator/database" "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" ) @@ -126,11 +125,7 @@ func FundAndGetEOATestAccount(t testing.TB, led atree.Ledger, flowEVMRootAddress account := GetTestEOAAccount(t, EOATestAccount1KeyHex) // fund account - db, err := database.NewDatabase(led, flowEVMRootAddress) - require.NoError(t, err) - - e := emulator.NewEmulator(db) - require.NoError(t, err) + e := emulator.NewEmulator(led, flowEVMRootAddress) blk, err := e.NewBlockView(types.NewDefaultBlockContext(2)) require.NoError(t, err) diff --git a/fvm/evm/testutils/backend.go b/fvm/evm/testutils/backend.go index 308b46f41e2..fc08d84aad8 100644 --- a/fvm/evm/testutils/backend.go +++ b/fvm/evm/testutils/backend.go @@ -52,24 +52,34 @@ func fullKey(owner, key []byte) string { func GetSimpleValueStore() *TestValueStore { data := make(map[string][]byte) allocator := make(map[string]uint64) - + bytesRead := 0 + bytesWritten := 0 return &TestValueStore{ GetValueFunc: func(owner, key []byte) ([]byte, error) { - return data[fullKey(owner, key)], nil + fk := fullKey(owner, key) + value := data[fk] + bytesRead += len(fk) + len(value) + return value, nil }, SetValueFunc: func(owner, key, value []byte) error { - data[fullKey(owner, key)] = value + fk := fullKey(owner, key) + data[fk] = value + bytesWritten += len(fk) + len(value) return nil }, ValueExistsFunc: func(owner, key []byte) (bool, error) { - return len(data[fullKey(owner, key)]) > 0, nil - + fk := fullKey(owner, key) + value := data[fk] + bytesRead += len(fk) + len(value) + return len(value) > 0, nil }, AllocateStorageIndexFunc: func(owner []byte) (atree.StorageIndex, error) { index := allocator[string(owner)] var data [8]byte allocator[string(owner)] = index + 1 binary.BigEndian.PutUint64(data[:], index) + bytesRead += len(owner) + 8 + bytesWritten += len(owner) + 8 return atree.StorageIndex(data), nil }, TotalStorageSizeFunc: func() int { @@ -82,9 +92,19 @@ func GetSimpleValueStore() *TestValueStore { } return size }, + TotalBytesReadFunc: func() int { + return bytesRead + }, + TotalBytesWrittenFunc: func() int { + return bytesWritten + }, TotalStorageItemsFunc: func() int { return len(maps.Keys(data)) + len(maps.Keys(allocator)) }, + ResetStatsFunc: func() { + bytesRead = 0 + bytesWritten = 0 + }, } } @@ -155,7 +175,10 @@ type TestValueStore struct { ValueExistsFunc func(owner, key []byte) (bool, error) AllocateStorageIndexFunc func(owner []byte) (atree.StorageIndex, error) TotalStorageSizeFunc func() int + TotalBytesReadFunc func() int + TotalBytesWrittenFunc func() int TotalStorageItemsFunc func() int + ResetStatsFunc func() } var _ environment.ValueStore = &TestValueStore{} @@ -188,6 +211,20 @@ func (vs *TestValueStore) AllocateStorageIndex(owner []byte) (atree.StorageIndex return vs.AllocateStorageIndexFunc(owner) } +func (vs *TestValueStore) TotalBytesRead() int { + if vs.TotalBytesReadFunc == nil { + panic("method not set") + } + return vs.TotalBytesReadFunc() +} + +func (vs *TestValueStore) TotalBytesWritten() int { + if vs.TotalBytesWrittenFunc == nil { + panic("method not set") + } + return vs.TotalBytesWrittenFunc() +} + func (vs *TestValueStore) TotalStorageSize() int { if vs.TotalStorageSizeFunc == nil { panic("method not set") @@ -202,6 +239,13 @@ func (vs *TestValueStore) TotalStorageItems() int { return vs.TotalStorageItemsFunc() } +func (vs *TestValueStore) ResetStats() { + if vs.ResetStatsFunc == nil { + panic("method not set") + } + vs.ResetStatsFunc() +} + type testMeter struct { meterComputation func(common.ComputationKind, uint) error hasComputationCapacity func(common.ComputationKind, uint) bool diff --git a/fvm/evm/testutils/contract.go b/fvm/evm/testutils/contract.go index 78316f44cd2..f67cced4c94 100644 --- a/fvm/evm/testutils/contract.go +++ b/fvm/evm/testutils/contract.go @@ -14,7 +14,6 @@ import ( "github.com/onflow/atree" "github.com/onflow/flow-go/fvm/evm/emulator" - "github.com/onflow/flow-go/fvm/evm/emulator/database" "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" ) @@ -387,10 +386,7 @@ func RunWithDeployedContract(t testing.TB, tc *TestContract, led atree.Ledger, f func DeployContract(t testing.TB, tc *TestContract, led atree.Ledger, flowEVMRootAddress flow.Address) { // deploy contract - db, err := database.NewDatabase(led, flowEVMRootAddress) - require.NoError(t, err) - - e := emulator.NewEmulator(db) + e := emulator.NewEmulator(led, flowEVMRootAddress) blk, err := e.NewBlockView(types.NewDefaultBlockContext(2)) require.NoError(t, err) From cc4f9db6fd491d8bd590b01d82c9f5fb5bf894d5 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 13:26:57 -0800 Subject: [PATCH 12/72] remove old emulator database --- fvm/evm/emulator/database/database.go | 424 ------------------ fvm/evm/emulator/database/database_test.go | 177 -------- fvm/evm/emulator/database/metered_database.go | 42 -- fvm/evm/emulator/database/storable.go | 242 ---------- fvm/evm/types/emulator.go | 16 - 5 files changed, 901 deletions(-) delete mode 100644 fvm/evm/emulator/database/database.go delete mode 100644 fvm/evm/emulator/database/database_test.go delete mode 100644 fvm/evm/emulator/database/metered_database.go delete mode 100644 fvm/evm/emulator/database/storable.go diff --git a/fvm/evm/emulator/database/database.go b/fvm/evm/emulator/database/database.go deleted file mode 100644 index 8b18e56e7bc..00000000000 --- a/fvm/evm/emulator/database/database.go +++ /dev/null @@ -1,424 +0,0 @@ -package database - -import ( - stdErrors "errors" - "runtime" - "sync" - - gethCommon "github.com/ethereum/go-ethereum/common" - gethTypes "github.com/ethereum/go-ethereum/core/types" - gethDB "github.com/ethereum/go-ethereum/ethdb" - "github.com/onflow/atree" - - "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/evm/types" - "github.com/onflow/flow-go/model/flow" -) - -const ( - FlowEVMRootSlabKey = "RootSlabKey" - FlowEVMRootHashKey = "RootHash" - StorageIDSize = 16 -) - -// Database is where EVM data is stored. -// -// under the hood, databases uses an Atree map -// stored under account `flowEVMRootAddress` -// each key value pairs inserted into this map is -// of type of ByteStringValue; we use this type instead -// of atree array, given the EVM environment is not smart enough -// to interact with a portion of the value and would load everything under a key -// before opearting on it. This means it could lead to having large slabs for a single value. -type Database struct { - flowEVMRootAddress flow.Address - led atree.Ledger - storage *atree.PersistentSlabStorage - baseStorage *atree.LedgerBaseStorage - atreemap *atree.OrderedMap - rootIDBytesToBeStored []byte // if is empty means we don't need to store anything - // Ramtin: other database implementations for EVM uses a lock - // to protect the storage against concurrent operations - // though one might do more research to see if we need - // these type of locking if the underlying structure (atree) - // has such protections or if EVM really needs it - lock sync.RWMutex -} - -var _ types.Database = &Database{} - -// NewDatabase returns a wrapped map that implements all the required database interface methods. -func NewDatabase(led atree.Ledger, flowEVMRootAddress flow.Address) (*Database, error) { - baseStorage := atree.NewLedgerBaseStorage(led) - - storage, err := NewPersistentSlabStorage(baseStorage) - if err != nil { - return nil, handleError(err) - } - - db := &Database{ - led: led, - baseStorage: baseStorage, - flowEVMRootAddress: flowEVMRootAddress, - storage: storage, - } - - err = db.retrieveOrCreateMapRoot() - if err != nil { - return nil, handleError(err) - } - return db, nil -} - -func (db *Database) retrieveOrCreateMapRoot() error { - rootIDBytes, err := db.led.GetValue(db.flowEVMRootAddress.Bytes(), []byte(FlowEVMRootSlabKey)) - if err != nil { - return err - } - - var m *atree.OrderedMap - if len(rootIDBytes) == 0 { - m, err = atree.NewMap(db.storage, atree.Address(db.flowEVMRootAddress), atree.NewDefaultDigesterBuilder(), emptyTypeInfo{}) - if err != nil { - return err - } - rootIDBytes := make([]byte, StorageIDSize) - _, err := m.StorageID().ToRawBytes(rootIDBytes) - if err != nil { - return err - } - db.rootIDBytesToBeStored = rootIDBytes - } else { - storageID, err := atree.NewStorageIDFromRawBytes(rootIDBytes) - if err != nil { - return err - } - m, err = atree.NewMapWithRootID(db.storage, storageID, atree.NewDefaultDigesterBuilder()) - if err != nil { - return err - } - } - db.atreemap = m - return nil -} - -// Get retrieves the given key if it's present in the key-value store. -func (db *Database) Get(key []byte) ([]byte, error) { - db.lock.RLock() - defer db.lock.RUnlock() - - value, err := db.get(key) - return value, handleError(err) -} - -func (db *Database) get(key []byte) ([]byte, error) { - data, err := db.atreemap.Get(compare, hashInputProvider, NewByteStringValue(key)) - if err != nil { - return nil, err - } - - v, err := data.StoredValue(db.atreemap.Storage) - if err != nil { - return nil, err - } - - return v.(ByteStringValue).Bytes(), nil -} - -// Put inserts the given value into the key-value store. -func (db *Database) Put(key []byte, value []byte) error { - db.lock.Lock() - defer db.lock.Unlock() - - err := db.put(key, value) - return handleError(err) -} - -func (db *Database) put(key []byte, value []byte) error { - existingValueStorable, err := db.atreemap.Set(compare, hashInputProvider, NewByteStringValue(key), NewByteStringValue(value)) - if err != nil { - return err - } - - if id, ok := existingValueStorable.(atree.StorageIDStorable); ok { - // NOTE: deep remove isn't necessary because value is ByteStringValue (not container) - err := db.storage.Remove(atree.StorageID(id)) - if err != nil { - return err - } - } - - return nil -} - -// Has checks if a key is present in the key-value store. -func (db *Database) Has(key []byte) (bool, error) { - db.lock.RLock() - defer db.lock.RUnlock() - has, err := db.has(key) - return has, handleError(err) -} - -func (db *Database) has(key []byte) (bool, error) { - has, err := db.atreemap.Has(compare, hashInputProvider, NewByteStringValue(key)) - if err != nil { - return false, err - } - return has, nil -} - -// Delete removes the key from the key-value store. -func (db *Database) Delete(key []byte) error { - db.lock.Lock() - defer db.lock.Unlock() - err := db.delete(key) - return handleError(err) -} - -func (db *Database) delete(key []byte) error { - removedMapKeyStorable, removedMapValueStorable, err := db.atreemap.Remove(compare, hashInputProvider, NewByteStringValue(key)) - if err != nil { - return err - } - - if id, ok := removedMapKeyStorable.(atree.StorageIDStorable); ok { - // NOTE: deep remove isn't necessary because key is ByteStringValue (not container) - err := db.storage.Remove(atree.StorageID(id)) - if err != nil { - return err - } - } - - if id, ok := removedMapValueStorable.(atree.StorageIDStorable); ok { - // NOTE: deep remove isn't necessary because value is ByteStringValue (not container) - err := db.storage.Remove(atree.StorageID(id)) - if err != nil { - return err - } - } - return nil -} - -// ApplyBatch applys changes from a batch into the database -func (db *Database) ApplyBatch(b *batch) error { - db.lock.Lock() - defer db.lock.Unlock() - err := db.applyBatch(b) - return err -} - -func (db *Database) applyBatch(b *batch) error { - var err error - for _, keyvalue := range b.writes { - if err != nil { - return err - } - if keyvalue.delete { - err = db.delete(keyvalue.key) - continue - } - err = db.put(keyvalue.key, keyvalue.value) - } - return err -} - -// GetRootHash returns the active root hash -func (db *Database) GetRootHash() (gethCommon.Hash, error) { - db.lock.Lock() - defer db.lock.Unlock() - - hash, err := db.getRootHash() - return hash, handleError(err) -} - -func (db *Database) getRootHash() (gethCommon.Hash, error) { - data, err := db.led.GetValue(db.flowEVMRootAddress[:], []byte(FlowEVMRootHashKey)) - if err != nil { - return gethCommon.Hash{}, handleError(err) - } - if len(data) == 0 { - return gethTypes.EmptyRootHash, nil - } - return gethCommon.BytesToHash(data), nil -} - -// Commits the changes from atree into the underlying storage -// -// this method can be merged as part of batcher -func (db *Database) Commit(root gethCommon.Hash) error { - db.lock.Lock() - defer db.lock.Unlock() - - err := db.commit(root) - return handleError(err) -} - -func (db *Database) commit(root gethCommon.Hash) error { - err := db.storage.FastCommit(runtime.NumCPU()) - if err != nil { - return err - } - - // check if we have to store the rootID - if len(db.rootIDBytesToBeStored) > 0 { - err = db.led.SetValue(db.flowEVMRootAddress.Bytes(), []byte(FlowEVMRootSlabKey), db.rootIDBytesToBeStored[:]) - if err != nil { - return err - } - } - - err = db.led.SetValue(db.flowEVMRootAddress[:], []byte(FlowEVMRootHashKey), root[:]) - if err != nil { - return err - } - return nil -} - -// Close is a no-op -func (db *Database) Close() error { - return nil -} - -// NewBatch creates a write-only key-value store that buffers changes to its host -// database until a final write is called. -func (db *Database) NewBatch() gethDB.Batch { - return &batch{ - db: db, - } -} - -// NewBatchWithSize creates a write-only database batch with pre-allocated buffer. -func (db *Database) NewBatchWithSize(size int) gethDB.Batch { - return &batch{ - db: db, - writes: make([]keyvalue, 0, size), - } -} - -// NewIterator is not supported in this database -// if needed in the future we could implement it using atree iterators -func (db *Database) NewIterator(prefix []byte, start []byte) gethDB.Iterator { - panic(types.ErrNotImplemented) -} - -// NewSnapshot is not supported -func (db *Database) NewSnapshot() (gethDB.Snapshot, error) { - return nil, types.ErrNotImplemented -} - -// Stat method is not supported -func (db *Database) Stat(property string) (string, error) { - return "", types.ErrNotImplemented -} - -func (db *Database) BytesRetrieved() int { - return db.baseStorage.BytesRetrieved() -} - -func (db *Database) BytesStored() int { - return db.baseStorage.BytesStored() -} -func (db *Database) ResetReporter() { - db.baseStorage.ResetReporter() -} - -// Compact is not supported on a memory database, but there's no need either as -// a memory database doesn't waste space anyway. -// Compact is a no op -func (db *Database) Compact(start []byte, limit []byte) error { - return nil -} - -// Len returns the number of entries currently present in the memory database. -// -// Note, this method is only used for testing (i.e. not public in general) and -// does not have explicit checks for closed-ness to allow simpler testing code. -func (db *Database) Len() int { - db.lock.RLock() - defer db.lock.RUnlock() - - return int(db.atreemap.Count()) -} - -// keyvalue is a key-value tuple tagged with a deletion field to allow creating -// memory-database write batches. -type keyvalue struct { - key []byte - value []byte - delete bool -} - -// batch is a write-only memory batch that commits changes to its host -// database when Write is called. A batch cannot be used concurrently. -type batch struct { - db *Database - writes []keyvalue - size int -} - -// Put inserts the given value into the batch for later committing. -func (b *batch) Put(key, value []byte) error { - return b.set(key, value, false) -} - -// Delete inserts the a key removal into the batch for later committing. -func (b *batch) Delete(key []byte) error { - return b.set(key, nil, true) -} - -func (b *batch) set(key []byte, value []byte, delete bool) error { - b.writes = append(b.writes, keyvalue{gethCommon.CopyBytes(key), gethCommon.CopyBytes(value), delete}) - b.size += len(key) + len(value) - return nil -} - -// DropCache drops the database read cache -func (db *Database) DropCache() { - db.storage.DropCache() -} - -// ValueSize retrieves the amount of data queued up for writing. -func (b *batch) ValueSize() int { - return b.size -} - -// Write flushes any accumulated data to the memory database. -func (b *batch) Write() error { - return b.db.ApplyBatch(b) -} - -// Reset resets the batch for reuse. -func (b *batch) Reset() { - // TODO: reset writes elements to release memory if value is large. - b.writes = b.writes[:0] - b.size = 0 -} - -// Replay replays the batch contents. -func (b *batch) Replay(w gethDB.KeyValueWriter) error { - for _, keyvalue := range b.writes { - if keyvalue.delete { - if err := w.Delete(keyvalue.key); err != nil { - return err - } - continue - } - if err := w.Put(keyvalue.key, keyvalue.value); err != nil { - return err - } - } - return nil -} - -func handleError(err error) error { - if err == nil { - return nil - } - var atreeFatalError *atree.FatalError - // if is a atree fatal error or fvm fatal error (the second one captures external errors) - if stdErrors.As(err, &atreeFatalError) || errors.IsFailure(err) { - return types.NewFatalError(err) - } - // wrap the non-fatal error with DB error - return types.NewDatabaseError(err) -} diff --git a/fvm/evm/emulator/database/database_test.go b/fvm/evm/emulator/database/database_test.go deleted file mode 100644 index a23e38d4295..00000000000 --- a/fvm/evm/emulator/database/database_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package database_test - -import ( - "fmt" - "testing" - - gethCommon "github.com/ethereum/go-ethereum/common" - gethTypes "github.com/ethereum/go-ethereum/core/types" - "github.com/onflow/atree" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/evm/emulator/database" - "github.com/onflow/flow-go/fvm/evm/testutils" - "github.com/onflow/flow-go/fvm/evm/types" - "github.com/onflow/flow-go/model/flow" -) - -func TestDatabase(t *testing.T) { - - key1 := []byte("ABC") - key2 := []byte("DEF") - value1 := []byte{1, 2, 3, 4, 5, 6, 7, 8} - value2 := []byte{9, 10, 11} - - t.Run("test basic database functionality", func(t *testing.T) { - testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { - testutils.RunWithTestFlowEVMRootAddress(t, backend, func(flowEVMRoot flow.Address) { - db, err := database.NewDatabase(backend, flowEVMRoot) - require.NoError(t, err) - - err = db.Put(key1, value1) - require.NoError(t, err) - - err = db.Commit(gethTypes.EmptyRootHash) - require.NoError(t, err) - - newdb, err := database.NewDatabase(backend, flowEVMRoot) - require.NoError(t, err) - - has, err := newdb.Has(key1) - require.NoError(t, err) - require.True(t, has) - - retValue, err := newdb.Get(key1) - require.NoError(t, err) - - require.Equal(t, value1, retValue) - - err = newdb.Delete(key1) - require.NoError(t, err) - - has, err = newdb.Has(key1) - require.NoError(t, err) - require.False(t, has) - - h, err := newdb.GetRootHash() - require.NoError(t, err) - require.Equal(t, gethTypes.EmptyRootHash, h) - - newRoot := gethCommon.Hash{1, 2, 3} - err = newdb.Commit(newRoot) - require.NoError(t, err) - - h, err = newdb.GetRootHash() - require.NoError(t, err) - require.Equal(t, newRoot, h) - }) - }) - }) - - t.Run("test batch functionality", func(t *testing.T) { - testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { - testutils.RunWithTestFlowEVMRootAddress(t, backend, func(flowEVMRoot flow.Address) { - db, err := database.NewDatabase(backend, flowEVMRoot) - require.NoError(t, err) - - err = db.Put(key1, value1) - require.NoError(t, err) - - has, err := db.Has(key1) - require.NoError(t, err) - require.True(t, has) - - batch := db.NewBatch() - err = batch.Delete(key1) - require.NoError(t, err) - - err = batch.Put(key2, value2) - require.NoError(t, err) - - has, err = db.Has(key2) - require.NoError(t, err) - require.False(t, has) - - err = batch.Write() - require.NoError(t, err) - - retVal, err := db.Get(key2) - require.NoError(t, err) - require.Equal(t, value2, retVal) - - has, err = db.Has(key1) - require.NoError(t, err) - require.False(t, has) - }) - }) - }) - - t.Run("test non fatal error", func(t *testing.T) { - ledger := &testutils.TestValueStore{ - GetValueFunc: func(_, _ []byte) ([]byte, error) { - return nil, errors.NewLedgerInteractionLimitExceededError(0, 0) - }, - SetValueFunc: func(owner, key, value []byte) error { - return nil - }, - } - testutils.RunWithTestFlowEVMRootAddress(t, ledger, func(flowEVMRoot flow.Address) { - _, err := database.NewDatabase(ledger, flowEVMRoot) - require.Error(t, err) - require.True(t, types.IsADatabaseError(err)) - }) - }) - - t.Run("test fatal error (get value)", func(t *testing.T) { - ledger := &testutils.TestValueStore{ - GetValueFunc: func(_, _ []byte) ([]byte, error) { - return nil, fmt.Errorf("a fatal error") - }, - SetValueFunc: func(owner, key, value []byte) error { - return nil - }, - } - testutils.RunWithTestFlowEVMRootAddress(t, ledger, func(flowEVMRoot flow.Address) { - _, err := database.NewDatabase(ledger, flowEVMRoot) - require.Error(t, err) - require.True(t, types.IsAFatalError(err)) - }) - }) - - t.Run("test fatal error (storage id allocation)", func(t *testing.T) { - ledger := &testutils.TestValueStore{ - AllocateStorageIndexFunc: func(_ []byte) (atree.StorageIndex, error) { - return atree.StorageIndex{}, fmt.Errorf("a fatal error") - }, - GetValueFunc: func(_, _ []byte) ([]byte, error) { - return nil, nil - }, - SetValueFunc: func(owner, key, value []byte) error { - return nil - }, - } - testutils.RunWithTestFlowEVMRootAddress(t, ledger, func(flowEVMRoot flow.Address) { - _, err := database.NewDatabase(ledger, flowEVMRoot) - require.Error(t, err) - require.True(t, types.IsAFatalError(err)) - }) - }) - - t.Run("test fatal error (not implemented methods)", func(t *testing.T) { - testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { - testutils.RunWithTestFlowEVMRootAddress(t, backend, func(flowEVMRoot flow.Address) { - db, err := database.NewDatabase(backend, flowEVMRoot) - require.NoError(t, err) - - _, err = db.Stat("") - require.Error(t, err) - require.True(t, types.IsAFatalError(err)) - - _, err = db.NewSnapshot() - require.Error(t, err) - require.True(t, types.IsAFatalError(err)) - }) - }) - }) -} diff --git a/fvm/evm/emulator/database/metered_database.go b/fvm/evm/emulator/database/metered_database.go deleted file mode 100644 index ab8853c5ae7..00000000000 --- a/fvm/evm/emulator/database/metered_database.go +++ /dev/null @@ -1,42 +0,0 @@ -package database - -import ( - "github.com/onflow/atree" - - "github.com/onflow/flow-go/model/flow" -) - -// MeteredDatabase wrapper around the database purposely built for testing and benchmarking. -type MeteredDatabase struct { - *Database -} - -// NewMeteredDatabase create a database wrapper purposely built for testing and benchmarking. -func NewMeteredDatabase(led atree.Ledger, flowEVMRootAddress flow.Address) (*MeteredDatabase, error) { - database, err := NewDatabase(led, flowEVMRootAddress) - if err != nil { - return nil, err - } - - return &MeteredDatabase{ - Database: database, - }, nil -} - -func (m *MeteredDatabase) DropCache() { - m.storage.DropCache() -} - -func (m *MeteredDatabase) BytesRead() int { - return m.baseStorage.BytesRetrieved() -} - -func (m *MeteredDatabase) BytesWritten() int { - return m.baseStorage.BytesStored() -} - -func (m *MeteredDatabase) ResetReporter() { - m.baseStorage.ResetReporter() - m.baseStorage.Size() - m.storage.Count() -} diff --git a/fvm/evm/emulator/database/storable.go b/fvm/evm/emulator/database/storable.go deleted file mode 100644 index 520c47573f1..00000000000 --- a/fvm/evm/emulator/database/storable.go +++ /dev/null @@ -1,242 +0,0 @@ -package database - -import ( - "bytes" - "encoding/binary" - "fmt" - "math" - - "github.com/fxamacker/cbor/v2" - - "github.com/onflow/atree" -) - -type ByteStringValue struct { - data []byte - size uint32 -} - -var _ atree.Value = &ByteStringValue{} -var _ atree.Storable = &ByteStringValue{} - -func NewByteStringValue(data []byte) ByteStringValue { - size := atree.GetUintCBORSize(uint64(len(data))) + uint32(len(data)) - return ByteStringValue{data: data, size: size} -} - -func (v ByteStringValue) ChildStorables() []atree.Storable { - return nil -} - -func (v ByteStringValue) StoredValue(_ atree.SlabStorage) (atree.Value, error) { - return v, nil -} - -func (v ByteStringValue) Storable(storage atree.SlabStorage, address atree.Address, maxInlineSize uint64) (atree.Storable, error) { - if uint64(v.ByteSize()) > maxInlineSize { - - // Create StorableSlab - id, err := storage.GenerateStorageID(address) - if err != nil { - return nil, err - } - - slab := &atree.StorableSlab{ - StorageID: id, - Storable: v, - } - - // Store StorableSlab in storage - err = storage.Store(id, slab) - if err != nil { - return nil, err - } - - // Return storage id as storable - return atree.StorageIDStorable(id), nil - } - - return v, nil -} - -func (v ByteStringValue) Encode(enc *atree.Encoder) error { - return enc.CBOR.EncodeBytes(v.data) -} - -func (v ByteStringValue) getHashInput(scratch []byte) ([]byte, error) { - - const cborTypeByteString = 0x40 - - buf := scratch - if uint32(len(buf)) < v.size { - buf = make([]byte, v.size) - } else { - buf = buf[:v.size] - } - - slen := len(v.data) - - if slen <= 23 { - buf[0] = cborTypeByteString | byte(slen) - copy(buf[1:], v.data) - return buf, nil - } - - if slen <= math.MaxUint8 { - buf[0] = cborTypeByteString | byte(24) - buf[1] = byte(slen) - copy(buf[2:], v.data) - return buf, nil - } - - if slen <= math.MaxUint16 { - buf[0] = cborTypeByteString | byte(25) - binary.BigEndian.PutUint16(buf[1:], uint16(slen)) - copy(buf[3:], v.data) - return buf, nil - } - - if slen <= math.MaxUint32 { - buf[0] = cborTypeByteString | byte(26) - binary.BigEndian.PutUint32(buf[1:], uint32(slen)) - copy(buf[5:], v.data) - return buf, nil - } - - buf[0] = cborTypeByteString | byte(27) - binary.BigEndian.PutUint64(buf[1:], uint64(slen)) - copy(buf[9:], v.data) - return buf, nil -} - -func (v ByteStringValue) ByteSize() uint32 { - return v.size -} - -func (v ByteStringValue) String() string { - return string(v.data) -} - -func (v ByteStringValue) Bytes() []byte { - return v.data -} - -func decodeStorable(dec *cbor.StreamDecoder, _ atree.StorageID) (atree.Storable, error) { - t, err := dec.NextType() - if err != nil { - return nil, err - } - - switch t { - case cbor.ByteStringType: - s, err := dec.DecodeBytes() - if err != nil { - return nil, err - } - return NewByteStringValue(s), nil - - case cbor.TagType: - tagNumber, err := dec.DecodeTagNumber() - if err != nil { - return nil, err - } - - switch tagNumber { - - case atree.CBORTagStorageID: - return atree.DecodeStorageIDStorable(dec) - - default: - return nil, fmt.Errorf("invalid tag number %d", tagNumber) - } - - default: - return nil, fmt.Errorf("invalid cbor type %s for storable", t) - } -} - -func compare(storage atree.SlabStorage, value atree.Value, storable atree.Storable) (bool, error) { - switch v := value.(type) { - - case ByteStringValue: - other, ok := storable.(ByteStringValue) - if ok { - return bytes.Equal(other.data, v.data), nil - } - - // Retrieve value from storage - otherValue, err := storable.StoredValue(storage) - if err != nil { - return false, err - } - other, ok = otherValue.(ByteStringValue) - if ok { - return bytes.Equal(other.data, v.data), nil - } - - return false, nil - } - - return false, fmt.Errorf("value %T not supported for comparison", value) -} - -func hashInputProvider(value atree.Value, buffer []byte) ([]byte, error) { - switch v := value.(type) { - case ByteStringValue: - return v.getHashInput(buffer) - } - - return nil, fmt.Errorf("value %T not supported for hash input", value) -} - -func NewPersistentSlabStorage(baseStorage atree.BaseStorage) (*atree.PersistentSlabStorage, error) { - - encMode, err := cbor.EncOptions{}.EncMode() - if err != nil { - return nil, err - } - - decMode, err := cbor.DecOptions{}.DecMode() - if err != nil { - return nil, err - } - - return atree.NewPersistentSlabStorage( - baseStorage, - encMode, - decMode, - decodeStorable, - decodeTypeInfo, - ), nil - -} - -type emptyTypeInfo struct{} - -var _ atree.TypeInfo = emptyTypeInfo{} - -func (emptyTypeInfo) Encode(e *cbor.StreamEncoder) error { - return e.EncodeNil() -} - -func (i emptyTypeInfo) Equal(other atree.TypeInfo) bool { - _, ok := other.(emptyTypeInfo) - return ok -} - -func decodeTypeInfo(dec *cbor.StreamDecoder) (atree.TypeInfo, error) { - ty, err := dec.NextType() - if err != nil { - return nil, err - } - switch ty { - case cbor.NilType: - err := dec.DecodeNil() - if err != nil { - return nil, err - } - return emptyTypeInfo{}, nil - } - - return nil, fmt.Errorf("not supported type info") -} diff --git a/fvm/evm/types/emulator.go b/fvm/evm/types/emulator.go index 1b86e06fe7b..73eef076a5b 100644 --- a/fvm/evm/types/emulator.go +++ b/fvm/evm/types/emulator.go @@ -3,9 +3,7 @@ package types import ( "math/big" - gethCommon "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" ) var ( @@ -65,17 +63,3 @@ type Emulator interface { // constructs a new block NewBlockView(ctx BlockContext) (BlockView, error) } - -// Database provides what Emulator needs for storing tries and accounts -// Errors returned by the methods are one of the followings: -// - Fatal error -// - Database error (non-fatal) -type Database interface { - ethdb.KeyValueStore - - // Commit commits the changes - Commit(rootHash gethCommon.Hash) error - - // GetRootHash returns the active root hash - GetRootHash() (gethCommon.Hash, error) -} From 07e2457bf06bd14b010006e8750a04dd1d4859bf Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 14:08:18 -0800 Subject: [PATCH 13/72] add log tests --- fvm/evm/emulator/state/stateDB.go | 128 ++++++++++++++++--------- fvm/evm/emulator/state/stateDB_test.go | 31 ++++++ fvm/evm/types/errors.go | 28 ++++++ 3 files changed, 140 insertions(+), 47 deletions(-) diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index 4581dda99ef..ee76cecd1fa 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -17,31 +17,38 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// TODO: question, does stateDB has to be thread safe ? -// error handling +// StateDB implements a types.StateDB interface // -// StateDB should not be used for running multiple transactions +// stateDB interface defined by the Geth doesn't support returning errors +// when state calls are happening, and requires stateDB to cache the error +// and return it at a later time (when commit is called). Only the first error +// is expected to be returned. +// Warning: current implementation of the StateDB is considered +// to be used for a single EVM transaction execution and is not +// thread safe.yet the current design supports addition of these properties +// future if needed type StateDB struct { - ledger atree.Ledger - root flow.Address - baseView types.BaseView - views []*DeltaView - dbErr error + ledger atree.Ledger + root flow.Address + baseView types.BaseView + views []*DeltaView + cachedError error } var _ types.StateDB = &StateDB{} +// NewStateDB constructs a new StateDB func NewStateDB(ledger atree.Ledger, root flow.Address) (*StateDB, error) { bv, err := NewBaseView(ledger, root) if err != nil { return nil, err } return &StateDB{ - ledger: ledger, - root: root, - baseView: bv, - views: []*DeltaView{NewDeltaView(bv)}, - dbErr: nil, + ledger: ledger, + root: root, + baseView: bv, + views: []*DeltaView{NewDeltaView(bv)}, + cachedError: nil, }, nil } @@ -125,68 +132,83 @@ func (db *StateDB) SetNonce(addr gethCommon.Address, nonce uint64) { db.lastestView().SetNonce(addr, nonce) } +// GetCodeHash returns the code hash of the given address func (db *StateDB) GetCodeHash(addr gethCommon.Address) gethCommon.Hash { hash, err := db.lastestView().GetCodeHash(addr) db.handleError(err) return hash } +// GetCode returns the code for the given address func (db *StateDB) GetCode(addr gethCommon.Address) []byte { code, err := db.lastestView().GetCode(addr) db.handleError(err) return code } -func (db *StateDB) SetCode(addr gethCommon.Address, code []byte) { - db.lastestView().SetCode(addr, code) -} - +// GetCodeSize returns the size of the code for the given address func (db *StateDB) GetCodeSize(addr gethCommon.Address) int { codeSize, err := db.lastestView().GetCodeSize(addr) db.handleError(err) return codeSize } +// SetCode sets the code for the given address +func (db *StateDB) SetCode(addr gethCommon.Address, code []byte) { + db.lastestView().SetCode(addr, code) +} + +// AddRefund adds an amount to the total (gas) refund func (db *StateDB) AddRefund(amount uint64) { db.lastestView().AddRefund(amount) } +// AddRefund subtracts an amount from the total (gas) refund func (db *StateDB) SubRefund(amount uint64) { db.lastestView().SubRefund(amount) } +// GetRefund returns the total (gas) refund func (db *StateDB) GetRefund() uint64 { return db.lastestView().GetRefund() } +// GetCommittedState returns the value for the given storage slot considering only the commited state and not +// changes in the scope of current transaction. func (db *StateDB) GetCommittedState(addr gethCommon.Address, key gethCommon.Hash) gethCommon.Hash { value, err := db.baseView.GetState(types.SlotAddress{Address: addr, Key: key}) db.handleError(err) return value } +// GetState returns the value for the given storage slot func (db *StateDB) GetState(addr gethCommon.Address, key gethCommon.Hash) gethCommon.Hash { state, err := db.lastestView().GetState(types.SlotAddress{Address: addr, Key: key}) db.handleError(err) return state } +// SetState sets a value for the given storage slot func (db *StateDB) SetState(addr gethCommon.Address, key gethCommon.Hash, value gethCommon.Hash) { db.lastestView().SetState(types.SlotAddress{Address: addr, Key: key}, value) } +// GetState returns the value for the given key of the transient storage func (db *StateDB) GetTransientState(addr gethCommon.Address, key gethCommon.Hash) gethCommon.Hash { return db.lastestView().GetTransientState(types.SlotAddress{Address: addr, Key: key}) } +// SetTransientState sets a value for the given key of the transient storage func (db *StateDB) SetTransientState(addr gethCommon.Address, key, value gethCommon.Hash) { db.lastestView().SetTransientState(types.SlotAddress{Address: addr, Key: key}, value) } +// AddressInAccessList checks if an address is in the access list func (db *StateDB) AddressInAccessList(addr gethCommon.Address) bool { return db.lastestView().AddressInAccessList(addr) } +// AddressInAccessList checks if the given (address,slot) is in the access list func (db *StateDB) SlotInAccessList(addr gethCommon.Address, key gethCommon.Hash) (addressOk bool, slotOk bool) { return db.lastestView().SlotInAccessList(types.SlotAddress{Address: addr, Key: key}) } @@ -201,32 +223,35 @@ func (db *StateDB) AddSlotToAccessList(addr gethCommon.Address, key gethCommon.H db.lastestView().AddSlotToAccessList(types.SlotAddress{Address: addr, Key: key}) } +// AddLog appends a lot to the collection of logs func (db *StateDB) AddLog(log *gethTypes.Log) { db.lastestView().AddLog(log) } +// AddPreimage adds a preimage to the collection of preimages func (db *StateDB) AddPreimage(hash gethCommon.Hash, data []byte) { db.lastestView().AddPreimage(hash, data) } +// RevertToSnapshot reverts the changes until we reach the given snaptshot func (db *StateDB) RevertToSnapshot(index int) { if index > len(db.views) { - db.dbErr = fmt.Errorf("invalid revert") + db.cachedError = fmt.Errorf("invalid revert") return } db.views = db.views[:index] } +// Snapshot takes an snapshot of the state and returns an int +// that can be used later for revert calls. func (db *StateDB) Snapshot() int { newView := db.lastestView().NewChildView() db.views = append(db.views, newView) return len(db.views) - 1 } -func (db *StateDB) lastestView() *DeltaView { - return db.views[len(db.views)-1] -} - +// Logs returns the list of logs +// it also update each log with the block and tx info func (db *StateDB) Logs( blockHash gethCommon.Hash, blockNumber uint64, @@ -246,6 +271,7 @@ func (db *StateDB) Logs( return allLogs } +// Preimages returns a set of preimages func (db *StateDB) Preimages() map[gethCommon.Hash][]byte { preImages := make(map[gethCommon.Hash][]byte, 0) for _, view := range db.views { @@ -256,10 +282,11 @@ func (db *StateDB) Preimages() map[gethCommon.Hash][]byte { return preImages } +// Commit commits state changes back to the underlying func (db *StateDB) Commit() error { // return error if any has been acumulated - if db.dbErr != nil { - return db.dbErr + if db.cachedError != nil { + return db.cachedError } var err error @@ -349,29 +376,9 @@ func (db *StateDB) Commit() error { return db.baseView.Commit() } -// Error returns the memorized database failure occurred earlier. -func (s *StateDB) Error() error { - return s.dbErr -} - -// handleError capture the first non-nil error it is called with. -func (s *StateDB) handleError(err error) { - if err == nil { - return - } - - var atreeFatalError *atree.FatalError - // if is a atree fatal error or fvm fatal error (the second one captures external errors) - if stdErrors.As(err, &atreeFatalError) || errors.IsFailure(err) { - panic(types.NewFatalError(err)) - } - - // already no error is captured - if s.dbErr == nil { - s.dbErr = types.NewDatabaseError(err) - } -} - +// Prepare is a highlevel logic that sadly is considered to be part of the +// stateDB interface and not on the layers above. +// based on parameters that are passed it updates accesslists func (db *StateDB) Prepare(rules gethParams.Rules, sender, coinbase gethCommon.Address, dest *gethCommon.Address, precompiles []gethCommon.Address, txAccesses gethTypes.AccessList) { if rules.IsBerlin { // no need for mutation @@ -395,3 +402,30 @@ func (db *StateDB) Prepare(rules gethParams.Rules, sender, coinbase gethCommon.A } } } + +// Error returns the memorized database failure occurred earlier. +func (s *StateDB) Error() error { + return s.cachedError +} + +func (db *StateDB) lastestView() *DeltaView { + return db.views[len(db.views)-1] +} + +// handleError capture the first non-nil error it is called with. +func (s *StateDB) handleError(err error) { + if err == nil { + return + } + + var atreeFatalError *atree.FatalError + // if is a atree fatal error or fvm fatal error (the second one captures external errors) + if stdErrors.As(err, &atreeFatalError) || errors.IsFailure(err) { + panic(types.NewFatalError(err)) + } + + // already no error is captured + if s.cachedError == nil { + s.cachedError = types.NewStateError(err) + } +} diff --git a/fvm/evm/emulator/state/stateDB_test.go b/fvm/evm/emulator/state/stateDB_test.go index c78aa054856..46f8318b2b6 100644 --- a/fvm/evm/emulator/state/stateDB_test.go +++ b/fvm/evm/emulator/state/stateDB_test.go @@ -4,6 +4,8 @@ import ( "math/big" "testing" + "github.com/ethereum/go-ethereum/common" + gethTypes "github.com/ethereum/go-ethereum/core/types" "github.com/onflow/flow-go/fvm/evm/emulator/state" "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/model/flow" @@ -111,4 +113,33 @@ func TestStateDB(t *testing.T) { db.RevertToSnapshot(10) require.Error(t, db.Error()) }) + + t.Run("test log functionality", func(t *testing.T) { + ledger := testutils.GetSimpleValueStore() + rootAddr := flow.Address{1, 2, 3, 4, 5, 6, 7, 8} + db, err := state.NewStateDB(ledger, rootAddr) + require.NoError(t, err) + + logs := []*gethTypes.Log{ + testutils.GetRandomLogFixture(t), + testutils.GetRandomLogFixture(t), + testutils.GetRandomLogFixture(t), + testutils.GetRandomLogFixture(t), + } + + db.AddLog(logs[0]) + db.AddLog(logs[1]) + + _ = db.Snapshot() + + db.AddLog(logs[2]) + db.AddLog(logs[3]) + + snapshot := db.Snapshot() + db.AddLog(testutils.GetRandomLogFixture(t)) + db.RevertToSnapshot(snapshot) + + ret := db.Logs(common.Hash{}, 1, common.Hash{}, 1) + require.Equal(t, ret, logs) + }) } diff --git a/fvm/evm/types/errors.go b/fvm/evm/types/errors.go index fa008b741a2..cee5ad2e723 100644 --- a/fvm/evm/types/errors.go +++ b/fvm/evm/types/errors.go @@ -117,6 +117,34 @@ func IsADatabaseError(err error) bool { return errors.As(err, &DatabaseError{}) } +// StateError is a non-fatal error, returned when a state operation +// has failed (e.g. reaching storage interaction limit) +type StateError struct { + err error +} + +// NewStateError returns a new StateError +func NewStateError(rootCause error) StateError { + return StateError{ + err: rootCause, + } +} + +// Unwrap unwraps the underlying evm error +func (err StateError) Unwrap() error { + return err.err +} + +func (err StateError) Error() string { + return fmt.Sprintf("state error: %v", err.err) +} + +// IsAStateError returns true if the error or any underlying errors +// is of the type EVM validation error +func IsAStateError(err error) bool { + return errors.As(err, &StateError{}) +} + // FatalError is user for any error that is not user related and something // unusual has happend. Usually we stop the node when this happens // given it might have a non-deterministic root. From 86117e64138088cf0ca192fba69a30c235857eb7 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 21:48:50 -0800 Subject: [PATCH 14/72] revert unwanted changes --- fvm/evm/emulator/state/base.go | 32 +++++++++++++++----------- fvm/evm/emulator/state/base_test.go | 7 +++--- fvm/evm/emulator/state/stateDB_test.go | 6 ++--- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/fvm/evm/emulator/state/base.go b/fvm/evm/emulator/state/base.go index 17342f6c6c6..4ec609de439 100644 --- a/fvm/evm/emulator/state/base.go +++ b/fvm/evm/emulator/state/base.go @@ -426,7 +426,12 @@ func (v *BaseView) getSlot(sk types.SlotAddress) (gethCommon.Hash, error) { return value, nil } - col, err := v.getSlotCollection(sk.Address) + acc, err := v.getAccount(sk.Address) + if err != nil || acc == nil || len(acc.CollectionID) == 0 { + return gethCommon.Hash{}, nil + } + + col, err := v.getSlotCollection(acc) if err != nil { return gethCommon.Hash{}, err } @@ -441,25 +446,26 @@ func (v *BaseView) getSlot(sk types.SlotAddress) (gethCommon.Hash, error) { } func (v *BaseView) storeSlot(sk types.SlotAddress, data gethCommon.Hash) error { - col, err := v.getSlotCollection(sk.Address) + acc, err := v.getAccount(sk.Address) if err != nil { return err } - v.cachedSlots[sk] = data - return col.Set(sk.Key.Bytes(), data.Bytes()) -} - -func (v *BaseView) getSlotCollection(addr gethCommon.Address) (*Collection, error) { - acc, err := v.getAccount(addr) - if err != nil { - return nil, err - } if acc == nil { - return nil, fmt.Errorf("slot belongs to a non-existing account") + return fmt.Errorf("slot belongs to a non-existing account") } if len(acc.CollectionID) == 0 { - return nil, fmt.Errorf("slot belongs to a non-smart contract account") + return fmt.Errorf("slot belongs to a non-smart contract account") + } + col, err := v.getSlotCollection(acc) + if err != nil { + return err } + v.cachedSlots[sk] = data + return col.Set(sk.Key.Bytes(), data.Bytes()) +} + +func (v *BaseView) getSlotCollection(acc *Account) (*Collection, error) { + var err error col, found := v.slots[acc.Address] if !found { col, err = v.collectionProvider.CollectionByID(acc.CollectionID) diff --git a/fvm/evm/emulator/state/base_test.go b/fvm/evm/emulator/state/base_test.go index 59c2ecf6b8f..25d54fb807f 100644 --- a/fvm/evm/emulator/state/base_test.go +++ b/fvm/evm/emulator/state/base_test.go @@ -158,8 +158,9 @@ func TestBaseView(t *testing.T) { } // non-existent account - _, err = view.GetState(slot1) - require.Error(t, err) + value, err := view.GetState(slot1) + require.NoError(t, err) + require.Equal(t, value, gethCommon.Hash{}) // store a new value newValue := testutils.RandomCommonHash(t) @@ -176,7 +177,7 @@ func TestBaseView(t *testing.T) { require.NoError(t, err) // return result from the cache - value, err := view.GetState(slot1) + value, err = view.GetState(slot1) require.NoError(t, err) require.Equal(t, newValue, value) diff --git a/fvm/evm/emulator/state/stateDB_test.go b/fvm/evm/emulator/state/stateDB_test.go index 46f8318b2b6..329133e686d 100644 --- a/fvm/evm/emulator/state/stateDB_test.go +++ b/fvm/evm/emulator/state/stateDB_test.go @@ -12,10 +12,6 @@ import ( "github.com/stretchr/testify/require" ) -// Test revert -// TODO: add test for error handling -// Test logs - func TestStateDB(t *testing.T) { t.Parallel() @@ -142,4 +138,6 @@ func TestStateDB(t *testing.T) { ret := db.Logs(common.Hash{}, 1, common.Hash{}, 1) require.Equal(t, ret, logs) }) + + // TODO: add test for error handling } From 0a10e03afda66ebbb11c1183d4e27a14a6856c17 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 22:20:49 -0800 Subject: [PATCH 15/72] add test for handling errors --- fvm/evm/emulator/state/delta.go | 8 ++++ fvm/evm/emulator/state/stateDB.go | 51 +++++++++++++++++-------- fvm/evm/emulator/state/stateDB_test.go | 53 +++++++++++++++++++++++++- 3 files changed, 95 insertions(+), 17 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index 8080c6c9f42..7a4c19957b2 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -97,9 +97,17 @@ func (d *DeltaView) Exist(addr gethCommon.Address) (bool, error) { // CreateAccount creates a new account for the given address func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { + // If a address already exists the balance is carried over to the new account. + // Carrying over the balance ensures that Ether doesn't disappear. (legacy behaviour of the Geth stateDB) + bal, err := d.GetBalance(addr) + if err != nil { + return err + } + d.created[addr] = struct{}{} // flag the address as dirty d.dirtyAddresses[addr] = struct{}{} + d.AddBalance(addr, bal) return nil } diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index ee76cecd1fa..9f51ade2497 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -76,8 +76,6 @@ func (db *StateDB) Empty(addr gethCommon.Address) bool { // CreateAccount creates a new account for the given address // it sets the nonce to zero func (db *StateDB) CreateAccount(addr gethCommon.Address) { - // TODO: If a state object with the address already exists the balance is carried over to the new account. - // Carrying over the balance ensures that Ether doesn't disappear. db.lastestView().CreateAccount(addr) } @@ -286,7 +284,7 @@ func (db *StateDB) Preimages() map[gethCommon.Hash][]byte { func (db *StateDB) Commit() error { // return error if any has been acumulated if db.cachedError != nil { - return db.cachedError + return wrapError(db.cachedError) } var err error @@ -319,7 +317,7 @@ func (db *StateDB) Commit() error { if db.HasSuicided(addr) { err = db.baseView.DeleteAccount(addr) if err != nil { - return err + return wrapError(err) } continue } @@ -332,7 +330,7 @@ func (db *StateDB) Commit() error { db.GetCodeHash(addr), ) if err != nil { - return err + return wrapError(err) } continue } @@ -344,7 +342,7 @@ func (db *StateDB) Commit() error { db.GetCodeHash(addr), ) if err != nil { - return err + return wrapError(err) } } @@ -368,12 +366,16 @@ func (db *StateDB) Commit() error { db.GetState(sk.Address, sk.Key), ) if err != nil { - return err + return wrapError(err) } } // don't purge views yet, people might call the logs etc - return db.baseView.Commit() + err = db.baseView.Commit() + if err != nil { + return wrapError(err) + } + return nil } // Prepare is a highlevel logic that sadly is considered to be part of the @@ -405,27 +407,44 @@ func (db *StateDB) Prepare(rules gethParams.Rules, sender, coinbase gethCommon.A // Error returns the memorized database failure occurred earlier. func (s *StateDB) Error() error { - return s.cachedError + return wrapError(s.cachedError) } func (db *StateDB) lastestView() *DeltaView { return db.views[len(db.views)-1] } -// handleError capture the first non-nil error it is called with. +// set error captures the first non-nil error it is called with. func (s *StateDB) handleError(err error) { if err == nil { return } + if s.cachedError == nil { + s.cachedError = err + } +} + +func wrapError(err error) error { + if err == nil { + return nil + } + + var atreeUserError *atree.UserError + // if is an atree user error + if stdErrors.As(err, &atreeUserError) { + return types.NewStateError(err) + } var atreeFatalError *atree.FatalError - // if is a atree fatal error or fvm fatal error (the second one captures external errors) - if stdErrors.As(err, &atreeFatalError) || errors.IsFailure(err) { - panic(types.NewFatalError(err)) + // if is a atree fatal error or + if stdErrors.As(err, &atreeFatalError) { + return types.NewFatalError(err) } - // already no error is captured - if s.cachedError == nil { - s.cachedError = types.NewStateError(err) + // if is fvm fatal error + if errors.IsFailure(err) { + return types.NewFatalError(err) } + + return types.NewStateError(err) } diff --git a/fvm/evm/emulator/state/stateDB_test.go b/fvm/evm/emulator/state/stateDB_test.go index 329133e686d..306727cdaf9 100644 --- a/fvm/evm/emulator/state/stateDB_test.go +++ b/fvm/evm/emulator/state/stateDB_test.go @@ -1,13 +1,16 @@ package state_test import ( + "fmt" "math/big" "testing" "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" + "github.com/onflow/atree" "github.com/onflow/flow-go/fvm/evm/emulator/state" "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" "github.com/stretchr/testify/require" ) @@ -139,5 +142,53 @@ func TestStateDB(t *testing.T) { require.Equal(t, ret, logs) }) - // TODO: add test for error handling + t.Run("test non-fatal error handling", func(t *testing.T) { + ledger := &testutils.TestValueStore{ + GetValueFunc: func(owner, key []byte) ([]byte, error) { + return nil, nil + }, + SetValueFunc: func(owner, key, value []byte) error { + return atree.NewUserError(fmt.Errorf("key not found")) + }, + AllocateStorageIndexFunc: func(owner []byte) (atree.StorageIndex, error) { + return atree.StorageIndex{}, nil + }, + } + rootAddr := flow.Address{1, 2, 3, 4, 5, 6, 7, 8} + db, err := state.NewStateDB(ledger, rootAddr) + require.NoError(t, err) + + db.CreateAccount(testutils.RandomCommonAddress(t)) + + err = db.Commit() + // ret := db.Error() + require.Error(t, err) + // check wrapping + require.True(t, types.IsAStateError(err)) + }) + + t.Run("test fatal error handling", func(t *testing.T) { + ledger := &testutils.TestValueStore{ + GetValueFunc: func(owner, key []byte) ([]byte, error) { + return nil, nil + }, + SetValueFunc: func(owner, key, value []byte) error { + return atree.NewFatalError(fmt.Errorf("key not found")) + }, + AllocateStorageIndexFunc: func(owner []byte) (atree.StorageIndex, error) { + return atree.StorageIndex{}, nil + }, + } + rootAddr := flow.Address{1, 2, 3, 4, 5, 6, 7, 8} + db, err := state.NewStateDB(ledger, rootAddr) + require.NoError(t, err) + + db.CreateAccount(testutils.RandomCommonAddress(t)) + + err = db.Commit() + // ret := db.Error() + require.Error(t, err) + // check wrapping + require.True(t, types.IsAFatalError(err)) + }) } From 12d1c0085d53dd29bfdcd86a70b7bdf0467cb391 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 22:24:46 -0800 Subject: [PATCH 16/72] fix test --- fvm/evm/emulator/state/delta.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index 7a4c19957b2..b18e84dd23d 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -99,15 +99,25 @@ func (d *DeltaView) Exist(addr gethCommon.Address) (bool, error) { func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { // If a address already exists the balance is carried over to the new account. // Carrying over the balance ensures that Ether doesn't disappear. (legacy behaviour of the Geth stateDB) - bal, err := d.GetBalance(addr) + exist, err := d.Exist(addr) if err != nil { return err } + var bal *big.Int + if exist { + bal, err = d.GetBalance(addr) + if err != nil { + return err + } + } d.created[addr] = struct{}{} // flag the address as dirty d.dirtyAddresses[addr] = struct{}{} - d.AddBalance(addr, bal) + if exist { + d.AddBalance(addr, bal) + } + return nil } From 9753ed064d1743a709f1eb574e359aea650a22df Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 22:35:44 -0800 Subject: [PATCH 17/72] add refund test --- fvm/evm/emulator/state/stateDB_test.go | 27 +++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/fvm/evm/emulator/state/stateDB_test.go b/fvm/evm/emulator/state/stateDB_test.go index 306727cdaf9..59652f04379 100644 --- a/fvm/evm/emulator/state/stateDB_test.go +++ b/fvm/evm/emulator/state/stateDB_test.go @@ -15,12 +15,13 @@ import ( "github.com/stretchr/testify/require" ) +var rootAddr = flow.Address{1, 2, 3, 4, 5, 6, 7, 8} + func TestStateDB(t *testing.T) { t.Parallel() t.Run("test commit functionality", func(t *testing.T) { ledger := testutils.GetSimpleValueStore() - rootAddr := flow.Address{1, 2, 3, 4, 5, 6, 7, 8} db, err := state.NewStateDB(ledger, rootAddr) require.NoError(t, err) @@ -58,7 +59,6 @@ func TestStateDB(t *testing.T) { t.Run("test snapshot and revert functionality", func(t *testing.T) { ledger := testutils.GetSimpleValueStore() - rootAddr := flow.Address{1, 2, 3, 4, 5, 6, 7, 8} db, err := state.NewStateDB(ledger, rootAddr) require.NoError(t, err) @@ -115,7 +115,6 @@ func TestStateDB(t *testing.T) { t.Run("test log functionality", func(t *testing.T) { ledger := testutils.GetSimpleValueStore() - rootAddr := flow.Address{1, 2, 3, 4, 5, 6, 7, 8} db, err := state.NewStateDB(ledger, rootAddr) require.NoError(t, err) @@ -142,6 +141,25 @@ func TestStateDB(t *testing.T) { require.Equal(t, ret, logs) }) + t.Run("test refund functionality", func(t *testing.T) { + ledger := testutils.GetSimpleValueStore() + db, err := state.NewStateDB(ledger, rootAddr) + require.NoError(t, err) + + require.Equal(t, uint64(0), db.GetRefund()) + db.AddRefund(10) + require.Equal(t, uint64(10), db.GetRefund()) + db.SubRefund(3) + require.Equal(t, uint64(7), db.GetRefund()) + + snap1 := db.Snapshot() + db.AddRefund(10) + require.Equal(t, uint64(17), db.GetRefund()) + + db.RevertToSnapshot(snap1) + require.Equal(t, uint64(7), db.GetRefund()) + }) + t.Run("test non-fatal error handling", func(t *testing.T) { ledger := &testutils.TestValueStore{ GetValueFunc: func(owner, key []byte) ([]byte, error) { @@ -154,7 +172,6 @@ func TestStateDB(t *testing.T) { return atree.StorageIndex{}, nil }, } - rootAddr := flow.Address{1, 2, 3, 4, 5, 6, 7, 8} db, err := state.NewStateDB(ledger, rootAddr) require.NoError(t, err) @@ -179,7 +196,6 @@ func TestStateDB(t *testing.T) { return atree.StorageIndex{}, nil }, } - rootAddr := flow.Address{1, 2, 3, 4, 5, 6, 7, 8} db, err := state.NewStateDB(ledger, rootAddr) require.NoError(t, err) @@ -191,4 +207,5 @@ func TestStateDB(t *testing.T) { // check wrapping require.True(t, types.IsAFatalError(err)) }) + } From 44b6d8727d8e0a560bc87fc299eadc4e4f499bc1 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 23:04:20 -0800 Subject: [PATCH 18/72] add more tests --- fvm/evm/emulator/state/stateDB_test.go | 70 ++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/fvm/evm/emulator/state/stateDB_test.go b/fvm/evm/emulator/state/stateDB_test.go index 59652f04379..a394dc96e31 100644 --- a/fvm/evm/emulator/state/stateDB_test.go +++ b/fvm/evm/emulator/state/stateDB_test.go @@ -6,7 +6,9 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + gethCommon "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" + gethParams "github.com/ethereum/go-ethereum/params" "github.com/onflow/atree" "github.com/onflow/flow-go/fvm/evm/emulator/state" "github.com/onflow/flow-go/fvm/evm/testutils" @@ -20,6 +22,28 @@ var rootAddr = flow.Address{1, 2, 3, 4, 5, 6, 7, 8} func TestStateDB(t *testing.T) { t.Parallel() + t.Run("test Empty method", func(t *testing.T) { + ledger := testutils.GetSimpleValueStore() + db, err := state.NewStateDB(ledger, rootAddr) + require.NoError(t, err) + + addr1 := testutils.RandomCommonAddress(t) + // non-existent account + require.True(t, db.Empty(addr1)) + require.NoError(t, db.Error()) + + db.CreateAccount(addr1) + require.NoError(t, db.Error()) + + require.True(t, db.Empty(addr1)) + require.NoError(t, db.Error()) + + db.AddBalance(addr1, big.NewInt(10)) + require.NoError(t, db.Error()) + + require.False(t, db.Empty(addr1)) + }) + t.Run("test commit functionality", func(t *testing.T) { ledger := testutils.GetSimpleValueStore() db, err := state.NewStateDB(ledger, rootAddr) @@ -160,6 +184,52 @@ func TestStateDB(t *testing.T) { require.Equal(t, uint64(7), db.GetRefund()) }) + t.Run("test Prepare functionality", func(t *testing.T) { + ledger := testutils.GetSimpleValueStore() + db, err := state.NewStateDB(ledger, rootAddr) + + sender := testutils.RandomCommonAddress(t) + coinbase := testutils.RandomCommonAddress(t) + dest := testutils.RandomCommonAddress(t) + precompiles := []gethCommon.Address{ + testutils.RandomCommonAddress(t), + testutils.RandomCommonAddress(t), + } + + txAccesses := gethTypes.AccessList([]gethTypes.AccessTuple{ + {Address: testutils.RandomCommonAddress(t), + StorageKeys: []gethCommon.Hash{ + testutils.RandomCommonHash(t), + testutils.RandomCommonHash(t), + }, + }, + }) + + rules := gethParams.Rules{ + IsBerlin: true, + IsShanghai: true, + } + + require.NoError(t, err) + db.Prepare(rules, sender, coinbase, &dest, precompiles, txAccesses) + + require.True(t, db.AddressInAccessList(sender)) + require.True(t, db.AddressInAccessList(coinbase)) + require.True(t, db.AddressInAccessList(dest)) + + for _, add := range precompiles { + require.True(t, db.AddressInAccessList(add)) + } + + for _, el := range txAccesses { + for _, key := range el.StorageKeys { + addrFound, slotFound := db.SlotInAccessList(el.Address, key) + require.True(t, addrFound) + require.True(t, slotFound) + } + } + }) + t.Run("test non-fatal error handling", func(t *testing.T) { ledger := &testutils.TestValueStore{ GetValueFunc: func(owner, key []byte) ([]byte, error) { From f7b72125972f1de0bf0416e8cede179e3c8509d0 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 23:05:38 -0800 Subject: [PATCH 19/72] clean up --- fvm/evm/emulator/state/delta.go | 5 ----- fvm/evm/types/state.go | 3 --- 2 files changed, 8 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index b18e84dd23d..c2248cddc9d 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -416,11 +416,6 @@ func (d *DeltaView) Preimages() map[gethCommon.Hash][]byte { return d.preimages } -// Commit for deltaview is a no-op -func (d *DeltaView) Commit() error { - return nil -} - // DirtyAddresses returns a set of addresses that has been updated in this view func (d *DeltaView) DirtyAddresses() map[gethCommon.Address]interface{} { return d.dirtyAddresses diff --git a/fvm/evm/types/state.go b/fvm/evm/types/state.go index 8d0182fa0b1..3d097bf4d00 100644 --- a/fvm/evm/types/state.go +++ b/fvm/evm/types/state.go @@ -93,9 +93,6 @@ type HotView interface { AddLog(*gethTypes.Log) // AddPreimage adds a preimage to the list of preimages (input -> hash mapping) AddPreimage(gethCommon.Hash, []byte) - - // Commit finalizes and commits the changes - Commit() error } // BaseView is a low-level mutable view of the state From 30c46b181a2e8e06fb480f68b1754cbdfcf0f4b2 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 23:09:20 -0800 Subject: [PATCH 20/72] adding more tests --- fvm/evm/emulator/state/stateDB_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fvm/evm/emulator/state/stateDB_test.go b/fvm/evm/emulator/state/stateDB_test.go index a394dc96e31..b93fd1aa312 100644 --- a/fvm/evm/emulator/state/stateDB_test.go +++ b/fvm/evm/emulator/state/stateDB_test.go @@ -65,9 +65,18 @@ func TestStateDB(t *testing.T) { db.SetState(addr1, key1, value1) + ret := db.GetState(addr1, key1) + require.Equal(t, value1, ret) + + ret = db.GetCommittedState(addr1, key1) + require.Equal(t, gethCommon.Hash{}, ret) + err = db.Commit() require.NoError(t, err) + ret = db.GetCommittedState(addr1, key1) + require.Equal(t, value1, ret) + // create a new db db, err = state.NewStateDB(ledger, rootAddr) require.NoError(t, err) From bc729af2f90c25ad7363a0ea739a15b1e70ac540 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 19 Dec 2023 23:24:41 -0800 Subject: [PATCH 21/72] remove database --- fvm/evm/emulator/emulator.go | 10 +-- fvm/evm/emulator/state_test.go | 62 +++++------------ fvm/evm/testutils/database.go | 117 --------------------------------- fvm/evm/types/errors.go | 28 -------- 4 files changed, 20 insertions(+), 197 deletions(-) delete mode 100644 fvm/evm/testutils/database.go diff --git a/fvm/evm/emulator/emulator.go b/fvm/evm/emulator/emulator.go index 1576a5531ad..87858f873f8 100644 --- a/fvm/evm/emulator/emulator.go +++ b/fvm/evm/emulator/emulator.go @@ -159,15 +159,15 @@ type procedure struct { // commit commits the changes to the state. func (proc *procedure) commit() error { - return proc.state.Commit() + return handleCommitError(proc.state.Commit()) } func handleCommitError(err error) error { if err == nil { return nil } - // if known types (database errors) don't do anything and return - if types.IsAFatalError(err) || types.IsADatabaseError(err) { + // if known types (state errors) don't do anything and return + if types.IsAFatalError(err) || types.IsAStateError(err) { return err } @@ -243,8 +243,8 @@ func (proc *procedure) run(msg *gethCore.Message, txType uint8) (*types.Result, ).TransitionDb() if err != nil { res.Failed = true - // if the error is a fatal error or a non-fatal database error return it - if types.IsAFatalError(err) || types.IsADatabaseError(err) { + // if the error is a fatal error or a non-fatal state error return it + if types.IsAFatalError(err) || types.IsAStateError(err) { return &res, err } // otherwise is a validation error (pre-check failure) diff --git a/fvm/evm/emulator/state_test.go b/fvm/evm/emulator/state_test.go index a7341ba6c97..368578b6d3d 100644 --- a/fvm/evm/emulator/state_test.go +++ b/fvm/evm/emulator/state_test.go @@ -10,15 +10,12 @@ import ( "github.com/onflow/flow-go/utils/io" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/common" - gethRawDB "github.com/ethereum/go-ethereum/core/rawdb" - gethState "github.com/ethereum/go-ethereum/core/state" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/fvm/evm/emulator/database" + "github.com/onflow/flow-go/fvm/evm/emulator/state" "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" ) @@ -36,37 +33,16 @@ const ( type storageTest struct { store *testutils.TestValueStore - db *database.MeteredDatabase - ethDB ethdb.Database - stateDB gethState.Database addressIndex uint64 - hash common.Hash metrics *metrics } func newStorageTest() (*storageTest, error) { simpleStore := testutils.GetSimpleValueStore() - db, err := database.NewMeteredDatabase(simpleStore, flow.Address{0x01}) - if err != nil { - return nil, err - } - - hash, err := db.GetRootHash() - if err != nil { - return nil, err - } - - rawDB := gethRawDB.NewDatabase(db) - stateDB := gethState.NewDatabase(rawDB) - return &storageTest{ store: simpleStore, - db: db, - ethDB: rawDB, - stateDB: stateDB, addressIndex: 100, - hash: hash, metrics: newMetrics(), }, nil } @@ -80,33 +56,22 @@ func (s *storageTest) newAddress() common.Address { // run the provided runner with a newly created state which gets comitted after the runner // is finished. Storage metrics are being recorded with each run. -func (s *storageTest) run(runner func(state *gethState.StateDB)) error { - state, err := gethState.New(s.hash, s.stateDB, nil) +func (s *storageTest) run(runner func(state types.StateDB)) error { + state, err := state.NewStateDB(s.store, flow.Address{0x01}) if err != nil { return err } runner(state) - s.hash, err = state.Commit(true) - if err != nil { - return err - } - - err = state.Database().TrieDB().Commit(s.hash, true) + err = state.Commit() if err != nil { return err } - err = s.db.Commit(s.hash) - if err != nil { - return err - } - - s.db.DropCache() - - s.metrics.add(bytesWrittenMetric, s.db.BytesStored()) - s.metrics.add(bytesReadMetric, s.db.BytesRetrieved()) + // TODO: figure out us + // s.metrics.add(bytesWrittenMetric, s.db.BytesStored()) + // s.metrics.add(bytesReadMetric, s.db.BytesRetrieved()) s.metrics.add(storageItemsMetric, s.store.TotalStorageItems()) s.metrics.add(storageBytesMetric, s.store.TotalStorageSize()) @@ -168,7 +133,7 @@ func Test_AccountCreations(t *testing.T) { accountChart := "accounts,storage_size" maxAccounts := 50_000 for i := 0; i < maxAccounts; i++ { - err = tester.run(func(state *gethState.StateDB) { + err = tester.run(func(state types.StateDB) { state.AddBalance(tester.newAddress(), big.NewInt(100)) }) require.NoError(t, err) @@ -205,16 +170,19 @@ func Test_AccountContractInteraction(t *testing.T) { interactions := 50000 for i := 0; i < interactions; i++ { - err = tester.run(func(state *gethState.StateDB) { + err = tester.run(func(state types.StateDB) { // create a new account accAddr := tester.newAddress() state.AddBalance(accAddr, big.NewInt(100)) // create a contract contractAddr := tester.newAddress() - state.SetBalance(contractAddr, big.NewInt(int64(i))) + state.AddBalance(contractAddr, big.NewInt(int64(i))) state.SetCode(contractAddr, code) - state.SetStorage(contractAddr, contractState) + + for k, v := range contractState { + state.SetState(contractAddr, k, v) + } // simulate interaction with contract state and account balance for fees state.SetState(contractAddr, common.HexToHash("0x03"), common.HexToHash("0x40")) diff --git a/fvm/evm/testutils/database.go b/fvm/evm/testutils/database.go deleted file mode 100644 index fcc4ef3d117..00000000000 --- a/fvm/evm/testutils/database.go +++ /dev/null @@ -1,117 +0,0 @@ -package testutils - -import ( - gethCommon "github.com/ethereum/go-ethereum/common" - gethDB "github.com/ethereum/go-ethereum/ethdb" - - "github.com/onflow/flow-go/fvm/evm/types" -) - -type TestDatabase struct { - GetFunc func(key []byte) ([]byte, error) - HasFunc func(key []byte) (bool, error) - PutFunc func(key []byte, value []byte) error - DeleteFunc func(key []byte) error - StatFunc func(property string) (string, error) - NewBatchFunc func() gethDB.Batch - NewBatchWithSizeFunc func(size int) gethDB.Batch - NewIteratorFunc func(prefix []byte, start []byte) gethDB.Iterator - CompactFunc func(start []byte, limit []byte) error - NewSnapshotFunc func() (gethDB.Snapshot, error) - CloseFunc func() error - GetRootHashFunc func() (gethCommon.Hash, error) - CommitFunc func(roothash gethCommon.Hash) error -} - -var _ types.Database = &TestDatabase{} - -func (db *TestDatabase) Get(key []byte) ([]byte, error) { - if db.GetFunc == nil { - panic("method not set") - } - return db.GetFunc(key) -} - -func (db *TestDatabase) Has(key []byte) (bool, error) { - if db.HasFunc == nil { - panic("method not set") - } - return db.HasFunc(key) -} - -func (db *TestDatabase) Put(key []byte, value []byte) error { - if db.PutFunc == nil { - panic("method not set") - } - return db.PutFunc(key, value) -} - -func (db *TestDatabase) Delete(key []byte) error { - if db.DeleteFunc == nil { - panic("method not set") - } - return db.DeleteFunc(key) -} - -func (db *TestDatabase) Commit(root gethCommon.Hash) error { - if db.CommitFunc == nil { - panic("method not set") - } - return db.CommitFunc(root) -} - -func (db *TestDatabase) GetRootHash() (gethCommon.Hash, error) { - if db.GetRootHashFunc == nil { - panic("method not set") - } - return db.GetRootHashFunc() -} - -func (db *TestDatabase) Stat(property string) (string, error) { - if db.StatFunc == nil { - panic("method not set") - } - return db.StatFunc(property) -} - -func (db *TestDatabase) NewBatch() gethDB.Batch { - if db.NewBatchFunc == nil { - panic("method not set") - } - return db.NewBatchFunc() -} - -func (db *TestDatabase) NewBatchWithSize(size int) gethDB.Batch { - if db.NewBatchWithSizeFunc == nil { - panic("method not set") - } - return db.NewBatchWithSizeFunc(size) -} - -func (db *TestDatabase) NewIterator(prefix []byte, start []byte) gethDB.Iterator { - if db.NewIteratorFunc == nil { - panic("method not set") - } - return db.NewIteratorFunc(prefix, start) -} - -func (db *TestDatabase) Compact(start []byte, limit []byte) error { - if db.CompactFunc == nil { - panic("method not set") - } - return db.CompactFunc(start, limit) -} - -func (db *TestDatabase) NewSnapshot() (gethDB.Snapshot, error) { - if db.NewSnapshotFunc == nil { - panic("method not set") - } - return db.NewSnapshotFunc() -} - -func (db *TestDatabase) Close() error { - if db.CloseFunc == nil { - panic("method not set") - } - return db.CloseFunc() -} diff --git a/fvm/evm/types/errors.go b/fvm/evm/types/errors.go index cee5ad2e723..bed9987206a 100644 --- a/fvm/evm/types/errors.go +++ b/fvm/evm/types/errors.go @@ -89,34 +89,6 @@ func IsEVMValidationError(err error) bool { return errors.As(err, &EVMValidationError{}) } -// DatabaseError is a non-fatal error, returned when a database operation -// has failed (e.g. reaching storage interaction limit) -type DatabaseError struct { - err error -} - -// NewDatabaseError returns a new DatabaseError -func NewDatabaseError(rootCause error) DatabaseError { - return DatabaseError{ - err: rootCause, - } -} - -// Unwrap unwraps the underlying evm error -func (err DatabaseError) Unwrap() error { - return err.err -} - -func (err DatabaseError) Error() string { - return fmt.Sprintf("database error: %v", err.err) -} - -// IsADatabaseError returns true if the error or any underlying errors -// is of the type EVM validation error -func IsADatabaseError(err error) bool { - return errors.As(err, &DatabaseError{}) -} - // StateError is a non-fatal error, returned when a state operation // has failed (e.g. reaching storage interaction limit) type StateError struct { From 0a166f3c557fd31a8edc9d590ba45fe033fdb8fd Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 20 Dec 2023 11:10:37 -0800 Subject: [PATCH 22/72] update test --- fvm/evm/emulator/state_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fvm/evm/emulator/state_test.go b/fvm/evm/emulator/state_test.go index 368578b6d3d..042c3007245 100644 --- a/fvm/evm/emulator/state_test.go +++ b/fvm/evm/emulator/state_test.go @@ -69,9 +69,8 @@ func (s *storageTest) run(runner func(state types.StateDB)) error { return err } - // TODO: figure out us - // s.metrics.add(bytesWrittenMetric, s.db.BytesStored()) - // s.metrics.add(bytesReadMetric, s.db.BytesRetrieved()) + s.metrics.add(bytesWrittenMetric, s.store.TotalBytesWritten()) + s.metrics.add(bytesReadMetric, s.store.TotalBytesRead()) s.metrics.add(storageItemsMetric, s.store.TotalStorageItems()) s.metrics.add(storageBytesMetric, s.store.TotalStorageSize()) From 4b82fcb890c51c455e2289f8f29068b16edbb711 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 20 Dec 2023 11:12:29 -0800 Subject: [PATCH 23/72] move state benchmark test to the state package --- fvm/evm/emulator/{ => state}/state_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename fvm/evm/emulator/{ => state}/state_test.go (99%) diff --git a/fvm/evm/emulator/state_test.go b/fvm/evm/emulator/state/state_test.go similarity index 99% rename from fvm/evm/emulator/state_test.go rename to fvm/evm/emulator/state/state_test.go index 042c3007245..b7502728d8b 100644 --- a/fvm/evm/emulator/state_test.go +++ b/fvm/evm/emulator/state/state_test.go @@ -1,4 +1,4 @@ -package emulator_test +package state_test import ( "encoding/binary" From 4894916f46541bdd4fc761b6e474ad7b0fbefa7c Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 20 Dec 2023 11:12:54 -0800 Subject: [PATCH 24/72] rename state_test to benchmark --- fvm/evm/emulator/state/{state_test.go => benchmark_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fvm/evm/emulator/state/{state_test.go => benchmark_test.go} (100%) diff --git a/fvm/evm/emulator/state/state_test.go b/fvm/evm/emulator/state/benchmark_test.go similarity index 100% rename from fvm/evm/emulator/state/state_test.go rename to fvm/evm/emulator/state/benchmark_test.go From b4808c5b350d283cf38cf54068d44d4b9b3f2ab9 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 20 Dec 2023 11:14:33 -0800 Subject: [PATCH 25/72] rename the test --- .../emulator/state/{benchmark_test.go => state_growth_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fvm/evm/emulator/state/{benchmark_test.go => state_growth_test.go} (100%) diff --git a/fvm/evm/emulator/state/benchmark_test.go b/fvm/evm/emulator/state/state_growth_test.go similarity index 100% rename from fvm/evm/emulator/state/benchmark_test.go rename to fvm/evm/emulator/state/state_growth_test.go From c69a7fa761693eb210aefd33a4153f7ac6931978 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 20 Dec 2023 11:15:27 -0800 Subject: [PATCH 26/72] make the test lighter --- fvm/evm/handler/handler_benchmark_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/handler/handler_benchmark_test.go b/fvm/evm/handler/handler_benchmark_test.go index 6614819ef22..6165b26c29d 100644 --- a/fvm/evm/handler/handler_benchmark_test.go +++ b/fvm/evm/handler/handler_benchmark_test.go @@ -10,7 +10,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -func BenchmarkStorage(b *testing.B) { benchmarkStorageGrowth(b, 100, 1_000_000) } +func BenchmarkStorage(b *testing.B) { benchmarkStorageGrowth(b, 100, 100) } // benchmark func benchmarkStorageGrowth(b *testing.B, accountCount, setupKittyCount int) { From 4d6e1b159207ef1c9022373ff6e6aa41fd6844db Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 20 Dec 2023 11:23:23 -0800 Subject: [PATCH 27/72] remove state root --- fvm/evm/handler/handler.go | 2 -- fvm/evm/handler/handler_test.go | 1 - fvm/evm/types/block.go | 5 ----- fvm/evm/types/events.go | 4 ---- fvm/evm/types/result.go | 4 ---- 5 files changed, 16 deletions(-) diff --git a/fvm/evm/handler/handler.go b/fvm/evm/handler/handler.go index 4ed802e08a8..acbfc1c173e 100644 --- a/fvm/evm/handler/handler.go +++ b/fvm/evm/handler/handler.go @@ -103,7 +103,6 @@ func (h *ContractHandler) Run(rlpEncodedTx []byte, coinbase types.Address) { bp, err := h.blockstore.BlockProposal() handleError(err) - bp.StateRoot = res.StateRootHash txHash := tx.Hash() bp.AppendTxHash(txHash) @@ -301,7 +300,6 @@ func (a *Account) executeAndHandleCall( bp, err := a.fch.blockstore.BlockProposal() handleError(err) bp.AppendTxHash(callHash) - bp.StateRoot = res.StateRootHash if deductSupplyDiff { bp.TotalSupply -= totalSupplyDiff } else { diff --git a/fvm/evm/handler/handler_test.go b/fvm/evm/handler/handler_test.go index 5f87664202e..0acf3d3ceff 100644 --- a/fvm/evm/handler/handler_test.go +++ b/fvm/evm/handler/handler_test.go @@ -51,7 +51,6 @@ func TestHandler_TransactionRun(t *testing.T) { require.NoError(t, err) result := &types.Result{ - StateRootHash: testutils.RandomCommonHash(t), DeployedContractAddress: types.Address(testutils.RandomAddress(t)), ReturnedValue: testutils.RandomData(t), GasConsumed: testutils.RandomGas(1000), diff --git a/fvm/evm/types/block.go b/fvm/evm/types/block.go index 6c20c6a4d90..88d7f930502 100644 --- a/fvm/evm/types/block.go +++ b/fvm/evm/types/block.go @@ -18,9 +18,6 @@ type Block struct { // holds the total amount of the native token deposited in the evm side. TotalSupply uint64 - // StateRoot returns the EVM root hash of the state after executing this block - StateRoot gethCommon.Hash - // ReceiptRoot returns the root hash of the receipts emitted in this block ReceiptRoot gethCommon.Hash @@ -52,7 +49,6 @@ func NewBlock(height, uuidIndex, totalSupply uint64, return &Block{ Height: height, TotalSupply: totalSupply, - StateRoot: stateRoot, ReceiptRoot: receiptRoot, TransactionHashes: txHashes, } @@ -69,6 +65,5 @@ func NewBlockFromBytes(encoded []byte) (*Block, error) { var GenesisBlock = &Block{ ParentBlockHash: gethCommon.Hash{}, Height: uint64(0), - StateRoot: gethTypes.EmptyRootHash, ReceiptRoot: gethTypes.EmptyRootHash, } diff --git a/fvm/evm/types/events.go b/fvm/evm/types/events.go index 148c3f59ede..4ba65294d86 100644 --- a/fvm/evm/types/events.go +++ b/fvm/evm/types/events.go @@ -44,7 +44,6 @@ var transactionExecutedEventCadenceType = &cadence.EventType{ cadence.NewField("failed", cadence.BoolType{}), cadence.NewField("transactionType", cadence.UInt8Type{}), cadence.NewField("gasConsumed", cadence.UInt64Type{}), - cadence.NewField("stateRootHash", cadence.StringType{}), cadence.NewField("deployedContractAddress", cadence.StringType{}), cadence.NewField("returnedValue", cadence.StringType{}), cadence.NewField("logs", cadence.StringType{}), @@ -70,7 +69,6 @@ func (p *TransactionExecutedPayload) CadenceEvent() (cadence.Event, error) { cadence.NewBool(p.Result.Failed), cadence.NewUInt8(p.Result.TxType), cadence.NewUInt64(p.Result.GasConsumed), - cadence.String(p.Result.StateRootHash.String()), cadence.String(hex.EncodeToString(p.Result.DeployedContractAddress.Bytes())), cadence.String(hex.EncodeToString(p.Result.ReturnedValue)), cadence.String(hex.EncodeToString(encodedLogs)), @@ -105,7 +103,6 @@ var blockExecutedEventCadenceType = &cadence.EventType{ cadence.NewField("height", cadence.UInt64Type{}), cadence.NewField("totalSupply", cadence.UInt64Type{}), cadence.NewField("parentHash", cadence.StringType{}), - cadence.NewField("stateRoot", cadence.StringType{}), cadence.NewField("receiptRoot", cadence.StringType{}), cadence.NewField( "transactionHashes", @@ -129,7 +126,6 @@ func (p *BlockExecutedEventPayload) CadenceEvent() (cadence.Event, error) { cadence.NewUInt64(p.Block.TotalSupply), cadence.String(p.Block.ReceiptRoot.String()), cadence.String(p.Block.ParentBlockHash.String()), - cadence.String(p.Block.StateRoot.String()), cadence.NewArray(hashes).WithType(cadence.NewVariableSizedArrayType(cadence.StringType{})), } diff --git a/fvm/evm/types/result.go b/fvm/evm/types/result.go index 6e4248b2d58..fb7321a5847 100644 --- a/fvm/evm/types/result.go +++ b/fvm/evm/types/result.go @@ -1,7 +1,6 @@ package types import ( - gethCommon "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" ) @@ -20,8 +19,6 @@ type Result struct { TxType uint8 // total gas consumed during an opeartion GasConsumed uint64 - // the root hash of the state after execution - StateRootHash gethCommon.Hash // the address where the contract is deployed (if any) DeployedContractAddress Address // returned value from a function call @@ -35,7 +32,6 @@ type Result struct { func (res *Result) Receipt() *gethTypes.ReceiptForStorage { receipt := &gethTypes.Receipt{ Type: res.TxType, - PostState: res.StateRootHash[:], CumulativeGasUsed: res.GasConsumed, // TODO: update to capture cumulative Logs: res.Logs, ContractAddress: res.DeployedContractAddress.ToCommon(), From 0596ab06b1225f92b02f539234d292d2a02f619a Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 2 Jan 2024 15:47:33 +0100 Subject: [PATCH 28/72] merge fix and added test --- .../migrations/storage_fees_migration.go | 63 ------------------- cmd/util/ledger/util/payload_grouping_test.go | 18 ++++++ cmd/util/ledger/util/util.go | 8 +-- 3 files changed, 22 insertions(+), 67 deletions(-) delete mode 100644 cmd/util/ledger/migrations/storage_fees_migration.go diff --git a/cmd/util/ledger/migrations/storage_fees_migration.go b/cmd/util/ledger/migrations/storage_fees_migration.go deleted file mode 100644 index b7dd49a1ec6..00000000000 --- a/cmd/util/ledger/migrations/storage_fees_migration.go +++ /dev/null @@ -1,63 +0,0 @@ -package migrations - -import ( - fvm "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/ledger" - "github.com/onflow/flow-go/ledger/common/convert" - "github.com/onflow/flow-go/ledger/common/utils" - "github.com/onflow/flow-go/model/flow" -) - -// iterates through registers keeping a map of register sizes -// after it has reached the end it add storage used and storage capacity for each address -func StorageFeesMigration(payload []ledger.Payload) ([]ledger.Payload, error) { - storageUsed := make(map[string]uint64) - newPayload := make([]ledger.Payload, len(payload)) - - for i, p := range payload { - err := incrementStorageUsed(p, storageUsed) - if err != nil { - return nil, err - } - newPayload[i] = p - } - - for s, u := range storageUsed { - // this is the storage used by the storage_used register we are about to add - id := flow.NewRegisterID( - flow.BytesToAddress([]byte(s)), - "storage_used") - storageUsedByStorageUsed := fvm.RegisterSize(id, make([]byte, 8)) - u = u + uint64(storageUsedByStorageUsed) - - newPayload = append(newPayload, *ledger.NewPayload( - convert.RegisterIDToLedgerKey(id), - utils.Uint64ToBinary(u), - )) - } - return newPayload, nil -} - -func incrementStorageUsed(p ledger.Payload, used map[string]uint64) error { - k, err := p.Key() - if err != nil { - return err - } - id, err := convert.LedgerKeyToRegisterID(k) - if err != nil { - return err - } - if len([]byte(id.Owner)) != flow.AddressLength { - // not an address - return nil - } - if _, ok := used[id.Owner]; !ok { - used[id.Owner] = 0 - } - used[id.Owner] = used[id.Owner] + uint64(registerSize(id, p)) - return nil -} - -func registerSize(id flow.RegisterID, p ledger.Payload) int { - return fvm.RegisterSize(id, p.Value()) -} diff --git a/cmd/util/ledger/util/payload_grouping_test.go b/cmd/util/ledger/util/payload_grouping_test.go index e2efe0654ef..6924f0c598f 100644 --- a/cmd/util/ledger/util/payload_grouping_test.go +++ b/cmd/util/ledger/util/payload_grouping_test.go @@ -10,6 +10,8 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "github.com/onflow/cadence/runtime/common" + "github.com/onflow/flow-go/cmd/util/ledger/util" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" @@ -38,7 +40,22 @@ func TestGroupPayloadsByAccountCompareResults(t *testing.T) { groups1 := util.GroupPayloadsByAccount(log, tmp1, 0) groups2 := util.GroupPayloadsByAccount(log, tmp2, runtime.NumCPU()) + groups3 := map[common.Address][]*ledger.Payload{} + for _, payload := range payloads { + key, err := payload.Key() + require.NoError(t, err) + registerID, err := convert.LedgerKeyToRegisterID(key) + require.NoError(t, err) + address, err := common.BytesToAddress([]byte(registerID.Owner)) + require.NoError(t, err) + if _, ok := groups3[address]; !ok { + groups3[address] = []*ledger.Payload{} + } + groups3[address] = append(groups3[address], payload) + } + require.Equal(t, groups1.Len(), groups2.Len()) + require.Equal(t, groups1.Len(), len(groups3)) for { group1, err1 := groups1.Next() group2, err2 := groups2.Next() @@ -53,6 +70,7 @@ func TestGroupPayloadsByAccountCompareResults(t *testing.T) { require.Equal(t, group1.Address, group2.Address) require.Equal(t, len(group1.Payloads), len(group2.Payloads)) + require.Equal(t, len(group1.Payloads), len(groups3[group1.Address])) } } diff --git a/cmd/util/ledger/util/util.go b/cmd/util/ledger/util/util.go index f7e127a0daa..46cc54e6850 100644 --- a/cmd/util/ledger/util/util.go +++ b/cmd/util/ledger/util/util.go @@ -26,7 +26,7 @@ var _ atree.Ledger = &AccountsAtreeLedger{} func (a *AccountsAtreeLedger) GetValue(owner, key []byte) ([]byte, error) { v, err := a.Accounts.GetValue( flow.NewRegisterID( - string(flow.BytesToAddress(owner).Bytes()), + flow.BytesToAddress(owner), string(key))) if err != nil { return nil, fmt.Errorf("getting value failed: %w", err) @@ -37,7 +37,7 @@ func (a *AccountsAtreeLedger) GetValue(owner, key []byte) ([]byte, error) { func (a *AccountsAtreeLedger) SetValue(owner, key, value []byte) error { err := a.Accounts.SetValue( flow.NewRegisterID( - string(flow.BytesToAddress(owner).Bytes()), + flow.BytesToAddress(owner), string(key)), value) if err != nil { @@ -113,7 +113,7 @@ type PayloadsReadonlyLedger struct { } func (p *PayloadsReadonlyLedger) GetValue(owner, key []byte) (value []byte, err error) { - v, err := p.Snapshot.Get(flow.NewRegisterID(string(owner), string(key))) + v, err := p.Snapshot.Get(flow.NewRegisterID(flow.BytesToAddress(owner), string(key))) if err != nil { return nil, fmt.Errorf("getting value failed: %w", err) } @@ -129,7 +129,7 @@ func (p *PayloadsReadonlyLedger) SetValue(owner, key, value []byte) (err error) } func (p *PayloadsReadonlyLedger) ValueExists(owner, key []byte) (exists bool, err error) { - _, ok := p.Snapshot.Payloads[flow.NewRegisterID(string(owner), string(key))] + _, ok := p.Snapshot.Payloads[flow.NewRegisterID(flow.BytesToAddress(owner), string(key))] return ok, nil } From 41f89793c2fe6d8c35e058ab0119bcc7e77dd547 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Wed, 13 Dec 2023 18:39:45 +0100 Subject: [PATCH 29/72] Deduplicate contract names migration --- .../deduplicate_contract_names_migration.go | 134 ++++++++++++++++++ ...duplicate_contract_names_migration_test.go | 90 ++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 cmd/util/ledger/migrations/deduplicate_contract_names_migration.go create mode 100644 cmd/util/ledger/migrations/deduplicate_contract_names_migration_test.go diff --git a/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go b/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go new file mode 100644 index 00000000000..5d79da0590c --- /dev/null +++ b/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go @@ -0,0 +1,134 @@ +package migrations + +import ( + "bytes" + "context" + "fmt" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + "github.com/fxamacker/cbor/v2" + + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/fvm/storage/state" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" +) + +// DeduplicateContractNamesMigration checks if the contract names have been duplicated and +// removes the duplicate ones. +type DeduplicateContractNamesMigration struct { + log zerolog.Logger +} + +func (d *DeduplicateContractNamesMigration) InitMigration( + log zerolog.Logger, + _ []*ledger.Payload, + _ int, +) error { + d.log = log. + With(). + Str("migration", "DeduplicateContractNamesMigration"). + Logger() + + return nil +} + +func (d *DeduplicateContractNamesMigration) MigrateAccount( + ctx context.Context, + address common.Address, + payloads []*ledger.Payload, +) ([]*ledger.Payload, error) { + snapshot, err := util.NewPayloadSnapshot(payloads) + if err != nil { + return nil, fmt.Errorf("failed to create payload snapshot: %w", err) + } + transactionState := state.NewTransactionState(snapshot, state.DefaultParameters()) + accounts := environment.NewAccounts(transactionState) + flowAddress := flow.ConvertAddress(address) + + contractNames, err := accounts.GetContractNames(flowAddress) + if err != nil { + return nil, fmt.Errorf("failed to get contract names: %w", err) + } + if len(contractNames) == 0 { + return payloads, nil + } + + contractNamesSet := make(map[string]struct{}) + removeIndexes := make([]int, 0) + for i, name := range contractNames { + if _, ok := contractNamesSet[name]; ok { + // duplicate contract name + removeIndexes = append(removeIndexes, i) + continue + } + + contractNamesSet[name] = struct{}{} + } + + if len(removeIndexes) == 0 { + return payloads, nil + } + + log.Info(). + Str("address", address.Hex()). + Strs("contract_names", contractNames). + Msg("removing duplicate contract names") + + // remove the duplicate contract names, keeping the original order + for i := len(removeIndexes) - 1; i >= 0; i-- { + contractNames = append(contractNames[:removeIndexes[i]], contractNames[removeIndexes[i]+1:]...) + } + + var buf bytes.Buffer + cborEncoder := cbor.NewEncoder(&buf) + err = cborEncoder.Encode(contractNames) + if err != nil { + return nil, fmt.Errorf( + "cannot encode contract names: %s", + contractNames, + ) + } + newContractNames := buf.Bytes() + + id := flow.ContractNamesRegisterID(flowAddress) + err = accounts.SetValue(id, newContractNames) + + if err != nil { + return nil, fmt.Errorf("setting value failed: %w", err) + } + + // finalize the transaction + result, err := transactionState.FinalizeMainTransaction() + if err != nil { + return nil, fmt.Errorf("failed to finalize main transaction: %w", err) + } + + for id, value := range result.WriteSet { + if value == nil { + delete(snapshot.Payloads, id) + continue + } + + snapshot.Payloads[id] = ledger.NewPayload( + convert.RegisterIDToLedgerKey(id), + value, + ) + } + + newPayloads := make([]*ledger.Payload, 0, len(snapshot.Payloads)) + for _, payload := range snapshot.Payloads { + newPayloads = append(newPayloads, payload) + } + + return newPayloads, nil + +} + +var _ AccountBasedMigration = &DeduplicateContractNamesMigration{} diff --git a/cmd/util/ledger/migrations/deduplicate_contract_names_migration_test.go b/cmd/util/ledger/migrations/deduplicate_contract_names_migration_test.go new file mode 100644 index 00000000000..a826d635ad4 --- /dev/null +++ b/cmd/util/ledger/migrations/deduplicate_contract_names_migration_test.go @@ -0,0 +1,90 @@ +package migrations_test + +import ( + "bytes" + "context" + "testing" + + "github.com/fxamacker/cbor/v2" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/cmd/util/ledger/migrations" + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" +) + +func TestDeduplicateContractNamesMigration(t *testing.T) { + migration := migrations.DeduplicateContractNamesMigration{} + + log := zerolog.New(zerolog.NewTestWriter(t)) + + err := migration.InitMigration(log, nil, 0) + require.NoError(t, err) + + ctx := context.Background() + + address, err := common.HexToAddress("0x1") + require.NoError(t, err) + + contractNames := []string{"test", "test", "test2", "test3", "test3"} + + var buf bytes.Buffer + cborEncoder := cbor.NewEncoder(&buf) + err = cborEncoder.Encode(contractNames) + require.NoError(t, err) + newContractNames := buf.Bytes() + + accountStatus := environment.NewAccountStatus() + + accountStatus.SetStorageUsed(1000) + + payloads, err := migration.MigrateAccount(ctx, address, + []*ledger.Payload{ + + ledger.NewPayload( + convert.RegisterIDToLedgerKey( + flow.RegisterID{ + Owner: string(address.Bytes()), + Key: flow.ContractNamesKey, + }, + ), + newContractNames, + ), + ledger.NewPayload( + convert.RegisterIDToLedgerKey( + flow.AccountStatusRegisterID(flow.ConvertAddress(address)), + ), + accountStatus.ToBytes(), + ), + }, + ) + + require.NoError(t, err) + require.Equal(t, 2, len(payloads)) + + for _, payload := range payloads { + key, err := payload.Key() + require.NoError(t, err) + id, err := convert.LedgerKeyToRegisterID(key) + require.NoError(t, err) + + if id.Key != flow.ContractNamesKey { + continue + } + + value := payload.Value() + + contracts := make([]string, 0) + buf := bytes.NewReader(value) + cborDecoder := cbor.NewDecoder(buf) + err = cborDecoder.Decode(&contracts) + require.NoError(t, err) + + require.Equal(t, 3, len(contracts)) + } +} From 2faa3880086216cc3b871d851082207115f48e46 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 2 Jan 2024 16:27:43 +0100 Subject: [PATCH 30/72] address review comments --- .../deduplicate_contract_names_migration.go | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go b/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go index 5d79da0590c..80707e39d95 100644 --- a/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go +++ b/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go @@ -1,14 +1,11 @@ package migrations import ( - "bytes" "context" "fmt" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/fxamacker/cbor/v2" + "github.com/rs/zerolog" "github.com/onflow/cadence/runtime/common" @@ -26,6 +23,10 @@ type DeduplicateContractNamesMigration struct { log zerolog.Logger } +func (d *DeduplicateContractNamesMigration) Close() error { + return nil +} + func (d *DeduplicateContractNamesMigration) InitMigration( log zerolog.Logger, _ []*ledger.Payload, @@ -56,46 +57,40 @@ func (d *DeduplicateContractNamesMigration) MigrateAccount( if err != nil { return nil, fmt.Errorf("failed to get contract names: %w", err) } - if len(contractNames) == 0 { + if len(contractNames) == 1 { return payloads, nil } - contractNamesSet := make(map[string]struct{}) - removeIndexes := make([]int, 0) - for i, name := range contractNames { - if _, ok := contractNamesSet[name]; ok { - // duplicate contract name - removeIndexes = append(removeIndexes, i) + var foundDuplicate bool + i := 1 + for i < len(contractNames) { + if contractNames[i-1] != contractNames[i] { + i++ continue } - - contractNamesSet[name] = struct{}{} + // Found duplicate (contactNames[i-1] == contactNames[i]) + // Remove contractNames[i] + copy(contractNames[i:], contractNames[i+1:]) + contractNames = contractNames[:len(contractNames)-1] + foundDuplicate = true } - if len(removeIndexes) == 0 { + if !foundDuplicate { return payloads, nil } - log.Info(). + d.log.Info(). Str("address", address.Hex()). Strs("contract_names", contractNames). Msg("removing duplicate contract names") - // remove the duplicate contract names, keeping the original order - for i := len(removeIndexes) - 1; i >= 0; i-- { - contractNames = append(contractNames[:removeIndexes[i]], contractNames[removeIndexes[i]+1:]...) - } - - var buf bytes.Buffer - cborEncoder := cbor.NewEncoder(&buf) - err = cborEncoder.Encode(contractNames) + newContractNames, err := cbor.Marshal(contractNames) if err != nil { return nil, fmt.Errorf( "cannot encode contract names: %s", contractNames, ) } - newContractNames := buf.Bytes() id := flow.ContractNamesRegisterID(flowAddress) err = accounts.SetValue(id, newContractNames) From 4a23ca1be0bf47450582299037998970af3e2650 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 2 Jan 2024 16:52:29 +0100 Subject: [PATCH 31/72] dont use snapshot to get one register --- .../deduplicate_contract_names_migration.go | 70 +++++++------------ 1 file changed, 26 insertions(+), 44 deletions(-) diff --git a/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go b/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go index 80707e39d95..4669f8e4b2b 100644 --- a/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go +++ b/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go @@ -9,9 +9,6 @@ import ( "github.com/onflow/cadence/runtime/common" - "github.com/onflow/flow-go/cmd/util/ledger/util" - "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" "github.com/onflow/flow-go/model/flow" @@ -45,21 +42,35 @@ func (d *DeduplicateContractNamesMigration) MigrateAccount( address common.Address, payloads []*ledger.Payload, ) ([]*ledger.Payload, error) { - snapshot, err := util.NewPayloadSnapshot(payloads) - if err != nil { - return nil, fmt.Errorf("failed to create payload snapshot: %w", err) - } - transactionState := state.NewTransactionState(snapshot, state.DefaultParameters()) - accounts := environment.NewAccounts(transactionState) flowAddress := flow.ConvertAddress(address) + contractNamesID := flow.ContractNamesRegisterID(flowAddress) + + var contractNamesPayload *ledger.Payload + contractNamesPayloadIndex := 0 + for i, payload := range payloads { + key, err := payload.Key() + if err != nil { + return nil, err + } + id, err := convert.LedgerKeyToRegisterID(key) + if err != nil { + return nil, err + } + if id == contractNamesID { + contractNamesPayload = payload + contractNamesPayloadIndex = i + break + } + } + if contractNamesPayload == nil { + return payloads, nil + } - contractNames, err := accounts.GetContractNames(flowAddress) + var contractNames []string + err := cbor.Unmarshal(contractNamesPayload.Value(), &contractNames) if err != nil { return nil, fmt.Errorf("failed to get contract names: %w", err) } - if len(contractNames) == 1 { - return payloads, nil - } var foundDuplicate bool i := 1 @@ -92,37 +103,8 @@ func (d *DeduplicateContractNamesMigration) MigrateAccount( ) } - id := flow.ContractNamesRegisterID(flowAddress) - err = accounts.SetValue(id, newContractNames) - - if err != nil { - return nil, fmt.Errorf("setting value failed: %w", err) - } - - // finalize the transaction - result, err := transactionState.FinalizeMainTransaction() - if err != nil { - return nil, fmt.Errorf("failed to finalize main transaction: %w", err) - } - - for id, value := range result.WriteSet { - if value == nil { - delete(snapshot.Payloads, id) - continue - } - - snapshot.Payloads[id] = ledger.NewPayload( - convert.RegisterIDToLedgerKey(id), - value, - ) - } - - newPayloads := make([]*ledger.Payload, 0, len(snapshot.Payloads)) - for _, payload := range snapshot.Payloads { - newPayloads = append(newPayloads, payload) - } - - return newPayloads, nil + payloads[contractNamesPayloadIndex] = ledger.NewPayload(convert.RegisterIDToLedgerKey(contractNamesID), newContractNames) + return payloads, nil } From 05fbf9867febaabf5eaa42304f128a7cc537bc05 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 2 Jan 2024 19:01:00 +0100 Subject: [PATCH 32/72] add more tests --- .../deduplicate_contract_names_migration.go | 17 +- ...duplicate_contract_names_migration_test.go | 223 ++++++++++++++---- 2 files changed, 193 insertions(+), 47 deletions(-) diff --git a/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go b/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go index 4669f8e4b2b..16bfda771ce 100644 --- a/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go +++ b/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go @@ -66,8 +66,12 @@ func (d *DeduplicateContractNamesMigration) MigrateAccount( return payloads, nil } + value := contractNamesPayload.Value() + if len(value) == 0 { + return payloads, nil + } var contractNames []string - err := cbor.Unmarshal(contractNamesPayload.Value(), &contractNames) + err := cbor.Unmarshal(value, &contractNames) if err != nil { return nil, fmt.Errorf("failed to get contract names: %w", err) } @@ -76,6 +80,17 @@ func (d *DeduplicateContractNamesMigration) MigrateAccount( i := 1 for i < len(contractNames) { if contractNames[i-1] != contractNames[i] { + + if contractNames[i-1] > contractNames[i] { + // this is not a valid state and we should fail. + // Contract names must be sorted by definition. + return nil, fmt.Errorf( + "contract names for account %s are not sorted: %s", + address.Hex(), + contractNames, + ) + } + i++ continue } diff --git a/cmd/util/ledger/migrations/deduplicate_contract_names_migration_test.go b/cmd/util/ledger/migrations/deduplicate_contract_names_migration_test.go index a826d635ad4..ba81bc826cd 100644 --- a/cmd/util/ledger/migrations/deduplicate_contract_names_migration_test.go +++ b/cmd/util/ledger/migrations/deduplicate_contract_names_migration_test.go @@ -1,8 +1,10 @@ package migrations_test import ( - "bytes" "context" + "fmt" + "math/rand" + "sort" "testing" "github.com/fxamacker/cbor/v2" @@ -20,71 +22,200 @@ import ( func TestDeduplicateContractNamesMigration(t *testing.T) { migration := migrations.DeduplicateContractNamesMigration{} - log := zerolog.New(zerolog.NewTestWriter(t)) - err := migration.InitMigration(log, nil, 0) require.NoError(t, err) - ctx := context.Background() - address, err := common.HexToAddress("0x1") require.NoError(t, err) - contractNames := []string{"test", "test", "test2", "test3", "test3"} - - var buf bytes.Buffer - cborEncoder := cbor.NewEncoder(&buf) - err = cborEncoder.Encode(contractNames) - require.NoError(t, err) - newContractNames := buf.Bytes() + ctx := context.Background() accountStatus := environment.NewAccountStatus() - accountStatus.SetStorageUsed(1000) + accountStatusPayload := ledger.NewPayload( + convert.RegisterIDToLedgerKey( + flow.AccountStatusRegisterID(flow.ConvertAddress(address)), + ), + accountStatus.ToBytes(), + ) - payloads, err := migration.MigrateAccount(ctx, address, - []*ledger.Payload{ - - ledger.NewPayload( - convert.RegisterIDToLedgerKey( - flow.RegisterID{ - Owner: string(address.Bytes()), - Key: flow.ContractNamesKey, - }, - ), - newContractNames, + contractNamesPayload := func(contractNames []byte) *ledger.Payload { + return ledger.NewPayload( + convert.RegisterIDToLedgerKey( + flow.RegisterID{ + Owner: string(address.Bytes()), + Key: flow.ContractNamesKey, + }, ), - ledger.NewPayload( - convert.RegisterIDToLedgerKey( - flow.AccountStatusRegisterID(flow.ConvertAddress(address)), - ), - accountStatus.ToBytes(), - ), - }, - ) + contractNames, + ) + } - require.NoError(t, err) - require.Equal(t, 2, len(payloads)) + requireContractNames := func(payloads []*ledger.Payload, f func([]string)) { + for _, payload := range payloads { + key, err := payload.Key() + require.NoError(t, err) + id, err := convert.LedgerKeyToRegisterID(key) + require.NoError(t, err) + + if id.Key != flow.ContractNamesKey { + continue + } + + contracts := make([]string, 0) + err = cbor.Unmarshal(payload.Value(), &contracts) + require.NoError(t, err) + + f(contracts) + + } + } + + t.Run("no contract names", func(t *testing.T) { + payloads, err := migration.MigrateAccount(ctx, address, + []*ledger.Payload{ + accountStatusPayload, + }, + ) + + require.NoError(t, err) + require.Equal(t, 1, len(payloads)) + }) - for _, payload := range payloads { - key, err := payload.Key() + t.Run("one contract", func(t *testing.T) { + contractNames := []string{"test"} + newContractNames, err := cbor.Marshal(contractNames) require.NoError(t, err) - id, err := convert.LedgerKeyToRegisterID(key) + + payloads, err := migration.MigrateAccount(ctx, address, + []*ledger.Payload{ + accountStatusPayload, + contractNamesPayload(newContractNames), + }, + ) + require.NoError(t, err) + require.Equal(t, 2, len(payloads)) - if id.Key != flow.ContractNamesKey { - continue + requireContractNames(payloads, func(contracts []string) { + require.Equal(t, 1, len(contracts)) + require.Equal(t, "test", contracts[0]) + }) + }) + + t.Run("two unique contracts", func(t *testing.T) { + contractNames := []string{"test", "test2"} + newContractNames, err := cbor.Marshal(contractNames) + require.NoError(t, err) + + payloads, err := migration.MigrateAccount(ctx, address, + []*ledger.Payload{ + accountStatusPayload, + contractNamesPayload(newContractNames), + }, + ) + + require.NoError(t, err) + require.Equal(t, 2, len(payloads)) + + requireContractNames(payloads, func(contracts []string) { + require.Equal(t, 2, len(contracts)) + require.Equal(t, "test", contracts[0]) + require.Equal(t, "test2", contracts[1]) + }) + }) + + t.Run("two contracts", func(t *testing.T) { + contractNames := []string{"test", "test"} + newContractNames, err := cbor.Marshal(contractNames) + require.NoError(t, err) + + payloads, err := migration.MigrateAccount(ctx, address, + []*ledger.Payload{ + accountStatusPayload, + contractNamesPayload(newContractNames), + }, + ) + + require.NoError(t, err) + require.Equal(t, 2, len(payloads)) + + requireContractNames(payloads, func(contracts []string) { + require.Equal(t, 1, len(contracts)) + require.Equal(t, "test", contracts[0]) + }) + }) + + t.Run("not sorted contracts", func(t *testing.T) { + contractNames := []string{"test2", "test"} + newContractNames, err := cbor.Marshal(contractNames) + require.NoError(t, err) + + _, err = migration.MigrateAccount(ctx, address, + []*ledger.Payload{ + accountStatusPayload, + contractNamesPayload(newContractNames), + }, + ) + + require.Error(t, err) + }) + + t.Run("duplicate contracts", func(t *testing.T) { + contractNames := []string{"test", "test", "test2", "test3", "test3"} + newContractNames, err := cbor.Marshal(contractNames) + require.NoError(t, err) + + payloads, err := migration.MigrateAccount(ctx, address, + []*ledger.Payload{ + accountStatusPayload, + contractNamesPayload(newContractNames), + }, + ) + + require.NoError(t, err) + require.Equal(t, 2, len(payloads)) + + requireContractNames(payloads, func(contracts []string) { + require.Equal(t, 3, len(contracts)) + require.Equal(t, "test", contracts[0]) + require.Equal(t, "test2", contracts[1]) + require.Equal(t, "test3", contracts[2]) + }) + }) + + t.Run("random contracts", func(t *testing.T) { + contractNames := make([]string, 1000) + uniqueContracts := 1 + for i := 0; i < 1000; i++ { + // i > 0 so it's easier to know how many unique contracts there are + if i > 0 && rand.Float32() < 0.5 { + uniqueContracts++ + } + contractNames[i] = fmt.Sprintf("test%d", uniqueContracts) } - value := payload.Value() + // sort contractNames alphabetically, because they are not sorted + sort.Slice(contractNames, func(i, j int) bool { + return contractNames[i] < contractNames[j] + }) - contracts := make([]string, 0) - buf := bytes.NewReader(value) - cborDecoder := cbor.NewDecoder(buf) - err = cborDecoder.Decode(&contracts) + newContractNames, err := cbor.Marshal(contractNames) require.NoError(t, err) - require.Equal(t, 3, len(contracts)) - } + payloads, err := migration.MigrateAccount(ctx, address, + []*ledger.Payload{ + accountStatusPayload, + contractNamesPayload(newContractNames), + }, + ) + + require.NoError(t, err) + require.Equal(t, 2, len(payloads)) + + requireContractNames(payloads, func(contracts []string) { + require.Equal(t, uniqueContracts, len(contracts)) + }) + }) } From bd7e9199dcedafe1837126d19fd18b418c3dd881 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 2 Jan 2024 21:50:37 +0100 Subject: [PATCH 33/72] remove empty payload --- .../migrations/deduplicate_contract_names_migration.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go b/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go index 16bfda771ce..ab35e04d8a3 100644 --- a/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go +++ b/cmd/util/ledger/migrations/deduplicate_contract_names_migration.go @@ -16,6 +16,8 @@ import ( // DeduplicateContractNamesMigration checks if the contract names have been duplicated and // removes the duplicate ones. +// +// This migration de-syncs storage used, so it should be run before the StorageUsedMigration. type DeduplicateContractNamesMigration struct { log zerolog.Logger } @@ -68,8 +70,13 @@ func (d *DeduplicateContractNamesMigration) MigrateAccount( value := contractNamesPayload.Value() if len(value) == 0 { + // Remove the empty payload + copy(payloads[contractNamesPayloadIndex:], payloads[contractNamesPayloadIndex+1:]) + payloads = payloads[:len(payloads)-1] + return payloads, nil } + var contractNames []string err := cbor.Unmarshal(value, &contractNames) if err != nil { From d1998b19f0c2a32dbf0b96752a68e1390f435fcc Mon Sep 17 00:00:00 2001 From: Peter Argue <89119817+peterargue@users.noreply.github.com> Date: Wed, 3 Jan 2024 10:12:58 -0800 Subject: [PATCH 34/72] [Access] Log script exec mismatches as error --- engine/access/rpc/backend/script_comparer.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/engine/access/rpc/backend/script_comparer.go b/engine/access/rpc/backend/script_comparer.go index 4bd219c4c96..e7ec9f3c489 100644 --- a/engine/access/rpc/backend/script_comparer.go +++ b/engine/access/rpc/backend/script_comparer.go @@ -53,7 +53,7 @@ func (c *scriptResultComparison) compare(execResult, localResult *scriptResult) if isOutOfRangeError(localResult.err) { c.metrics.ScriptExecutionNotIndexed() c.logComparison(execResult, localResult, - "script execution results do not match EN because data is not indexed yet") + "script execution results do not match EN because data is not indexed yet", false) return false } @@ -66,7 +66,7 @@ func (c *scriptResultComparison) compare(execResult, localResult *scriptResult) c.metrics.ScriptExecutionErrorMismatch() c.logComparison(execResult, localResult, - "cadence errors from local execution do not match and EN") + "cadence errors from local execution do not match EN", true) return false } @@ -77,12 +77,12 @@ func (c *scriptResultComparison) compare(execResult, localResult *scriptResult) c.metrics.ScriptExecutionResultMismatch() c.logComparison(execResult, localResult, - "script execution results from local execution do not match EN") + "script execution results from local execution do not match EN", true) return false } // logScriptExecutionComparison logs the script execution comparison between local execution and execution node -func (c *scriptResultComparison) logComparison(execResult, localResult *scriptResult, msg string) { +func (c *scriptResultComparison) logComparison(execResult, localResult *scriptResult, msg string, useError bool) { args := make([]string, len(c.request.arguments)) for i, arg := range c.request.arguments { args[i] = string(arg) @@ -109,7 +109,11 @@ func (c *scriptResultComparison) logComparison(execResult, localResult *scriptRe lgCtx = lgCtx.Dur("local_duration_ms", localResult.duration) lg := lgCtx.Logger() - lg.Debug().Msg(msg) + if useError { + lg.Error().Msg(msg) + } else { + lg.Debug().Msg(msg) + } } func isOutOfRangeError(err error) bool { From 3983134e89fbee2373de91a9122e0d9d147a7a27 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 12:28:06 -0800 Subject: [PATCH 35/72] PR feedbacks --- fvm/evm/emulator/state/delta.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index 8080c6c9f42..df038824ef2 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -55,20 +55,20 @@ func NewDeltaView(parent types.ReadOnlyView) *DeltaView { return &DeltaView{ parent: parent, - dirtyAddresses: make(map[gethCommon.Address]interface{}, 0), - created: make(map[gethCommon.Address]interface{}, 0), - suicided: make(map[gethCommon.Address]interface{}, 0), - balances: make(map[gethCommon.Address]*big.Int, 0), - nonces: make(map[gethCommon.Address]uint64, 0), - codes: make(map[gethCommon.Address][]byte, 0), - codeHashes: make(map[gethCommon.Address]gethCommon.Hash, 0), - dirtySlots: make(map[types.SlotAddress]interface{}, 0), - states: make(map[types.SlotAddress]gethCommon.Hash, 0), - transient: make(map[types.SlotAddress]gethCommon.Hash, 0), - accessListAddresses: make(map[gethCommon.Address]interface{}, 0), - accessListSlots: make(map[types.SlotAddress]interface{}, 0), + dirtyAddresses: make(map[gethCommon.Address]interface{}), + created: make(map[gethCommon.Address]interface{}), + suicided: make(map[gethCommon.Address]interface{}), + balances: make(map[gethCommon.Address]*big.Int), + nonces: make(map[gethCommon.Address]uint64), + codes: make(map[gethCommon.Address][]byte), + codeHashes: make(map[gethCommon.Address]gethCommon.Hash), + dirtySlots: make(map[types.SlotAddress]interface{}), + states: make(map[types.SlotAddress]gethCommon.Hash), + transient: make(map[types.SlotAddress]gethCommon.Hash), + accessListAddresses: make(map[gethCommon.Address]interface{}), + accessListSlots: make(map[types.SlotAddress]interface{}), logs: make([]*gethTypes.Log, 0), - preimages: make(map[gethCommon.Hash][]byte, 0), + preimages: make(map[gethCommon.Hash][]byte), // for refund we just copy the data refund: parent.GetRefund(), @@ -280,7 +280,7 @@ func (d *DeltaView) SetCode(addr gethCommon.Address, code []byte) error { return nil } -// GetTransientState returns the value of the slot of the main state +// GetState returns the value of the slot of the main state func (d *DeltaView) GetState(sk types.SlotAddress) (gethCommon.Hash, error) { val, found := d.states[sk] if found { @@ -315,7 +315,7 @@ func (d *DeltaView) GetTransientState(sk types.SlotAddress) gethCommon.Hash { return d.parent.GetTransientState(sk) } -// SetState adds sets a value for the given slot of the transient storage +// SetTransientState adds sets a value for the given slot of the transient storage func (d *DeltaView) SetTransientState(sk types.SlotAddress, value gethCommon.Hash) { d.transient[sk] = value } From 7a570cb619a2ab8d7af28c011038c4a2fcd76f88 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 14:52:59 -0800 Subject: [PATCH 36/72] update logic to handle creation after destruction --- fvm/evm/emulator/state/delta.go | 34 +++++++++- fvm/evm/emulator/state/delta_test.go | 97 +++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 4 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index df038824ef2..3aed174c8d7 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -22,6 +22,7 @@ type DeltaView struct { dirtyAddresses map[gethCommon.Address]interface{} created map[gethCommon.Address]interface{} suicided map[gethCommon.Address]interface{} + deleted map[gethCommon.Address]interface{} balances map[gethCommon.Address]*big.Int nonces map[gethCommon.Address]uint64 codes map[gethCommon.Address][]byte @@ -58,6 +59,7 @@ func NewDeltaView(parent types.ReadOnlyView) *DeltaView { dirtyAddresses: make(map[gethCommon.Address]interface{}), created: make(map[gethCommon.Address]interface{}), suicided: make(map[gethCommon.Address]interface{}), + deleted: make(map[gethCommon.Address]interface{}), balances: make(map[gethCommon.Address]*big.Int), nonces: make(map[gethCommon.Address]uint64), codes: make(map[gethCommon.Address][]byte), @@ -100,6 +102,25 @@ func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { d.created[addr] = struct{}{} // flag the address as dirty d.dirtyAddresses[addr] = struct{}{} + + // if has already suicided + if d.HasSuicided(addr) { + // balance has already been set to zero + d.nonces[addr] = 0 + d.codes[addr] = nil + d.codeHashes[addr] = gethTypes.EmptyCodeHash + + // flag addr as deleted, this flag helps with postponing deletion of slabs + // otherwise we have to iterate over all slabs of this account and set the to nil + d.deleted[addr] = struct{}{} + + // remove slabs from cache related to this account + for k := range d.states { + if k.Address == addr { + delete(d.states, k) + } + } + } return nil } @@ -134,6 +155,9 @@ func (d *DeltaView) Suicide(addr gethCommon.Address) (bool, error) { // flag the account for deletion d.suicided[addr] = struct{}{} + // set balance to zero + d.balances[addr] = new(big.Int) + // flag the address as dirty d.dirtyAddresses[addr] = struct{}{} return true, nil @@ -141,9 +165,6 @@ func (d *DeltaView) Suicide(addr gethCommon.Address) (bool, error) { // GetBalance returns the balance of the given address func (d *DeltaView) GetBalance(addr gethCommon.Address) (*big.Int, error) { - if d.HasSuicided(addr) { - return big.NewInt(0), nil - } val, found := d.balances[addr] if found { return val, nil @@ -286,6 +307,13 @@ func (d *DeltaView) GetState(sk types.SlotAddress) (gethCommon.Hash, error) { if found { return val, nil } + // if address is deleted in the scope of this delta view, + // don't go backward. this has been done to skip the step to iterate + // over all the state slabs and delete them. + _, deleted := d.deleted[sk.Address] + if deleted { + return gethCommon.Hash{}, nil + } return d.parent.GetState(sk) } diff --git a/fvm/evm/emulator/state/delta_test.go b/fvm/evm/emulator/state/delta_test.go index b6c81ab660a..e5b7ce542b6 100644 --- a/fvm/evm/emulator/state/delta_test.go +++ b/fvm/evm/emulator/state/delta_test.go @@ -192,7 +192,9 @@ func TestDeltaView(t *testing.T) { return false, nil } }, - + HasSuicidedFunc: func(a gethCommon.Address) bool { + return false + }, GetNonceFunc: func(addr gethCommon.Address) (uint64, error) { switch addr { case addr1: @@ -248,6 +250,9 @@ func TestDeltaView(t *testing.T) { return false, nil } }, + HasSuicidedFunc: func(a gethCommon.Address) bool { + return false + }, GetCodeFunc: func(addr gethCommon.Address) ([]byte, error) { switch addr { case addr1: @@ -627,6 +632,96 @@ func TestDeltaView(t *testing.T) { } }) + t.Run("test account creation after suicide call", func(t *testing.T) { + addr1 := testutils.RandomCommonAddress(t) + + view := state.NewDeltaView( + &MockedReadOnlyView{ + // we need get refund for parent + GetRefundFunc: emptyRefund, + ExistFunc: func(addr gethCommon.Address) (bool, error) { + return true, nil + }, + HasSuicidedFunc: func(gethCommon.Address) bool { + return true + }, + GetBalanceFunc: func(addr gethCommon.Address) (*big.Int, error) { + return new(big.Int), nil + }, + GetStateFunc: func(sa types.SlotAddress) (gethCommon.Hash, error) { + return gethCommon.Hash{}, nil + }, + }) + + found, err := view.Exist(addr1) + require.NoError(t, err) + require.True(t, found) + + // set balance + initBalance := big.NewInt(10) + err = view.AddBalance(addr1, initBalance) + require.NoError(t, err) + + bal, err := view.GetBalance(addr1) + require.NoError(t, err) + require.Equal(t, initBalance, bal) + + // set code + code := []byte{1, 2, 3} + err = view.SetCode(addr1, code) + require.NoError(t, err) + + ret, err := view.GetCode(addr1) + require.NoError(t, err) + require.Equal(t, code, ret) + + // set key values + key := testutils.RandomCommonHash(t) + value := testutils.RandomCommonHash(t) + sk := types.SlotAddress{Address: addr1, Key: key} + err = view.SetState(sk, value) + require.NoError(t, err) + + vret, err := view.GetState(sk) + require.NoError(t, err) + require.Equal(t, value, vret) + + success, err := view.Suicide(addr1) + require.NoError(t, err) + require.True(t, success) + + // balance should be returned zero + bal, err = view.GetBalance(addr1) + require.NoError(t, err) + require.Equal(t, new(big.Int), bal) + + // get code should still work + ret, err = view.GetCode(addr1) + require.NoError(t, err) + require.Equal(t, code, ret) + + // get state should also still work + vret, err = view.GetState(sk) + require.NoError(t, err) + require.Equal(t, value, vret) + + // now re-create account + + err = view.CreateAccount(addr1) + require.NoError(t, err) + + bal, err = view.GetBalance(addr1) + require.NoError(t, err) + require.Equal(t, new(big.Int), bal) + + ret, err = view.GetCode(addr1) + require.NoError(t, err) + require.Equal(t, nil, ret) + + vret, err = view.GetState(sk) + require.NoError(t, err) + require.Len(t, vret, 0) + }) } type MockedReadOnlyView struct { From fb0ab2a283b4ce7ffb1449b912a65e3b95cfd588 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 15:18:55 -0800 Subject: [PATCH 37/72] handle case for delete and create --- fvm/evm/emulator/state/stateDB.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index 9f51ade2497..a4191b9fbe1 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -313,14 +313,17 @@ func (db *StateDB) Commit() error { // update accounts for _, addr := range sortedAddresses { - // TODO check if address is + deleted := false + // First we need to delete accounts if db.HasSuicided(addr) { err = db.baseView.DeleteAccount(addr) if err != nil { return wrapError(err) } - continue + deleted = true } + // then create new ones + // an account might be in a single transaction be deleted and recreated if db.IsCreated(addr) { err = db.baseView.CreateAccount( addr, @@ -334,6 +337,9 @@ func (db *StateDB) Commit() error { } continue } + if deleted { + continue + } err = db.baseView.UpdateAccount( addr, db.GetBalance(addr), From 5625564100658440a905d349e6ea92f6bb62c132 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 15:26:15 -0800 Subject: [PATCH 38/72] minor fixes --- fvm/evm/emulator/state/stateDB.go | 34 ++++++++++++++++++------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index a4191b9fbe1..31b8b296f64 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -76,7 +76,8 @@ func (db *StateDB) Empty(addr gethCommon.Address) bool { // CreateAccount creates a new account for the given address // it sets the nonce to zero func (db *StateDB) CreateAccount(addr gethCommon.Address) { - db.lastestView().CreateAccount(addr) + err := db.lastestView().CreateAccount(addr) + db.handleError(err) } // IsCreated returns true if address is recently created (context of a transaction) @@ -105,7 +106,7 @@ func (db *StateDB) SubBalance(addr gethCommon.Address, amount *big.Int) { db.handleError(err) } -// SubBalance adds the amount from the balance of the given address +// AddBalance adds the amount from the balance of the given address func (db *StateDB) AddBalance(addr gethCommon.Address, amount *big.Int) { err := db.lastestView().AddBalance(addr, amount) db.handleError(err) @@ -127,7 +128,8 @@ func (db *StateDB) GetNonce(addr gethCommon.Address) uint64 { // SetNonce sets the nonce value for the given address func (db *StateDB) SetNonce(addr gethCommon.Address, nonce uint64) { - db.lastestView().SetNonce(addr, nonce) + err := db.lastestView().SetNonce(addr, nonce) + db.handleError(err) } // GetCodeHash returns the code hash of the given address @@ -153,17 +155,20 @@ func (db *StateDB) GetCodeSize(addr gethCommon.Address) int { // SetCode sets the code for the given address func (db *StateDB) SetCode(addr gethCommon.Address, code []byte) { - db.lastestView().SetCode(addr, code) + err := db.lastestView().SetCode(addr, code) + db.handleError(err) } -// AddRefund adds an amount to the total (gas) refund +// AddRefund adds the amount to the total (gas) refund func (db *StateDB) AddRefund(amount uint64) { - db.lastestView().AddRefund(amount) + err := db.lastestView().AddRefund(amount) + db.handleError(err) } -// AddRefund subtracts an amount from the total (gas) refund +// SubRefund subtracts the amount from the total (gas) refund func (db *StateDB) SubRefund(amount uint64) { - db.lastestView().SubRefund(amount) + err := db.lastestView().SubRefund(amount) + db.handleError(err) } // GetRefund returns the total (gas) refund @@ -188,10 +193,11 @@ func (db *StateDB) GetState(addr gethCommon.Address, key gethCommon.Hash) gethCo // SetState sets a value for the given storage slot func (db *StateDB) SetState(addr gethCommon.Address, key gethCommon.Hash, value gethCommon.Hash) { - db.lastestView().SetState(types.SlotAddress{Address: addr, Key: key}, value) + err := db.lastestView().SetState(types.SlotAddress{Address: addr, Key: key}, value) + db.handleError(err) } -// GetState returns the value for the given key of the transient storage +// GetTransientState returns the value for the given key of the transient storage func (db *StateDB) GetTransientState(addr gethCommon.Address, key gethCommon.Hash) gethCommon.Hash { return db.lastestView().GetTransientState(types.SlotAddress{Address: addr, Key: key}) } @@ -206,7 +212,7 @@ func (db *StateDB) AddressInAccessList(addr gethCommon.Address) bool { return db.lastestView().AddressInAccessList(addr) } -// AddressInAccessList checks if the given (address,slot) is in the access list +// SlotInAccessList checks if the given (address,slot) is in the access list func (db *StateDB) SlotInAccessList(addr gethCommon.Address, key gethCommon.Hash) (addressOk bool, slotOk bool) { return db.lastestView().SlotInAccessList(types.SlotAddress{Address: addr, Key: key}) } @@ -421,12 +427,12 @@ func (db *StateDB) lastestView() *DeltaView { } // set error captures the first non-nil error it is called with. -func (s *StateDB) handleError(err error) { +func (db *StateDB) handleError(err error) { if err == nil { return } - if s.cachedError == nil { - s.cachedError = err + if db.cachedError == nil { + db.cachedError = err } } From a45a4b6a9bc02d15fff5f2d2b5070ea398a009e6 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 15:31:29 -0800 Subject: [PATCH 39/72] lint fix --- fvm/evm/emulator/state/stateDB.go | 2 +- fvm/evm/emulator/state/stateDB_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index 31b8b296f64..263f15a8d29 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -10,8 +10,8 @@ import ( gethCommon "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" gethParams "github.com/ethereum/go-ethereum/params" - "github.com/onflow/atree" + "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" diff --git a/fvm/evm/emulator/state/stateDB_test.go b/fvm/evm/emulator/state/stateDB_test.go index b93fd1aa312..2d45395a72e 100644 --- a/fvm/evm/emulator/state/stateDB_test.go +++ b/fvm/evm/emulator/state/stateDB_test.go @@ -5,16 +5,16 @@ import ( "math/big" "testing" - "github.com/ethereum/go-ethereum/common" gethCommon "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" gethParams "github.com/ethereum/go-ethereum/params" "github.com/onflow/atree" + "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/fvm/evm/emulator/state" "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" - "github.com/stretchr/testify/require" ) var rootAddr = flow.Address{1, 2, 3, 4, 5, 6, 7, 8} @@ -170,7 +170,7 @@ func TestStateDB(t *testing.T) { db.AddLog(testutils.GetRandomLogFixture(t)) db.RevertToSnapshot(snapshot) - ret := db.Logs(common.Hash{}, 1, common.Hash{}, 1) + ret := db.Logs(gethCommon.Hash{}, 1, gethCommon.Hash{}, 1) require.Equal(t, ret, logs) }) From 61a99049190e16c94ff1cc78fcce3955a0404378 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 15:33:12 -0800 Subject: [PATCH 40/72] doc update --- fvm/evm/emulator/state/stateDB.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index 263f15a8d29..570d392599a 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -25,7 +25,7 @@ import ( // is expected to be returned. // Warning: current implementation of the StateDB is considered // to be used for a single EVM transaction execution and is not -// thread safe.yet the current design supports addition of these properties +// thread safe. yet the current design supports addition of concurrency in the // future if needed type StateDB struct { ledger atree.Ledger From aa0f58e0bc2a39d28c1dfa7a617507c541dbd8e5 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 15:34:23 -0800 Subject: [PATCH 41/72] hot fix --- fvm/evm/emulator/state/delta.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index b531d9d62ea..b966f5d3bf3 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -111,16 +111,13 @@ func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { if err != nil { return err } + d.AddBalance(addr, bal) } d.created[addr] = struct{}{} // flag the address as dirty d.dirtyAddresses[addr] = struct{}{} - if exist { - d.AddBalance(addr, bal) - } - // if has already suicided if d.HasSuicided(addr) { // balance has already been set to zero From 90e0c27391ee75a53b67b017403418c13bbba504 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 15:46:02 -0800 Subject: [PATCH 42/72] fix tests --- fvm/evm/emulator/emulator_test.go | 38 +------------------------------ 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/fvm/evm/emulator/emulator_test.go b/fvm/evm/emulator/emulator_test.go index fe6553a6cd0..25873941dd7 100644 --- a/fvm/evm/emulator/emulator_test.go +++ b/fvm/evm/emulator/emulator_test.go @@ -329,44 +329,8 @@ func TestTransfers(t *testing.T) { }) } -// TODO: fix me -// func TestDatabaseErrorHandling(t *testing.T) { - -// t.Run("test non-fatal db error handling", func(t *testing.T) { -// db := &testutils.TestDatabase{ -// GetRootHashFunc: func() (gethCommon.Hash, error) { -// return gethTypes.EmptyRootHash, types.NewDatabaseError(fmt.Errorf("some non-fatal error")) -// }, -// } - -// RunWithNewEmulator(t, backend, rootAddr, func(em *emulator.Emulator) { -// RunWithNewBlockView(t, em, func(blk types.BlockView) { -// _, err := blk.DirectCall(types.NewDepositCall(types.EmptyAddress, big.NewInt(1))) -// require.Error(t, err) -// require.True(t, types.IsADatabaseError(err)) -// }) -// }) -// }) - -// t.Run("test fatal db error handling", func(t *testing.T) { -// db := &testutils.TestDatabase{ -// GetRootHashFunc: func() (gethCommon.Hash, error) { -// return gethTypes.EmptyRootHash, types.NewFatalError(fmt.Errorf("some non-fatal error")) -// }, -// } - -// RunWithNewEmulator(t, db, func(em *emulator.Emulator) { -// RunWithNewBlockView(t, em, func(blk types.BlockView) { -// _, err := blk.DirectCall(types.NewDepositCall(types.EmptyAddress, big.NewInt(1))) -// require.Error(t, err) -// require.True(t, types.IsAFatalError(err)) -// }) -// }) -// }) -// } - func TestStorageNoSideEffect(t *testing.T) { - t.Skip("we need to fix this issue ") + // TODO fix me testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { testutils.RunWithTestFlowEVMRootAddress(t, backend, func(flowEVMRoot flow.Address) { From a9a2989b5a6e94f55f031626c2b78392ef7ca047 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 15:47:04 -0800 Subject: [PATCH 43/72] fix test --- fvm/evm/emulator/emulator_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fvm/evm/emulator/emulator_test.go b/fvm/evm/emulator/emulator_test.go index 25873941dd7..3cca27b0906 100644 --- a/fvm/evm/emulator/emulator_test.go +++ b/fvm/evm/emulator/emulator_test.go @@ -330,15 +330,13 @@ func TestTransfers(t *testing.T) { } func TestStorageNoSideEffect(t *testing.T) { - // TODO fix me - testutils.RunWithTestBackend(t, func(backend *testutils.TestBackend) { testutils.RunWithTestFlowEVMRootAddress(t, backend, func(flowEVMRoot flow.Address) { var err error em := emulator.NewEmulator(backend, flowEVMRoot) testAccount := types.NewAddressFromString("test") - amount := big.NewInt(100) + amount := big.NewInt(10) RunWithNewBlockView(t, em, func(blk types.BlockView) { _, err = blk.DirectCall(types.NewDepositCall(testAccount, amount)) require.NoError(t, err) From b05b5a31d60f6511622399d0f589eb13c7c82ffd Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 16:20:06 -0800 Subject: [PATCH 44/72] update geth version --- fvm/evm/emulator/state/base.go | 4 +- fvm/evm/emulator/state/base_test.go | 2 +- fvm/evm/emulator/state/delta.go | 34 ++++----- fvm/evm/emulator/state/delta_test.go | 47 ++++++------ fvm/evm/emulator/state/stateDB.go | 28 ++++--- fvm/evm/types/state.go | 10 ++- go.mod | 52 +++++++------ go.sum | 110 ++++++++++++++++----------- insecure/go.mod | 52 +++++++------ insecure/go.sum | 110 ++++++++++++++++----------- integration/go.mod | 51 +++++++------ integration/go.sum | 105 +++++++++++++++---------- 12 files changed, 347 insertions(+), 258 deletions(-) diff --git a/fvm/evm/emulator/state/base.go b/fvm/evm/emulator/state/base.go index ae098bc77ef..ec283b2565f 100644 --- a/fvm/evm/emulator/state/base.go +++ b/fvm/evm/emulator/state/base.go @@ -94,8 +94,8 @@ func (v *BaseView) IsCreated(gethCommon.Address) bool { return false } -// HasSuicided returns true if an address has suicided in the context of this transaction -func (v *BaseView) HasSuicided(gethCommon.Address) bool { +// HasSelfDestructed returns true if an address has self destructed in the context of this transaction +func (v *BaseView) HasSelfDestructed(gethCommon.Address) bool { return false } diff --git a/fvm/evm/emulator/state/base_test.go b/fvm/evm/emulator/state/base_test.go index 25d54fb807f..f80a59e3023 100644 --- a/fvm/evm/emulator/state/base_test.go +++ b/fvm/evm/emulator/state/base_test.go @@ -199,7 +199,7 @@ func TestBaseView(t *testing.T) { view, err := state.NewBaseView(testutils.GetSimpleValueStore(), flow.Address{1, 2, 3, 4}) require.NoError(t, err) - require.Equal(t, false, view.HasSuicided(gethCommon.Address{})) + require.Equal(t, false, view.HasSelfDestructed(gethCommon.Address{})) require.Equal(t, false, view.IsCreated(gethCommon.Address{})) require.Equal(t, uint64(0), view.GetRefund()) require.Equal(t, gethCommon.Hash{}, view.GetTransientState(types.SlotAddress{})) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index b966f5d3bf3..afb9ad1c221 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -21,7 +21,7 @@ type DeltaView struct { // account changes dirtyAddresses map[gethCommon.Address]interface{} created map[gethCommon.Address]interface{} - suicided map[gethCommon.Address]interface{} + selfDestructed map[gethCommon.Address]interface{} deleted map[gethCommon.Address]interface{} balances map[gethCommon.Address]*big.Int nonces map[gethCommon.Address]uint64 @@ -58,7 +58,7 @@ func NewDeltaView(parent types.ReadOnlyView) *DeltaView { dirtyAddresses: make(map[gethCommon.Address]interface{}), created: make(map[gethCommon.Address]interface{}), - suicided: make(map[gethCommon.Address]interface{}), + selfDestructed: make(map[gethCommon.Address]interface{}), deleted: make(map[gethCommon.Address]interface{}), balances: make(map[gethCommon.Address]*big.Int), nonces: make(map[gethCommon.Address]uint64), @@ -90,7 +90,7 @@ func (d *DeltaView) Exist(addr gethCommon.Address) (bool, error) { if found { return true, nil } - _, found = d.suicided[addr] + _, found = d.selfDestructed[addr] if found { return true, nil } @@ -118,8 +118,8 @@ func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { // flag the address as dirty d.dirtyAddresses[addr] = struct{}{} - // if has already suicided - if d.HasSuicided(addr) { + // if has already self destructed + if d.HasSelfDestructed(addr) { // balance has already been set to zero d.nonces[addr] = 0 d.codes[addr] = nil @@ -148,34 +148,26 @@ func (d *DeltaView) IsCreated(addr gethCommon.Address) bool { return d.parent.IsCreated(addr) } -// HasSuicided returns true if address has been flagged for deletion -func (d *DeltaView) HasSuicided(addr gethCommon.Address) bool { - _, found := d.suicided[addr] +// HasSelfDestructed returns true if address has been flagged for deletion +func (d *DeltaView) HasSelfDestructed(addr gethCommon.Address) bool { + _, found := d.selfDestructed[addr] if found { return true } - return d.parent.HasSuicided(addr) + return d.parent.HasSelfDestructed(addr) } -// Suicide sets a flag to delete the account at the end of transaction -func (d *DeltaView) Suicide(addr gethCommon.Address) (bool, error) { - // if it doesn't exist, return false - exists, err := d.Exist(addr) - if err != nil { - return false, err - } - if !exists { - return false, nil - } +// SelfDestruct sets a flag to delete the account at the end of transaction +func (d *DeltaView) SelfDestruct(addr gethCommon.Address) error { // flag the account for deletion - d.suicided[addr] = struct{}{} + d.selfDestructed[addr] = struct{}{} // set balance to zero d.balances[addr] = new(big.Int) // flag the address as dirty d.dirtyAddresses[addr] = struct{}{} - return true, nil + return nil } // GetBalance returns the balance of the given address diff --git a/fvm/evm/emulator/state/delta_test.go b/fvm/evm/emulator/state/delta_test.go index e5b7ce542b6..0e25b3825ff 100644 --- a/fvm/evm/emulator/state/delta_test.go +++ b/fvm/evm/emulator/state/delta_test.go @@ -5,7 +5,6 @@ import ( "math/big" "testing" - "github.com/ethereum/go-ethereum/common" gethCommon "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" gethCrypto "github.com/ethereum/go-ethereum/crypto" @@ -42,7 +41,7 @@ func TestDeltaView(t *testing.T) { return false, fmt.Errorf("some error") } }, - HasSuicidedFunc: func(gethCommon.Address) bool { + HasSelfDestructedFunc: func(gethCommon.Address) bool { return false }, }) @@ -71,17 +70,16 @@ func TestDeltaView(t *testing.T) { require.NoError(t, err) require.True(t, found) - // test HasSuicided first - success := view.HasSuicided(addr1) + // test HasSelfDestructed first + success := view.HasSelfDestructed(addr1) require.False(t, success) // set addr1 for deletion - success, err = view.Suicide(addr1) + err = view.SelfDestruct(addr1) require.NoError(t, err) - require.True(t, success) - // check HasSuicided now - success = view.HasSuicided(addr1) + // check HasSelfDestructed now + success = view.HasSelfDestructed(addr1) require.True(t, success) // addr1 should still exist after suicide call @@ -109,7 +107,7 @@ func TestDeltaView(t *testing.T) { return false, nil } }, - HasSuicidedFunc: func(gethCommon.Address) bool { + HasSelfDestructedFunc: func(gethCommon.Address) bool { return false }, GetBalanceFunc: func(addr gethCommon.Address) (*big.Int, error) { @@ -130,9 +128,8 @@ func TestDeltaView(t *testing.T) { require.Equal(t, addr1InitBal, bal) // call suicide on addr - success, err := view.Suicide(addr1) + err = view.SelfDestruct(addr1) require.NoError(t, err) - require.True(t, success) // now it should return balance of zero bal, err = view.GetBalance(addr1) @@ -192,7 +189,7 @@ func TestDeltaView(t *testing.T) { return false, nil } }, - HasSuicidedFunc: func(a gethCommon.Address) bool { + HasSelfDestructedFunc: func(a gethCommon.Address) bool { return false }, GetNonceFunc: func(addr gethCommon.Address) (uint64, error) { @@ -250,7 +247,7 @@ func TestDeltaView(t *testing.T) { return false, nil } }, - HasSuicidedFunc: func(a gethCommon.Address) bool { + HasSelfDestructedFunc: func(a gethCommon.Address) bool { return false }, GetCodeFunc: func(addr gethCommon.Address) ([]byte, error) { @@ -269,7 +266,7 @@ func TestDeltaView(t *testing.T) { return 0, fmt.Errorf("some error") } }, - GetCodeHashFunc: func(addr gethCommon.Address) (common.Hash, error) { + GetCodeHashFunc: func(addr gethCommon.Address) (gethCommon.Hash, error) { switch addr { case addr1: return addr1IntiCodeHash, nil @@ -590,7 +587,7 @@ func TestDeltaView(t *testing.T) { GetNonceFunc: func(addr gethCommon.Address) (uint64, error) { return 0, nil }, - HasSuicidedFunc: func(gethCommon.Address) bool { + HasSelfDestructedFunc: func(gethCommon.Address) bool { return false }, }) @@ -604,7 +601,7 @@ func TestDeltaView(t *testing.T) { require.NoError(t, err) // Suicide address 2 - _, err = view.Suicide(addresses[1]) + err = view.SelfDestruct(addresses[1]) require.NoError(t, err) // add balance for address 3 @@ -642,7 +639,7 @@ func TestDeltaView(t *testing.T) { ExistFunc: func(addr gethCommon.Address) (bool, error) { return true, nil }, - HasSuicidedFunc: func(gethCommon.Address) bool { + HasSelfDestructedFunc: func(gethCommon.Address) bool { return true }, GetBalanceFunc: func(addr gethCommon.Address) (*big.Int, error) { @@ -686,9 +683,8 @@ func TestDeltaView(t *testing.T) { require.NoError(t, err) require.Equal(t, value, vret) - success, err := view.Suicide(addr1) + err = view.SelfDestruct(addr1) require.NoError(t, err) - require.True(t, success) // balance should be returned zero bal, err = view.GetBalance(addr1) @@ -716,17 +712,18 @@ func TestDeltaView(t *testing.T) { ret, err = view.GetCode(addr1) require.NoError(t, err) - require.Equal(t, nil, ret) + require.Len(t, ret, 0) vret, err = view.GetState(sk) require.NoError(t, err) - require.Len(t, vret, 0) + emptyValue := gethCommon.Hash{} + require.Equal(t, emptyValue, vret) }) } type MockedReadOnlyView struct { ExistFunc func(gethCommon.Address) (bool, error) - HasSuicidedFunc func(gethCommon.Address) bool + HasSelfDestructedFunc func(gethCommon.Address) bool IsCreatedFunc func(gethCommon.Address) bool GetBalanceFunc func(gethCommon.Address) (*big.Int, error) GetNonceFunc func(gethCommon.Address) (uint64, error) @@ -756,11 +753,11 @@ func (v *MockedReadOnlyView) IsCreated(addr gethCommon.Address) bool { return v.IsCreatedFunc(addr) } -func (v *MockedReadOnlyView) HasSuicided(addr gethCommon.Address) bool { - if v.HasSuicidedFunc == nil { +func (v *MockedReadOnlyView) HasSelfDestructed(addr gethCommon.Address) bool { + if v.HasSelfDestructedFunc == nil { panic("HasSuicided is not set in this mocked view") } - return v.HasSuicidedFunc(addr) + return v.HasSelfDestructedFunc(addr) } func (v *MockedReadOnlyView) GetBalance(addr gethCommon.Address) (*big.Int, error) { diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index 570d392599a..0a376e93804 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -54,7 +54,7 @@ func NewStateDB(ledger atree.Ledger, root flow.Address) (*StateDB, error) { // Exist returns true if the given address exists in state. // -// this should also return true for suicided accounts during the transaction execution. +// this should also return true for self destructed accounts during the transaction execution. func (db *StateDB) Exist(addr gethCommon.Address) bool { exist, err := db.lastestView().Exist(addr) db.handleError(err) @@ -85,19 +85,25 @@ func (db *StateDB) IsCreated(addr gethCommon.Address) bool { return db.lastestView().IsCreated(addr) } -// Suicide flags the address for deletion. +// SelfDestruct flags the address for deletion. // // while this address exists for the rest of transaction, -// the balance of this account is return zero after the Suicide call. -func (db *StateDB) Suicide(addr gethCommon.Address) bool { - success, err := db.lastestView().Suicide(addr) +// the balance of this account is return zero after the SelfDestruct call. +func (db *StateDB) SelfDestruct(addr gethCommon.Address) { + err := db.lastestView().SelfDestruct(addr) db.handleError(err) - return success } -// HasSuicided returns true if address is flaged with suicide. -func (db *StateDB) HasSuicided(addr gethCommon.Address) bool { - return db.lastestView().HasSuicided(addr) +// Selfdestruct6780 would only follow the self destruct steps if account is created +func (db *StateDB) Selfdestruct6780(addr gethCommon.Address) { + if db.IsCreated(addr) { + db.SelfDestruct(addr) + } +} + +// HasSelfDestructed returns true if address is flaged with self destruct. +func (db *StateDB) HasSelfDestructed(addr gethCommon.Address) bool { + return db.lastestView().HasSelfDestructed(addr) } // SubBalance substitutes the amount from the balance of the given address @@ -320,8 +326,8 @@ func (db *StateDB) Commit() error { // update accounts for _, addr := range sortedAddresses { deleted := false - // First we need to delete accounts - if db.HasSuicided(addr) { + // first we need to delete accounts + if db.HasSelfDestructed(addr) { err = db.baseView.DeleteAccount(addr) if err != nil { return wrapError(err) diff --git a/fvm/evm/types/state.go b/fvm/evm/types/state.go index 25d032a53c7..d672c5cbf82 100644 --- a/fvm/evm/types/state.go +++ b/fvm/evm/types/state.go @@ -33,8 +33,9 @@ type ReadOnlyView interface { Exist(gethCommon.Address) (bool, error) // IsCreated returns true if address has been created in this tx IsCreated(gethCommon.Address) bool - // HasSuicided returns true if an address has suicided - HasSuicided(gethCommon.Address) bool + // HasSelfDestructed returns true if an address has self destructed + HasSelfDestructed(gethCommon.Address) bool + // GetBalance returns the balance of an address GetBalance(gethCommon.Address) (*big.Int, error) // GetNonce returns the nonce of an address @@ -63,8 +64,9 @@ type HotView interface { // CreateAccount creates a new account CreateAccount(gethCommon.Address) error - // Suicide set the flag for deletion of the account after execution - Suicide(gethCommon.Address) (success bool, err error) + // SelfDestruct set the flag for destruction of the account after execution + SelfDestruct(gethCommon.Address) error + // SubBalance subtracts the amount from the balance the given address SubBalance(gethCommon.Address, *big.Int) error // AddBalance adds the amount to the balance of the given address diff --git a/go.mod b/go.mod index 7f4082c93f0..bba38d192ea 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,14 @@ require ( cloud.google.com/go/profiler v0.3.0 cloud.google.com/go/storage v1.30.1 github.com/antihax/optional v1.0.0 - github.com/aws/aws-sdk-go-v2/config v1.18.19 + github.com/aws/aws-sdk-go-v2/config v1.18.45 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.1 github.com/aws/aws-sdk-go-v2/service/s3 v1.15.0 github.com/btcsuite/btcd/btcec/v2 v2.2.1 github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger/v2 v2.2007.4 github.com/ef-ds/deque v1.0.4 - github.com/ethereum/go-ethereum v1.12.0 + github.com/ethereum/go-ethereum v1.13.5 github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c github.com/gammazero/workerpool v1.1.2 github.com/gogo/protobuf v1.3.2 @@ -86,7 +86,7 @@ require ( golang.org/x/sync v0.5.0 golang.org/x/sys v0.15.0 golang.org/x/text v0.14.0 - golang.org/x/time v0.1.0 + golang.org/x/time v0.3.0 golang.org/x/tools v0.16.0 google.golang.org/api v0.126.0 google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 @@ -98,7 +98,7 @@ require ( ) require ( - github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 + github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 github.com/coreos/go-semver v0.3.0 github.com/go-playground/validator/v10 v10.14.1 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb @@ -118,21 +118,22 @@ require ( cloud.google.com/go/compute v1.21.0 // indirect cloud.google.com/go/iam v1.1.1 // indirect github.com/DataDog/zstd v1.5.2 // indirect - github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect - github.com/VictoriaMetrics/fastcache v1.6.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect + github.com/VictoriaMetrics/fastcache v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.21.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.43 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect - github.com/aws/smithy-go v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect + github.com/aws/smithy-go v1.15.0 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.7.0 // indirect @@ -144,8 +145,12 @@ require ( github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.3 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect @@ -156,6 +161,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elastic/gosigar v0.14.2 // indirect + github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect @@ -186,8 +192,8 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect - github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect - github.com/huin/goupnp v1.2.0 // indirect + github.com/holiman/uint256 v1.2.3 // indirect + github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/boxo v0.10.0 // indirect @@ -239,6 +245,7 @@ require ( github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -277,10 +284,11 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect + github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c // indirect - github.com/tklauser/go-sysconf v0.3.9 // indirect - github.com/tklauser/numcpus v0.3.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d // indirect github.com/vmihailenco/tagparser v0.1.1 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect @@ -304,8 +312,8 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect nhooyr.io/websocket v1.8.7 // indirect + rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index db6fda050e3..e75fe7be66a 100644 --- a/go.sum +++ b/go.sum @@ -100,17 +100,20 @@ github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwS github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= -github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= -github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= @@ -138,46 +141,46 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= -github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= github.com/aws/aws-sdk-go-v2/config v1.8.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY= -github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= -github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= +github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= +github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= github.com/aws/aws-sdk-go-v2/credentials v1.4.0/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0/go.mod h1:CpNzHK9VEFUCknu50kkB8z58AH2B5DvPP7ea1LHve/Y= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.1 h1:VGkV9KmhGqOQWnHyi4gLG98kE6OecT42fdrCGFWxJsc= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.1/go.mod h1:PLlnMiki//sGnCJiW+aVpvP/C8Kcm8mEj/IVm9+9qk4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2/go.mod h1:BQV0agm+JEhqR+2RT5e1XTFIDcAAV0eW6z2trp+iduw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 h1:gceOysEWNNwLd6cki65IMBZ4WAM0MwgBQq2n7kejoT8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0/go.mod h1:v8ygadNyATSm6elwJ/4gzJwcFhri9RqS8skgHKiwXPU= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.0 h1:HWsM0YQWX76V6MOp07YuTYacm8k7h69ObJuw7Nck+og= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.0/go.mod h1:LKb3cKNQIMh+itGnEpKGcnL/6OIjPZqrtYah1w5f+3o= github.com/aws/aws-sdk-go-v2/service/s3 v1.15.0 h1:nPLfLPfglacc29Y949sDxpr3X/blaY40s3B85WT2yZU= github.com/aws/aws-sdk-go-v2/service/s3 v1.15.0/go.mod h1:Iv2aJVtVSm/D22rFoX99cLG4q4uB7tppuCsulGe98k4= github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= github.com/aws/aws-sdk-go-v2/service/sts v1.7.0/go.mod h1:0qcSMCyASQPN2sk/1KQLQ2Fh6yq8wm0HSDAimPhzCoM= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -249,18 +252,24 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w= -github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= @@ -283,6 +292,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= +github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= @@ -350,9 +361,11 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= -github.com/ethereum/go-ethereum v1.12.0 h1:bdnhLPtqETd4m3mS8BGMNvBTf36bO5bx/hxE2zljOa0= -github.com/ethereum/go-ethereum v1.12.0/go.mod h1:/oo2X/dZLJjf2mJ6YT9wcWxa4nNJDBKDBU6sFIpx1Gs= +github.com/ethereum/go-ethereum v1.13.5 h1:U6TCRciCqZRe4FPXmy1sMGxTfuk8P7u2UoinF3VbaFk= +github.com/ethereum/go-ethereum v1.13.5/go.mod h1:yMTu38GSuyxaYzQMViqNmQ1s3cE84abZexQmTgenWk0= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -427,6 +440,7 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -456,7 +470,7 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= @@ -579,6 +593,7 @@ github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYa github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -666,14 +681,14 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= -github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= +github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= +github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= -github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -1216,6 +1231,9 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1612,6 +1630,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/supranational/blst v0.3.4/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -1619,10 +1639,12 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= -github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= -github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d h1:5JInRQbk5UBX8JfUvKh2oYTLMVwj3p6n+wapDDm7hko= @@ -2036,7 +2058,6 @@ golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2074,6 +2095,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2100,8 +2123,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2418,7 +2441,6 @@ gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -2466,6 +2488,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= diff --git a/insecure/go.mod b/insecure/go.mod index 5ffb8474d9b..3bb90a663fe 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -28,24 +28,25 @@ require ( cloud.google.com/go/iam v1.1.1 // indirect cloud.google.com/go/storage v1.30.1 // indirect github.com/DataDog/zstd v1.5.2 // indirect - github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect - github.com/VictoriaMetrics/fastcache v1.6.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect - github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect + github.com/VictoriaMetrics/fastcache v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.21.2 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.45 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.43 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.0 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.15.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect - github.com/aws/smithy-go v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect + github.com/aws/smithy-go v1.15.0 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.7.0 // indirect @@ -56,11 +57,15 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 // indirect + github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 // indirect github.com/cockroachdb/redact v1.1.3 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect @@ -74,7 +79,8 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/ef-ds/deque v1.0.4 // indirect github.com/elastic/gosigar v0.14.2 // indirect - github.com/ethereum/go-ethereum v1.12.0 // indirect + github.com/ethereum/c-kzg-4844 v0.4.0 // indirect + github.com/ethereum/go-ethereum v1.13.5 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -119,8 +125,8 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect - github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect - github.com/huin/goupnp v1.2.0 // indirect + github.com/holiman/uint256 v1.2.3 // indirect + github.com/huin/goupnp v1.3.0 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect @@ -185,6 +191,7 @@ require ( github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -246,10 +253,11 @@ require ( github.com/spf13/viper v1.15.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect + github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c // indirect - github.com/tklauser/go-sysconf v0.3.9 // indirect - github.com/tklauser/numcpus v0.3.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/vmihailenco/msgpack/v4 v4.3.11 // indirect @@ -280,7 +288,7 @@ require ( golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.1.0 // indirect + golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.16.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect @@ -291,10 +299,10 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect nhooyr.io/websocket v1.8.7 // indirect + rsc.io/tmplfunc v0.0.3 // indirect ) replace github.com/onflow/flow-go => ../ diff --git a/insecure/go.sum b/insecure/go.sum index 87413f90d27..612baac3a83 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -90,17 +90,20 @@ github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwS github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= -github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= -github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= @@ -127,46 +130,46 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= -github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= github.com/aws/aws-sdk-go-v2/config v1.8.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY= -github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= -github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= +github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= +github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= github.com/aws/aws-sdk-go-v2/credentials v1.4.0/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0/go.mod h1:CpNzHK9VEFUCknu50kkB8z58AH2B5DvPP7ea1LHve/Y= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.1 h1:VGkV9KmhGqOQWnHyi4gLG98kE6OecT42fdrCGFWxJsc= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.1/go.mod h1:PLlnMiki//sGnCJiW+aVpvP/C8Kcm8mEj/IVm9+9qk4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2/go.mod h1:BQV0agm+JEhqR+2RT5e1XTFIDcAAV0eW6z2trp+iduw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 h1:gceOysEWNNwLd6cki65IMBZ4WAM0MwgBQq2n7kejoT8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0/go.mod h1:v8ygadNyATSm6elwJ/4gzJwcFhri9RqS8skgHKiwXPU= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.0 h1:HWsM0YQWX76V6MOp07YuTYacm8k7h69ObJuw7Nck+og= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.0/go.mod h1:LKb3cKNQIMh+itGnEpKGcnL/6OIjPZqrtYah1w5f+3o= github.com/aws/aws-sdk-go-v2/service/s3 v1.15.0 h1:nPLfLPfglacc29Y949sDxpr3X/blaY40s3B85WT2yZU= github.com/aws/aws-sdk-go-v2/service/s3 v1.15.0/go.mod h1:Iv2aJVtVSm/D22rFoX99cLG4q4uB7tppuCsulGe98k4= github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= github.com/aws/aws-sdk-go-v2/service/sts v1.7.0/go.mod h1:0qcSMCyASQPN2sk/1KQLQ2Fh6yq8wm0HSDAimPhzCoM= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -237,18 +240,24 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w= -github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= @@ -271,6 +280,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= +github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= @@ -338,9 +349,11 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= -github.com/ethereum/go-ethereum v1.12.0 h1:bdnhLPtqETd4m3mS8BGMNvBTf36bO5bx/hxE2zljOa0= -github.com/ethereum/go-ethereum v1.12.0/go.mod h1:/oo2X/dZLJjf2mJ6YT9wcWxa4nNJDBKDBU6sFIpx1Gs= +github.com/ethereum/go-ethereum v1.13.5 h1:U6TCRciCqZRe4FPXmy1sMGxTfuk8P7u2UoinF3VbaFk= +github.com/ethereum/go-ethereum v1.13.5/go.mod h1:yMTu38GSuyxaYzQMViqNmQ1s3cE84abZexQmTgenWk0= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -412,6 +425,7 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -441,7 +455,7 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= @@ -561,6 +575,7 @@ github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYa github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -646,14 +661,14 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= -github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= +github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= +github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= -github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -1194,6 +1209,9 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1585,6 +1603,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/supranational/blst v0.3.4/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -1592,10 +1612,12 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= -github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= -github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d h1:5JInRQbk5UBX8JfUvKh2oYTLMVwj3p6n+wapDDm7hko= @@ -2003,7 +2025,6 @@ golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2034,6 +2055,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2060,8 +2083,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2346,7 +2369,6 @@ gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -2393,6 +2415,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= diff --git a/integration/go.mod b/integration/go.mod index fc395bfc52e..a76d64b7669 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -52,28 +52,28 @@ require ( github.com/DataDog/zstd v1.5.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect - github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect - github.com/VictoriaMetrics/fastcache v1.6.0 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect + github.com/VictoriaMetrics/fastcache v1.12.1 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/apache/arrow/go/v12 v12.0.0 // indirect github.com/apache/thrift v0.16.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect - github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.21.2 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.45 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.43 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.0 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.15.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect - github.com/aws/smithy-go v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect + github.com/aws/smithy-go v1.15.0 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.7.0 // indirect @@ -85,11 +85,15 @@ require ( github.com/cloudflare/circl v1.1.0 // indirect github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 // indirect + github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 // indirect github.com/cockroachdb/redact v1.1.3 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect @@ -108,7 +112,8 @@ require ( github.com/ef-ds/deque v1.0.4 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/ethereum/go-ethereum v1.12.0 // indirect + github.com/ethereum/c-kzg-4844 v0.4.0 // indirect + github.com/ethereum/go-ethereum v1.13.5 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -133,7 +138,7 @@ require ( github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/goccy/go-json v0.9.11 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -160,8 +165,8 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect - github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect - github.com/huin/goupnp v1.2.0 // indirect + github.com/holiman/uint256 v1.2.3 // indirect + github.com/huin/goupnp v1.3.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect @@ -230,6 +235,7 @@ require ( github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -296,10 +302,11 @@ require ( github.com/spf13/viper v1.15.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect + github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c // indirect - github.com/tklauser/go-sysconf v0.3.9 // indirect - github.com/tklauser/numcpus v0.3.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect @@ -342,7 +349,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect @@ -350,6 +356,7 @@ require ( modernc.org/mathutil v1.5.0 // indirect modernc.org/memory v1.5.0 // indirect modernc.org/sqlite v1.21.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect ) replace github.com/onflow/flow-go => ../ diff --git a/integration/go.sum b/integration/go.sum index 7db3a31f5c6..b4275f546d2 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -109,11 +109,12 @@ github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBY github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= -github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= -github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -154,46 +155,46 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= -github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= github.com/aws/aws-sdk-go-v2/config v1.8.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY= -github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= -github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= +github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= +github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= github.com/aws/aws-sdk-go-v2/credentials v1.4.0/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0/go.mod h1:CpNzHK9VEFUCknu50kkB8z58AH2B5DvPP7ea1LHve/Y= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.1 h1:VGkV9KmhGqOQWnHyi4gLG98kE6OecT42fdrCGFWxJsc= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.1/go.mod h1:PLlnMiki//sGnCJiW+aVpvP/C8Kcm8mEj/IVm9+9qk4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2/go.mod h1:BQV0agm+JEhqR+2RT5e1XTFIDcAAV0eW6z2trp+iduw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 h1:gceOysEWNNwLd6cki65IMBZ4WAM0MwgBQq2n7kejoT8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0/go.mod h1:v8ygadNyATSm6elwJ/4gzJwcFhri9RqS8skgHKiwXPU= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.0 h1:HWsM0YQWX76V6MOp07YuTYacm8k7h69ObJuw7Nck+og= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.0/go.mod h1:LKb3cKNQIMh+itGnEpKGcnL/6OIjPZqrtYah1w5f+3o= github.com/aws/aws-sdk-go-v2/service/s3 v1.15.0 h1:nPLfLPfglacc29Y949sDxpr3X/blaY40s3B85WT2yZU= github.com/aws/aws-sdk-go-v2/service/s3 v1.15.0/go.mod h1:Iv2aJVtVSm/D22rFoX99cLG4q4uB7tppuCsulGe98k4= github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= github.com/aws/aws-sdk-go-v2/service/sts v1.7.0/go.mod h1:0qcSMCyASQPN2sk/1KQLQ2Fh6yq8wm0HSDAimPhzCoM= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -266,18 +267,24 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w= -github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= @@ -303,6 +310,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= +github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -392,9 +401,11 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= -github.com/ethereum/go-ethereum v1.12.0 h1:bdnhLPtqETd4m3mS8BGMNvBTf36bO5bx/hxE2zljOa0= -github.com/ethereum/go-ethereum v1.12.0/go.mod h1:/oo2X/dZLJjf2mJ6YT9wcWxa4nNJDBKDBU6sFIpx1Gs= +github.com/ethereum/go-ethereum v1.13.5 h1:U6TCRciCqZRe4FPXmy1sMGxTfuk8P7u2UoinF3VbaFk= +github.com/ethereum/go-ethereum v1.13.5/go.mod h1:yMTu38GSuyxaYzQMViqNmQ1s3cE84abZexQmTgenWk0= github.com/fanliao/go-promise v0.0.0-20141029170127-1890db352a72/go.mod h1:PjfxuH4FZdUyfMdtBio2lsRr1AKEaVPwelzuHuh8Lqc= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -475,6 +486,7 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -498,8 +510,8 @@ github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaL github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= @@ -622,6 +634,7 @@ github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYa github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -709,15 +722,15 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= -github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= +github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= +github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= -github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -1284,6 +1297,9 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= @@ -1704,6 +1720,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/supranational/blst v0.3.4/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -1711,10 +1729,12 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= -github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= -github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d h1:5JInRQbk5UBX8JfUvKh2oYTLMVwj3p6n+wapDDm7hko= @@ -2148,7 +2168,6 @@ golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2186,6 +2205,7 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -2507,7 +2527,6 @@ gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -2565,6 +2584,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= From 497b5931f91f32bb3289b9ae49580a88ed62adc5 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 17:58:10 -0800 Subject: [PATCH 45/72] fix default tx context --- fvm/evm/emulator/config.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fvm/evm/emulator/config.go b/fvm/evm/emulator/config.go index 74f012839a9..b15ede521e9 100644 --- a/fvm/evm/emulator/config.go +++ b/fvm/evm/emulator/config.go @@ -69,7 +69,10 @@ func defaultConfig() *Config { EVMConfig: vm.Config{ NoBaseFee: true, }, - TxContext: &vm.TxContext{}, + TxContext: &vm.TxContext{ + GasPrice: new(big.Int), + BlobFeeCap: new(big.Int), + }, BlockContext: &vm.BlockContext{ CanTransfer: core.CanTransfer, Transfer: core.Transfer, From 16bcb53276a1116b10a2cde4973dda2ca99e222a Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 20:51:05 -0800 Subject: [PATCH 46/72] lint fix --- fvm/evm/emulator/state/delta.go | 5 ++++- storage/pebble/registers.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index afb9ad1c221..a8384d76aa3 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -111,7 +111,10 @@ func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { if err != nil { return err } - d.AddBalance(addr, bal) + err = d.AddBalance(addr, bal) + if err != nil { + return err + } } d.created[addr] = struct{}{} diff --git a/storage/pebble/registers.go b/storage/pebble/registers.go index a1f3bc20bed..40a4657d4af 100644 --- a/storage/pebble/registers.go +++ b/storage/pebble/registers.go @@ -59,9 +59,12 @@ func (s *Registers) Get( fmt.Sprintf("height %d not indexed, indexed range is [%d-%d]", height, s.firstHeight, latestHeight), ) } - iter := s.db.NewIter(&pebble.IterOptions{ + iter, err := s.db.NewIter(&pebble.IterOptions{ UseL6Filters: true, }) + if err != nil { + return nil, err + } defer iter.Close() From d163786dfc43406fc1258d11e1e5d9510e22eab0 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 21:25:28 -0800 Subject: [PATCH 47/72] minor improvements --- fvm/evm/emulator/state/delta.go | 23 ++++++++--------------- fvm/evm/types/state.go | 3 --- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index 3aed174c8d7..6a5325bed25 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -28,9 +28,9 @@ type DeltaView struct { codes map[gethCommon.Address][]byte codeHashes map[gethCommon.Address]gethCommon.Hash - // states changes + // slot changes dirtySlots map[types.SlotAddress]interface{} - states map[types.SlotAddress]gethCommon.Hash + slots map[types.SlotAddress]gethCommon.Hash // transient storage transient map[types.SlotAddress]gethCommon.Hash @@ -65,7 +65,7 @@ func NewDeltaView(parent types.ReadOnlyView) *DeltaView { codes: make(map[gethCommon.Address][]byte), codeHashes: make(map[gethCommon.Address]gethCommon.Hash), dirtySlots: make(map[types.SlotAddress]interface{}), - states: make(map[types.SlotAddress]gethCommon.Hash), + slots: make(map[types.SlotAddress]gethCommon.Hash), transient: make(map[types.SlotAddress]gethCommon.Hash), accessListAddresses: make(map[gethCommon.Address]interface{}), accessListSlots: make(map[types.SlotAddress]interface{}), @@ -115,9 +115,9 @@ func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { d.deleted[addr] = struct{}{} // remove slabs from cache related to this account - for k := range d.states { + for k := range d.slots { if k.Address == addr { - delete(d.states, k) + delete(d.slots, k) } } } @@ -303,7 +303,7 @@ func (d *DeltaView) SetCode(addr gethCommon.Address, code []byte) error { // GetState returns the value of the slot of the main state func (d *DeltaView) GetState(sk types.SlotAddress) (gethCommon.Hash, error) { - val, found := d.states[sk] + val, found := d.slots[sk] if found { return val, nil } @@ -323,13 +323,11 @@ func (d *DeltaView) SetState(sk types.SlotAddress, value gethCommon.Hash) error if err != nil { return err } - // we skip the value is the same - // this step might look not helping with performance but we kept it to - // act similar to the Geth StateDB behaviour + // if the value hasn't changed, skip if value == lastValue { return nil } - d.states[sk] = value + d.slots[sk] = value d.dirtySlots[sk] = struct{}{} return nil } @@ -426,11 +424,6 @@ func (d *DeltaView) Preimages() map[gethCommon.Hash][]byte { return d.preimages } -// Commit for deltaview is a no-op -func (d *DeltaView) Commit() error { - return nil -} - // DirtyAddresses returns a set of addresses that has been updated in this view func (d *DeltaView) DirtyAddresses() map[gethCommon.Address]interface{} { return d.dirtyAddresses diff --git a/fvm/evm/types/state.go b/fvm/evm/types/state.go index 5cde6d332a2..25d032a53c7 100644 --- a/fvm/evm/types/state.go +++ b/fvm/evm/types/state.go @@ -93,9 +93,6 @@ type HotView interface { AddLog(*gethTypes.Log) // AddPreimage adds a preimage to the list of preimages (input -> hash mapping) AddPreimage(gethCommon.Hash, []byte) - - // Commit finalizes and commits the changes - Commit() error } // BaseView is a low-level mutable view of the state From 38a22599ac521010fb28d28af3446b381f3fd134 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 3 Jan 2024 23:28:47 -0800 Subject: [PATCH 48/72] fix tests --- fvm/evm/emulator/state/delta_test.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fvm/evm/emulator/state/delta_test.go b/fvm/evm/emulator/state/delta_test.go index e5b7ce542b6..7b79e040567 100644 --- a/fvm/evm/emulator/state/delta_test.go +++ b/fvm/evm/emulator/state/delta_test.go @@ -5,7 +5,6 @@ import ( "math/big" "testing" - "github.com/ethereum/go-ethereum/common" gethCommon "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" gethCrypto "github.com/ethereum/go-ethereum/crypto" @@ -269,7 +268,7 @@ func TestDeltaView(t *testing.T) { return 0, fmt.Errorf("some error") } }, - GetCodeHashFunc: func(addr gethCommon.Address) (common.Hash, error) { + GetCodeHashFunc: func(addr gethCommon.Address) (gethCommon.Hash, error) { switch addr { case addr1: return addr1IntiCodeHash, nil @@ -706,7 +705,6 @@ func TestDeltaView(t *testing.T) { require.Equal(t, value, vret) // now re-create account - err = view.CreateAccount(addr1) require.NoError(t, err) @@ -716,11 +714,12 @@ func TestDeltaView(t *testing.T) { ret, err = view.GetCode(addr1) require.NoError(t, err) - require.Equal(t, nil, ret) + require.Len(t, ret, 0) vret, err = view.GetState(sk) require.NoError(t, err) - require.Len(t, vret, 0) + emptyValue := gethCommon.Hash{} + require.Equal(t, emptyValue, vret) }) } From 4ad9138f2c0a0488bbf50420ef540a793f63c6a0 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Thu, 4 Jan 2024 10:57:18 -0800 Subject: [PATCH 49/72] apply PR feedbacks --- fvm/evm/emulator/state/base.go | 10 ++++++++-- fvm/evm/emulator/state/stateDB.go | 11 ++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/fvm/evm/emulator/state/base.go b/fvm/evm/emulator/state/base.go index ae098bc77ef..6792bc2ca67 100644 --- a/fvm/evm/emulator/state/base.go +++ b/fvm/evm/emulator/state/base.go @@ -422,7 +422,10 @@ func (v *BaseView) getSlot(sk types.SlotAddress) (gethCommon.Hash, error) { } acc, err := v.getAccount(sk.Address) - if err != nil || acc == nil || len(acc.CollectionID) == 0 { + if err != nil { + return gethCommon.Hash{}, err + } + if acc == nil || len(acc.CollectionID) == 0 { return gethCommon.Hash{}, nil } @@ -477,7 +480,10 @@ func (v *BaseView) getSlotCollection(acc *Account) (*Collection, error) { // update account's collection ID acc.CollectionID = col.CollectionID() err = v.storeAccount(acc) - return col, err + if err != nil { + return nil, err + } + return col, nil } col, found := v.slots[acc.Address] diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index 570d392599a..94fd9be9c0c 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -4,6 +4,7 @@ import ( "bytes" stdErrors "errors" "fmt" + "maps" "math/big" "sort" @@ -299,12 +300,8 @@ func (db *StateDB) Commit() error { addresses := make(map[gethCommon.Address]interface{}) slots := make(map[types.SlotAddress]interface{}) for _, view := range db.views { - for key := range view.DirtyAddresses() { - addresses[key] = struct{}{} - } - for key := range view.DirtySlots() { - slots[key] = struct{}{} - } + maps.Copy(addresses, view.DirtyAddresses()) + maps.Copy(slots, view.DirtySlots()) } // sort addresses @@ -312,6 +309,7 @@ func (db *StateDB) Commit() error { for addr := range addresses { sortedAddresses = append(sortedAddresses, addr) } + sort.Slice(sortedAddresses, func(i, j int) bool { return bytes.Compare(sortedAddresses[i][:], sortedAddresses[j][:]) < 0 @@ -395,7 +393,6 @@ func (db *StateDB) Commit() error { // based on parameters that are passed it updates accesslists func (db *StateDB) Prepare(rules gethParams.Rules, sender, coinbase gethCommon.Address, dest *gethCommon.Address, precompiles []gethCommon.Address, txAccesses gethTypes.AccessList) { if rules.IsBerlin { - // no need for mutation db.AddAddressToAccessList(sender) if dest != nil { From bc0cba24257571c137195bc10c5c3f141467594c Mon Sep 17 00:00:00 2001 From: ramtinms Date: Thu, 4 Jan 2024 11:34:05 -0800 Subject: [PATCH 50/72] revert unwanted changes --- fvm/evm/stdlib/contract.go | 94 ++++++- fvm/evm/stdlib/contract_test.go | 436 +++++++++++++++++++++++++++++++- 2 files changed, 520 insertions(+), 10 deletions(-) diff --git a/fvm/evm/stdlib/contract.go b/fvm/evm/stdlib/contract.go index f39487982f4..d7fe8401d59 100644 --- a/fvm/evm/stdlib/contract.go +++ b/fvm/evm/stdlib/contract.go @@ -3,6 +3,7 @@ package stdlib import ( _ "embed" "fmt" + "math" "math/big" "reflect" "regexp" @@ -39,6 +40,8 @@ const ContractName = "EVM" const evmAddressTypeBytesFieldName = "bytes" const evmAddressTypeQualifiedIdentifier = "EVM.EVMAddress" +const abiEncodingByteSize = 32 + var EVMTransactionBytesCadenceType = cadence.NewVariableSizedArrayType(cadence.TheUInt8Type) var evmTransactionBytesType = sema.NewVariableSizedType(nil, sema.UInt8Type) @@ -97,6 +100,77 @@ func (e abiDecodingError) Error() string { return b.String() } +func reportABIEncodingComputation( + inter *interpreter.Interpreter, + values *interpreter.ArrayValue, + evmAddressTypeID common.TypeID, + reportComputation func(intensity uint), +) { + values.Iterate(inter, func(element interpreter.Value) (resume bool) { + switch value := element.(type) { + case *interpreter.StringValue: + // Dynamic variables, such as strings, are encoded + // in 2+ chunks of 32 bytes. The first chunk contains + // the index where information for the string begin, + // the second chunk contains the number of bytes the + // string occupies, and the third chunk contains the + // value of the string itself. + computation := uint(2 * abiEncodingByteSize) + stringLength := len(value.Str) + chunks := math.Ceil(float64(stringLength) / float64(abiEncodingByteSize)) + computation += uint(chunks * abiEncodingByteSize) + reportComputation(computation) + + case interpreter.BoolValue, + interpreter.UInt8Value, + interpreter.UInt16Value, + interpreter.UInt32Value, + interpreter.UInt64Value, + interpreter.UInt128Value, + interpreter.UInt256Value, + interpreter.Int8Value, + interpreter.Int16Value, + interpreter.Int32Value, + interpreter.Int64Value, + interpreter.Int128Value, + interpreter.Int256Value: + + // Numeric and bool variables are also static variables + // with a fixed size of 32 bytes. + reportComputation(abiEncodingByteSize) + + case *interpreter.CompositeValue: + if value.TypeID() == evmAddressTypeID { + // EVM addresses are static variables with a fixed + // size of 32 bytes. + reportComputation(abiEncodingByteSize) + } else { + panic(abiEncodingError{ + Type: value.StaticType(inter), + }) + } + case *interpreter.ArrayValue: + // Dynamic variables, such as arrays & slices, are encoded + // in 2+ chunks of 32 bytes. The first chunk contains + // the index where information for the array begin, + // the second chunk contains the number of bytes the + // array occupies, and the third chunk contains the + // values of the array itself. + computation := uint(2 * abiEncodingByteSize) + reportComputation(computation) + reportABIEncodingComputation(inter, value, evmAddressTypeID, reportComputation) + + default: + panic(abiEncodingError{ + Type: element.StaticType(inter), + }) + } + + // continue iteration + return true + }) +} + // EVM.encodeABI const internalEVMTypeEncodeABIFunctionName = "encodeABI" @@ -135,6 +209,15 @@ func newInternalEVMTypeEncodeABIFunction( panic(errors.NewUnreachableError()) } + reportABIEncodingComputation( + inter, + valuesArray, + evmAddressTypeID, + func(intensity uint) { + inter.ReportComputation(environment.ComputationKindEVMEncodeABI, intensity) + }, + ) + size := valuesArray.Count() values := make([]any, 0, size) @@ -159,19 +242,12 @@ func newInternalEVMTypeEncodeABIFunction( return true }) - hexData, err := arguments.Pack(values...) + encodedValues, err := arguments.Pack(values...) if err != nil { panic(abiEncodingError{}) } - encodedValues := interpreter.ByteSliceToByteArrayValue(inter, hexData) - - invocation.Interpreter.ReportComputation( - environment.ComputationKindEVMEncodeABI, - uint(encodedValues.Count()), - ) - - return encodedValues + return interpreter.ByteSliceToByteArrayValue(inter, encodedValues) }, ) } diff --git a/fvm/evm/stdlib/contract_test.go b/fvm/evm/stdlib/contract_test.go index 8dbe73ac6b9..a10f90913de 100644 --- a/fvm/evm/stdlib/contract_test.go +++ b/fvm/evm/stdlib/contract_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/fvm/blueprints" + "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/evm/stdlib" . "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/fvm/evm/types" @@ -262,6 +263,7 @@ func TestEVMEncodeABI(t *testing.T) { accountCodes := map[common.Location][]byte{} var events []cadence.Event + computation := uint(0) runtimeInterface := &TestRuntimeInterface{ Storage: NewTestLedger(nil, nil), OnGetSigningAccounts: func() ([]runtime.Address, error) { @@ -283,6 +285,12 @@ func TestEVMEncodeABI(t *testing.T) { OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { return json.Decode(nil, b) }, + OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { + if compKind == environment.ComputationKindEVMEncodeABI { + computation += intensity + } + return nil + }, } nextTransactionLocation := NewTransactionLocationGenerator() @@ -340,6 +348,311 @@ func TestEVMEncodeABI(t *testing.T) { encodedABI, result, ) + assert.Equal(t, computation, uint(len(cdcBytes))) +} + +func TestEVMEncodeABIComputation(t *testing.T) { + + t.Parallel() + + handler := &testContractHandler{} + + contractsAddress := flow.BytesToAddress([]byte{0x1}) + + transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) + scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) + + rt := runtime.NewInterpreterRuntime(runtime.Config{}) + + script := []byte(` + import EVM from 0x1 + + access(all) + fun main(): [UInt8] { + let address = EVM.EVMAddress( + bytes: [ + 122, 88, 192, 190, 114, 190, 33, 139, 65, 198, + 8, 183, 254, 124, 91, 182, 48, 115, 108, 113 + ] + ) + let arr: [UInt8] = [1, 2, 3, 4, 5] + + return EVM.encodeABI([ + "John Doe", + UInt64(33), + false, + address, + [arr], + ["one", "two", "three"] + ]) + } + `) + + accountCodes := map[common.Location][]byte{} + var events []cadence.Event + + computation := uint(0) + runtimeInterface := &TestRuntimeInterface{ + Storage: NewTestLedger(nil, nil), + OnGetSigningAccounts: func() ([]runtime.Address, error) { + return []runtime.Address{runtime.Address(contractsAddress)}, nil + }, + OnResolveLocation: SingleIdentifierLocationResolver(t), + OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { + accountCodes[location] = code + return nil + }, + OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { + code = accountCodes[location] + return code, nil + }, + OnEmitEvent: func(event cadence.Event) error { + events = append(events, event) + return nil + }, + OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { + return json.Decode(nil, b) + }, + OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { + if compKind == environment.ComputationKindEVMEncodeABI { + computation += intensity + } + return nil + }, + } + + nextTransactionLocation := NewTransactionLocationGenerator() + nextScriptLocation := NewScriptLocationGenerator() + + // Deploy contracts + + deployContracts( + t, + rt, + contractsAddress, + runtimeInterface, + transactionEnvironment, + nextTransactionLocation, + ) + + // Run script + + result, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: [][]byte{}, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) + require.NoError(t, err) + + cdcBytes, ok := result.(cadence.Array) + require.True(t, ok) + // computation & len(cdcBytes.Values) is equal to 832 + assert.Equal(t, computation, uint(len(cdcBytes.Values))) +} + +func TestEVMEncodeABIComputationEmptyDynamicVariables(t *testing.T) { + + t.Parallel() + + handler := &testContractHandler{} + + contractsAddress := flow.BytesToAddress([]byte{0x1}) + + transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) + scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) + + rt := runtime.NewInterpreterRuntime(runtime.Config{}) + + script := []byte(` + import EVM from 0x1 + + access(all) + fun main(): [UInt8] { + return EVM.encodeABI([ + "", + [[""], [] as [String]], + [] as [UInt8], + ["", "", ""] + ]) + } + `) + + accountCodes := map[common.Location][]byte{} + var events []cadence.Event + + computation := uint(0) + runtimeInterface := &TestRuntimeInterface{ + Storage: NewTestLedger(nil, nil), + OnGetSigningAccounts: func() ([]runtime.Address, error) { + return []runtime.Address{runtime.Address(contractsAddress)}, nil + }, + OnResolveLocation: SingleIdentifierLocationResolver(t), + OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { + accountCodes[location] = code + return nil + }, + OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { + code = accountCodes[location] + return code, nil + }, + OnEmitEvent: func(event cadence.Event) error { + events = append(events, event) + return nil + }, + OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { + return json.Decode(nil, b) + }, + OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { + if compKind == environment.ComputationKindEVMEncodeABI { + computation += intensity + } + return nil + }, + } + + nextTransactionLocation := NewTransactionLocationGenerator() + nextScriptLocation := NewScriptLocationGenerator() + + // Deploy contracts + + deployContracts( + t, + rt, + contractsAddress, + runtimeInterface, + transactionEnvironment, + nextTransactionLocation, + ) + + // Run script + + result, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: [][]byte{}, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) + require.NoError(t, err) + + cdcBytes, ok := result.(cadence.Array) + require.True(t, ok) + // computation & len(cdcBytes.Values) is equal to 832 + assert.Equal(t, computation, uint(len(cdcBytes.Values))) +} + +func TestEVMEncodeABIComputationDynamicVariablesAboveChunkSize(t *testing.T) { + + t.Parallel() + + handler := &testContractHandler{} + + contractsAddress := flow.BytesToAddress([]byte{0x1}) + + transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) + scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) + + rt := runtime.NewInterpreterRuntime(runtime.Config{}) + + script := []byte(` + import EVM from 0x1 + + access(all) + fun main(): [UInt8] { + let str = "abcdefghijklmnopqrstuvwxyz" + let arr: [UInt64] = [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53 + ] + + return EVM.encodeABI([ + str, + str.concat(str).concat(str), + [[str]], + arr, + [arr], + arr.concat(arr).concat(arr) + ]) + } + `) + + accountCodes := map[common.Location][]byte{} + var events []cadence.Event + + computation := uint(0) + runtimeInterface := &TestRuntimeInterface{ + Storage: NewTestLedger(nil, nil), + OnGetSigningAccounts: func() ([]runtime.Address, error) { + return []runtime.Address{runtime.Address(contractsAddress)}, nil + }, + OnResolveLocation: SingleIdentifierLocationResolver(t), + OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { + accountCodes[location] = code + return nil + }, + OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { + code = accountCodes[location] + return code, nil + }, + OnEmitEvent: func(event cadence.Event) error { + events = append(events, event) + return nil + }, + OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { + return json.Decode(nil, b) + }, + OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { + if compKind == environment.ComputationKindEVMEncodeABI { + computation += intensity + } + return nil + }, + } + + nextTransactionLocation := NewTransactionLocationGenerator() + nextScriptLocation := NewScriptLocationGenerator() + + // Deploy contracts + + deployContracts( + t, + rt, + contractsAddress, + runtimeInterface, + transactionEnvironment, + nextTransactionLocation, + ) + + // Run script + + result, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: [][]byte{}, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) + require.NoError(t, err) + + cdcBytes, ok := result.(cadence.Array) + require.True(t, ok) + // computation & len(cdcBytes.Values) is equal to 832 + assert.Equal(t, computation, uint(len(cdcBytes.Values))) } func TestEVMDecodeABI(t *testing.T) { @@ -375,6 +688,7 @@ func TestEVMDecodeABI(t *testing.T) { accountCodes := map[common.Location][]byte{} var events []cadence.Event + computation := uint(0) runtimeInterface := &TestRuntimeInterface{ Storage: NewTestLedger(nil, nil), OnGetSigningAccounts: func() ([]runtime.Address, error) { @@ -396,6 +710,12 @@ func TestEVMDecodeABI(t *testing.T) { OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { return json.Decode(nil, b) }, + OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { + if compKind == environment.ComputationKindEVMDecodeABI { + computation += intensity + } + return nil + }, } nextTransactionLocation := NewTransactionLocationGenerator() @@ -451,6 +771,120 @@ func TestEVMDecodeABI(t *testing.T) { require.NoError(t, err) assert.Equal(t, cadence.NewBool(true), result) + assert.Equal(t, computation, uint(len(cdcBytes))) +} + +func TestEVMDecodeABIComputation(t *testing.T) { + + t.Parallel() + + handler := &testContractHandler{} + + contractsAddress := flow.BytesToAddress([]byte{0x1}) + + transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) + scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) + + rt := runtime.NewInterpreterRuntime(runtime.Config{}) + + script := []byte(` + import EVM from 0x1 + + access(all) + fun main(): [UInt8] { + let address = EVM.EVMAddress( + bytes: [ + 122, 88, 192, 190, 114, 190, 33, 139, 65, 198, + 8, 183, 254, 124, 91, 182, 48, 115, 108, 113 + ] + ) + let arr: [UInt8] = [1, 2, 3, 4, 5] + + let data = EVM.encodeABI([ + "John Doe", + UInt64(33), + true, + address, + [arr], + ["one", "two", "three"] + ]) + + let types = [ + Type(), Type(), Type(), Type(), + Type<[[UInt8]]>(), Type<[String]>() + ] + let values = EVM.decodeABI(types: types, data: data) + + return data + } + `) + + accountCodes := map[common.Location][]byte{} + var events []cadence.Event + + computation := uint(0) + runtimeInterface := &TestRuntimeInterface{ + Storage: NewTestLedger(nil, nil), + OnGetSigningAccounts: func() ([]runtime.Address, error) { + return []runtime.Address{runtime.Address(contractsAddress)}, nil + }, + OnResolveLocation: SingleIdentifierLocationResolver(t), + OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { + accountCodes[location] = code + return nil + }, + OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { + code = accountCodes[location] + return code, nil + }, + OnEmitEvent: func(event cadence.Event) error { + events = append(events, event) + return nil + }, + OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { + return json.Decode(nil, b) + }, + OnMeterComputation: func(compKind common.ComputationKind, intensity uint) error { + if compKind == environment.ComputationKindEVMDecodeABI { + computation += intensity + } + return nil + }, + } + + nextTransactionLocation := NewTransactionLocationGenerator() + nextScriptLocation := NewScriptLocationGenerator() + + // Deploy contracts + + deployContracts( + t, + rt, + contractsAddress, + runtimeInterface, + transactionEnvironment, + nextTransactionLocation, + ) + + // Run script + + result, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: [][]byte{}, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) + require.NoError(t, err) + + cdcBytes, ok := result.(cadence.Array) + require.True(t, ok) + // computation & len(cdcBytes.Values) is equal to 832 + assert.Equal(t, computation, uint(len(cdcBytes.Values))) } func TestEVMEncodeDecodeABIRoundtrip(t *testing.T) { @@ -1069,7 +1503,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { assert.ErrorContains( t, err, - "failed to ABI encode value of type [Character]", + "failed to ABI encode value of type Character", ) }) From e613e85c16a4a31833119a84cd739968bed59988 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Thu, 4 Jan 2024 14:06:39 -0800 Subject: [PATCH 51/72] simplify creat logic --- fvm/evm/emulator/state/delta.go | 11 ++--------- fvm/evm/emulator/state/delta_test.go | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index b966f5d3bf3..2158e2eeb04 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -101,18 +101,11 @@ func (d *DeltaView) Exist(addr gethCommon.Address) (bool, error) { func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { // If a address already exists the balance is carried over to the new account. // Carrying over the balance ensures that Ether doesn't disappear. (legacy behaviour of the Geth stateDB) - exist, err := d.Exist(addr) + bal, err := d.GetBalance(addr) if err != nil { return err } - var bal *big.Int - if exist { - bal, err = d.GetBalance(addr) - if err != nil { - return err - } - d.AddBalance(addr, bal) - } + d.AddBalance(addr, bal) d.created[addr] = struct{}{} // flag the address as dirty diff --git a/fvm/evm/emulator/state/delta_test.go b/fvm/evm/emulator/state/delta_test.go index e5b7ce542b6..fe2376a5f14 100644 --- a/fvm/evm/emulator/state/delta_test.go +++ b/fvm/evm/emulator/state/delta_test.go @@ -5,7 +5,6 @@ import ( "math/big" "testing" - "github.com/ethereum/go-ethereum/common" gethCommon "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" gethCrypto "github.com/ethereum/go-ethereum/crypto" @@ -42,6 +41,9 @@ func TestDeltaView(t *testing.T) { return false, fmt.Errorf("some error") } }, + GetBalanceFunc: func(a gethCommon.Address) (*big.Int, error) { + return new(big.Int), nil + }, HasSuicidedFunc: func(gethCommon.Address) bool { return false }, @@ -164,15 +166,6 @@ func TestDeltaView(t *testing.T) { // handling error on the parent _, err = view.GetBalance(addr3) require.Error(t, err) - - // create a new account at addr3 - err = view.CreateAccount(addr3) - require.NoError(t, err) - - // now the balance should return 0 - bal, err = view.GetBalance(addr3) - require.NoError(t, err) - require.Equal(t, big.NewInt(0), bal) }) t.Run("test nonce functionality", func(t *testing.T) { @@ -195,6 +188,9 @@ func TestDeltaView(t *testing.T) { HasSuicidedFunc: func(a gethCommon.Address) bool { return false }, + GetBalanceFunc: func(gethCommon.Address) (*big.Int, error) { + return new(big.Int), nil + }, GetNonceFunc: func(addr gethCommon.Address) (uint64, error) { switch addr { case addr1: @@ -253,6 +249,9 @@ func TestDeltaView(t *testing.T) { HasSuicidedFunc: func(a gethCommon.Address) bool { return false }, + GetBalanceFunc: func(a gethCommon.Address) (*big.Int, error) { + return new(big.Int), nil + }, GetCodeFunc: func(addr gethCommon.Address) ([]byte, error) { switch addr { case addr1: @@ -269,7 +268,7 @@ func TestDeltaView(t *testing.T) { return 0, fmt.Errorf("some error") } }, - GetCodeHashFunc: func(addr gethCommon.Address) (common.Hash, error) { + GetCodeHashFunc: func(addr gethCommon.Address) (gethCommon.Hash, error) { switch addr { case addr1: return addr1IntiCodeHash, nil @@ -716,11 +715,12 @@ func TestDeltaView(t *testing.T) { ret, err = view.GetCode(addr1) require.NoError(t, err) - require.Equal(t, nil, ret) + require.Len(t, ret, 0) vret, err = view.GetState(sk) require.NoError(t, err) - require.Len(t, vret, 0) + emptyValue := gethCommon.Hash{} + require.Equal(t, emptyValue, vret) }) } From f07d7214de47fa5d58512ef5253b982cd4537e6e Mon Sep 17 00:00:00 2001 From: ramtinms Date: Thu, 4 Jan 2024 14:59:04 -0800 Subject: [PATCH 52/72] apply PR feedback --- fvm/evm/emulator/state/delta.go | 87 ++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 34 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index 6a5325bed25..f7e7dcd0e74 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -19,25 +19,25 @@ type DeltaView struct { parent types.ReadOnlyView // account changes - dirtyAddresses map[gethCommon.Address]interface{} - created map[gethCommon.Address]interface{} - suicided map[gethCommon.Address]interface{} - deleted map[gethCommon.Address]interface{} + dirtyAddresses map[gethCommon.Address]struct{} + created map[gethCommon.Address]struct{} + suicided map[gethCommon.Address]struct{} + deleted map[gethCommon.Address]struct{} balances map[gethCommon.Address]*big.Int nonces map[gethCommon.Address]uint64 codes map[gethCommon.Address][]byte codeHashes map[gethCommon.Address]gethCommon.Hash // slot changes - dirtySlots map[types.SlotAddress]interface{} + dirtySlots map[types.SlotAddress]struct{} slots map[types.SlotAddress]gethCommon.Hash // transient storage transient map[types.SlotAddress]gethCommon.Hash // access lists - accessListAddresses map[gethCommon.Address]interface{} - accessListSlots map[types.SlotAddress]interface{} + accessListAddresses map[gethCommon.Address]struct{} + accessListSlots map[types.SlotAddress]struct{} // logs logs []*gethTypes.Log @@ -56,21 +56,16 @@ func NewDeltaView(parent types.ReadOnlyView) *DeltaView { return &DeltaView{ parent: parent, - dirtyAddresses: make(map[gethCommon.Address]interface{}), - created: make(map[gethCommon.Address]interface{}), - suicided: make(map[gethCommon.Address]interface{}), - deleted: make(map[gethCommon.Address]interface{}), - balances: make(map[gethCommon.Address]*big.Int), - nonces: make(map[gethCommon.Address]uint64), - codes: make(map[gethCommon.Address][]byte), - codeHashes: make(map[gethCommon.Address]gethCommon.Hash), - dirtySlots: make(map[types.SlotAddress]interface{}), - slots: make(map[types.SlotAddress]gethCommon.Hash), - transient: make(map[types.SlotAddress]gethCommon.Hash), - accessListAddresses: make(map[gethCommon.Address]interface{}), - accessListSlots: make(map[types.SlotAddress]interface{}), - logs: make([]*gethTypes.Log, 0), - preimages: make(map[gethCommon.Hash][]byte), + dirtyAddresses: make(map[gethCommon.Address]struct{}), + created: make(map[gethCommon.Address]struct{}), + suicided: make(map[gethCommon.Address]struct{}), + deleted: make(map[gethCommon.Address]struct{}), + balances: make(map[gethCommon.Address]*big.Int), + nonces: make(map[gethCommon.Address]uint64), + codes: make(map[gethCommon.Address][]byte), + codeHashes: make(map[gethCommon.Address]gethCommon.Hash), + dirtySlots: make(map[types.SlotAddress]struct{}), + slots: make(map[types.SlotAddress]gethCommon.Hash), // for refund we just copy the data refund: parent.GetRefund(), @@ -334,15 +329,20 @@ func (d *DeltaView) SetState(sk types.SlotAddress, value gethCommon.Hash) error // GetTransientState returns the value of the slot of the transient state func (d *DeltaView) GetTransientState(sk types.SlotAddress) gethCommon.Hash { - val, found := d.transient[sk] - if found { - return val + if d.transient != nil { + val, found := d.transient[sk] + if found { + return val + } } return d.parent.GetTransientState(sk) } // SetTransientState adds sets a value for the given slot of the transient storage func (d *DeltaView) SetTransientState(sk types.SlotAddress, value gethCommon.Hash) { + if d.transient == nil { + d.transient = make(map[types.SlotAddress]gethCommon.Hash) + } d.transient[sk] = value } @@ -368,15 +368,21 @@ func (d *DeltaView) SubRefund(amount uint64) error { // AddressInAccessList checks if the address is in the access list func (d *DeltaView) AddressInAccessList(addr gethCommon.Address) bool { - _, addressFound := d.accessListAddresses[addr] - if !addressFound { - addressFound = d.parent.AddressInAccessList(addr) + if d.accessListAddresses != nil { + _, addressFound := d.accessListAddresses[addr] + if addressFound { + return true + } } - return addressFound + return d.parent.AddressInAccessList(addr) } // AddAddressToAccessList adds an address to the access list func (d *DeltaView) AddAddressToAccessList(addr gethCommon.Address) bool { + if d.accessListAddresses == nil { + d.accessListAddresses = make(map[gethCommon.Address]struct{}) + } + addrPresent := d.AddressInAccessList(addr) d.accessListAddresses[addr] = struct{}{} return !addrPresent @@ -385,16 +391,22 @@ func (d *DeltaView) AddAddressToAccessList(addr gethCommon.Address) bool { // SlotInAccessList checks if the slot is in the access list func (d *DeltaView) SlotInAccessList(sk types.SlotAddress) (addressOk bool, slotOk bool) { addressFound := d.AddressInAccessList(sk.Address) - _, slotFound := d.accessListSlots[sk] - if !slotFound { - _, slotFound = d.parent.SlotInAccessList(sk) + if d.accessListSlots != nil { + _, slotFound := d.accessListSlots[sk] + if slotFound { + return addressFound, true + } } + _, slotFound := d.parent.SlotInAccessList(sk) return addressFound, slotFound } // AddSlotToAccessList adds a slot to the access list // it also adds the address to the address list func (d *DeltaView) AddSlotToAccessList(sk types.SlotAddress) (addrAdded bool, slotAdded bool) { + if d.accessListSlots == nil { + d.accessListSlots = make(map[types.SlotAddress]struct{}) + } addrPresent, slotPresent := d.SlotInAccessList(sk) d.accessListAddresses[sk.Address] = struct{}{} d.accessListSlots[sk] = struct{}{} @@ -403,6 +415,9 @@ func (d *DeltaView) AddSlotToAccessList(sk types.SlotAddress) (addrAdded bool, s // AddLog appends a log to the log collection func (d *DeltaView) AddLog(log *gethTypes.Log) { + if d.logs == nil { + d.logs = make([]*gethTypes.Log, 0) + } d.logs = append(d.logs, log) } @@ -413,6 +428,10 @@ func (d *DeltaView) Logs() []*gethTypes.Log { // AddPreimage adds a preimage func (d *DeltaView) AddPreimage(hash gethCommon.Hash, preimage []byte) { + if d.preimages == nil { + d.preimages = make(map[gethCommon.Hash][]byte) + } + // make a copy (legacy behaviour) pi := make([]byte, len(preimage)) copy(pi, preimage) @@ -425,11 +444,11 @@ func (d *DeltaView) Preimages() map[gethCommon.Hash][]byte { } // DirtyAddresses returns a set of addresses that has been updated in this view -func (d *DeltaView) DirtyAddresses() map[gethCommon.Address]interface{} { +func (d *DeltaView) DirtyAddresses() map[gethCommon.Address]struct{} { return d.dirtyAddresses } // DirtySlots returns a set of slots that has been updated in this view -func (d *DeltaView) DirtySlots() map[types.SlotAddress]interface{} { +func (d *DeltaView) DirtySlots() map[types.SlotAddress]struct{} { return d.dirtySlots } From f5d2673c409b00e3cce39fe5f1a2cb1b06c353a6 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Thu, 4 Jan 2024 16:06:07 -0800 Subject: [PATCH 53/72] add a condition --- fvm/evm/emulator/state/delta.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index f7e7dcd0e74..4bdf46066cc 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -139,6 +139,9 @@ func (d *DeltaView) HasSuicided(addr gethCommon.Address) bool { // Suicide sets a flag to delete the account at the end of transaction func (d *DeltaView) Suicide(addr gethCommon.Address) (bool, error) { + if d.IsCreated(addr) { + return false, fmt.Errorf("invalid operation, can't suicide an account that is just created") + } // if it doesn't exist, return false exists, err := d.Exist(addr) if err != nil { From 72a15a20808fb427a07ac6ff0cea262a453dbf2c Mon Sep 17 00:00:00 2001 From: ramtinms Date: Thu, 4 Jan 2024 16:07:48 -0800 Subject: [PATCH 54/72] fix test --- fvm/evm/emulator/state/delta_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fvm/evm/emulator/state/delta_test.go b/fvm/evm/emulator/state/delta_test.go index 7b79e040567..c31c110f20e 100644 --- a/fvm/evm/emulator/state/delta_test.go +++ b/fvm/evm/emulator/state/delta_test.go @@ -41,6 +41,9 @@ func TestDeltaView(t *testing.T) { return false, fmt.Errorf("some error") } }, + IsCreatedFunc: func(a gethCommon.Address) bool { + return false + }, HasSuicidedFunc: func(gethCommon.Address) bool { return false }, @@ -111,6 +114,9 @@ func TestDeltaView(t *testing.T) { HasSuicidedFunc: func(gethCommon.Address) bool { return false }, + IsCreatedFunc: func(a gethCommon.Address) bool { + return false + }, GetBalanceFunc: func(addr gethCommon.Address) (*big.Int, error) { switch addr { case addr1: @@ -589,6 +595,9 @@ func TestDeltaView(t *testing.T) { GetNonceFunc: func(addr gethCommon.Address) (uint64, error) { return 0, nil }, + IsCreatedFunc: func(a gethCommon.Address) bool { + return false + }, HasSuicidedFunc: func(gethCommon.Address) bool { return false }, @@ -644,6 +653,9 @@ func TestDeltaView(t *testing.T) { HasSuicidedFunc: func(gethCommon.Address) bool { return true }, + IsCreatedFunc: func(a gethCommon.Address) bool { + return false + }, GetBalanceFunc: func(addr gethCommon.Address) (*big.Int, error) { return new(big.Int), nil }, From 036c3705608b0ed6882c22d6dd662151a416ae2b Mon Sep 17 00:00:00 2001 From: ramtinms Date: Thu, 4 Jan 2024 23:54:39 -0800 Subject: [PATCH 55/72] update suicide logic --- fvm/evm/emulator/state/base.go | 4 +- fvm/evm/emulator/state/base_test.go | 4 +- fvm/evm/emulator/state/delta.go | 112 ++++++++++++++++++--------- fvm/evm/emulator/state/delta_test.go | 43 ++++++---- fvm/evm/types/state.go | 2 +- 5 files changed, 109 insertions(+), 56 deletions(-) diff --git a/fvm/evm/emulator/state/base.go b/fvm/evm/emulator/state/base.go index 4ae9d11c590..eb1a1477813 100644 --- a/fvm/evm/emulator/state/base.go +++ b/fvm/evm/emulator/state/base.go @@ -94,8 +94,8 @@ func (v *BaseView) IsCreated(gethCommon.Address) bool { } // HasSuicided returns true if an address has suicided in the context of this transaction -func (v *BaseView) HasSuicided(gethCommon.Address) bool { - return false +func (v *BaseView) HasSuicided(gethCommon.Address) (bool, *big.Int) { + return false, new(big.Int) } // GetBalance returns the balance of an address diff --git a/fvm/evm/emulator/state/base_test.go b/fvm/evm/emulator/state/base_test.go index 59c2ecf6b8f..dd29dc29b18 100644 --- a/fvm/evm/emulator/state/base_test.go +++ b/fvm/evm/emulator/state/base_test.go @@ -198,7 +198,9 @@ func TestBaseView(t *testing.T) { view, err := state.NewBaseView(testutils.GetSimpleValueStore(), flow.Address{1, 2, 3, 4}) require.NoError(t, err) - require.Equal(t, false, view.HasSuicided(gethCommon.Address{})) + suicided, bal := view.HasSuicided(gethCommon.Address{}) + require.Equal(t, false, suicided) + require.Equal(t, new(big.Int), bal) require.Equal(t, false, view.IsCreated(gethCommon.Address{})) require.Equal(t, uint64(0), view.GetRefund()) require.Equal(t, gethCommon.Hash{}, view.GetTransientState(types.SlotAddress{})) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index 4bdf46066cc..98e47ce8767 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -18,19 +18,27 @@ import ( type DeltaView struct { parent types.ReadOnlyView - // account changes + // dirtyAddresses keeps a set of addresses with changes dirtyAddresses map[gethCommon.Address]struct{} - created map[gethCommon.Address]struct{} - suicided map[gethCommon.Address]struct{} - deleted map[gethCommon.Address]struct{} - balances map[gethCommon.Address]*big.Int - nonces map[gethCommon.Address]uint64 - codes map[gethCommon.Address][]byte - codeHashes map[gethCommon.Address]gethCommon.Hash - - // slot changes - dirtySlots map[types.SlotAddress]struct{} - slots map[types.SlotAddress]gethCommon.Hash + // created keeps a set of recently created addresses + created map[gethCommon.Address]struct{} + // suicided keeps a set of addresses flagged to be destructed at the + // end of transaction, it also keeps the balance of the addresses before destruction + toBeDestructed map[gethCommon.Address]*big.Int + // is a flag used to track accounts that has been flagged for + // destruction but recreated later + recreated map[gethCommon.Address]struct{} + // balances keeps the changes to the account balances + balances map[gethCommon.Address]*big.Int + // nonces keeps the changes to the account nonces + nonces map[gethCommon.Address]uint64 + // codes keeps the changes to the account codes + codes map[gethCommon.Address][]byte + // codeHashes keeps the changes to account code hashes + codeHashes map[gethCommon.Address]gethCommon.Hash + + // slots keeps a set of slots that has been changed in this view + slots map[types.SlotAddress]gethCommon.Hash // transient storage transient map[types.SlotAddress]gethCommon.Hash @@ -58,14 +66,14 @@ func NewDeltaView(parent types.ReadOnlyView) *DeltaView { dirtyAddresses: make(map[gethCommon.Address]struct{}), created: make(map[gethCommon.Address]struct{}), - suicided: make(map[gethCommon.Address]struct{}), - deleted: make(map[gethCommon.Address]struct{}), + toBeDestructed: make(map[gethCommon.Address]*big.Int), + recreated: make(map[gethCommon.Address]struct{}), balances: make(map[gethCommon.Address]*big.Int), nonces: make(map[gethCommon.Address]uint64), codes: make(map[gethCommon.Address][]byte), codeHashes: make(map[gethCommon.Address]gethCommon.Hash), - dirtySlots: make(map[types.SlotAddress]struct{}), - slots: make(map[types.SlotAddress]gethCommon.Hash), + + slots: make(map[types.SlotAddress]gethCommon.Hash), // for refund we just copy the data refund: parent.GetRefund(), @@ -85,7 +93,7 @@ func (d *DeltaView) Exist(addr gethCommon.Address) (bool, error) { if found { return true, nil } - _, found = d.suicided[addr] + _, found = d.toBeDestructed[addr] if found { return true, nil } @@ -93,21 +101,29 @@ func (d *DeltaView) Exist(addr gethCommon.Address) (bool, error) { } // CreateAccount creates a new account for the given address +// +// if address has been flaged earlier for destruction, carry over the balance +// and reset the data from the orginal account. func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { - d.created[addr] = struct{}{} - // flag the address as dirty + // if is already created return + if d.IsCreated(addr) { + return nil + } + d.dirtyAddresses[addr] = struct{}{} - // if has already suicided - if d.HasSuicided(addr) { - // balance has already been set to zero + // if it has flagged for destruction in this transction, + // reset everything and carry over the balance + destructed, balance := d.HasSuicided(addr) + if destructed { d.nonces[addr] = 0 d.codes[addr] = nil d.codeHashes[addr] = gethTypes.EmptyCodeHash + d.balances[addr] = balance - // flag addr as deleted, this flag helps with postponing deletion of slabs + // flag addr as recreated, this flag helps with postponing deletion of slabs // otherwise we have to iterate over all slabs of this account and set the to nil - d.deleted[addr] = struct{}{} + d.recreated[addr] = struct{}{} // remove slabs from cache related to this account for k := range d.slots { @@ -115,7 +131,16 @@ func (d *DeltaView) CreateAccount(addr gethCommon.Address) error { delete(d.slots, k) } } + return nil + } + + d.created[addr] = struct{}{} + // carry over balance + bal, err := d.GetBalance(addr) + if err != nil { + return err } + d.balances[addr] = bal return nil } @@ -128,20 +153,26 @@ func (d *DeltaView) IsCreated(addr gethCommon.Address) bool { return d.parent.IsCreated(addr) } -// HasSuicided returns true if address has been flagged for deletion -func (d *DeltaView) HasSuicided(addr gethCommon.Address) bool { - _, found := d.suicided[addr] +// HasSuicided returns true if address has been flagged for destruction +// it also returns the balance of the address before destruction +func (d *DeltaView) HasSuicided(addr gethCommon.Address) (bool, *big.Int) { + bal, found := d.toBeDestructed[addr] if found { - return true + return true, bal } return d.parent.HasSuicided(addr) } // Suicide sets a flag to delete the account at the end of transaction +// +// if an account has been created in this transaction, it would return an error func (d *DeltaView) Suicide(addr gethCommon.Address) (bool, error) { + // if it has been recently created, calling suicide is not + // a valid operation if d.IsCreated(addr) { return false, fmt.Errorf("invalid operation, can't suicide an account that is just created") } + // if it doesn't exist, return false exists, err := d.Exist(addr) if err != nil { @@ -150,14 +181,18 @@ func (d *DeltaView) Suicide(addr gethCommon.Address) (bool, error) { if !exists { return false, nil } - // flag the account for deletion - d.suicided[addr] = struct{}{} - - // set balance to zero - d.balances[addr] = new(big.Int) + // flag the account for destruction and capture the balance + // before destruction + d.toBeDestructed[addr], err = d.GetBalance(addr) + if err != nil { + return false, err + } // flag the address as dirty d.dirtyAddresses[addr] = struct{}{} + + // set balance to zero + d.balances[addr] = new(big.Int) return true, nil } @@ -308,8 +343,8 @@ func (d *DeltaView) GetState(sk types.SlotAddress) (gethCommon.Hash, error) { // if address is deleted in the scope of this delta view, // don't go backward. this has been done to skip the step to iterate // over all the state slabs and delete them. - _, deleted := d.deleted[sk.Address] - if deleted { + _, recreated := d.recreated[sk.Address] + if recreated { return gethCommon.Hash{}, nil } return d.parent.GetState(sk) @@ -326,7 +361,6 @@ func (d *DeltaView) SetState(sk types.SlotAddress, value gethCommon.Hash) error return nil } d.slots[sk] = value - d.dirtySlots[sk] = struct{}{} return nil } @@ -453,5 +487,9 @@ func (d *DeltaView) DirtyAddresses() map[gethCommon.Address]struct{} { // DirtySlots returns a set of slots that has been updated in this view func (d *DeltaView) DirtySlots() map[types.SlotAddress]struct{} { - return d.dirtySlots + dirtySlots := make(map[types.SlotAddress]struct{}) + for sk := range d.slots { + dirtySlots[sk] = struct{}{} + } + return dirtySlots } diff --git a/fvm/evm/emulator/state/delta_test.go b/fvm/evm/emulator/state/delta_test.go index c31c110f20e..1b80d51a12c 100644 --- a/fvm/evm/emulator/state/delta_test.go +++ b/fvm/evm/emulator/state/delta_test.go @@ -44,8 +44,11 @@ func TestDeltaView(t *testing.T) { IsCreatedFunc: func(a gethCommon.Address) bool { return false }, - HasSuicidedFunc: func(gethCommon.Address) bool { - return false + GetBalanceFunc: func(gethCommon.Address) (*big.Int, error) { + return new(big.Int), nil + }, + HasSuicidedFunc: func(gethCommon.Address) (bool, *big.Int) { + return false, new(big.Int) }, }) @@ -74,7 +77,7 @@ func TestDeltaView(t *testing.T) { require.True(t, found) // test HasSuicided first - success := view.HasSuicided(addr1) + success, _ := view.HasSuicided(addr1) require.False(t, success) // set addr1 for deletion @@ -83,7 +86,7 @@ func TestDeltaView(t *testing.T) { require.True(t, success) // check HasSuicided now - success = view.HasSuicided(addr1) + success, _ = view.HasSuicided(addr1) require.True(t, success) // addr1 should still exist after suicide call @@ -111,8 +114,8 @@ func TestDeltaView(t *testing.T) { return false, nil } }, - HasSuicidedFunc: func(gethCommon.Address) bool { - return false + HasSuicidedFunc: func(a gethCommon.Address) (bool, *big.Int) { + return false, new(big.Int) }, IsCreatedFunc: func(a gethCommon.Address) bool { return false @@ -197,9 +200,15 @@ func TestDeltaView(t *testing.T) { return false, nil } }, - HasSuicidedFunc: func(a gethCommon.Address) bool { + HasSuicidedFunc: func(a gethCommon.Address) (bool, *big.Int) { + return false, new(big.Int) + }, + IsCreatedFunc: func(a gethCommon.Address) bool { return false }, + GetBalanceFunc: func(a gethCommon.Address) (*big.Int, error) { + return new(big.Int), nil + }, GetNonceFunc: func(addr gethCommon.Address) (uint64, error) { switch addr { case addr1: @@ -255,7 +264,10 @@ func TestDeltaView(t *testing.T) { return false, nil } }, - HasSuicidedFunc: func(a gethCommon.Address) bool { + HasSuicidedFunc: func(a gethCommon.Address) (bool, *big.Int) { + return false, new(big.Int) + }, + IsCreatedFunc: func(a gethCommon.Address) bool { return false }, GetCodeFunc: func(addr gethCommon.Address) ([]byte, error) { @@ -598,8 +610,8 @@ func TestDeltaView(t *testing.T) { IsCreatedFunc: func(a gethCommon.Address) bool { return false }, - HasSuicidedFunc: func(gethCommon.Address) bool { - return false + HasSuicidedFunc: func(gethCommon.Address) (bool, *big.Int) { + return false, new(big.Int) }, }) @@ -650,8 +662,8 @@ func TestDeltaView(t *testing.T) { ExistFunc: func(addr gethCommon.Address) (bool, error) { return true, nil }, - HasSuicidedFunc: func(gethCommon.Address) bool { - return true + HasSuicidedFunc: func(gethCommon.Address) (bool, *big.Int) { + return true, big.NewInt(2) }, IsCreatedFunc: func(a gethCommon.Address) bool { return false @@ -720,9 +732,10 @@ func TestDeltaView(t *testing.T) { err = view.CreateAccount(addr1) require.NoError(t, err) + // it should carry over the balance bal, err = view.GetBalance(addr1) require.NoError(t, err) - require.Equal(t, new(big.Int), bal) + require.Equal(t, initBalance, bal) ret, err = view.GetCode(addr1) require.NoError(t, err) @@ -737,7 +750,7 @@ func TestDeltaView(t *testing.T) { type MockedReadOnlyView struct { ExistFunc func(gethCommon.Address) (bool, error) - HasSuicidedFunc func(gethCommon.Address) bool + HasSuicidedFunc func(gethCommon.Address) (bool, *big.Int) IsCreatedFunc func(gethCommon.Address) bool GetBalanceFunc func(gethCommon.Address) (*big.Int, error) GetNonceFunc func(gethCommon.Address) (uint64, error) @@ -767,7 +780,7 @@ func (v *MockedReadOnlyView) IsCreated(addr gethCommon.Address) bool { return v.IsCreatedFunc(addr) } -func (v *MockedReadOnlyView) HasSuicided(addr gethCommon.Address) bool { +func (v *MockedReadOnlyView) HasSuicided(addr gethCommon.Address) (bool, *big.Int) { if v.HasSuicidedFunc == nil { panic("HasSuicided is not set in this mocked view") } diff --git a/fvm/evm/types/state.go b/fvm/evm/types/state.go index 25d032a53c7..1e2d3a01325 100644 --- a/fvm/evm/types/state.go +++ b/fvm/evm/types/state.go @@ -34,7 +34,7 @@ type ReadOnlyView interface { // IsCreated returns true if address has been created in this tx IsCreated(gethCommon.Address) bool // HasSuicided returns true if an address has suicided - HasSuicided(gethCommon.Address) bool + HasSuicided(gethCommon.Address) (bool, *big.Int) // GetBalance returns the balance of an address GetBalance(gethCommon.Address) (*big.Int, error) // GetNonce returns the nonce of an address From ae9ef2e4c792bb21c7305f1332562f1b22ae29df Mon Sep 17 00:00:00 2001 From: ramtinms Date: Fri, 5 Jan 2024 00:21:29 -0800 Subject: [PATCH 56/72] apply PR feedbacks --- fvm/evm/emulator/state/stateDB.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index 373a116646c..f4e88903543 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -6,6 +6,7 @@ import ( "fmt" "maps" "math/big" + "slices" "sort" gethCommon "github.com/ethereum/go-ethereum/common" @@ -311,10 +312,9 @@ func (db *StateDB) Commit() error { sortedAddresses = append(sortedAddresses, addr) } - sort.Slice(sortedAddresses, - func(i, j int) bool { - return bytes.Compare(sortedAddresses[i][:], sortedAddresses[j][:]) < 0 - }) + slices.SortFunc(sortedAddresses, func(a, b gethCommon.Address) int { + return bytes.Compare(a[:], b[:]) + }) // update accounts for _, addr := range sortedAddresses { From fe783e33d61629f8f179c4cb93efc8a97488c26e Mon Sep 17 00:00:00 2001 From: ramtinms Date: Fri, 5 Jan 2024 08:57:16 -0800 Subject: [PATCH 57/72] no maps use --- fvm/evm/emulator/state/stateDB.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index f4e88903543..77dcf1dac40 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -4,7 +4,6 @@ import ( "bytes" stdErrors "errors" "fmt" - "maps" "math/big" "slices" "sort" @@ -302,8 +301,12 @@ func (db *StateDB) Commit() error { addresses := make(map[gethCommon.Address]struct{}) slots := make(map[types.SlotAddress]struct{}) for _, view := range db.views { - maps.Copy(addresses, view.DirtyAddresses()) - maps.Copy(slots, view.DirtySlots()) + for key := range view.DirtyAddresses() { + addresses[key] = struct{}{} + } + for key := range view.DirtySlots() { + slots[key] = struct{}{} + } } // sort addresses From bd17a8a4dc7315aa040f33dc8f9b15fe27ecca81 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Fri, 5 Jan 2024 09:09:02 -0800 Subject: [PATCH 58/72] fix lint issue --- fvm/evm/emulator/state/stateDB.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fvm/evm/emulator/state/stateDB.go b/fvm/evm/emulator/state/stateDB.go index 77dcf1dac40..61b3aca336a 100644 --- a/fvm/evm/emulator/state/stateDB.go +++ b/fvm/evm/emulator/state/stateDB.go @@ -5,7 +5,6 @@ import ( stdErrors "errors" "fmt" "math/big" - "slices" "sort" gethCommon "github.com/ethereum/go-ethereum/common" @@ -315,9 +314,10 @@ func (db *StateDB) Commit() error { sortedAddresses = append(sortedAddresses, addr) } - slices.SortFunc(sortedAddresses, func(a, b gethCommon.Address) int { - return bytes.Compare(a[:], b[:]) - }) + sort.Slice(sortedAddresses, + func(i, j int) bool { + return bytes.Compare(sortedAddresses[i][:], sortedAddresses[j][:]) < 0 + }) // update accounts for _, addr := range sortedAddresses { From e95f776b1c33f13278f780683b9188cd5596d4c7 Mon Sep 17 00:00:00 2001 From: Daniel Sainati Date: Fri, 5 Jan 2024 12:37:51 -0500 Subject: [PATCH 59/72] update version --- go.mod | 2 +- go.sum | 4 ++-- insecure/go.mod | 2 +- insecure/go.sum | 4 ++-- integration/go.mod | 2 +- integration/go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 7f4082c93f0..078fa48dd16 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/onflow/atree v0.6.0 - github.com/onflow/cadence v0.42.6 + github.com/onflow/cadence v0.42.7 github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v0.14.0 github.com/onflow/flow-core-contracts/lib/go/templates v0.14.0 diff --git a/go.sum b/go.sum index db6fda050e3..7f3594ac2ae 100644 --- a/go.sum +++ b/go.sum @@ -1325,8 +1325,8 @@ github.com/onflow/atree v0.1.0-beta1.0.20211027184039-559ee654ece9/go.mod h1:+6x github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= -github.com/onflow/cadence v0.42.6 h1:VtI0EpKrdbfqITRMsvyZC4dhgcW1x1LNUQuEpdMDzus= -github.com/onflow/cadence v0.42.6/go.mod h1:raU8va8QRyTa/eUbhej4mbyW2ETePfSaywoo36MddgE= +github.com/onflow/cadence v0.42.7 h1:Qp9VYX901saO7wPwF/rwV4cMS+0mfWxnm9EqbYElYy4= +github.com/onflow/cadence v0.42.7/go.mod h1:raU8va8QRyTa/eUbhej4mbyW2ETePfSaywoo36MddgE= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= github.com/onflow/flow-core-contracts/lib/go/contracts v0.14.0 h1:DpkgyNAP3SAe7dMQX/Tb7BWFstodqtREo4hxWFHwdS0= diff --git a/insecure/go.mod b/insecure/go.mod index 5ffb8474d9b..c4206564281 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -198,7 +198,7 @@ require ( github.com/nxadm/tail v1.4.8 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.6.0 // indirect - github.com/onflow/cadence v0.42.6 // indirect + github.com/onflow/cadence v0.42.7 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20231016154253-a00dbf7c061f // indirect github.com/onflow/flow-core-contracts/lib/go/templates v1.2.4-0.20231016154253-a00dbf7c061f // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 87413f90d27..a1387bdf17d 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1302,8 +1302,8 @@ github.com/onflow/atree v0.1.0-beta1.0.20211027184039-559ee654ece9/go.mod h1:+6x github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= -github.com/onflow/cadence v0.42.6 h1:VtI0EpKrdbfqITRMsvyZC4dhgcW1x1LNUQuEpdMDzus= -github.com/onflow/cadence v0.42.6/go.mod h1:raU8va8QRyTa/eUbhej4mbyW2ETePfSaywoo36MddgE= +github.com/onflow/cadence v0.42.7 h1:Qp9VYX901saO7wPwF/rwV4cMS+0mfWxnm9EqbYElYy4= +github.com/onflow/cadence v0.42.7/go.mod h1:raU8va8QRyTa/eUbhej4mbyW2ETePfSaywoo36MddgE= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20231016154253-a00dbf7c061f h1:S8yIZw9LFXfYD1V5H9BiixihHw3GrXVPrmfplSzYaww= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20231016154253-a00dbf7c061f/go.mod h1:jM6GMAL+m0hjusUgiYDNrixPQ6b9s8xjoJQoEu5bHQI= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.4-0.20231016154253-a00dbf7c061f h1:Ep+Mpo2miWMe4pjPGIaEvEzshRep30dvNgxqk+//FrQ= diff --git a/integration/go.mod b/integration/go.mod index fc395bfc52e..74f3455ab24 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -19,7 +19,7 @@ require ( github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ipfs-blockstore v1.3.0 - github.com/onflow/cadence v0.42.6 + github.com/onflow/cadence v0.42.7 github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20231016154253-a00dbf7c061f github.com/onflow/flow-core-contracts/lib/go/templates v1.2.4-0.20231016154253-a00dbf7c061f github.com/onflow/flow-emulator v0.58.1-0.20231130142844-f22e54339f85 diff --git a/integration/go.sum b/integration/go.sum index 7db3a31f5c6..996679a643d 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1394,8 +1394,8 @@ github.com/onflow/atree v0.1.0-beta1.0.20211027184039-559ee654ece9/go.mod h1:+6x github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= -github.com/onflow/cadence v0.42.6 h1:VtI0EpKrdbfqITRMsvyZC4dhgcW1x1LNUQuEpdMDzus= -github.com/onflow/cadence v0.42.6/go.mod h1:raU8va8QRyTa/eUbhej4mbyW2ETePfSaywoo36MddgE= +github.com/onflow/cadence v0.42.7 h1:Qp9VYX901saO7wPwF/rwV4cMS+0mfWxnm9EqbYElYy4= +github.com/onflow/cadence v0.42.7/go.mod h1:raU8va8QRyTa/eUbhej4mbyW2ETePfSaywoo36MddgE= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20231016154253-a00dbf7c061f h1:S8yIZw9LFXfYD1V5H9BiixihHw3GrXVPrmfplSzYaww= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20231016154253-a00dbf7c061f/go.mod h1:jM6GMAL+m0hjusUgiYDNrixPQ6b9s8xjoJQoEu5bHQI= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.4-0.20231016154253-a00dbf7c061f h1:Ep+Mpo2miWMe4pjPGIaEvEzshRep30dvNgxqk+//FrQ= From e62339174950a04d22aad8b498d2521395f5ff2b Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 21 Dec 2023 22:26:28 -0800 Subject: [PATCH 60/72] refactor checkers --- cmd/execution_builder.go | 6 +- engine/execution/checker/core.go | 170 +++++++++++++++++++++++++++++ engine/execution/checker/engine.go | 112 +++---------------- 3 files changed, 187 insertions(+), 101 deletions(-) create mode 100644 engine/execution/checker/core.go diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 564e7ce1d13..53a607bdb4b 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -941,12 +941,12 @@ func (exeNode *ExecutionNode) LoadCheckerEngine( module.ReadyDoneAware, error, ) { - exeNode.checkerEng = checker.New( + core := checker.NewCore( node.Logger, node.State, exeNode.executionState, - node.Storage.Seals, ) + exeNode.checkerEng = checker.NewEngine(core) return exeNode.checkerEng, nil } @@ -1056,8 +1056,6 @@ func (exeNode *ExecutionNode) LoadFollowerCore( return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - exeNode.followerDistributor.AddFinalizationConsumer(exeNode.checkerEng) - // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block exeNode.followerCore, err = consensus.NewFollower( diff --git a/engine/execution/checker/core.go b/engine/execution/checker/core.go new file mode 100644 index 00000000000..46112db51b2 --- /dev/null +++ b/engine/execution/checker/core.go @@ -0,0 +1,170 @@ +package checker + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine/execution/state" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" +) + +// trigger the check once every minute, +const DefaultTimeInterval = time.Minute * 1 + +// The checker engine checks if the execution result matches the sealed result. +type Core struct { + log zerolog.Logger + state protocol.State + execState state.ExecutionState +} + +func NewCore( + logger zerolog.Logger, + state protocol.State, + execState state.ExecutionState, +) *Core { + e := &Core{ + log: logger.With().Str("engine", "checker").Logger(), + state: state, + execState: execState, + } + + return e +} + +// The point of checker engine is to perform this following check +func checkMyCommitWithSealedCommit( + executedBlock *flow.Header, + myCommit flow.StateCommitment, + sealedCommit flow.StateCommitment, +) error { + if myCommit != sealedCommit { + // mismatch + return fmt.Errorf("execution result is different from the sealed result, height: %v, block_id: %v, sealed_commit: %v, my_commit: %v", + executedBlock.Height, + executedBlock.ID(), + sealedCommit, + myCommit, + ) + } + + // match + return nil +} + +// runLoop runs the check every minute. +// Why using a timer instead of listening to finalized and executed events? +// because it's simpler as it doesn't need to subscribe to those events. +// It also runs less checks, note: the checker doesn't need to find the +// first mismatched block, as long as it can find a mismatch, it's good enough. +// A timer could reduce the number of checks, as it only checks once every minute. +func (c *Core) runLoop(ctx context.Context, tickInterval time.Duration) error { + ticker := time.NewTicker(tickInterval) + for { + select { + case <-ticker.C: + err := c.runCheck() + if err != nil { + return err + } + case <-ctx.Done(): + ticker.Stop() // critical for ticker to be garbage collected + return nil + } + } +} + +// it skips when the last sealed has not been executed, and last executed has not been finalized. +func (c *Core) runCheck() error { + // find last sealed block + lastSealedBlock, lastFinal, seal, err := c.findLastSealedBlock() + if err != nil { + return err + } + + mycommitAtLastSealed, err := c.execState.StateCommitmentByBlockID(lastSealedBlock.ID()) + if err == nil { + // if last sealed block has been executed, then check if they match + return checkMyCommitWithSealedCommit(lastSealedBlock, mycommitAtLastSealed, seal.FinalState) + } + + // if last sealed block has not been executed, then check if recent executed block has + // been sealed already, if yes, check if they match. + lastExecutedHeight, err := c.findLastExecutedBlockHeight() + if err != nil { + return err + } + + if lastExecutedHeight > lastFinal.Height { + // last executed block has not been finalized yet, + // can't check since unfinalized block is also unsealed, skip + return nil + } + + // TODO: better to query seals from protocol state, + // switch to state.Final().LastSealed() when available + sealedExecuted, seal, err := c.findLatestSealedAtHeight(lastExecutedHeight) + + sealedCommit := seal.FinalState + + mycommit, err := c.execState.StateCommitmentByBlockID(seal.BlockID) + if errors.Is(err, storage.ErrNotFound) { + // have not executed the sealed block yet + // in other words, this can't detect execution fork, if the execution is behind + // the sealing + return nil + } + + if err != nil { + return fmt.Errorf("could not get my state commitment OnFinalizedBlock, blockID: %v", seal.BlockID) + } + + return checkMyCommitWithSealedCommit(sealedExecuted, mycommit, sealedCommit) +} + +// findLastSealedBlock finds the last sealed block +func (c *Core) findLastSealedBlock() (*flow.Header, *flow.Header, *flow.Seal, error) { + finalized := c.state.Final() + lastFinal, err := finalized.Head() + if err != nil { + return nil, nil, nil, err + } + + _, lastSeal, err := finalized.SealedResult() + if err != nil { + return nil, nil, nil, fmt.Errorf("could not get the last sealed for the finalized block: %w", err) + } + + lastSealed, err := c.state.AtBlockID(lastSeal.BlockID).Head() + if err != nil { + return nil, nil, nil, fmt.Errorf("could not get the last sealed block: %w", err) + } + + return lastSealed, lastFinal, lastSeal, err +} + +// findLastExecutedBlockHeight finds the last executed block height +func (c *Core) findLastExecutedBlockHeight() (uint64, error) { + height, _, err := c.execState.GetHighestExecutedBlockID(context.Background()) + if err != nil { + return 0, fmt.Errorf("could not get the last executed block: %w", err) + } + return height, nil +} + +// findLatestSealedAtHeight finds the latest sealed block at the given height +func (c *Core) findLatestSealedAtHeight(finalizedHeight uint64) (*flow.Header, *flow.Seal, error) { + _, seal, err := c.state.AtHeight(finalizedHeight).SealedResult() + + sealed, err := c.state.AtBlockID(seal.BlockID).Head() + if err != nil { + return nil, nil, fmt.Errorf("could not get the last sealed block: %w", err) + } + return sealed, seal, nil +} diff --git a/engine/execution/checker/engine.go b/engine/execution/checker/engine.go index a1a96184105..c0ac828c8f7 100644 --- a/engine/execution/checker/engine.go +++ b/engine/execution/checker/engine.go @@ -1,108 +1,26 @@ package checker import ( - "errors" - "fmt" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/consensus/hotstuff/notifications" - "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/execution/state" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" ) type Engine struct { - notifications.NoopConsumer // satisfy the FinalizationConsumer interface - - unit *engine.Unit - log zerolog.Logger - state protocol.State - execState state.ExecutionState - sealsDB storage.Seals -} - -func New( - logger zerolog.Logger, - state protocol.State, - execState state.ExecutionState, - sealsDB storage.Seals, -) *Engine { - return &Engine{ - unit: engine.NewUnit(), - log: logger.With().Str("engine", "checker").Logger(), - state: state, - execState: execState, - sealsDB: sealsDB, - } -} - -func (e *Engine) Ready() <-chan struct{} { - // make sure we will run into a crashloop if result gets inconsistent - // with sealed result. - - finalized, err := e.state.Final().Head() - - if err != nil { - e.log.Fatal().Err(err).Msg("could not get finalized block on startup") - } - - err = e.checkLastSealed(finalized.ID()) - if err != nil { - e.log.Fatal().Err(err).Msg("execution consistency check failed on startup") - } - return e.unit.Ready() -} - -func (e *Engine) Done() <-chan struct{} { - return e.unit.Done() -} - -// when a block is finalized check if the last sealed has been executed, -// if it has been executed, check whether if the sealed result is consistent -// with the executed result -func (e *Engine) OnFinalizedBlock(block *model.Block) { - err := e.checkLastSealed(block.BlockID) - if err != nil { - e.log.Fatal().Err(err).Msg("execution consistency check failed") - } + *component.ComponentManager } -func (e *Engine) checkLastSealed(finalizedID flow.Identifier) error { - // TODO: better to query seals from protocol state, - // switch to state.Final().LastSealed() when available - seal, err := e.sealsDB.HighestInFork(finalizedID) - if err != nil { - return fmt.Errorf("could not get the last sealed for the finalized block: %w", err) - } - - blockID := seal.BlockID - sealedCommit := seal.FinalState - - mycommit, err := e.execState.StateCommitmentByBlockID(blockID) - if errors.Is(err, storage.ErrNotFound) { - // have not executed the sealed block yet - // in other words, this can't detect execution fork, if the execution is behind - // the sealing - return nil - } - - if err != nil { - return fmt.Errorf("could not get my state commitment OnFinalizedBlock, blockID: %v", blockID) - } - - if mycommit != sealedCommit { - sealed, err := e.state.AtBlockID(blockID).Head() - if err != nil { - return fmt.Errorf("could not get sealed block when checkLastSealed: %v, err: %w", blockID, err) - } +func NewEngine(core *Core) *Engine { + e := &Engine{} - return fmt.Errorf("execution result is different from the sealed result, height: %v, block_id: %v, sealed_commit: %x, my_commit: %x", - sealed.Height, blockID, sealedCommit, mycommit) - } + e.ComponentManager = component.NewComponentManagerBuilder(). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + err := core.runLoop(ctx, DefaultTimeInterval) + if err != nil { + ctx.Throw(err) + } + }). + Build() - return nil + return e } From e6ff36ce912f45c4562a6c5f4f98447d09494b57 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Dec 2023 09:25:18 -0800 Subject: [PATCH 61/72] use defer to stop ticker --- engine/execution/checker/core.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/execution/checker/core.go b/engine/execution/checker/core.go index 46112db51b2..5607e6ad238 100644 --- a/engine/execution/checker/core.go +++ b/engine/execution/checker/core.go @@ -66,6 +66,7 @@ func checkMyCommitWithSealedCommit( // A timer could reduce the number of checks, as it only checks once every minute. func (c *Core) runLoop(ctx context.Context, tickInterval time.Duration) error { ticker := time.NewTicker(tickInterval) + defer ticker.Stop() // critical for ticker to be garbage collected for { select { case <-ticker.C: @@ -74,7 +75,6 @@ func (c *Core) runLoop(ctx context.Context, tickInterval time.Duration) error { return err } case <-ctx.Done(): - ticker.Stop() // critical for ticker to be garbage collected return nil } } From bbbcded18823414f00d361f4479a1b552f108d9c Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Dec 2023 11:47:59 -0800 Subject: [PATCH 62/72] fix lint --- engine/execution/checker/core.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/engine/execution/checker/core.go b/engine/execution/checker/core.go index 5607e6ad238..25d0ef4afbf 100644 --- a/engine/execution/checker/core.go +++ b/engine/execution/checker/core.go @@ -110,6 +110,9 @@ func (c *Core) runCheck() error { // TODO: better to query seals from protocol state, // switch to state.Final().LastSealed() when available sealedExecuted, seal, err := c.findLatestSealedAtHeight(lastExecutedHeight) + if err != nil { + return fmt.Errorf("could not get the last sealed block at height: %v, err: %w", lastExecutedHeight, err) + } sealedCommit := seal.FinalState @@ -146,7 +149,7 @@ func (c *Core) findLastSealedBlock() (*flow.Header, *flow.Header, *flow.Seal, er return nil, nil, nil, fmt.Errorf("could not get the last sealed block: %w", err) } - return lastSealed, lastFinal, lastSeal, err + return lastSealed, lastFinal, lastSeal, nil } // findLastExecutedBlockHeight finds the last executed block height @@ -161,6 +164,9 @@ func (c *Core) findLastExecutedBlockHeight() (uint64, error) { // findLatestSealedAtHeight finds the latest sealed block at the given height func (c *Core) findLatestSealedAtHeight(finalizedHeight uint64) (*flow.Header, *flow.Seal, error) { _, seal, err := c.state.AtHeight(finalizedHeight).SealedResult() + if err != nil { + return nil, nil, fmt.Errorf("could not get the last sealed for the finalized block: %w", err) + } sealed, err := c.state.AtBlockID(seal.BlockID).Head() if err != nil { From 78e24b63a622013e9f47400c4e27f2fea690530a Mon Sep 17 00:00:00 2001 From: Leo Zhang Date: Fri, 5 Jan 2024 10:33:58 -0800 Subject: [PATCH 63/72] Apply suggestions from code review Co-authored-by: Janez Podhostnik <67895329+janezpodhostnik@users.noreply.github.com> --- engine/execution/checker/core.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/execution/checker/core.go b/engine/execution/checker/core.go index 25d0ef4afbf..92d36ac8399 100644 --- a/engine/execution/checker/core.go +++ b/engine/execution/checker/core.go @@ -14,10 +14,10 @@ import ( "github.com/onflow/flow-go/storage" ) -// trigger the check once every minute, +// DefaultTimeInterval triggers the check once every minute, const DefaultTimeInterval = time.Minute * 1 -// The checker engine checks if the execution result matches the sealed result. +// Core is the core logic of the checker engine that checks if the execution result matches the sealed result. type Core struct { log zerolog.Logger state protocol.State @@ -38,7 +38,7 @@ func NewCore( return e } -// The point of checker engine is to perform this following check +// checkMyCommitWithSealedCommit is the main check of the checker engine func checkMyCommitWithSealedCommit( executedBlock *flow.Header, myCommit flow.StateCommitment, From 5a0e698f4baca2ea53c8ac1e1c20ce772e07afc9 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 5 Jan 2024 11:18:15 -0800 Subject: [PATCH 64/72] address review comments --- engine/execution/checker/core.go | 30 ++----------------------- engine/execution/checker/engine.go | 35 ++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/engine/execution/checker/core.go b/engine/execution/checker/core.go index 92d36ac8399..78ca7475dd9 100644 --- a/engine/execution/checker/core.go +++ b/engine/execution/checker/core.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "time" "github.com/rs/zerolog" @@ -14,9 +13,6 @@ import ( "github.com/onflow/flow-go/storage" ) -// DefaultTimeInterval triggers the check once every minute, -const DefaultTimeInterval = time.Minute * 1 - // Core is the core logic of the checker engine that checks if the execution result matches the sealed result. type Core struct { log zerolog.Logger @@ -58,30 +54,8 @@ func checkMyCommitWithSealedCommit( return nil } -// runLoop runs the check every minute. -// Why using a timer instead of listening to finalized and executed events? -// because it's simpler as it doesn't need to subscribe to those events. -// It also runs less checks, note: the checker doesn't need to find the -// first mismatched block, as long as it can find a mismatch, it's good enough. -// A timer could reduce the number of checks, as it only checks once every minute. -func (c *Core) runLoop(ctx context.Context, tickInterval time.Duration) error { - ticker := time.NewTicker(tickInterval) - defer ticker.Stop() // critical for ticker to be garbage collected - for { - select { - case <-ticker.C: - err := c.runCheck() - if err != nil { - return err - } - case <-ctx.Done(): - return nil - } - } -} - -// it skips when the last sealed has not been executed, and last executed has not been finalized. -func (c *Core) runCheck() error { +// RunCheck skips when the last sealed has not been executed, and last executed has not been finalized. +func (c *Core) RunCheck() error { // find last sealed block lastSealedBlock, lastFinal, seal, err := c.findLastSealedBlock() if err != nil { diff --git a/engine/execution/checker/engine.go b/engine/execution/checker/engine.go index c0ac828c8f7..5c9a5bc1404 100644 --- a/engine/execution/checker/engine.go +++ b/engine/execution/checker/engine.go @@ -1,21 +1,30 @@ package checker import ( + "context" + "time" + "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" ) type Engine struct { *component.ComponentManager + core *Core } +// DefaultTimeInterval triggers the check once every minute, +const DefaultTimeInterval = time.Minute * 1 + func NewEngine(core *Core) *Engine { - e := &Engine{} + e := &Engine{ + core: core, + } e.ComponentManager = component.NewComponentManagerBuilder(). AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { ready() - err := core.runLoop(ctx, DefaultTimeInterval) + err := e.runLoop(ctx, DefaultTimeInterval) if err != nil { ctx.Throw(err) } @@ -24,3 +33,25 @@ func NewEngine(core *Core) *Engine { return e } + +// runLoop runs the check every minute. +// Why using a timer instead of listening to finalized and executed events? +// because it's simpler as it doesn't need to subscribe to those events. +// It also runs less checks, note: the checker doesn't need to find the +// first mismatched block, as long as it can find a mismatch, it's good enough. +// A timer could reduce the number of checks, as it only checks once every minute. +func (e *Engine) runLoop(ctx context.Context, tickInterval time.Duration) error { + ticker := time.NewTicker(tickInterval) + defer ticker.Stop() // critical for ticker to be garbage collected + for { + select { + case <-ticker.C: + err := e.core.RunCheck() + if err != nil { + return err + } + case <-ctx.Done(): + return nil + } + } +} From 911976c2c047015ec85ca1a36299d5b88197a55b Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 5 Jan 2024 13:59:51 -0800 Subject: [PATCH 65/72] add tests for checker engine core --- engine/execution/checker/core_test.go | 156 ++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 engine/execution/checker/core_test.go diff --git a/engine/execution/checker/core_test.go b/engine/execution/checker/core_test.go new file mode 100644 index 00000000000..cd27c5cabdc --- /dev/null +++ b/engine/execution/checker/core_test.go @@ -0,0 +1,156 @@ +package checker_test + +import ( + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine/execution/checker" + stateMock "github.com/onflow/flow-go/engine/execution/state/mock" + "github.com/onflow/flow-go/model/flow" + protocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +func makeCore(t *testing.T) (*checker.Core, *protocol.State, *stateMock.ExecutionState) { + logger := unittest.Logger() + state := protocol.NewState(t) + execState := stateMock.NewExecutionState(t) + core := checker.NewCore(logger, state, execState) + return core, state, execState +} + +func mockFinalizedBlock(t *testing.T, state *protocol.State, finalized *flow.Header) *protocol.Snapshot { + finalizedSnapshot := protocol.NewSnapshot(t) + finalizedSnapshot.On("Head").Return(finalized, nil) + state.On("Final").Return(finalizedSnapshot) + return finalizedSnapshot +} + +func mockAtBlockID(t *testing.T, state *protocol.State, header *flow.Header) *protocol.Snapshot { + snapshot := protocol.NewSnapshot(t) + snapshot.On("Head").Return(header, nil) + state.On("AtBlockID", header.ID()).Return(snapshot) + return snapshot +} + +func mockSealedBlock(t *testing.T, state *protocol.State, finalized *protocol.Snapshot, sealed *flow.Header) (*flow.ExecutionResult, *flow.Seal) { + lastSealResult := unittest.ExecutionResultFixture(func(r *flow.ExecutionResult) { + r.BlockID = sealed.ID() + }) + lastSeal := unittest.Seal.Fixture(unittest.Seal.WithResult(lastSealResult)) + finalized.On("SealedResult").Return(lastSealResult, lastSeal, nil) + return lastSealResult, lastSeal +} + +func mockFinalizedSealedBlock(t *testing.T, state *protocol.State, finalized *flow.Header, sealed *flow.Header) (*flow.ExecutionResult, *flow.Seal) { + finalizedSnapshot := mockFinalizedBlock(t, state, finalized) + return mockSealedBlock(t, state, finalizedSnapshot, sealed) +} + +func mockSealedBlockAtHeight(t *testing.T, state *protocol.State, height uint64, lastSealed *flow.Header) (*flow.ExecutionResult, *flow.Seal) { + snapshotAtHeight := protocol.NewSnapshot(t) + lastSealedResultAtHeight := unittest.ExecutionResultFixture(func(r *flow.ExecutionResult) { + r.BlockID = lastSealed.ID() + }) + lastSealAtHeight := unittest.Seal.Fixture(unittest.Seal.WithResult(lastSealedResultAtHeight)) + snapshotAtHeight.On("SealedResult").Return(lastSealedResultAtHeight, lastSealAtHeight, nil) + state.On("AtHeight", height).Return(snapshotAtHeight, nil) + return lastSealedResultAtHeight, lastSealAtHeight +} + +func mockExecutedBlock(t *testing.T, es *stateMock.ExecutionState, executed *flow.Header, result *flow.ExecutionResult) { + commit, err := result.FinalStateCommitment() + require.NoError(t, err) + es.On("StateCommitmentByBlockID", executed.ID()).Return(commit, nil) +} + +func mockUnexecutedBlock(t *testing.T, es *stateMock.ExecutionState, unexecuted *flow.Header) { + es.On("StateCommitmentByBlockID", unexecuted.ID()).Return(nil, storage.ErrNotFound) +} + +func TestCheckPassIfLastSealedIsExecutedAndMatch(t *testing.T) { + // ..<- LastSealed(executed) <- .. <- LastFinalized <- .. <- LastExecuted <- ... + chain, _, _ := unittest.ChainFixture(10) + lastFinal := chain[7].Header + lastSealed := chain[5].Header + + core, state, es := makeCore(t) + lastSealedResult, _ := mockFinalizedSealedBlock(t, state, lastFinal, lastSealed) + mockAtBlockID(t, state, lastSealed) + mockExecutedBlock(t, es, lastSealed, lastSealedResult) + + require.NoError(t, core.RunCheck()) +} + +func TestCheckFailIfLastSealedIsExecutedButMismatch(t *testing.T) { + // ..<- LastSealed(executed) <- .. <- LastFinalized <- .. <- LastExecuted <- ... + chain, _, _ := unittest.ChainFixture(10) + lastFinal := chain[7].Header + lastSealed := chain[5].Header + + core, state, es := makeCore(t) + _, _ = mockFinalizedSealedBlock(t, state, lastFinal, lastSealed) + mockAtBlockID(t, state, lastSealed) + + mismatchingResult := unittest.ExecutionResultFixture() + + mockExecutedBlock(t, es, lastSealed, mismatchingResult) + + require.Error(t, core.RunCheck()) + require.Contains(t, core.RunCheck().Error(), "execution result is different from the sealed result") +} + +func TestCheckPassIfLastSealedIsNotExecutedAndLastExecutedMatch(t *testing.T) { + // LastSealedExecuted (sealed) <..<- LastExecuted(finalized) <..<- LastSealed(not executed) <..<- LastFinalized + chain, _, _ := unittest.ChainFixture(10) + lastFinal := chain[7].Header + lastSealed := chain[5].Header + lastExecuted := chain[3].Header + lastSealedExecuted := chain[1].Header + + core, state, es := makeCore(t) + // mock that last sealed is not executed + mockFinalizedSealedBlock(t, state, lastFinal, lastSealed) + mockAtBlockID(t, state, lastSealed) + mockUnexecutedBlock(t, es, lastSealed) + + // mock the last sealed and is also executed + es.On("GetHighestExecutedBlockID", mock.Anything).Return(lastExecuted.Height, lastExecuted.ID(), nil) + lastSealedResultAtExecutedHeight, _ := mockSealedBlockAtHeight(t, state, lastExecuted.Height, lastSealedExecuted) + mockAtBlockID(t, state, lastSealedExecuted) + + // mock with matching result + mockExecutedBlock(t, es, lastSealedExecuted, lastSealedResultAtExecutedHeight) + + require.NoError(t, core.RunCheck()) +} + +func TestCheckFailIfLastSealedIsNotExecutedAndLastExecutedMismatch(t *testing.T) { + // LastSealedExecuted (sealed) <..<- LastExecuted(finalized) <..<- LastSealed(not executed) <..<- LastFinalized + chain, _, _ := unittest.ChainFixture(10) + lastFinal := chain[7].Header + lastSealed := chain[5].Header + lastExecuted := chain[3].Header + lastSealedExecuted := chain[1].Header + + core, state, es := makeCore(t) + // mock that last sealed is not executed + mockFinalizedSealedBlock(t, state, lastFinal, lastSealed) + mockAtBlockID(t, state, lastSealed) + mockUnexecutedBlock(t, es, lastSealed) + + // mock the last sealed and is also executed + es.On("GetHighestExecutedBlockID", mock.Anything).Return(lastExecuted.Height, lastExecuted.ID(), nil) + mockSealedBlockAtHeight(t, state, lastExecuted.Height, lastSealedExecuted) + mockAtBlockID(t, state, lastSealedExecuted) + + // mock with mismatching result + mismatchingResult := unittest.ExecutionResultFixture() + mockExecutedBlock(t, es, lastSealedExecuted, mismatchingResult) + + require.Error(t, core.RunCheck()) + require.Contains(t, core.RunCheck().Error(), "execution result is different from the sealed result") +} From 836c81cb834b551d40aff2f207578093e34e61a1 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 5 Jan 2024 14:59:03 -0800 Subject: [PATCH 66/72] add flag to enable / disable checker --- cmd/execution_builder.go | 7 +++++++ cmd/execution_config.go | 2 ++ 2 files changed, 9 insertions(+) diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 53a607bdb4b..224c5cba2b9 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -941,6 +941,13 @@ func (exeNode *ExecutionNode) LoadCheckerEngine( module.ReadyDoneAware, error, ) { + if !exeNode.exeConf.enableChecker { + node.Logger.Warn().Msgf("checker engine is disabled") + return &module.NoopReadyDoneAware{}, nil + } + + node.Logger.Info().Msgf("checker engine is enabled") + core := checker.NewCore( node.Logger, node.State, diff --git a/cmd/execution_config.go b/cmd/execution_config.go index 033bf2c4d87..3a16e5e6a3a 100644 --- a/cmd/execution_config.go +++ b/cmd/execution_config.go @@ -65,6 +65,7 @@ type ExecutionConfig struct { // file descriptors causing connection failures. onflowOnlyLNs bool enableStorehouse bool + enableChecker bool } func (exeConf *ExecutionConfig) SetupFlags(flags *pflag.FlagSet) { @@ -118,6 +119,7 @@ func (exeConf *ExecutionConfig) SetupFlags(flags *pflag.FlagSet) { flags.BoolVar(&exeConf.onflowOnlyLNs, "temp-onflow-only-lns", false, "do not use unless required. forces node to only request collections from onflow collection nodes") flags.BoolVar(&exeConf.enableStorehouse, "enable-storehouse", false, "enable storehouse to store registers on disk, default is false") + flags.BoolVar(&exeConf.enableChecker, "enable-checker", true, "enable checker to check the correctness of the execution result, default is true") } From 5aa1c8fe7f7ae60d14b8c1710c17c983786f2070 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Fri, 5 Jan 2024 16:07:09 -0800 Subject: [PATCH 67/72] lint fix --- fvm/evm/emulator/emulator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/emulator/emulator.go b/fvm/evm/emulator/emulator.go index 87858f873f8..69ad180d043 100644 --- a/fvm/evm/emulator/emulator.go +++ b/fvm/evm/emulator/emulator.go @@ -8,8 +8,8 @@ import ( gethTypes "github.com/ethereum/go-ethereum/core/types" gethVM "github.com/ethereum/go-ethereum/core/vm" gethCrypto "github.com/ethereum/go-ethereum/crypto" - "github.com/onflow/atree" + "github.com/onflow/flow-go/fvm/evm/emulator/state" "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" From c572fdf45edc5fc95b73086a25ca4faa19bae71d Mon Sep 17 00:00:00 2001 From: ramtinms Date: Fri, 5 Jan 2024 19:03:00 -0800 Subject: [PATCH 68/72] fix lazy allocation --- fvm/evm/emulator/state/delta.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fvm/evm/emulator/state/delta.go b/fvm/evm/emulator/state/delta.go index 005bbbc9164..3eb3747f4b2 100644 --- a/fvm/evm/emulator/state/delta.go +++ b/fvm/evm/emulator/state/delta.go @@ -441,11 +441,14 @@ func (d *DeltaView) SlotInAccessList(sk types.SlotAddress) (addressOk bool, slot // AddSlotToAccessList adds a slot to the access list // it also adds the address to the address list func (d *DeltaView) AddSlotToAccessList(sk types.SlotAddress) (addrAdded bool, slotAdded bool) { + addrPresent, slotPresent := d.SlotInAccessList(sk) + if d.accessListAddresses == nil { + d.accessListAddresses = make(map[gethCommon.Address]struct{}) + } + d.accessListAddresses[sk.Address] = struct{}{} if d.accessListSlots == nil { d.accessListSlots = make(map[types.SlotAddress]struct{}) } - addrPresent, slotPresent := d.SlotInAccessList(sk) - d.accessListAddresses[sk.Address] = struct{}{} d.accessListSlots[sk] = struct{}{} return !addrPresent, !slotPresent } From 24d30dc765ebcea65d2a369ef4977ec11d92ba40 Mon Sep 17 00:00:00 2001 From: Daniel Sainati Date: Mon, 8 Jan 2024 10:51:15 -0500 Subject: [PATCH 69/72] update to flow-go-sdk v0.41.18 --- go.mod | 2 +- go.sum | 4 ++-- insecure/go.mod | 2 +- insecure/go.sum | 4 ++-- integration/go.mod | 2 +- integration/go.sum | 4 ++-- network/mocknetwork/connector.go | 11 +++++------ network/mocknetwork/connector_factory.go | 11 +++++------ network/mocknetwork/connector_host.go | 11 +++++------ 9 files changed, 24 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 078fa48dd16..b44d54bcf45 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v0.14.0 github.com/onflow/flow-core-contracts/lib/go/templates v0.14.0 - github.com/onflow/flow-go-sdk v0.41.17 + github.com/onflow/flow-go-sdk v0.41.18 github.com/onflow/flow-go/crypto v0.25.0 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231213135419-ae911cc351a2 github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d diff --git a/go.sum b/go.sum index 7f3594ac2ae..8df64001ab3 100644 --- a/go.sum +++ b/go.sum @@ -1336,8 +1336,8 @@ github.com/onflow/flow-core-contracts/lib/go/templates v0.14.0/go.mod h1:ZeLxwaB github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13 h1:B4ll7e3j+MqTJv2122Enq3RtDNzmIGRu9xjV7fo7un0= github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= -github.com/onflow/flow-go-sdk v0.41.17 h1:HpNn3j2fqLGA6H3HGfAuh2A+TsPBv8gWO3kvK9Hvtic= -github.com/onflow/flow-go-sdk v0.41.17/go.mod h1:ZIj2XBI9R0QiKzbI6iPwOeqyIy/M4+atczoMOEWdKYw= +github.com/onflow/flow-go-sdk v0.41.18 h1:hyT7wEvwCaxIfxF32Mv9M0jXjU1uuVurTidfdsQIJTw= +github.com/onflow/flow-go-sdk v0.41.18/go.mod h1:Iaw7+wKET7ILH8ih27X31kVplgIItDXLVX+hfTunZh8= github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= github.com/onflow/flow-go/crypto v0.25.0 h1:6lmoiAQ3APCF+nV7f4f2AXL3PuDKqQiWqRJXmjrMEq4= github.com/onflow/flow-go/crypto v0.25.0/go.mod h1:OOb2vYcS8AOCajBClhHTJ0NKftFl1RQgTQ0+Vh4nbqk= diff --git a/insecure/go.mod b/insecure/go.mod index c4206564281..e881266f746 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -202,7 +202,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20231016154253-a00dbf7c061f // indirect github.com/onflow/flow-core-contracts/lib/go/templates v1.2.4-0.20231016154253-a00dbf7c061f // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13 // indirect - github.com/onflow/flow-go-sdk v0.41.17 // indirect + github.com/onflow/flow-go-sdk v0.41.18 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231213135419-ae911cc351a2 // indirect github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d // indirect diff --git a/insecure/go.sum b/insecure/go.sum index a1387bdf17d..b5a3f820b01 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1311,8 +1311,8 @@ github.com/onflow/flow-core-contracts/lib/go/templates v1.2.4-0.20231016154253-a github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13 h1:B4ll7e3j+MqTJv2122Enq3RtDNzmIGRu9xjV7fo7un0= github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= -github.com/onflow/flow-go-sdk v0.41.17 h1:HpNn3j2fqLGA6H3HGfAuh2A+TsPBv8gWO3kvK9Hvtic= -github.com/onflow/flow-go-sdk v0.41.17/go.mod h1:ZIj2XBI9R0QiKzbI6iPwOeqyIy/M4+atczoMOEWdKYw= +github.com/onflow/flow-go-sdk v0.41.18 h1:hyT7wEvwCaxIfxF32Mv9M0jXjU1uuVurTidfdsQIJTw= +github.com/onflow/flow-go-sdk v0.41.18/go.mod h1:Iaw7+wKET7ILH8ih27X31kVplgIItDXLVX+hfTunZh8= github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= github.com/onflow/flow-go/crypto v0.25.0 h1:6lmoiAQ3APCF+nV7f4f2AXL3PuDKqQiWqRJXmjrMEq4= github.com/onflow/flow-go/crypto v0.25.0/go.mod h1:OOb2vYcS8AOCajBClhHTJ0NKftFl1RQgTQ0+Vh4nbqk= diff --git a/integration/go.mod b/integration/go.mod index 74f3455ab24..c80b42a946e 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -24,7 +24,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/templates v1.2.4-0.20231016154253-a00dbf7c061f github.com/onflow/flow-emulator v0.58.1-0.20231130142844-f22e54339f85 github.com/onflow/flow-go v0.32.7 - github.com/onflow/flow-go-sdk v0.41.17 + github.com/onflow/flow-go-sdk v0.41.18 github.com/onflow/flow-go/crypto v0.25.0 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231213135419-ae911cc351a2 diff --git a/integration/go.sum b/integration/go.sum index 996679a643d..8dd48a6fcf2 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1405,8 +1405,8 @@ github.com/onflow/flow-emulator v0.58.1-0.20231130142844-f22e54339f85/go.mod h1: github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13 h1:B4ll7e3j+MqTJv2122Enq3RtDNzmIGRu9xjV7fo7un0= github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= -github.com/onflow/flow-go-sdk v0.41.17 h1:HpNn3j2fqLGA6H3HGfAuh2A+TsPBv8gWO3kvK9Hvtic= -github.com/onflow/flow-go-sdk v0.41.17/go.mod h1:ZIj2XBI9R0QiKzbI6iPwOeqyIy/M4+atczoMOEWdKYw= +github.com/onflow/flow-go-sdk v0.41.18 h1:hyT7wEvwCaxIfxF32Mv9M0jXjU1uuVurTidfdsQIJTw= +github.com/onflow/flow-go-sdk v0.41.18/go.mod h1:Iaw7+wKET7ILH8ih27X31kVplgIItDXLVX+hfTunZh8= github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= github.com/onflow/flow-go/crypto v0.25.0 h1:6lmoiAQ3APCF+nV7f4f2AXL3PuDKqQiWqRJXmjrMEq4= github.com/onflow/flow-go/crypto v0.25.0/go.mod h1:OOb2vYcS8AOCajBClhHTJ0NKftFl1RQgTQ0+Vh4nbqk= diff --git a/network/mocknetwork/connector.go b/network/mocknetwork/connector.go index deedbd4f815..c4ef0abf89d 100644 --- a/network/mocknetwork/connector.go +++ b/network/mocknetwork/connector.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. +// Code generated by mockery v2.30.10. DO NOT EDIT. package mocknetwork @@ -20,13 +20,12 @@ func (_m *Connector) Connect(ctx context.Context, peerChan <-chan peer.AddrInfo) _m.Called(ctx, peerChan) } -type mockConstructorTestingTNewConnector interface { +// NewConnector creates a new instance of Connector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewConnector(t interface { mock.TestingT Cleanup(func()) -} - -// NewConnector creates a new instance of Connector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewConnector(t mockConstructorTestingTNewConnector) *Connector { +}) *Connector { mock := &Connector{} mock.Mock.Test(t) diff --git a/network/mocknetwork/connector_factory.go b/network/mocknetwork/connector_factory.go index b1baeb4f749..78b9b1059f3 100644 --- a/network/mocknetwork/connector_factory.go +++ b/network/mocknetwork/connector_factory.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. +// Code generated by mockery v2.30.10. DO NOT EDIT. package mocknetwork @@ -40,13 +40,12 @@ func (_m *ConnectorFactory) Execute(_a0 host.Host) (p2p.Connector, error) { return r0, r1 } -type mockConstructorTestingTNewConnectorFactory interface { +// NewConnectorFactory creates a new instance of ConnectorFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewConnectorFactory(t interface { mock.TestingT Cleanup(func()) -} - -// NewConnectorFactory creates a new instance of ConnectorFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewConnectorFactory(t mockConstructorTestingTNewConnectorFactory) *ConnectorFactory { +}) *ConnectorFactory { mock := &ConnectorFactory{} mock.Mock.Test(t) diff --git a/network/mocknetwork/connector_host.go b/network/mocknetwork/connector_host.go index e656391a11f..e117183003f 100644 --- a/network/mocknetwork/connector_host.go +++ b/network/mocknetwork/connector_host.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. +// Code generated by mockery v2.30.10. DO NOT EDIT. package mocknetwork @@ -100,13 +100,12 @@ func (_m *ConnectorHost) PeerInfo(peerId peer.ID) peer.AddrInfo { return r0 } -type mockConstructorTestingTNewConnectorHost interface { +// NewConnectorHost creates a new instance of ConnectorHost. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewConnectorHost(t interface { mock.TestingT Cleanup(func()) -} - -// NewConnectorHost creates a new instance of ConnectorHost. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewConnectorHost(t mockConstructorTestingTNewConnectorHost) *ConnectorHost { +}) *ConnectorHost { mock := &ConnectorHost{} mock.Mock.Test(t) From 73bbc9d7222d710767a46bb7696b39d04bf74419 Mon Sep 17 00:00:00 2001 From: Daniel Sainati Date: Mon, 8 Jan 2024 12:10:27 -0500 Subject: [PATCH 70/72] revert mock changes --- network/mocknetwork/connector.go | 11 ++++++----- network/mocknetwork/connector_factory.go | 11 ++++++----- network/mocknetwork/connector_host.go | 11 ++++++----- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/network/mocknetwork/connector.go b/network/mocknetwork/connector.go index c4ef0abf89d..deedbd4f815 100644 --- a/network/mocknetwork/connector.go +++ b/network/mocknetwork/connector.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.30.10. DO NOT EDIT. +// Code generated by mockery v2.21.4. DO NOT EDIT. package mocknetwork @@ -20,12 +20,13 @@ func (_m *Connector) Connect(ctx context.Context, peerChan <-chan peer.AddrInfo) _m.Called(ctx, peerChan) } -// NewConnector creates a new instance of Connector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewConnector(t interface { +type mockConstructorTestingTNewConnector interface { mock.TestingT Cleanup(func()) -}) *Connector { +} + +// NewConnector creates a new instance of Connector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnector(t mockConstructorTestingTNewConnector) *Connector { mock := &Connector{} mock.Mock.Test(t) diff --git a/network/mocknetwork/connector_factory.go b/network/mocknetwork/connector_factory.go index 78b9b1059f3..b1baeb4f749 100644 --- a/network/mocknetwork/connector_factory.go +++ b/network/mocknetwork/connector_factory.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.30.10. DO NOT EDIT. +// Code generated by mockery v2.21.4. DO NOT EDIT. package mocknetwork @@ -40,12 +40,13 @@ func (_m *ConnectorFactory) Execute(_a0 host.Host) (p2p.Connector, error) { return r0, r1 } -// NewConnectorFactory creates a new instance of ConnectorFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewConnectorFactory(t interface { +type mockConstructorTestingTNewConnectorFactory interface { mock.TestingT Cleanup(func()) -}) *ConnectorFactory { +} + +// NewConnectorFactory creates a new instance of ConnectorFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnectorFactory(t mockConstructorTestingTNewConnectorFactory) *ConnectorFactory { mock := &ConnectorFactory{} mock.Mock.Test(t) diff --git a/network/mocknetwork/connector_host.go b/network/mocknetwork/connector_host.go index e117183003f..e656391a11f 100644 --- a/network/mocknetwork/connector_host.go +++ b/network/mocknetwork/connector_host.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.30.10. DO NOT EDIT. +// Code generated by mockery v2.21.4. DO NOT EDIT. package mocknetwork @@ -100,12 +100,13 @@ func (_m *ConnectorHost) PeerInfo(peerId peer.ID) peer.AddrInfo { return r0 } -// NewConnectorHost creates a new instance of ConnectorHost. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewConnectorHost(t interface { +type mockConstructorTestingTNewConnectorHost interface { mock.TestingT Cleanup(func()) -}) *ConnectorHost { +} + +// NewConnectorHost creates a new instance of ConnectorHost. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnectorHost(t mockConstructorTestingTNewConnectorHost) *ConnectorHost { mock := &ConnectorHost{} mock.Mock.Test(t) From 18339166d3a20035e302dc62bd03c74c7aa67a7a Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Mon, 8 Jan 2024 18:33:44 +0100 Subject: [PATCH 71/72] add elementsmatch to test --- cmd/util/ledger/util/payload_grouping_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/util/ledger/util/payload_grouping_test.go b/cmd/util/ledger/util/payload_grouping_test.go index 6924f0c598f..96b50bd4e5b 100644 --- a/cmd/util/ledger/util/payload_grouping_test.go +++ b/cmd/util/ledger/util/payload_grouping_test.go @@ -70,7 +70,9 @@ func TestGroupPayloadsByAccountCompareResults(t *testing.T) { require.Equal(t, group1.Address, group2.Address) require.Equal(t, len(group1.Payloads), len(group2.Payloads)) + require.ElementsMatch(t, group1.Payloads, group2.Payloads) require.Equal(t, len(group1.Payloads), len(groups3[group1.Address])) + require.ElementsMatch(t, group1.Payloads, groups3[group1.Address]) } } From a98f4c2a0e42d68d77f19093eb9b346753a87fd4 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Mon, 8 Jan 2024 21:05:06 +0100 Subject: [PATCH 72/72] fix lint --- fvm/transactionStorageLimiter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/transactionStorageLimiter_test.go b/fvm/transactionStorageLimiter_test.go index 0c86289a876..d73701bdaee 100644 --- a/fvm/transactionStorageLimiter_test.go +++ b/fvm/transactionStorageLimiter_test.go @@ -187,7 +187,7 @@ func TestTransactionStorageLimiter(t *testing.T) { executionSnapshot := &snapshot.ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ - flow.NewRegisterID(string(evm.Bytes()), "a"): flow.RegisterValue("foo"), + flow.NewRegisterID(evm, "a"): flow.RegisterValue("foo"), }, }