From b6d677f2940bbb5c09c86c2af2e130c03443ccc6 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 2 Apr 2024 15:31:31 +0200 Subject: [PATCH 01/14] Revert "Revert "Optimise merge registers for migrations"" This reverts commit 7503026c1da3519c2e9e31177a0502426474a6c8. --- .../migrations/account_storage_migration.go | 4 +- .../migrations/atree_register_migration.go | 8 +- .../migrations/cadence_value_validation.go | 2 +- .../migrations/cadence_values_migration.go | 4 +- .../cadence_values_migration_test.go | 8 +- cmd/util/ledger/migrations/merge.go | 72 ------ .../ledger/migrations/migrator_runtime.go | 4 +- .../migrations/transaction_migration.go | 6 +- cmd/util/ledger/util/payload_snapshot.go | 239 ++++++++++++++++++ cmd/util/ledger/util/payload_snapshot_test.go | 184 ++++++++++++++ cmd/util/ledger/util/util.go | 43 +--- 11 files changed, 441 insertions(+), 133 deletions(-) delete mode 100644 cmd/util/ledger/migrations/merge.go create mode 100644 cmd/util/ledger/util/payload_snapshot.go create mode 100644 cmd/util/ledger/util/payload_snapshot_test.go diff --git a/cmd/util/ledger/migrations/account_storage_migration.go b/cmd/util/ledger/migrations/account_storage_migration.go index 060d41cf168..8bd2c527dfb 100644 --- a/cmd/util/ledger/migrations/account_storage_migration.go +++ b/cmd/util/ledger/migrations/account_storage_migration.go @@ -58,11 +58,9 @@ func NewAccountStorageMigration( flow.Address(address): {}, } - newPayloads, err := MergeRegisterChanges( - migrationRuntime.Snapshot.Payloads, + newPayloads, err := migrationRuntime.Snapshot.ApplyChangesAndGetNewPayloads( result.WriteSet, expectedAddresses, - nil, log, ) if err != nil { diff --git a/cmd/util/ledger/migrations/atree_register_migration.go b/cmd/util/ledger/migrations/atree_register_migration.go index 4de94d0fae4..0931fa61bcf 100644 --- a/cmd/util/ledger/migrations/atree_register_migration.go +++ b/cmd/util/ledger/migrations/atree_register_migration.go @@ -129,7 +129,7 @@ func (m *AtreeRegisterMigrator) MigrateAccount( m.rw.Write(migrationProblem{ Address: address.Hex(), Key: "", - Size: len(mr.Snapshot.Payloads), + Size: mr.Snapshot.Len(), Kind: "more_registers_after_migration", Msg: fmt.Sprintf("original: %d, new: %d", originalLen, newLen), }) @@ -227,7 +227,7 @@ func (m *AtreeRegisterMigrator) convertStorageDomain( m.rw.Write(migrationProblem{ Address: mr.Address.Hex(), - Size: len(mr.Snapshot.Payloads), + Size: mr.Snapshot.Len(), Key: string(key), Kind: "migration_failure", Msg: err.Error(), @@ -245,7 +245,7 @@ func (m *AtreeRegisterMigrator) validateChangesAndCreateNewRegisters( storageMapIds map[string]struct{}, ) ([]*ledger.Payload, error) { originalPayloadsSnapshot := mr.Snapshot - originalPayloads := originalPayloadsSnapshot.Payloads + originalPayloads := originalPayloadsSnapshot.PayloadMap() newPayloads := make([]*ledger.Payload, 0, len(originalPayloads)) // store state payload so that it can be updated @@ -331,7 +331,7 @@ func (m *AtreeRegisterMigrator) validateChangesAndCreateNewRegisters( m.rw.Write(migrationProblem{ Address: mr.Address.Hex(), Key: id.String(), - Size: len(mr.Snapshot.Payloads), + Size: mr.Snapshot.Len(), Kind: "not_migrated", Msg: fmt.Sprintf("%x", value.Value()), }) diff --git a/cmd/util/ledger/migrations/cadence_value_validation.go b/cmd/util/ledger/migrations/cadence_value_validation.go index e21183683dd..6ec9435b9a3 100644 --- a/cmd/util/ledger/migrations/cadence_value_validation.go +++ b/cmd/util/ledger/migrations/cadence_value_validation.go @@ -363,7 +363,7 @@ func newReadonlyStorageRuntime(payloads []*ledger.Payload) ( *readonlyStorageRuntime, error, ) { - snapshot, err := util.NewPayloadSnapshot(payloads) + snapshot, err := util.NewMapBasedPayloadSnapshot(payloads) if err != nil { return nil, fmt.Errorf("failed to create payload snapshot: %w", err) } diff --git a/cmd/util/ledger/migrations/cadence_values_migration.go b/cmd/util/ledger/migrations/cadence_values_migration.go index 10963b22a44..4b8a6b3ccd8 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration.go +++ b/cmd/util/ledger/migrations/cadence_values_migration.go @@ -191,11 +191,9 @@ func (m *CadenceBaseMigrator) MigrateAccount( flow.Address(address): {}, } - newPayloads, err := MergeRegisterChanges( - migrationRuntime.Snapshot.Payloads, + newPayloads, err := migrationRuntime.Snapshot.ApplyChangesAndGetNewPayloads( result.WriteSet, expectedAddresses, - expectedAddresses, m.log, ) if err != nil { diff --git a/cmd/util/ledger/migrations/cadence_values_migration_test.go b/cmd/util/ledger/migrations/cadence_values_migration_test.go index 3771c51bb91..f34d44c2a4b 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration_test.go +++ b/cmd/util/ledger/migrations/cadence_values_migration_test.go @@ -798,11 +798,9 @@ func TestProgramParsingError(t *testing.T) { flow.Address(testAddress): {}, } - payloads, err = MergeRegisterChanges( - runtime.Snapshot.Payloads, + payloads, err = runtime.Snapshot.ApplyChangesAndGetNewPayloads( result.WriteSet, expectedAddresses, - nil, logger, ) require.NoError(t, err) @@ -937,11 +935,9 @@ func TestCoreContractUsage(t *testing.T) { flow.Address(testAddress): {}, } - payloads, err = MergeRegisterChanges( - runtime.Snapshot.Payloads, + payloads, err = runtime.Snapshot.ApplyChangesAndGetNewPayloads( result.WriteSet, expectedAddresses, - nil, logger, ) require.NoError(t, err) diff --git a/cmd/util/ledger/migrations/merge.go b/cmd/util/ledger/migrations/merge.go deleted file mode 100644 index 81efe0a9fa5..00000000000 --- a/cmd/util/ledger/migrations/merge.go +++ /dev/null @@ -1,72 +0,0 @@ -package migrations - -import ( - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/ledger" - "github.com/onflow/flow-go/ledger/common/convert" - "github.com/onflow/flow-go/model/flow" -) - -func MergeRegisterChanges( - originalPayloads map[flow.RegisterID]*ledger.Payload, - changes map[flow.RegisterID]flow.RegisterValue, - expectedChangeAddresses map[flow.Address]struct{}, - expectedOriginalAddresses map[flow.Address]struct{}, - logger zerolog.Logger, -) ([]*ledger.Payload, error) { - - newPayloads := make([]*ledger.Payload, 0, len(originalPayloads)) - - // Add all new payloads. - for id, value := range changes { - delete(originalPayloads, id) - if len(value) == 0 { - continue - } - - if expectedChangeAddresses != nil { - ownerAddress := flow.BytesToAddress([]byte(id.Owner)) - - if _, ok := expectedChangeAddresses[ownerAddress]; !ok { - // something was changed that does not belong to this account. Log it. - logger.Error(). - Str("key", id.String()). - Str("actual_address", ownerAddress.Hex()). - Interface("expected_addresses", expectedChangeAddresses). - Hex("value", value). - Msg("key is part of the change set, but is for a different account") - } - } - - key := convert.RegisterIDToLedgerKey(id) - newPayloads = append(newPayloads, ledger.NewPayload(key, value)) - } - - // Add any old payload that wasn't updated. - for id, value := range originalPayloads { - if len(value.Value()) == 0 { - // This is strange, but we don't want to add empty values. Log it. - logger.Warn().Msgf("empty value for key %s", id) - continue - } - - if expectedOriginalAddresses != nil { - ownerAddress := flow.BytesToAddress([]byte(id.Owner)) - - if _, ok := expectedOriginalAddresses[ownerAddress]; !ok { - // something was changed that does not belong to this account. Log it. - logger.Error(). - Str("key", id.String()). - Str("actual_address", ownerAddress.Hex()). - Interface("expected_addresses", expectedOriginalAddresses). - Hex("value", value.Value()). - Msg("key is part of the original set, but is for a different account") - } - } - - newPayloads = append(newPayloads, value) - } - - return newPayloads, nil -} diff --git a/cmd/util/ledger/migrations/migrator_runtime.go b/cmd/util/ledger/migrations/migrator_runtime.go index 074d0428d77..92da7ff36f0 100644 --- a/cmd/util/ledger/migrations/migrator_runtime.go +++ b/cmd/util/ledger/migrations/migrator_runtime.go @@ -34,7 +34,7 @@ func NewMigratorRuntime( *migratorRuntime, error, ) { - snapshot, err := util.NewPayloadSnapshot(payloads) + snapshot, err := util.NewMapBasedPayloadSnapshot(payloads) if err != nil { return nil, fmt.Errorf("failed to create payload snapshot: %w", err) } @@ -107,7 +107,7 @@ func NewMigratorRuntime( } type migratorRuntime struct { - Snapshot *util.PayloadSnapshot + Snapshot util.MigrationStorageSnapshot TransactionState state.NestedTransactionPreparer Interpreter *interpreter.Interpreter Storage *runtime.Storage diff --git a/cmd/util/ledger/migrations/transaction_migration.go b/cmd/util/ledger/migrations/transaction_migration.go index 16ad691c4ed..59e8d023e2c 100644 --- a/cmd/util/ledger/migrations/transaction_migration.go +++ b/cmd/util/ledger/migrations/transaction_migration.go @@ -27,7 +27,7 @@ func NewTransactionBasedMigration( fvm.WithTransactionFeesEnabled(false)) ctx := fvm.NewContext(options...) - snapshot, err := util.NewPayloadSnapshot(payloads) + snapshot, err := util.NewMapBasedPayloadSnapshot(payloads) if err != nil { return nil, err } @@ -48,11 +48,9 @@ func NewTransactionBasedMigration( return nil, res.Err } - return MergeRegisterChanges( - snapshot.Payloads, + return snapshot.ApplyChangesAndGetNewPayloads( executionSnapshot.WriteSet, expectedWriteAddresses, - nil, logger, ) } diff --git a/cmd/util/ledger/util/payload_snapshot.go b/cmd/util/ledger/util/payload_snapshot.go new file mode 100644 index 00000000000..59e3eca47f7 --- /dev/null +++ b/cmd/util/ledger/util/payload_snapshot.go @@ -0,0 +1,239 @@ +package util + +import ( + "sort" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" +) + +type MigrationStorageSnapshot interface { + snapshot.StorageSnapshot + + Exists(id flow.RegisterID) bool + ApplyChangesAndGetNewPayloads( + changes map[flow.RegisterID]flow.RegisterValue, + expectedChangeAddresses map[flow.Address]struct{}, + logger zerolog.Logger, + ) ([]*ledger.Payload, error) + + Len() int + PayloadMap() map[flow.RegisterID]*ledger.Payload +} + +type PayloadSnapshot struct { + Payloads map[flow.RegisterID]*ledger.Payload +} + +var _ MigrationStorageSnapshot = PayloadSnapshot{} + +func NewPayloadSnapshot(payloads []*ledger.Payload) (*PayloadSnapshot, error) { + l := &PayloadSnapshot{ + Payloads: make(map[flow.RegisterID]*ledger.Payload, len(payloads)), + } + for _, payload := range payloads { + key, err := payload.Key() + if err != nil { + return nil, err + } + id, err := convert.LedgerKeyToRegisterID(key) + if err != nil { + return nil, err + } + l.Payloads[id] = payload + } + return l, nil +} + +func (p PayloadSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { + value, exists := p.Payloads[id] + if !exists { + return nil, nil + } + return value.Value(), nil +} + +func (p PayloadSnapshot) Exists(id flow.RegisterID) bool { + _, exists := p.Payloads[id] + return exists +} + +func (p PayloadSnapshot) Len() int { + return len(p.Payloads) +} + +func (p PayloadSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { + return p.Payloads +} + +// ApplyChangesAndGetNewPayloads applies the given changes to the snapshot and returns the new payloads. +// the snapshot is destroyed. +func (p PayloadSnapshot) ApplyChangesAndGetNewPayloads( + changes map[flow.RegisterID]flow.RegisterValue, + expectedChangeAddresses map[flow.Address]struct{}, + logger zerolog.Logger, +) ([]*ledger.Payload, error) { + originalPayloads := p.Payloads + + newPayloads := make([]*ledger.Payload, 0, len(originalPayloads)) + + // Add all new payloads. + for id, value := range changes { + delete(originalPayloads, id) + if len(value) == 0 { + continue + } + + if expectedChangeAddresses != nil { + ownerAddress := flow.BytesToAddress([]byte(id.Owner)) + + if _, ok := expectedChangeAddresses[ownerAddress]; !ok { + // something was changed that does not belong to this account. Log it. + logger.Error(). + Str("key", id.String()). + Str("actual_address", ownerAddress.Hex()). + Interface("expected_addresses", expectedChangeAddresses). + Hex("value", value). + Msg("key is part of the change set, but is for a different account") + } + } + + key := convert.RegisterIDToLedgerKey(id) + newPayloads = append(newPayloads, ledger.NewPayload(key, value)) + } + + // Add any old payload that wasn't updated. + for id, value := range originalPayloads { + if len(value.Value()) == 0 { + // This is strange, but we don't want to add empty values. Log it. + logger.Warn().Msgf("empty value for key %s", id) + continue + } + + newPayloads = append(newPayloads, value) + } + + return newPayloads, nil +} + +type MapBasedPayloadSnapshot struct { + reverseMap map[flow.RegisterID]int + payloads []*ledger.Payload +} + +var _ MigrationStorageSnapshot = (*MapBasedPayloadSnapshot)(nil) + +func NewMapBasedPayloadSnapshot(payloads []*ledger.Payload) (*MapBasedPayloadSnapshot, error) { + payloadsCopy := make([]*ledger.Payload, len(payloads)) + copy(payloadsCopy, payloads) + l := &MapBasedPayloadSnapshot{ + reverseMap: make(map[flow.RegisterID]int, len(payloads)), + payloads: payloadsCopy, + } + for i, payload := range payloadsCopy { + key, err := payload.Key() + if err != nil { + return nil, err + } + id, err := convert.LedgerKeyToRegisterID(key) + if err != nil { + return nil, err + } + l.reverseMap[id] = i + } + return l, nil +} + +func (p *MapBasedPayloadSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { + index, exists := p.reverseMap[id] + if !exists { + return nil, nil + } + return p.payloads[index].Value(), nil +} + +func (p *MapBasedPayloadSnapshot) Exists(id flow.RegisterID) bool { + _, exists := p.reverseMap[id] + return exists +} + +func (p *MapBasedPayloadSnapshot) Len() int { + return len(p.payloads) +} + +func (p *MapBasedPayloadSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { + result := make(map[flow.RegisterID]*ledger.Payload, len(p.payloads)) + for id, index := range p.reverseMap { + result[id] = p.payloads[index] + } + return result +} + +// ApplyChangesAndGetNewPayloads applies the given changes to the snapshot and returns the new payloads. +// the snapshot is destroyed. +func (p *MapBasedPayloadSnapshot) ApplyChangesAndGetNewPayloads( + changes map[flow.RegisterID]flow.RegisterValue, + expectedChangeAddresses map[flow.Address]struct{}, + logger zerolog.Logger, +) ([]*ledger.Payload, error) { + + // append all new payloads at once at the end + newPayloads := make([]*ledger.Payload, 0, len(changes)) + deletedPayloads := make([]int, 0, len(changes)) + + for id, value := range changes { + if expectedChangeAddresses != nil { + ownerAddress := flow.BytesToAddress([]byte(id.Owner)) + + if _, ok := expectedChangeAddresses[ownerAddress]; !ok { + // something was changed that does not belong to this account. Log it. + logger.Error(). + Str("key", id.String()). + Str("actual_address", ownerAddress.Hex()). + Interface("expected_addresses", expectedChangeAddresses). + Hex("value", value). + Msg("key is part of the change set, but is for a different account") + } + } + + existingItemIndex, exists := p.reverseMap[id] + if !exists { + if len(value) == 0 { + // do not add new empty values + continue + } + newPayloads = append(newPayloads, ledger.NewPayload(convert.RegisterIDToLedgerKey(id), value)) + } else { + if len(value) == 0 { + // do not add new empty values + deletedPayloads = append(deletedPayloads, existingItemIndex) + continue + } + + // update existing payload + p.payloads[existingItemIndex] = ledger.NewPayload(convert.RegisterIDToLedgerKey(id), value) + } + } + + // remove deleted payloads by moving the last ones to the deleted positions + // and then re-slicing the array + sort.Ints(deletedPayloads) + for i := len(deletedPayloads) - 1; i >= 0; i-- { + index := deletedPayloads[i] + // take items from the very end of the array + p.payloads[index] = p.payloads[len(p.payloads)-1-(len(deletedPayloads)-1-i)] + } + p.payloads = p.payloads[:len(p.payloads)-len(deletedPayloads)] + + result := append(p.payloads, newPayloads...) + + // destroy the snapshot to prevent further use + p.payloads = nil + p.reverseMap = nil + + return result, nil +} diff --git a/cmd/util/ledger/util/payload_snapshot_test.go b/cmd/util/ledger/util/payload_snapshot_test.go new file mode 100644 index 00000000000..0b909e2af42 --- /dev/null +++ b/cmd/util/ledger/util/payload_snapshot_test.go @@ -0,0 +1,184 @@ +package util_test + +import ( + "crypto/rand" + "math/big" + "strconv" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" +) + +func Benchmark_PayloadSnapshot(b *testing.B) { + benchMerge := func( + b *testing.B, + payloadsNum int, changesNum []int, + ) { + b.Run("merge_"+strconv.Itoa(payloadsNum), func(b *testing.B) { + benchmarkMerge(b, payloadsNum, changesNum) + }) + } + + benchCreate := func( + b *testing.B, + payloadsNum int, + ) { + b.Run("create_"+strconv.Itoa(payloadsNum), func(b *testing.B) { + benchmarkCreate(b, payloadsNum) + }) + } + + benchCreate(b, 1000) + benchCreate(b, 100000) + benchCreate(b, 10000000) + + benchMerge(b, 1000, []int{10, 100, 1000}) + benchMerge(b, 100000, []int{10, 1000, 100000}) + benchMerge(b, 10000000, []int{10, 10000, 10000000}) + +} + +func randomPayload(size int) []byte { + payload := make([]byte, size) + _, _ = rand.Read(payload) + return payload +} + +func randomInt(max int) int { + n, _ := rand.Int(rand.Reader, big.NewInt(int64(max))) + return int(n.Int64()) +} + +func createPayloads(payloadsNum int) []*ledger.Payload { + // 1kb payloads + payload := randomPayload(1024) + + payloads := make([]*ledger.Payload, payloadsNum) + for i := 0; i < payloadsNum; i++ { + p := make([]byte, len(payload)) + copy(p, payload) + payloads[i] = ledger.NewPayload( + convert.RegisterIDToLedgerKey(flow.RegisterID{ + Owner: flow.EmptyAddress.String(), + Key: strconv.Itoa(i), + }), + p, + ) + } + return payloads +} + +func benchmarkMerge(b *testing.B, payloadsNum int, changes []int) { + + creatChanges := func( + existingPayloadCount int, + changePayloads int, + deletePayloads int, + createPayloads int, + ) map[flow.RegisterID]flow.RegisterValue { + // half of the changes should be new payloads + changes := make(map[flow.RegisterID]flow.RegisterValue) + payload := randomPayload(1024) + for i := 0; i < changePayloads; i++ { + p := make([]byte, len(payload)) + copy(p, payload) + + // get random index + index := randomInt(existingPayloadCount) + + changes[flow.RegisterID{ + Owner: flow.EmptyAddress.String(), + Key: strconv.Itoa(index), + }] = p + } + + for i := 0; i < deletePayloads; i++ { + + // get random index + index := randomInt(existingPayloadCount) + + changes[flow.RegisterID{ + Owner: flow.EmptyAddress.String(), + Key: strconv.Itoa(index), + }] = []byte{} + } + + payload = randomPayload(1024) + for i := 0; i < createPayloads; i++ { + p := make([]byte, len(payload)) + copy(p, payload) + changes[flow.RegisterID{ + Owner: flow.EmptyAddress.String(), + Key: strconv.Itoa(i + existingPayloadCount), + }] = p + } + + return changes + } + + payloads := createPayloads(payloadsNum) + + for _, changesNum := range changes { + b.Run("changes_"+strconv.Itoa(changesNum), func(b *testing.B) { + + // third of the changes should be new payloads + // third of the changes should be existing payloads + // third of the changes should be removals + change, remove, add := changesNum/3, changesNum/3, changesNum-2*(changesNum/3) + + changes := creatChanges(len(payloads), change, remove, add) + + b.Run("MapBasedPayloadSnapshot", func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + snapshot, err := util.NewMapBasedPayloadSnapshot(payloads) + require.NoError(b, err) + b.StartTimer() + + _, err = snapshot.ApplyChangesAndGetNewPayloads(changes, nil, zerolog.Nop()) + require.NoError(b, err) + } + }) + + b.Run("PayloadSnapshot", func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + snapshot, err := util.NewPayloadSnapshot(payloads) + require.NoError(b, err) + b.StartTimer() + + _, err = snapshot.ApplyChangesAndGetNewPayloads(changes, nil, zerolog.Nop()) + require.NoError(b, err) + } + }) + }) + } +} + +func benchmarkCreate( + b *testing.B, + payloadsNum int, +) { + + payloads := createPayloads(payloadsNum) + + b.Run("MapBasedPayloadSnapshot", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := util.NewMapBasedPayloadSnapshot(payloads) + require.NoError(b, err) + } + }) + + b.Run("PayloadSnapshot", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := util.NewPayloadSnapshot(payloads) + require.NoError(b, err) + } + }) +} diff --git a/cmd/util/ledger/util/util.go b/cmd/util/ledger/util/util.go index b2248efb513..c1da55fe6d9 100644 --- a/cmd/util/ledger/util/util.go +++ b/cmd/util/ledger/util/util.go @@ -12,7 +12,6 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" "github.com/onflow/flow-go/model/flow" @@ -69,38 +68,6 @@ func (a *AccountsAtreeLedger) AllocateStorageIndex(owner []byte) (atree.StorageI return v, nil } -type PayloadSnapshot struct { - Payloads map[flow.RegisterID]*ledger.Payload -} - -var _ snapshot.StorageSnapshot = (*PayloadSnapshot)(nil) - -func NewPayloadSnapshot(payloads []*ledger.Payload) (*PayloadSnapshot, error) { - l := &PayloadSnapshot{ - Payloads: make(map[flow.RegisterID]*ledger.Payload, len(payloads)), - } - for _, payload := range payloads { - key, err := payload.Key() - if err != nil { - return nil, err - } - id, err := convert.LedgerKeyToRegisterID(key) - if err != nil { - return nil, err - } - l.Payloads[id] = payload - } - return l, nil -} - -func (p PayloadSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { - value, exists := p.Payloads[id] - if !exists { - return nil, nil - } - return value.Value(), nil -} - // NopMemoryGauge is a no-op implementation of the MemoryGauge interface type NopMemoryGauge struct{} @@ -111,7 +78,7 @@ func (n NopMemoryGauge) MeterMemory(common.MemoryUsage) error { var _ common.MemoryGauge = (*NopMemoryGauge)(nil) type PayloadsReadonlyLedger struct { - Snapshot *PayloadSnapshot + Snapshot MigrationStorageSnapshot AllocateStorageIndexFunc func(owner []byte) (atree.StorageIndex, error) SetValueFunc func(owner, key, value []byte) (err error) @@ -133,9 +100,9 @@ func (p *PayloadsReadonlyLedger) SetValue(owner, key, value []byte) (err error) panic("SetValue not expected to be called") } -func (p *PayloadsReadonlyLedger) ValueExists(owner, key []byte) (exists bool, err error) { - _, ok := p.Snapshot.Payloads[flow.NewRegisterID(flow.BytesToAddress(owner), string(key))] - return ok, nil +func (p *PayloadsReadonlyLedger) ValueExists(owner, key []byte) (bool, error) { + exists := p.Snapshot.Exists(flow.NewRegisterID(flow.BytesToAddress(owner), string(key))) + return exists, nil } func (p *PayloadsReadonlyLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { @@ -146,7 +113,7 @@ func (p *PayloadsReadonlyLedger) AllocateStorageIndex(owner []byte) (atree.Stora panic("AllocateStorageIndex not expected to be called") } -func NewPayloadsReadonlyLedger(snapshot *PayloadSnapshot) *PayloadsReadonlyLedger { +func NewPayloadsReadonlyLedger(snapshot MigrationStorageSnapshot) *PayloadsReadonlyLedger { return &PayloadsReadonlyLedger{Snapshot: snapshot} } From bce5dc5ee0c9dbad9be0f3c66e911dac86e63e45 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 2 Apr 2024 16:23:31 +0200 Subject: [PATCH 02/14] fix tests --- cmd/util/ledger/util/util.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/util/ledger/util/util.go b/cmd/util/ledger/util/util.go index c1da55fe6d9..71a353230ea 100644 --- a/cmd/util/ledger/util/util.go +++ b/cmd/util/ledger/util/util.go @@ -141,7 +141,7 @@ func PayloadsAndAccountsFromEmulatorSnapshot(db *sql.DB) ( []common.Address, error, ) { - rows, err := db.Query("SELECT key, value, version, height FROM ledger") + rows, err := db.Query("SELECT key, value, version, height FROM ledger ORDER BY height DESC") if err != nil { return nil, nil, nil, err } @@ -185,6 +185,10 @@ func PayloadsAndAccountsFromEmulatorSnapshot(db *sql.DB) ( value, ) + if _, ok := payloadSet[registerId]; ok { + continue + } + payloads = append(payloads, payload) payloadSet[registerId] = PayloadMetaInfo{ Height: height, From f14f23a2bbe0218248c1403fd6144cf1a74839a7 Mon Sep 17 00:00:00 2001 From: Josh Hannan Date: Tue, 2 Apr 2024 14:05:16 -0500 Subject: [PATCH 03/14] update core contract dependencies and state commitments --- .../state/bootstrap/bootstrap_test.go | 2 +- go.mod | 12 +++++----- go.sum | 24 +++++++++---------- insecure/go.mod | 12 +++++----- insecure/go.sum | 24 +++++++++---------- integration/go.mod | 12 +++++----- integration/go.sum | 24 +++++++++---------- utils/unittest/execution_state.go | 6 ++--- 8 files changed, 58 insertions(+), 58 deletions(-) diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index 1974bd51b57..67eaaff65f2 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("91638e2c14bf2b3d0b6e1833045dcb96cc5884e4064d96b1a66fb6ae22ffdecf") + expectedStateCommitmentBytes, _ := hex.DecodeString("cea75198bee790788d14738230263d97bb3cbfff354bc1e05abd4c40fd2e8155") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/go.mod b/go.mod index 9ec875a5d39..a3ad9b4eff1 100644 --- a/go.mod +++ b/go.mod @@ -54,8 +54,8 @@ require ( github.com/onflow/cadence v1.0.0-preview.18 github.com/onflow/crypto v0.25.0 github.com/onflow/flow v0.3.4 - github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240327170024-10241fffd864 - github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240327170024-10241fffd864 + github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e + github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e github.com/onflow/flow-go-sdk v1.0.0-preview.16 github.com/onflow/flow/protobuf/go/flow v0.3.7 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 @@ -259,10 +259,10 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240327162334-bd133114f87a // indirect - github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240327162334-bd133114f87a // indirect - github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240326155818-c01c72c091c0 // indirect - github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240326155818-c01c72c091c0 // indirect + github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956 // indirect + github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956 // indirect + github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d // indirect + github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d // indirect github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba // indirect github.com/onsi/ginkgo/v2 v2.13.2 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect diff --git a/go.sum b/go.sum index 5b418ed7f1c..85f714859cd 100644 --- a/go.sum +++ b/go.sum @@ -2494,21 +2494,21 @@ github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240327170024-10241fffd864 h1:Tym6HXbuhoTwef+EfHWDaVgFs/gwmJ0yKD+zjJH1O1s= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240327170024-10241fffd864/go.mod h1:Br3AdtgZlvvk2h4YFJcbCHCJHsM8y0LX1CK+2EdkYR4= -github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240327170024-10241fffd864 h1:shtPP46wORFIX3XE+gnvwVUtwejAZm3wye7Y1nJD75A= -github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240327170024-10241fffd864/go.mod h1:v+SKoOBjMd8c8jspqcnyM1iTejCQq8JaGr6B1XW8Vsw= -github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240327162334-bd133114f87a h1:7cMT6DGMqCw98hXzuDU69WxUZxsqsRxNMMlWXG/yPOc= -github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240327162334-bd133114f87a/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= -github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240327162334-bd133114f87a h1:oe3ErYY8qds7IER/zmNGKT3zz6cFcjC0doX2Q0WOUv0= -github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240327162334-bd133114f87a/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e h1:z0U8ZnjhYVoaZn193tGTDuIb2uGn2rMTC0Szoaqrda8= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e/go.mod h1:i2qeJgBl/88UsZ4anUtvW7jPlPAD3izA2iERg4SRgmU= +github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e h1:NSZs3LVVfeimWP7BErgUT/HDg5Pu9kKMBqJWXVQP9SI= +github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e/go.mod h1:CA6ON8bUZWkZnlhYNsvs3Pox1G6dFtBp/wPZSQA7ivQ= +github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956 h1:WbG97gmdbgfYZT8YCyye0fAwz4k5vditXcPGoy63m9M= +github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= +github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956 h1:Ef9UKtwNcHVG2R8YskYiwRoaTZFhAVmQ0ZN3c0eDUGU= +github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= github.com/onflow/flow-go-sdk v1.0.0-preview.16 h1:m5Dj5XLUTHIFgjWPDsapaIFKIheBlhFLwZ9aXxwX6hQ= github.com/onflow/flow-go-sdk v1.0.0-preview.16/go.mod h1:zQwbb+mHfV7R+xa03xr6RoU13bZOF2uKgdf7dOGELvc= -github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240326155818-c01c72c091c0 h1:xekR2FttJpwtpRfH1BFvf9Kztm9be8grT1GT9bpQK8M= -github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240326155818-c01c72c091c0/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= -github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240326155818-c01c72c091c0 h1:Qk24unKC2ncZ+U+fd63pC5BdWyMwkKEOsjMdIrj70O0= -github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240326155818-c01c72c091c0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d h1:CeM0F/t8f6UYKIP/PzHApt0BfX5FLCbFiSW3tkXHLyA= +github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= +github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d h1:9BUEgH1oFUMeOab++UNgok9Jk+rejQIrIHYKNe/TD20= +github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/flow/protobuf/go/flow v0.3.7 h1:+6sBdlE/u4ZMTVB9U1lA6Xn2Bd48lOOX96Bv9dNubsk= github.com/onflow/flow/protobuf/go/flow v0.3.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= diff --git a/insecure/go.mod b/insecure/go.mod index 53bcc2a5812..2308ba9ff89 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -206,13 +206,13 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 // indirect github.com/onflow/cadence v1.0.0-preview.18 // indirect - github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240327170024-10241fffd864 // indirect - github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240327170024-10241fffd864 // indirect - github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240327162334-bd133114f87a // indirect - github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240327162334-bd133114f87a // indirect + github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e // indirect + github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e // indirect + github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956 // indirect + github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956 // indirect github.com/onflow/flow-go-sdk v1.0.0-preview.16 // indirect - github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240326155818-c01c72c091c0 // indirect - github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240326155818-c01c72c091c0 // indirect + github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d // indirect + github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d // indirect github.com/onflow/flow/protobuf/go/flow v0.3.7 // indirect github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba // indirect github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 502230574f0..b9305e3c1e0 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2477,21 +2477,21 @@ github.com/onflow/cadence v1.0.0-preview.18 h1:1gN+suBexuu1gZz0JjWDC9dcGBI/GIMP8 github.com/onflow/cadence v1.0.0-preview.18/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240327170024-10241fffd864 h1:Tym6HXbuhoTwef+EfHWDaVgFs/gwmJ0yKD+zjJH1O1s= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240327170024-10241fffd864/go.mod h1:Br3AdtgZlvvk2h4YFJcbCHCJHsM8y0LX1CK+2EdkYR4= -github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240327170024-10241fffd864 h1:shtPP46wORFIX3XE+gnvwVUtwejAZm3wye7Y1nJD75A= -github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240327170024-10241fffd864/go.mod h1:v+SKoOBjMd8c8jspqcnyM1iTejCQq8JaGr6B1XW8Vsw= -github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240327162334-bd133114f87a h1:7cMT6DGMqCw98hXzuDU69WxUZxsqsRxNMMlWXG/yPOc= -github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240327162334-bd133114f87a/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= -github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240327162334-bd133114f87a h1:oe3ErYY8qds7IER/zmNGKT3zz6cFcjC0doX2Q0WOUv0= -github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240327162334-bd133114f87a/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e h1:z0U8ZnjhYVoaZn193tGTDuIb2uGn2rMTC0Szoaqrda8= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e/go.mod h1:i2qeJgBl/88UsZ4anUtvW7jPlPAD3izA2iERg4SRgmU= +github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e h1:NSZs3LVVfeimWP7BErgUT/HDg5Pu9kKMBqJWXVQP9SI= +github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e/go.mod h1:CA6ON8bUZWkZnlhYNsvs3Pox1G6dFtBp/wPZSQA7ivQ= +github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956 h1:WbG97gmdbgfYZT8YCyye0fAwz4k5vditXcPGoy63m9M= +github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= +github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956 h1:Ef9UKtwNcHVG2R8YskYiwRoaTZFhAVmQ0ZN3c0eDUGU= +github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= github.com/onflow/flow-go-sdk v1.0.0-preview.16 h1:m5Dj5XLUTHIFgjWPDsapaIFKIheBlhFLwZ9aXxwX6hQ= github.com/onflow/flow-go-sdk v1.0.0-preview.16/go.mod h1:zQwbb+mHfV7R+xa03xr6RoU13bZOF2uKgdf7dOGELvc= -github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240326155818-c01c72c091c0 h1:xekR2FttJpwtpRfH1BFvf9Kztm9be8grT1GT9bpQK8M= -github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240326155818-c01c72c091c0/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= -github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240326155818-c01c72c091c0 h1:Qk24unKC2ncZ+U+fd63pC5BdWyMwkKEOsjMdIrj70O0= -github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240326155818-c01c72c091c0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d h1:CeM0F/t8f6UYKIP/PzHApt0BfX5FLCbFiSW3tkXHLyA= +github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= +github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d h1:9BUEgH1oFUMeOab++UNgok9Jk+rejQIrIHYKNe/TD20= +github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/flow/protobuf/go/flow v0.3.7 h1:+6sBdlE/u4ZMTVB9U1lA6Xn2Bd48lOOX96Bv9dNubsk= github.com/onflow/flow/protobuf/go/flow v0.3.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= diff --git a/integration/go.mod b/integration/go.mod index eeea3b2884a..8b25ed77cd2 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -23,8 +23,8 @@ require ( github.com/libp2p/go-libp2p v0.32.2 github.com/onflow/cadence v1.0.0-preview.18 github.com/onflow/crypto v0.25.0 - github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240327170024-10241fffd864 - github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240327170024-10241fffd864 + github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e + github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e github.com/onflow/flow-emulator v1.0.0-M7.0.20240227020422-2ec59747f9be github.com/onflow/flow-go v0.34.0-crescendo-preview.2.0.20240227001756-cb6311412b78 github.com/onflow/flow-go-sdk v1.0.0-preview.16 @@ -252,10 +252,10 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 // indirect - github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240327162334-bd133114f87a // indirect - github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240327162334-bd133114f87a // indirect - github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240326155818-c01c72c091c0 // indirect - github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240326155818-c01c72c091c0 // indirect + github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956 // indirect + github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956 // indirect + github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d // indirect + github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d // indirect github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba // indirect github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b // indirect github.com/onsi/ginkgo/v2 v2.13.2 // indirect diff --git a/integration/go.sum b/integration/go.sum index 464041dc8ec..672c505019f 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2553,23 +2553,23 @@ github.com/onflow/cadence v1.0.0-preview.18 h1:1gN+suBexuu1gZz0JjWDC9dcGBI/GIMP8 github.com/onflow/cadence v1.0.0-preview.18/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240327170024-10241fffd864 h1:Tym6HXbuhoTwef+EfHWDaVgFs/gwmJ0yKD+zjJH1O1s= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240327170024-10241fffd864/go.mod h1:Br3AdtgZlvvk2h4YFJcbCHCJHsM8y0LX1CK+2EdkYR4= -github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240327170024-10241fffd864 h1:shtPP46wORFIX3XE+gnvwVUtwejAZm3wye7Y1nJD75A= -github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240327170024-10241fffd864/go.mod h1:v+SKoOBjMd8c8jspqcnyM1iTejCQq8JaGr6B1XW8Vsw= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e h1:z0U8ZnjhYVoaZn193tGTDuIb2uGn2rMTC0Szoaqrda8= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e/go.mod h1:i2qeJgBl/88UsZ4anUtvW7jPlPAD3izA2iERg4SRgmU= +github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e h1:NSZs3LVVfeimWP7BErgUT/HDg5Pu9kKMBqJWXVQP9SI= +github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e/go.mod h1:CA6ON8bUZWkZnlhYNsvs3Pox1G6dFtBp/wPZSQA7ivQ= github.com/onflow/flow-emulator v1.0.0-M7.0.20240227020422-2ec59747f9be h1:n6Zi/9uRQhI9Lpzi87tozuOzuV9KZhUGI7QI7H4LnMc= github.com/onflow/flow-emulator v1.0.0-M7.0.20240227020422-2ec59747f9be/go.mod h1:lr5R66ZAhCaXnotdyXe0iI7bi8s/cIM/4mu0if2JXEQ= -github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240327162334-bd133114f87a h1:7cMT6DGMqCw98hXzuDU69WxUZxsqsRxNMMlWXG/yPOc= -github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240327162334-bd133114f87a/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= -github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240327162334-bd133114f87a h1:oe3ErYY8qds7IER/zmNGKT3zz6cFcjC0doX2Q0WOUv0= -github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240327162334-bd133114f87a/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= +github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956 h1:WbG97gmdbgfYZT8YCyye0fAwz4k5vditXcPGoy63m9M= +github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= +github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956 h1:Ef9UKtwNcHVG2R8YskYiwRoaTZFhAVmQ0ZN3c0eDUGU= +github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= github.com/onflow/flow-go-sdk v1.0.0-preview.16 h1:m5Dj5XLUTHIFgjWPDsapaIFKIheBlhFLwZ9aXxwX6hQ= github.com/onflow/flow-go-sdk v1.0.0-preview.16/go.mod h1:zQwbb+mHfV7R+xa03xr6RoU13bZOF2uKgdf7dOGELvc= -github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240326155818-c01c72c091c0 h1:xekR2FttJpwtpRfH1BFvf9Kztm9be8grT1GT9bpQK8M= -github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240326155818-c01c72c091c0/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= -github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240326155818-c01c72c091c0 h1:Qk24unKC2ncZ+U+fd63pC5BdWyMwkKEOsjMdIrj70O0= -github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240326155818-c01c72c091c0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d h1:CeM0F/t8f6UYKIP/PzHApt0BfX5FLCbFiSW3tkXHLyA= +github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= +github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d h1:9BUEgH1oFUMeOab++UNgok9Jk+rejQIrIHYKNe/TD20= +github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/flow/protobuf/go/flow v0.3.7 h1:+6sBdlE/u4ZMTVB9U1lA6Xn2Bd48lOOX96Bv9dNubsk= github.com/onflow/flow/protobuf/go/flow v0.3.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 546e80f9051..52e159dc1ad 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -23,7 +23,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "6f8688833c626e2ebb55c09e1dfc806c351f4f0f4c67af1e956d7aab188f396a" +const GenesisStateCommitmentHex = "324e5cad4c66d8f714c9d24252eaecafc015898401abfd38f0afc37d0d2f4818" var GenesisStateCommitment flow.StateCommitment @@ -87,10 +87,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "1371cc7b9a594e47c4e328d32b7157f0bbcbc662e974d488dd0f015e74584fd7" + return "8e9cebdbed75af974a7a918f2f22f6f7145f739bd05ef02926e0afff878a29ee" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "fb11cecf7cdf6b9d7fa190fef57f02c02d4264f6cd4abb9b4b00d613230d3ae1" + return "a686891ebdccf40eb70bbbb610fef2e36c321fd162f8e1e4631c09a1597dddb1" } From eda3cafd5a8acf5ff3c2c08bfc3c8a61e5c6f66c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 3 Apr 2024 14:23:03 -0700 Subject: [PATCH 04/14] Update to Cadence v1.0.0-preview.19 --- go.mod | 6 +++--- go.sum | 11 ++++++----- insecure/go.mod | 6 +++--- insecure/go.sum | 11 ++++++----- integration/go.mod | 6 +++--- integration/go.sum | 11 ++++++----- 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index a3ad9b4eff1..202f545c6d5 100644 --- a/go.mod +++ b/go.mod @@ -51,12 +51,12 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 - github.com/onflow/cadence v1.0.0-preview.18 - github.com/onflow/crypto v0.25.0 + github.com/onflow/cadence v1.0.0-preview.19 + github.com/onflow/crypto v0.25.1 github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e - github.com/onflow/flow-go-sdk v1.0.0-preview.16 + github.com/onflow/flow-go-sdk v1.0.0-preview.17 github.com/onflow/flow/protobuf/go/flow v0.3.7 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible diff --git a/go.sum b/go.sum index 85f714859cd..0e4d292a8a7 100644 --- a/go.sum +++ b/go.sum @@ -2488,10 +2488,11 @@ github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2/go.mod h1:xvP61FoOs github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483 h1:LpiQhTAfM9CAmNVEs0n//cBBgCg+vJSiIxTHYUklZ84= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.18 h1:1gN+suBexuu1gZz0JjWDC9dcGBI/GIMP8R0Tyou9mzA= -github.com/onflow/cadence v1.0.0-preview.18/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= -github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= +github.com/onflow/cadence v1.0.0-preview.19 h1:EowxTD6g1lDAvdh5VK++YVTncj46r4MhYeEpm6L7GnQ= +github.com/onflow/cadence v1.0.0-preview.19/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= +github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= +github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e h1:z0U8ZnjhYVoaZn193tGTDuIb2uGn2rMTC0Szoaqrda8= @@ -2503,8 +2504,8 @@ github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956/ github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956 h1:Ef9UKtwNcHVG2R8YskYiwRoaTZFhAVmQ0ZN3c0eDUGU= github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.16 h1:m5Dj5XLUTHIFgjWPDsapaIFKIheBlhFLwZ9aXxwX6hQ= -github.com/onflow/flow-go-sdk v1.0.0-preview.16/go.mod h1:zQwbb+mHfV7R+xa03xr6RoU13bZOF2uKgdf7dOGELvc= +github.com/onflow/flow-go-sdk v1.0.0-preview.17 h1:RDW+FCKWJBCX7U6bK7ZEBVqBrhdNS9CIAV+xvVeLLYY= +github.com/onflow/flow-go-sdk v1.0.0-preview.17/go.mod h1:2XygOoc/RN2c28o3vebEc3TLhzfAviOEwpn2yQJHojQ= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d h1:CeM0F/t8f6UYKIP/PzHApt0BfX5FLCbFiSW3tkXHLyA= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d h1:9BUEgH1oFUMeOab++UNgok9Jk+rejQIrIHYKNe/TD20= diff --git a/insecure/go.mod b/insecure/go.mod index 2308ba9ff89..3b86e4aaf66 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -10,7 +10,7 @@ require ( github.com/libp2p/go-libp2p v0.32.2 github.com/libp2p/go-libp2p-pubsub v0.10.0 github.com/multiformats/go-multiaddr-dns v0.3.1 - github.com/onflow/crypto v0.25.0 + github.com/onflow/crypto v0.25.1 github.com/onflow/flow-go v0.33.2-0.20240122190738-254af677b873 github.com/rs/zerolog v1.29.0 github.com/spf13/pflag v1.0.5 @@ -205,12 +205,12 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 // indirect - github.com/onflow/cadence v1.0.0-preview.18 // indirect + github.com/onflow/cadence v1.0.0-preview.19 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e // indirect github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956 // indirect github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956 // indirect - github.com/onflow/flow-go-sdk v1.0.0-preview.16 // indirect + github.com/onflow/flow-go-sdk v1.0.0-preview.17 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d // indirect github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d // indirect github.com/onflow/flow/protobuf/go/flow v0.3.7 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index b9305e3c1e0..96ca14f7644 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2473,10 +2473,11 @@ github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 h1:jJLDswfAVB0bHCu1y1FPdKukPcTNmN+jYEX9S9phbv0= github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.18 h1:1gN+suBexuu1gZz0JjWDC9dcGBI/GIMP8R0Tyou9mzA= -github.com/onflow/cadence v1.0.0-preview.18/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= -github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= +github.com/onflow/cadence v1.0.0-preview.19 h1:EowxTD6g1lDAvdh5VK++YVTncj46r4MhYeEpm6L7GnQ= +github.com/onflow/cadence v1.0.0-preview.19/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= +github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= +github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e h1:z0U8ZnjhYVoaZn193tGTDuIb2uGn2rMTC0Szoaqrda8= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e/go.mod h1:i2qeJgBl/88UsZ4anUtvW7jPlPAD3izA2iERg4SRgmU= github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e h1:NSZs3LVVfeimWP7BErgUT/HDg5Pu9kKMBqJWXVQP9SI= @@ -2486,8 +2487,8 @@ github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956/ github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956 h1:Ef9UKtwNcHVG2R8YskYiwRoaTZFhAVmQ0ZN3c0eDUGU= github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.16 h1:m5Dj5XLUTHIFgjWPDsapaIFKIheBlhFLwZ9aXxwX6hQ= -github.com/onflow/flow-go-sdk v1.0.0-preview.16/go.mod h1:zQwbb+mHfV7R+xa03xr6RoU13bZOF2uKgdf7dOGELvc= +github.com/onflow/flow-go-sdk v1.0.0-preview.17 h1:RDW+FCKWJBCX7U6bK7ZEBVqBrhdNS9CIAV+xvVeLLYY= +github.com/onflow/flow-go-sdk v1.0.0-preview.17/go.mod h1:2XygOoc/RN2c28o3vebEc3TLhzfAviOEwpn2yQJHojQ= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d h1:CeM0F/t8f6UYKIP/PzHApt0BfX5FLCbFiSW3tkXHLyA= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d h1:9BUEgH1oFUMeOab++UNgok9Jk+rejQIrIHYKNe/TD20= diff --git a/integration/go.mod b/integration/go.mod index 8b25ed77cd2..5b6acb45ea1 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -21,13 +21,13 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ipfs-blockstore v1.3.0 github.com/libp2p/go-libp2p v0.32.2 - github.com/onflow/cadence v1.0.0-preview.18 - github.com/onflow/crypto v0.25.0 + github.com/onflow/cadence v1.0.0-preview.19 + github.com/onflow/crypto v0.25.1 github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e github.com/onflow/flow-emulator v1.0.0-M7.0.20240227020422-2ec59747f9be github.com/onflow/flow-go v0.34.0-crescendo-preview.2.0.20240227001756-cb6311412b78 - github.com/onflow/flow-go-sdk v1.0.0-preview.16 + github.com/onflow/flow-go-sdk v1.0.0-preview.17 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.3.7 github.com/plus3it/gorecurcopy v0.0.1 diff --git a/integration/go.sum b/integration/go.sum index 672c505019f..034d0c4d83b 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2549,10 +2549,11 @@ github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 h1:jJLDswfAVB0bHCu1y1FPdKukPcTNmN+jYEX9S9phbv0= github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.18 h1:1gN+suBexuu1gZz0JjWDC9dcGBI/GIMP8R0Tyou9mzA= -github.com/onflow/cadence v1.0.0-preview.18/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= -github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= +github.com/onflow/cadence v1.0.0-preview.19 h1:EowxTD6g1lDAvdh5VK++YVTncj46r4MhYeEpm6L7GnQ= +github.com/onflow/cadence v1.0.0-preview.19/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= +github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= +github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e h1:z0U8ZnjhYVoaZn193tGTDuIb2uGn2rMTC0Szoaqrda8= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240402184019-90048578066e/go.mod h1:i2qeJgBl/88UsZ4anUtvW7jPlPAD3izA2iERg4SRgmU= github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240402184019-90048578066e h1:NSZs3LVVfeimWP7BErgUT/HDg5Pu9kKMBqJWXVQP9SI= @@ -2564,8 +2565,8 @@ github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240402160548-a9c331660956/ github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956 h1:Ef9UKtwNcHVG2R8YskYiwRoaTZFhAVmQ0ZN3c0eDUGU= github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240402160548-a9c331660956/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.16 h1:m5Dj5XLUTHIFgjWPDsapaIFKIheBlhFLwZ9aXxwX6hQ= -github.com/onflow/flow-go-sdk v1.0.0-preview.16/go.mod h1:zQwbb+mHfV7R+xa03xr6RoU13bZOF2uKgdf7dOGELvc= +github.com/onflow/flow-go-sdk v1.0.0-preview.17 h1:RDW+FCKWJBCX7U6bK7ZEBVqBrhdNS9CIAV+xvVeLLYY= +github.com/onflow/flow-go-sdk v1.0.0-preview.17/go.mod h1:2XygOoc/RN2c28o3vebEc3TLhzfAviOEwpn2yQJHojQ= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d h1:CeM0F/t8f6UYKIP/PzHApt0BfX5FLCbFiSW3tkXHLyA= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240402163945-74687e7a5b9d/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240402163945-74687e7a5b9d h1:9BUEgH1oFUMeOab++UNgok9Jk+rejQIrIHYKNe/TD20= From 4d2b9f8e3aa73f81a7e584dff0e3deac35ac563c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 3 Apr 2024 14:42:57 -0700 Subject: [PATCH 05/14] adjust Cadence API usage --- .../migrations/atree_register_migration.go | 4 +- .../ledger/migrations/cadence_value_diff.go | 4 +- .../migrations/cadence_value_validation.go | 55 ++++++++++++------- .../ledger/migrations/migrator_runtime.go | 5 +- .../reporters/fungible_token_tracker.go | 4 +- cmd/util/ledger/util/util.go | 32 +++++++++++ 6 files changed, 77 insertions(+), 27 deletions(-) diff --git a/cmd/util/ledger/migrations/atree_register_migration.go b/cmd/util/ledger/migrations/atree_register_migration.go index 0931fa61bcf..6a57fd53aa0 100644 --- a/cmd/util/ledger/migrations/atree_register_migration.go +++ b/cmd/util/ledger/migrations/atree_register_migration.go @@ -178,7 +178,7 @@ func (m *AtreeRegisterMigrator) convertStorageDomain( } storageMapIds[string(atree.SlabIndexToLedgerKey(storageMap.StorageID().Index))] = struct{}{} - iterator := storageMap.Iterator(util.NopMemoryGauge{}) + iterator := storageMap.Iterator(nil) keys := make([]interpreter.StringStorageMapKey, 0, storageMap.Count()) // to be safe avoid modifying the map while iterating for { @@ -200,7 +200,7 @@ func (m *AtreeRegisterMigrator) convertStorageDomain( var value interpreter.Value err := capturePanic(func() { - value = storageMap.ReadValue(util.NopMemoryGauge{}, key) + value = storageMap.ReadValue(nil, key) }) if err != nil { return fmt.Errorf("failed to read value for key %s: %w", key, err) diff --git a/cmd/util/ledger/migrations/cadence_value_diff.go b/cmd/util/ledger/migrations/cadence_value_diff.go index 94e148920e3..abe800bae01 100644 --- a/cmd/util/ledger/migrations/cadence_value_diff.go +++ b/cmd/util/ledger/migrations/cadence_value_diff.go @@ -232,9 +232,9 @@ func (dr *CadenceValueDiffReporter) diffStorageDomain(oldRuntime, newRuntime *re continue } - oldValue := oldStorageMap.ReadValue(nopMemoryGauge, mapKey) + oldValue := oldStorageMap.ReadValue(nil, mapKey) - newValue := newStorageMap.ReadValue(nopMemoryGauge, mapKey) + newValue := newStorageMap.ReadValue(nil, mapKey) hasDifference := dr.diffValues( oldRuntime.Interpreter, diff --git a/cmd/util/ledger/migrations/cadence_value_validation.go b/cmd/util/ledger/migrations/cadence_value_validation.go index 6ec9435b9a3..1aa8e94b2bd 100644 --- a/cmd/util/ledger/migrations/cadence_value_validation.go +++ b/cmd/util/ledger/migrations/cadence_value_validation.go @@ -17,8 +17,6 @@ import ( "github.com/onflow/flow-go/ledger" ) -var nopMemoryGauge = util.NopMemoryGauge{} - // TODO: optimize memory by reusing payloads snapshot created for migration func validateCadenceValues( address common.Address, @@ -79,7 +77,7 @@ func validateStorageDomain( return fmt.Errorf("old storage map count %d, new storage map count %d", oldStorageMap.Count(), newStorageMap.Count()) } - oldIterator := oldStorageMap.Iterator(nopMemoryGauge) + oldIterator := oldStorageMap.Iterator(nil) for { key, oldValue := oldIterator.Next() if key == nil { @@ -91,7 +89,7 @@ func validateStorageDomain( return fmt.Errorf("invalid key type %T, expected interpreter.StringAtreeValue", key) } - newValue := newStorageMap.ReadValue(nopMemoryGauge, interpreter.StringStorageMapKey(stringKey)) + newValue := newStorageMap.ReadValue(nil, interpreter.StringStorageMapKey(stringKey)) err := cadenceValueEqual(oldRuntime.Interpreter, oldValue, newRuntime.Interpreter, newValue) if err != nil { @@ -274,25 +272,42 @@ func cadenceCompositeValueEqual( var err *validationError vFieldNames := make([]string, 0, 10) // v's field names - v.ForEachField(nopMemoryGauge, func(fieldName string, fieldValue interpreter.Value) bool { - otherFieldValue := otherComposite.GetField(otherInterpreter, interpreter.EmptyLocationRange, fieldName) + v.ForEachField( + vInterpreter, + func(fieldName string, fieldValue interpreter.Value) bool { + otherFieldValue := otherComposite.GetField( + otherInterpreter, + interpreter.EmptyLocationRange, + fieldName, + ) - err = cadenceValueEqual(vInterpreter, fieldValue, otherInterpreter, otherFieldValue) - if err != nil { - err.addTrace(fmt.Sprintf("(%s.%s)", v.TypeID(), fieldName)) - return false - } + err = cadenceValueEqual( + vInterpreter, + fieldValue, + otherInterpreter, + otherFieldValue, + ) + if err != nil { + err.addTrace(fmt.Sprintf("(%s.%s)", v.TypeID(), fieldName)) + return false + } - vFieldNames = append(vFieldNames, fieldName) - return true - }) + vFieldNames = append(vFieldNames, fieldName) + return true + }, + interpreter.EmptyLocationRange, + ) // TODO: Use CompositeValue.FieldCount() from Cadence after it is merged and available. otherFieldNames := make([]string, 0, len(vFieldNames)) // otherComposite's field names - otherComposite.ForEachField(nopMemoryGauge, func(fieldName string, _ interpreter.Value) bool { - otherFieldNames = append(otherFieldNames, fieldName) - return true - }) + otherComposite.ForEachField( + otherInterpreter, + func(fieldName string, _ interpreter.Value) bool { + otherFieldNames = append(otherFieldNames, fieldName) + return true + }, + interpreter.EmptyLocationRange, + ) if len(vFieldNames) != len(otherFieldNames) { return newValidationErrorf( @@ -327,7 +342,7 @@ func cadenceDictionaryValueEqual( oldIterator := v.Iterator() for { - key := oldIterator.NextKey(nopMemoryGauge) + key := oldIterator.NextKey(nil) if key == nil { break } @@ -370,7 +385,7 @@ func newReadonlyStorageRuntime(payloads []*ledger.Payload) ( readonlyLedger := util.NewPayloadsReadonlyLedger(snapshot) - storage := runtime.NewStorage(readonlyLedger, nopMemoryGauge) + storage := runtime.NewStorage(readonlyLedger, nil) env := runtime.NewBaseInterpreterEnvironment(runtime.Config{ // Attachments are enabled everywhere except for Mainnet diff --git a/cmd/util/ledger/migrations/migrator_runtime.go b/cmd/util/ledger/migrations/migrator_runtime.go index 92da7ff36f0..65d07e9d653 100644 --- a/cmd/util/ledger/migrations/migrator_runtime.go +++ b/cmd/util/ledger/migrations/migrator_runtime.go @@ -43,7 +43,7 @@ func NewMigratorRuntime( accounts := environment.NewAccounts(transactionState) accountsAtreeLedger := util.NewAccountsAtreeLedger(accounts) - runtimeStorage := runtime.NewStorage(accountsAtreeLedger, util.NopMemoryGauge{}) + runtimeStorage := runtime.NewStorage(accountsAtreeLedger, nil) derivedChainData, err := derived.NewDerivedChainData(derived.DefaultDerivedDataCacheSize) if err != nil { @@ -122,7 +122,8 @@ type migratorRuntime struct { var _ stdlib.AccountContractNamesProvider = &migratorRuntime{} func (mr *migratorRuntime) GetReadOnlyStorage() *runtime.Storage { - return runtime.NewStorage(util.NewPayloadsReadonlyLedger(mr.Snapshot), util.NopMemoryGauge{}) + readonlyLedger := util.NewPayloadsReadonlyLedger(mr.Snapshot) + return runtime.NewStorage(readonlyLedger, nil) } func (mr *migratorRuntime) GetAccountContractNames(address common.Address) ([]string, error) { diff --git a/cmd/util/ledger/reporters/fungible_token_tracker.go b/cmd/util/ledger/reporters/fungible_token_tracker.go index a84d8282b7a..175c533e358 100644 --- a/cmd/util/ledger/reporters/fungible_token_tracker.go +++ b/cmd/util/ledger/reporters/fungible_token_tracker.go @@ -219,6 +219,8 @@ func (r *FungibleTokenTracker) iterateChildren(tr trace, addr flow.Address, valu // continue iteration return true - }) + }, + interpreter.EmptyLocationRange, + ) } } diff --git a/cmd/util/ledger/util/util.go b/cmd/util/ledger/util/util.go index 71a353230ea..92a9c8e40c8 100644 --- a/cmd/util/ledger/util/util.go +++ b/cmd/util/ledger/util/util.go @@ -68,6 +68,38 @@ func (a *AccountsAtreeLedger) AllocateStorageIndex(owner []byte) (atree.StorageI return v, nil } +type PayloadSnapshot struct { + Payloads map[flow.RegisterID]*ledger.Payload +} + +var _ snapshot.StorageSnapshot = (*PayloadSnapshot)(nil) + +func NewPayloadSnapshot(payloads []*ledger.Payload) (*PayloadSnapshot, error) { + l := &PayloadSnapshot{ + Payloads: make(map[flow.RegisterID]*ledger.Payload, len(payloads)), + } + for _, payload := range payloads { + key, err := payload.Key() + if err != nil { + return nil, err + } + id, err := convert.LedgerKeyToRegisterID(key) + if err != nil { + return nil, err + } + l.Payloads[id] = payload + } + return l, nil +} + +func (p PayloadSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { + value, exists := p.Payloads[id] + if !exists { + return nil, nil + } + return value.Value(), nil +} + // NopMemoryGauge is a no-op implementation of the MemoryGauge interface type NopMemoryGauge struct{} From 9f3a8562251d2a061085b79caf6f6e49485cff87 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:17:56 -0500 Subject: [PATCH 06/14] Fix storage health check in Cadence 1.0 migration Currently, storage health check always fails with "slabs not referenced from account storage" because storage maps are not loaded in Cadence runtime storage even though payloads are loaded. This commit fixes this problem by loading storage map explicitly after loading payloads in storage. --- cmd/util/ledger/migrations/cadence_values_migration.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/util/ledger/migrations/cadence_values_migration.go b/cmd/util/ledger/migrations/cadence_values_migration.go index 4b8a6b3ccd8..9cfac91b916 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration.go +++ b/cmd/util/ledger/migrations/cadence_values_migration.go @@ -133,6 +133,11 @@ func (m *CadenceBaseMigrator) MigrateAccount( } } + // Load storage map. + for _, domain := range domains { + _ = storage.GetStorageMap(address, domain, false) + } + storageHealthErrorBefore = storage.CheckHealth() if storageHealthErrorBefore != nil { m.log.Warn(). From 450a6acd6ecc84775563979983626f9a3b01df43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 2 Apr 2024 15:56:59 -0700 Subject: [PATCH 07/14] switch contract update problems from failures to logged errors --- .../change_contract_code_migration_test.go | 151 +++++++++++++----- .../migrations/staged_contracts_migration.go | 34 ++-- .../staged_contracts_migration_test.go | 8 +- 3 files changed, 137 insertions(+), 56 deletions(-) diff --git a/cmd/util/ledger/migrations/change_contract_code_migration_test.go b/cmd/util/ledger/migrations/change_contract_code_migration_test.go index 53147a7bb70..c192e3ec5e3 100644 --- a/cmd/util/ledger/migrations/change_contract_code_migration_test.go +++ b/cmd/util/ledger/migrations/change_contract_code_migration_test.go @@ -2,10 +2,12 @@ package migrations_test import ( "context" + "io" "testing" "github.com/onflow/cadence/runtime/common" "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/cmd/util/ledger/migrations" @@ -43,13 +45,27 @@ access(all) contract B { access(all) fun bar() {} }` +type logWriter struct { + logs []string +} + +var _ io.Writer = &logWriter{} + +func (l *logWriter) Write(bytes []byte) (int, error) { + l.logs = append(l.logs, string(bytes)) + return len(bytes), nil +} + func TestChangeContractCodeMigration(t *testing.T) { t.Parallel() - address1, err := common.HexToAddress("0x1") + chainID := flow.Emulator + addressGenerator := chainID.Chain().NewAddressGenerator() + + address1, err := addressGenerator.NextAddress() require.NoError(t, err) - address2, err := common.HexToAddress("0x2") + address2, err := addressGenerator.NextAddress() require.NoError(t, err) ctx := context.Background() @@ -57,14 +73,16 @@ func TestChangeContractCodeMigration(t *testing.T) { t.Run("no contracts", func(t *testing.T) { t.Parallel() - log := zerolog.New(zerolog.NewTestWriter(t)) + writer := &logWriter{} + log := zerolog.New(writer) migration := migrations.NewChangeContractCodeMigration(flow.Emulator, log) err := migration.InitMigration(log, nil, 0) require.NoError(t, err) - _, err = migration.MigrateAccount(ctx, address1, + _, err = migration.MigrateAccount(ctx, + common.Address(address1), []*ledger.Payload{}, ) @@ -72,21 +90,30 @@ func TestChangeContractCodeMigration(t *testing.T) { err = migration.Close() require.NoError(t, err) + + require.Empty(t, writer.logs) }) t.Run("1 contract - dont migrate", func(t *testing.T) { t.Parallel() - log := zerolog.New(zerolog.NewTestWriter(t)) + writer := &logWriter{} + log := zerolog.New(writer) migration := migrations.NewChangeContractCodeMigration(flow.Emulator, log) err := migration.InitMigration(log, nil, 0) require.NoError(t, err) - payloads, err := migration.MigrateAccount(ctx, address1, + payloads, err := migration.MigrateAccount( + ctx, + common.Address(address1), []*ledger.Payload{ - newContractPayload(address1, "A", []byte(contractA)), + newContractPayload( + common.Address(address1), + "A", + []byte(contractA), + ), }, ) @@ -96,12 +123,15 @@ func TestChangeContractCodeMigration(t *testing.T) { err = migration.Close() require.NoError(t, err) + + require.Empty(t, writer.logs) }) t.Run("1 contract - migrate", func(t *testing.T) { t.Parallel() - log := zerolog.New(zerolog.NewTestWriter(t)) + writer := &logWriter{} + log := zerolog.New(writer) migration := migrations.NewChangeContractCodeMigration(flow.Emulator, log) @@ -110,7 +140,7 @@ func TestChangeContractCodeMigration(t *testing.T) { migration.RegisterContractChange( migrations.StagedContract{ - Address: address1, + Address: common.Address(address1), Contract: migrations.Contract{ Name: "A", Code: []byte(updatedContractA), @@ -118,9 +148,15 @@ func TestChangeContractCodeMigration(t *testing.T) { }, ) - payloads, err := migration.MigrateAccount(ctx, address1, + payloads, err := migration.MigrateAccount( + ctx, + common.Address(address1), []*ledger.Payload{ - newContractPayload(address1, "A", []byte(contractA)), + newContractPayload( + common.Address(address1), + "A", + []byte(contractA), + ), }, ) @@ -130,12 +166,15 @@ func TestChangeContractCodeMigration(t *testing.T) { err = migration.Close() require.NoError(t, err) + + require.Empty(t, writer.logs) }) t.Run("2 contracts - migrate 1", func(t *testing.T) { t.Parallel() - log := zerolog.New(zerolog.NewTestWriter(t)) + writer := &logWriter{} + log := zerolog.New(writer) migration := migrations.NewChangeContractCodeMigration(flow.Emulator, log) @@ -144,7 +183,7 @@ func TestChangeContractCodeMigration(t *testing.T) { migration.RegisterContractChange( migrations.StagedContract{ - Address: address1, + Address: common.Address(address1), Contract: migrations.Contract{ Name: "A", Code: []byte(updatedContractA), @@ -152,10 +191,12 @@ func TestChangeContractCodeMigration(t *testing.T) { }, ) - payloads, err := migration.MigrateAccount(ctx, address1, + payloads, err := migration.MigrateAccount( + ctx, + common.Address(address1), []*ledger.Payload{ - newContractPayload(address1, "A", []byte(contractA)), - newContractPayload(address1, "B", []byte(contractB)), + newContractPayload(common.Address(address1), "A", []byte(contractA)), + newContractPayload(common.Address(address1), "B", []byte(contractB)), }, ) @@ -166,12 +207,15 @@ func TestChangeContractCodeMigration(t *testing.T) { err = migration.Close() require.NoError(t, err) + + require.Empty(t, writer.logs) }) t.Run("2 contracts - migrate 2", func(t *testing.T) { t.Parallel() - log := zerolog.New(zerolog.NewTestWriter(t)) + writer := &logWriter{} + log := zerolog.New(writer) migration := migrations.NewChangeContractCodeMigration(flow.Emulator, log) @@ -180,7 +224,7 @@ func TestChangeContractCodeMigration(t *testing.T) { migration.RegisterContractChange( migrations.StagedContract{ - Address: address1, + Address: common.Address(address1), Contract: migrations.Contract{ Name: "A", Code: []byte(updatedContractA), @@ -189,7 +233,7 @@ func TestChangeContractCodeMigration(t *testing.T) { ) migration.RegisterContractChange( migrations.StagedContract{ - Address: address1, + Address: common.Address(address1), Contract: migrations.Contract{ Name: "B", Code: []byte(updatedContractB), @@ -197,10 +241,12 @@ func TestChangeContractCodeMigration(t *testing.T) { }, ) - payloads, err := migration.MigrateAccount(ctx, address1, + payloads, err := migration.MigrateAccount( + ctx, + common.Address(address1), []*ledger.Payload{ - newContractPayload(address1, "A", []byte(contractA)), - newContractPayload(address1, "B", []byte(contractB)), + newContractPayload(common.Address(address1), "A", []byte(contractA)), + newContractPayload(common.Address(address1), "B", []byte(contractB)), }, ) @@ -211,12 +257,15 @@ func TestChangeContractCodeMigration(t *testing.T) { err = migration.Close() require.NoError(t, err) + + require.Empty(t, writer.logs) }) t.Run("2 contracts on different accounts - migrate 1", func(t *testing.T) { t.Parallel() - log := zerolog.New(zerolog.NewTestWriter(t)) + writer := &logWriter{} + log := zerolog.New(writer) migration := migrations.NewChangeContractCodeMigration(flow.Emulator, log) @@ -225,7 +274,7 @@ func TestChangeContractCodeMigration(t *testing.T) { migration.RegisterContractChange( migrations.StagedContract{ - Address: address1, + Address: common.Address(address1), Contract: migrations.Contract{ Name: "A", Code: []byte(updatedContractA), @@ -233,10 +282,12 @@ func TestChangeContractCodeMigration(t *testing.T) { }, ) - payloads, err := migration.MigrateAccount(ctx, address1, + payloads, err := migration.MigrateAccount( + ctx, + common.Address(address1), []*ledger.Payload{ - newContractPayload(address1, "A", []byte(contractA)), - newContractPayload(address2, "A", []byte(contractA)), + newContractPayload(common.Address(address1), "A", []byte(contractA)), + newContractPayload(common.Address(address2), "A", []byte(contractA)), }, ) @@ -247,12 +298,19 @@ func TestChangeContractCodeMigration(t *testing.T) { err = migration.Close() require.NoError(t, err) + + require.Len(t, writer.logs, 1) + assert.Contains(t, + writer.logs[0], + "payload address ee82856bf20e2aa6 does not match expected address f8d6e0586b0a20c7", + ) }) t.Run("not all contracts on one account migrated", func(t *testing.T) { t.Parallel() - log := zerolog.New(zerolog.NewTestWriter(t)) + writer := &logWriter{} + log := zerolog.New(writer) migration := migrations.NewChangeContractCodeMigration(flow.Emulator, log) @@ -261,7 +319,7 @@ func TestChangeContractCodeMigration(t *testing.T) { migration.RegisterContractChange( migrations.StagedContract{ - Address: address1, + Address: common.Address(address1), Contract: migrations.Contract{ Name: "A", Code: []byte(updatedContractA), @@ -270,7 +328,7 @@ func TestChangeContractCodeMigration(t *testing.T) { ) migration.RegisterContractChange( migrations.StagedContract{ - Address: address1, + Address: common.Address(address1), Contract: migrations.Contract{ Name: "B", Code: []byte(updatedContractB), @@ -278,19 +336,28 @@ func TestChangeContractCodeMigration(t *testing.T) { }, ) - _, err = migration.MigrateAccount(ctx, address1, + _, err = migration.MigrateAccount( + ctx, + common.Address(address1), []*ledger.Payload{ - newContractPayload(address1, "A", []byte(contractA)), + newContractPayload(common.Address(address1), "A", []byte(contractA)), }, ) - require.Error(t, err) + require.NoError(t, err) + + require.Len(t, writer.logs, 1) + assert.Contains(t, + writer.logs[0], + `"failed to find all contract registers that need to be changed for address"`, + ) }) t.Run("not all accounts migrated", func(t *testing.T) { t.Parallel() - log := zerolog.New(zerolog.NewTestWriter(t)) + writer := &logWriter{} + log := zerolog.New(writer) migration := migrations.NewChangeContractCodeMigration(flow.Emulator, log) @@ -299,7 +366,7 @@ func TestChangeContractCodeMigration(t *testing.T) { migration.RegisterContractChange( migrations.StagedContract{ - Address: address2, + Address: common.Address(address2), Contract: migrations.Contract{ Name: "A", Code: []byte(updatedContractA), @@ -307,15 +374,23 @@ func TestChangeContractCodeMigration(t *testing.T) { }, ) - _, err = migration.MigrateAccount(ctx, address1, + _, err = migration.MigrateAccount( + ctx, + common.Address(address1), []*ledger.Payload{ - newContractPayload(address1, "A", []byte(contractA)), + newContractPayload(common.Address(address1), "A", []byte(contractA)), }, ) require.NoError(t, err) err = migration.Close() - require.Error(t, err) + require.NoError(t, err) + + require.Len(t, writer.logs, 1) + assert.Contains(t, + writer.logs[0], + `"failed to find all contract registers that need to be changed"`, + ) }) } diff --git a/cmd/util/ledger/migrations/staged_contracts_migration.go b/cmd/util/ledger/migrations/staged_contracts_migration.go index b3f4b02ea22..95457b9d357 100644 --- a/cmd/util/ledger/migrations/staged_contracts_migration.go +++ b/cmd/util/ledger/migrations/staged_contracts_migration.go @@ -3,14 +3,11 @@ package migrations import ( "context" "encoding/csv" - "fmt" "io" "os" "strings" "sync" - "github.com/rs/zerolog" - "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" @@ -18,6 +15,7 @@ import ( "github.com/onflow/cadence/runtime/pretty" "github.com/onflow/cadence/runtime/sema" "github.com/onflow/cadence/runtime/stdlib" + "github.com/rs/zerolog" "github.com/onflow/flow-go/cmd/util/ledger/util" "github.com/onflow/flow-go/ledger" @@ -79,16 +77,20 @@ func (m *StagedContractsMigration) Close() error { defer m.mutex.RUnlock() if len(m.stagedContracts) > 0 { - var sb strings.Builder - sb.WriteString("failed to find all contract registers that need to be changed:\n") + dict := zerolog.Dict() for address, contracts := range m.stagedContracts { - _, _ = fmt.Fprintf(&sb, "- address: %s\n", address.HexWithPrefix()) + arr := zerolog.Arr() for registerID := range contracts { - _, _ = fmt.Fprintf(&sb, " - %s\n", flow.RegisterIDContractName(registerID)) + arr = arr.Str(flow.RegisterIDContractName(registerID)) } + dict = dict.Array( + address.HexWithPrefix(), + arr, + ) } - return fmt.Errorf(sb.String()) - + m.log.Error(). + Dict("contracts", dict). + Msg("failed to find all contract registers that need to be changed") } return nil @@ -291,16 +293,14 @@ func (m *StagedContractsMigration) MigrateAccount( } if len(contractUpdates) > 0 { - var sb strings.Builder - _, _ = fmt.Fprintf( - &sb, - "failed to find all contract registers that need to be changed for address %s:\n", - address.HexWithPrefix(), - ) + arr := zerolog.Arr() for registerID := range contractUpdates { - _, _ = fmt.Fprintf(&sb, " - %s\n", flow.RegisterIDContractName(registerID)) + arr = arr.Str(flow.RegisterIDContractName(registerID)) } - return nil, fmt.Errorf(sb.String()) + m.log.Error(). + Array("contracts", arr). + Str("address", address.HexWithPrefix()). + Msg("failed to find all contract registers that need to be changed for address") } return oldPayloads, nil diff --git a/cmd/util/ledger/migrations/staged_contracts_migration_test.go b/cmd/util/ledger/migrations/staged_contracts_migration_test.go index 4ff19f94bce..d8ce53e592b 100644 --- a/cmd/util/ledger/migrations/staged_contracts_migration_test.go +++ b/cmd/util/ledger/migrations/staged_contracts_migration_test.go @@ -397,7 +397,13 @@ func TestStagedContractsMigration(t *testing.T) { common.Address(address1), nil, ) - require.ErrorContains(t, err, "failed to find all contract registers that need to be changed") + require.NoError(t, err) + + require.Len(t, logWriter.logs, 1) + assert.Contains(t, + logWriter.logs[0], + `"failed to find all contract registers that need to be changed for address"`, + ) }) } From 1de9f75ecb76a1acef31eb91df66d2cbdd154261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 2 Apr 2024 16:10:55 -0700 Subject: [PATCH 08/14] fix storage health check --- cmd/util/ledger/migrations/cadence.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/util/ledger/migrations/cadence.go b/cmd/util/ledger/migrations/cadence.go index 3f620b3d123..e011e1f0554 100644 --- a/cmd/util/ledger/migrations/cadence.go +++ b/cmd/util/ledger/migrations/cadence.go @@ -433,8 +433,8 @@ func NewCadence1Migrations( nWorker, chainID, diffMigrations, - checkStorageHealthBeforeMigration, logVerboseDiff, + checkStorageHealthBeforeMigration, )..., ) From e47d0485b08a029ffabe38bc80a8d17c836df922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 2 Apr 2024 16:45:34 -0700 Subject: [PATCH 09/14] clean up options --- cmd/util/cmd/execution-state-extract/cmd.go | 35 +++--- .../execution_state_extract.go | 71 ++--------- .../execution_state_extract_test.go | 21 ++-- cmd/util/ledger/migrations/cadence.go | 110 ++++++++---------- .../migrations/cadence_values_migration.go | 30 ++--- .../cadence_values_migration_test.go | 65 ++++------- .../staged_contracts_migration_test.go | 17 ++- 7 files changed, 124 insertions(+), 225 deletions(-) diff --git a/cmd/util/cmd/execution-state-extract/cmd.go b/cmd/util/cmd/execution-state-extract/cmd.go index fc25f93d653..6665bcc3031 100644 --- a/cmd/util/cmd/execution-state-extract/cmd.go +++ b/cmd/util/cmd/execution-state-extract/cmd.go @@ -181,7 +181,7 @@ func run(*cobra.Command, []string) { cache := &metrics.NoopCollector{} commits := badger.NewCommits(cache, db) - stateCommitment, err = getStateCommitment(commits, blockID) + stateCommitment, err = commits.ByBlockID(blockID) if err != nil { log.Fatal().Err(err).Msgf("cannot get state commitment for block %v", blockID) } @@ -340,6 +340,19 @@ func run(*cobra.Command, []string) { log.Fatal().Err(err).Msgf("error loading staged contracts: %s", err.Error()) } + opts := migrations.Options{ + NWorker: flagNWorker, + DiffMigrations: flagDiffMigration, + LogVerboseDiff: flagLogVerboseDiff, + CheckStorageHealthBeforeMigration: flagCheckStorageHealthBeforeMigration, + ChainID: chainID, + EVMContractChange: evmContractChange, + BurnerContractChange: burnerContractChange, + StagedContracts: stagedContracts, + Prune: flagPrune, + MaxAccountSize: flagMaxAccountSize, + } + if len(flagInputPayloadFileName) > 0 { err = extractExecutionStateFromPayloads( log.Logger, @@ -347,19 +360,11 @@ func run(*cobra.Command, []string) { flagOutputDir, flagNWorker, !flagNoMigration, - flagDiffMigration, - flagLogVerboseDiff, - flagCheckStorageHealthBeforeMigration, - chainID, - evmContractChange, - burnerContractChange, - stagedContracts, flagInputPayloadFileName, flagOutputPayloadFileName, exportedAddresses, flagSortPayloads, - flagPrune, - flagMaxAccountSize, + opts, ) } else { err = extractExecutionState( @@ -369,18 +374,10 @@ func run(*cobra.Command, []string) { flagOutputDir, flagNWorker, !flagNoMigration, - flagDiffMigration, - flagLogVerboseDiff, - flagCheckStorageHealthBeforeMigration, - chainID, - evmContractChange, - burnerContractChange, - stagedContracts, flagOutputPayloadFileName, exportedAddresses, flagSortPayloads, - flagPrune, - flagMaxAccountSize, + opts, ) } diff --git a/cmd/util/cmd/execution-state-extract/execution_state_extract.go b/cmd/util/cmd/execution-state-extract/execution_state_extract.go index 36f884c312c..638cd87e5e7 100644 --- a/cmd/util/cmd/execution-state-extract/execution_state_extract.go +++ b/cmd/util/cmd/execution-state-extract/execution_state_extract.go @@ -23,13 +23,8 @@ import ( "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/storage" ) -func getStateCommitment(commits storage.Commits, blockHash flow.Identifier) (flow.StateCommitment, error) { - return commits.ByBlockID(blockHash) -} - func extractExecutionState( log zerolog.Logger, dir string, @@ -37,18 +32,10 @@ func extractExecutionState( outputDir string, nWorker int, // number of concurrent worker to migration payloads runMigrations bool, - diffMigrations bool, - logVerboseDiff bool, - checkStorageHealthBeforeMigration bool, - chainID flow.ChainID, - evmContractChange migrators.EVMContractChange, - burnerContractChange migrators.BurnerContractChange, - stagedContracts []migrators.StagedContract, outputPayloadFile string, exportPayloadsByAddresses []common.Address, sortPayloads bool, - prune bool, - maxAccountSize uint64, + opts migrators.Options, ) error { log.Info().Msg("init WAL") @@ -111,17 +98,8 @@ func extractExecutionState( migrations := newMigrations( log, dir, - nWorker, runMigrations, - diffMigrations, - logVerboseDiff, - checkStorageHealthBeforeMigration, - chainID, - evmContractChange, - burnerContractChange, - stagedContracts, - prune, - maxAccountSize, + opts, ) newState := ledger.State(targetHash) @@ -227,21 +205,13 @@ func extractExecutionStateFromPayloads( log zerolog.Logger, dir string, outputDir string, - nWorker int, // number of concurrent worker to migation payloads + nWorker int, // number of concurrent worker to migration payloads runMigrations bool, - diffMigrations bool, - logVerboseDiff bool, - checkStorageHealthBeforeMigration bool, - chainID flow.ChainID, - evmContractChange migrators.EVMContractChange, - burnerContractChange migrators.BurnerContractChange, - stagedContracts []migrators.StagedContract, inputPayloadFile string, outputPayloadFile string, exportPayloadsByAddresses []common.Address, sortPayloads bool, - prune bool, - maxAccountSize uint64, + opts migrators.Options, ) error { inputPayloadsFromPartialState, payloads, err := util.ReadPayloadFile(log, inputPayloadFile) @@ -254,17 +224,8 @@ func extractExecutionStateFromPayloads( migrations := newMigrations( log, dir, - nWorker, runMigrations, - diffMigrations, - logVerboseDiff, - checkStorageHealthBeforeMigration, - chainID, - evmContractChange, - burnerContractChange, - stagedContracts, - prune, - maxAccountSize, + opts, ) payloads, err = migratePayloads(log, payloads, migrations) @@ -412,17 +373,8 @@ func createTrieFromPayloads(logger zerolog.Logger, payloads []*ledger.Payload) ( func newMigrations( log zerolog.Logger, dir string, - nWorker int, runMigrations bool, - diffMigrations bool, - logVerboseDiff bool, - checkStorageHealthBeforeMigration bool, - chainID flow.ChainID, - evmContractChange migrators.EVMContractChange, - burnerContractChange migrators.BurnerContractChange, - stagedContracts []migrators.StagedContract, - prune bool, - maxAccountSize uint64, + opts migrators.Options, ) []ledger.Migration { if !runMigrations { return nil @@ -435,16 +387,7 @@ func newMigrations( namedMigrations := migrators.NewCadence1Migrations( log, rwf, - nWorker, - chainID, - diffMigrations, - logVerboseDiff, - checkStorageHealthBeforeMigration, - evmContractChange, - burnerContractChange, - stagedContracts, - prune, - maxAccountSize, + opts, ) migrations := make([]ledger.Migration, 0, len(namedMigrations)) diff --git a/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go b/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go index f65e1d90cd8..738c251543a 100644 --- a/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go +++ b/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go @@ -41,7 +41,7 @@ func TestExtractExecutionState(t *testing.T) { db := common.InitStorage(datadir) commits := badger.NewCommits(metr, db) - _, err := getStateCommitment(commits, unittest.IdentifierFixture()) + _, err := commits.ByBlockID(unittest.IdentifierFixture()) require.Error(t, err) }) }) @@ -58,7 +58,7 @@ func TestExtractExecutionState(t *testing.T) { err := commits.Store(blockID, stateCommitment) require.NoError(t, err) - retrievedStateCommitment, err := getStateCommitment(commits, blockID) + retrievedStateCommitment, err := commits.ByBlockID(blockID) require.NoError(t, err) require.Equal(t, stateCommitment, retrievedStateCommitment) }) @@ -66,6 +66,13 @@ func TestExtractExecutionState(t *testing.T) { t.Run("empty WAL doesn't find anything", func(t *testing.T) { withDirs(t, func(datadir, execdir, outdir string) { + opts := migrations.Options{ + NWorker: 10, + ChainID: flow.Emulator, + EVMContractChange: migrations.EVMContractChangeNone, + BurnerContractChange: migrations.BurnerContractChangeDeploy, + } + err := extractExecutionState( zerolog.Nop(), execdir, @@ -73,18 +80,10 @@ func TestExtractExecutionState(t *testing.T) { outdir, 10, false, - false, - false, - false, - flow.Emulator, - migrations.EVMContractChangeNone, - migrations.BurnerContractChangeDeploy, - nil, "", nil, false, - false, - 0, + opts, ) require.Error(t, err) }) diff --git a/cmd/util/ledger/migrations/cadence.go b/cmd/util/ledger/migrations/cadence.go index e011e1f0554..91e4425239f 100644 --- a/cmd/util/ledger/migrations/cadence.go +++ b/cmd/util/ledger/migrations/cadence.go @@ -218,11 +218,7 @@ type NamedMigration struct { func NewCadence1ValueMigrations( log zerolog.Logger, rwf reporters.ReportWriterFactory, - nWorker int, - chainID flow.ChainID, - diffMigrations bool, - logVerboseDiff bool, - checkStorageHealthBeforeMigration bool, + opts Options, ) (migrations []NamedMigration) { // Populated by CadenceLinkValueMigrator, @@ -247,46 +243,41 @@ func NewCadence1ValueMigrations( }, } - for index, migrationConstructor := range []func(checkStorageHealthBeforeMigration bool) *CadenceBaseMigrator{ - func(checkStorageHealthBeforeMigration bool) *CadenceBaseMigrator { + for index, migrationConstructor := range []func(opts Options) *CadenceBaseMigrator{ + func(opts Options) *CadenceBaseMigrator { return NewCadence1ValueMigrator( rwf, - diffMigrations, - logVerboseDiff, - checkStorageHealthBeforeMigration, errorMessageHandler, contracts, - NewCadence1CompositeStaticTypeConverter(chainID), - NewCadence1InterfaceStaticTypeConverter(chainID), + NewCadence1CompositeStaticTypeConverter(opts.ChainID), + NewCadence1InterfaceStaticTypeConverter(opts.ChainID), + opts, ) }, - func(checkStorageHealthBeforeMigration bool) *CadenceBaseMigrator { + func(opts Options) *CadenceBaseMigrator { return NewCadence1LinkValueMigrator( rwf, - diffMigrations, - logVerboseDiff, - checkStorageHealthBeforeMigration, errorMessageHandler, contracts, capabilityMapping, + opts, ) }, - func(checkStorageHealthBeforeMigration bool) *CadenceBaseMigrator { + func(opts Options) *CadenceBaseMigrator { return NewCadence1CapabilityValueMigrator( rwf, - diffMigrations, - logVerboseDiff, - checkStorageHealthBeforeMigration, errorMessageHandler, contracts, capabilityMapping, + opts, ) }, } { - accountBasedMigration := migrationConstructor( - // Only check storage health before the first migration - checkStorageHealthBeforeMigration && index == 0, - ) + opts := opts + // Only check storage health before the first migration + opts.CheckStorageHealthBeforeMigration = opts.CheckStorageHealthBeforeMigration && index == 0 + + accountBasedMigration := migrationConstructor(opts) migrations = append( migrations, @@ -294,7 +285,8 @@ func NewCadence1ValueMigrations( Name: accountBasedMigration.name, Migrate: NewAccountBasedMigration( log, - nWorker, []AccountBasedMigration{ + opts.NWorker, + []AccountBasedMigration{ accountBasedMigration, }, ), @@ -307,31 +299,27 @@ func NewCadence1ValueMigrations( func NewCadence1ContractsMigrations( log zerolog.Logger, - nWorker int, - chainID flow.ChainID, - evmContractChange EVMContractChange, - burnerContractChange BurnerContractChange, - stagedContracts []StagedContract, + opts Options, ) []NamedMigration { systemContractsMigration := NewSystemContractsMigration( - chainID, + opts.ChainID, log, SystemContractChangesOptions{ - EVM: evmContractChange, - Burner: burnerContractChange, + EVM: opts.EVMContractChange, + Burner: opts.BurnerContractChange, }, ) - stagedContractsMigration := NewStagedContractsMigration(chainID, log). + stagedContractsMigration := NewStagedContractsMigration(opts.ChainID, log). WithContractUpdateValidation() - stagedContractsMigration.RegisterContractUpdates(stagedContracts) + stagedContractsMigration.RegisterContractUpdates(opts.StagedContracts) toAccountBasedMigration := func(migration AccountBasedMigration) ledger.Migration { return NewAccountBasedMigration( log, - nWorker, + opts.NWorker, []AccountBasedMigration{ migration, }, @@ -340,12 +328,12 @@ func NewCadence1ContractsMigrations( var migrations []NamedMigration - if burnerContractChange == BurnerContractChangeDeploy { + if opts.BurnerContractChange == BurnerContractChangeDeploy { migrations = append( migrations, NamedMigration{ Name: "burner-deployment-migration", - Migrate: NewBurnerDeploymentMigration(chainID, log), + Migrate: NewBurnerDeploymentMigration(opts.ChainID, log), }, ) } @@ -365,28 +353,32 @@ func NewCadence1ContractsMigrations( return migrations } +type Options struct { + NWorker int + DiffMigrations bool + LogVerboseDiff bool + CheckStorageHealthBeforeMigration bool + ChainID flow.ChainID + EVMContractChange EVMContractChange + BurnerContractChange BurnerContractChange + StagedContracts []StagedContract + Prune bool + MaxAccountSize uint64 +} + func NewCadence1Migrations( log zerolog.Logger, rwf reporters.ReportWriterFactory, - nWorker int, - chainID flow.ChainID, - diffMigrations bool, - logVerboseDiff bool, - checkStorageHealthBeforeMigration bool, - evmContractChange EVMContractChange, - burnerContractChange BurnerContractChange, - stagedContracts []StagedContract, - prune bool, - maxAccountSize uint64, + opts Options, ) []NamedMigration { var migrations []NamedMigration - if maxAccountSize > 0 { + if opts.MaxAccountSize > 0 { maxSizeExceptions := map[string]struct{}{} - systemContracts := systemcontracts.SystemContractsForChain(chainID) + systemContracts := systemcontracts.SystemContractsForChain(opts.ChainID) for _, systemContract := range systemContracts.All() { maxSizeExceptions[string(systemContract.Address.Bytes())] = struct{}{} } @@ -395,13 +387,13 @@ func NewCadence1Migrations( migrations, NamedMigration{ Name: "account-size-filter-migration", - Migrate: NewAccountSizeFilterMigration(maxAccountSize, maxSizeExceptions, log), + Migrate: NewAccountSizeFilterMigration(opts.MaxAccountSize, maxSizeExceptions, log), }, ) } - if prune { - migration := NewCadence1PruneMigration(chainID, log) + if opts.Prune { + migration := NewCadence1PruneMigration(opts.ChainID, log) if migration != nil { migrations = append( migrations, @@ -417,11 +409,7 @@ func NewCadence1Migrations( migrations, NewCadence1ContractsMigrations( log, - nWorker, - chainID, - evmContractChange, - burnerContractChange, - stagedContracts, + opts, )..., ) @@ -430,11 +418,7 @@ func NewCadence1Migrations( NewCadence1ValueMigrations( log, rwf, - nWorker, - chainID, - diffMigrations, - logVerboseDiff, - checkStorageHealthBeforeMigration, + opts, )..., ) diff --git a/cmd/util/ledger/migrations/cadence_values_migration.go b/cmd/util/ledger/migrations/cadence_values_migration.go index 9cfac91b916..92062b2f7bb 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration.go +++ b/cmd/util/ledger/migrations/cadence_values_migration.go @@ -251,17 +251,15 @@ func checkPayloadOwnership(payload *ledger.Payload, address common.Address, log // which runs some of the Cadence value migrations (static types, entitlements, strings) func NewCadence1ValueMigrator( rwf reporters.ReportWriterFactory, - diffMigrations bool, - logVerboseDiff bool, - checkStorageHealthBeforeMigration bool, errorMessageHandler *errorMessageHandler, contracts map[common.AddressLocation][]byte, compositeTypeConverter statictypes.CompositeTypeConverterFunc, interfaceTypeConverter statictypes.InterfaceTypeConverterFunc, + opts Options, ) *CadenceBaseMigrator { var diffReporter reporters.ReportWriter - if diffMigrations { + if opts.DiffMigrations { diffReporter = rwf.ReportWriter("cadence-value-migration-diff") } @@ -269,8 +267,8 @@ func NewCadence1ValueMigrator( name: "cadence-value-migration", reporter: rwf.ReportWriter("cadence-value-migrator"), diffReporter: diffReporter, - logVerboseDiff: logVerboseDiff, - checkStorageHealthBeforeMigration: checkStorageHealthBeforeMigration, + logVerboseDiff: opts.LogVerboseDiff, + checkStorageHealthBeforeMigration: opts.CheckStorageHealthBeforeMigration, valueMigrations: func( inter *interpreter.Interpreter, _ environment.Accounts, @@ -294,15 +292,13 @@ func NewCadence1ValueMigrator( // It populates the given map with the IDs of the capability controller it issues. func NewCadence1LinkValueMigrator( rwf reporters.ReportWriterFactory, - diffMigrations bool, - logVerboseDiff bool, - checkStorageHealthBeforeMigration bool, errorMessageHandler *errorMessageHandler, contracts map[common.AddressLocation][]byte, capabilityMapping *capcons.CapabilityMapping, + opts Options, ) *CadenceBaseMigrator { var diffReporter reporters.ReportWriter - if diffMigrations { + if opts.DiffMigrations { diffReporter = rwf.ReportWriter("cadence-link-value-migration-diff") } @@ -310,8 +306,8 @@ func NewCadence1LinkValueMigrator( name: "cadence-link-value-migration", reporter: rwf.ReportWriter("cadence-link-value-migrator"), diffReporter: diffReporter, - logVerboseDiff: logVerboseDiff, - checkStorageHealthBeforeMigration: checkStorageHealthBeforeMigration, + logVerboseDiff: opts.LogVerboseDiff, + checkStorageHealthBeforeMigration: opts.CheckStorageHealthBeforeMigration, valueMigrations: func( _ *interpreter.Interpreter, accounts environment.Accounts, @@ -341,15 +337,13 @@ func NewCadence1LinkValueMigrator( // generated by the link value migration. func NewCadence1CapabilityValueMigrator( rwf reporters.ReportWriterFactory, - diffMigrations bool, - logVerboseDiff bool, - checkStorageHealthBeforeMigration bool, errorMessageHandler *errorMessageHandler, contracts map[common.AddressLocation][]byte, capabilityMapping *capcons.CapabilityMapping, + opts Options, ) *CadenceBaseMigrator { var diffReporter reporters.ReportWriter - if diffMigrations { + if opts.DiffMigrations { diffReporter = rwf.ReportWriter("cadence-capability-value-migration-diff") } @@ -357,8 +351,8 @@ func NewCadence1CapabilityValueMigrator( name: "cadence-capability-value-migration", reporter: rwf.ReportWriter("cadence-capability-value-migrator"), diffReporter: diffReporter, - logVerboseDiff: logVerboseDiff, - checkStorageHealthBeforeMigration: checkStorageHealthBeforeMigration, + logVerboseDiff: opts.LogVerboseDiff, + checkStorageHealthBeforeMigration: opts.CheckStorageHealthBeforeMigration, valueMigrations: func( _ *interpreter.Interpreter, _ environment.Accounts, diff --git a/cmd/util/ledger/migrations/cadence_values_migration_test.go b/cmd/util/ledger/migrations/cadence_values_migration_test.go index f34d44c2a4b..812b14343b1 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration_test.go +++ b/cmd/util/ledger/migrations/cadence_values_migration_test.go @@ -80,16 +80,13 @@ func TestCadenceValuesMigration(t *testing.T) { migrations := NewCadence1Migrations( logger, rwf, - nWorker, - chainID, - false, - false, - false, - evmContractChange, - burnerContractChange, - stagedContracts, - false, - 0, + Options{ + NWorker: nWorker, + ChainID: chainID, + EVMContractChange: evmContractChange, + BurnerContractChange: burnerContractChange, + StagedContracts: stagedContracts, + }, ) for _, migration := range migrations { @@ -699,16 +696,12 @@ func TestBootstrappedStateMigration(t *testing.T) { migrations := NewCadence1Migrations( logger, rwf, - nWorker, - chainID, - false, - false, - false, - evmContractChange, - burnerContractChange, - nil, - false, - 0, + Options{ + NWorker: nWorker, + ChainID: chainID, + EVMContractChange: evmContractChange, + BurnerContractChange: burnerContractChange, + }, ) for _, migration := range migrations { @@ -826,16 +819,12 @@ func TestProgramParsingError(t *testing.T) { migrations := NewCadence1Migrations( logger, rwf, - nWorker, - chainID, - false, - false, - false, - evmContractChange, - burnerContractChange, - nil, - false, - 0, + Options{ + NWorker: nWorker, + ChainID: chainID, + EVMContractChange: evmContractChange, + BurnerContractChange: burnerContractChange, + }, ) for _, migration := range migrations { @@ -952,16 +941,12 @@ func TestCoreContractUsage(t *testing.T) { migrations := NewCadence1Migrations( logger, rwf, - nWorker, - chainID, - false, - false, - false, - evmContractChange, - burnerContractChange, - nil, - false, - 0, + Options{ + NWorker: nWorker, + ChainID: chainID, + EVMContractChange: evmContractChange, + BurnerContractChange: burnerContractChange, + }, ) for _, migration := range migrations { diff --git a/cmd/util/ledger/migrations/staged_contracts_migration_test.go b/cmd/util/ledger/migrations/staged_contracts_migration_test.go index d8ce53e592b..39422af68c9 100644 --- a/cmd/util/ledger/migrations/staged_contracts_migration_test.go +++ b/cmd/util/ledger/migrations/staged_contracts_migration_test.go @@ -1478,16 +1478,13 @@ func TestConcurrentContractUpdate(t *testing.T) { migrations := NewCadence1Migrations( logger, rwf, - nWorker, - chainID, - false, - false, - false, - evmContractChange, - burnerContractChange, - stagedContracts, - false, - 0, + Options{ + NWorker: nWorker, + ChainID: chainID, + EVMContractChange: evmContractChange, + BurnerContractChange: burnerContractChange, + StagedContracts: stagedContracts, + }, ) for _, migration := range migrations { From 9a3ce016c65aca829757efc2a7ab5476d14b684e Mon Sep 17 00:00:00 2001 From: Jordan Ribbink Date: Fri, 5 Apr 2024 08:21:54 -0700 Subject: [PATCH 10/14] Export user defined type change checker --- cmd/util/ledger/migrations/staged_contracts_migration.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/util/ledger/migrations/staged_contracts_migration.go b/cmd/util/ledger/migrations/staged_contracts_migration.go index 95457b9d357..dad890389d7 100644 --- a/cmd/util/ledger/migrations/staged_contracts_migration.go +++ b/cmd/util/ledger/migrations/staged_contracts_migration.go @@ -63,7 +63,7 @@ func NewStagedContractsMigration(chainID flow.ChainID, log zerolog.Logger) *Stag func (m *StagedContractsMigration) WithContractUpdateValidation() *StagedContractsMigration { m.enableUpdateValidation = true - m.userDefinedTypeChangeCheckFunc = newUserDefinedTypeChangeCheckerFunc(m.chainID) + m.userDefinedTypeChangeCheckFunc = NewUserDefinedTypeChangeCheckerFunc(m.chainID) return m } @@ -403,7 +403,7 @@ func StagedContractsFromCSV(path string) ([]StagedContract, error) { return contracts, nil } -func newUserDefinedTypeChangeCheckerFunc( +func NewUserDefinedTypeChangeCheckerFunc( chainID flow.ChainID, ) func(oldTypeID common.TypeID, newTypeID common.TypeID) (checked, valid bool) { From 201861a5c9262bc071e52489e0a2d8d8b6d3488e Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 9 Apr 2024 15:23:41 +0200 Subject: [PATCH 11/14] cleanup --- .../migrations/account_storage_migration.go | 2 + .../migrations/atree_register_migration.go | 11 +-- .../migrations/cadence_value_diff_test.go | 6 ++ .../migrations/cadence_value_validation.go | 3 +- .../cadence_value_validation_test.go | 3 + .../migrations/cadence_values_migration.go | 2 + .../cadence_values_migration_test.go | 7 +- .../ledger/migrations/migrator_runtime.go | 21 +++--- .../migrations/staged_contracts_migration.go | 7 +- .../migrations/transaction_migration.go | 4 +- .../util/{ => snapshot}/payload_snapshot.go | 67 +++++++++++++------ .../{ => snapshot}/payload_snapshot_test.go | 22 +++--- cmd/util/ledger/util/util.go | 5 +- 13 files changed, 108 insertions(+), 52 deletions(-) rename cmd/util/ledger/util/{ => snapshot}/payload_snapshot.go (75%) rename cmd/util/ledger/util/{ => snapshot}/payload_snapshot_test.go (85%) diff --git a/cmd/util/ledger/migrations/account_storage_migration.go b/cmd/util/ledger/migrations/account_storage_migration.go index 8bd2c527dfb..fb503929a80 100644 --- a/cmd/util/ledger/migrations/account_storage_migration.go +++ b/cmd/util/ledger/migrations/account_storage_migration.go @@ -9,6 +9,7 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" ) @@ -24,6 +25,7 @@ func NewAccountStorageMigration( address, payloads, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) if err != nil { return nil, fmt.Errorf("failed to create migrator runtime: %w", err) diff --git a/cmd/util/ledger/migrations/atree_register_migration.go b/cmd/util/ledger/migrations/atree_register_migration.go index 6a57fd53aa0..831cda34bf2 100644 --- a/cmd/util/ledger/migrations/atree_register_migration.go +++ b/cmd/util/ledger/migrations/atree_register_migration.go @@ -18,6 +18,7 @@ import ( "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" @@ -87,7 +88,7 @@ func (m *AtreeRegisterMigrator) MigrateAccount( oldPayloads []*ledger.Payload, ) ([]*ledger.Payload, error) { // create all the runtime components we need for the migration - mr, err := NewMigratorRuntime(address, oldPayloads, util.RuntimeInterfaceConfig{}) + mr, err := NewMigratorRuntime(address, oldPayloads, util.RuntimeInterfaceConfig{}, snapshot.MapBased) if err != nil { return nil, fmt.Errorf("failed to create migrator runtime: %w", err) } @@ -139,7 +140,7 @@ func (m *AtreeRegisterMigrator) MigrateAccount( } func (m *AtreeRegisterMigrator) migrateAccountStorage( - mr *migratorRuntime, + mr *MigratorRuntime, storageMapIds map[string]struct{}, ) (map[flow.RegisterID]flow.RegisterValue, error) { @@ -166,7 +167,7 @@ func (m *AtreeRegisterMigrator) migrateAccountStorage( } func (m *AtreeRegisterMigrator) convertStorageDomain( - mr *migratorRuntime, + mr *MigratorRuntime, storageMapIds map[string]struct{}, domain string, ) error { @@ -240,7 +241,7 @@ func (m *AtreeRegisterMigrator) convertStorageDomain( } func (m *AtreeRegisterMigrator) validateChangesAndCreateNewRegisters( - mr *migratorRuntime, + mr *MigratorRuntime, changes map[flow.RegisterID]flow.RegisterValue, storageMapIds map[string]struct{}, ) ([]*ledger.Payload, error) { @@ -375,7 +376,7 @@ func (m *AtreeRegisterMigrator) validateChangesAndCreateNewRegisters( } func (m *AtreeRegisterMigrator) cloneValue( - mr *migratorRuntime, + mr *MigratorRuntime, value interpreter.Value, ) (interpreter.Value, error) { diff --git a/cmd/util/ledger/migrations/cadence_value_diff_test.go b/cmd/util/ledger/migrations/cadence_value_diff_test.go index 20e5899810b..730f8dd7141 100644 --- a/cmd/util/ledger/migrations/cadence_value_diff_test.go +++ b/cmd/util/ledger/migrations/cadence_value_diff_test.go @@ -8,6 +8,7 @@ import ( "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" @@ -174,6 +175,7 @@ func TestDiffCadenceValues(t *testing.T) { address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) @@ -274,6 +276,7 @@ func TestDiffCadenceValues(t *testing.T) { address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) @@ -381,6 +384,7 @@ func TestDiffCadenceValues(t *testing.T) { address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) @@ -499,6 +503,7 @@ func TestDiffCadenceValues(t *testing.T) { address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) @@ -624,6 +629,7 @@ func createStorageMapPayloads(t *testing.T, address common.Address, domain strin address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) diff --git a/cmd/util/ledger/migrations/cadence_value_validation.go b/cmd/util/ledger/migrations/cadence_value_validation.go index 1aa8e94b2bd..ae576d04c5f 100644 --- a/cmd/util/ledger/migrations/cadence_value_validation.go +++ b/cmd/util/ledger/migrations/cadence_value_validation.go @@ -14,6 +14,7 @@ import ( "go.opentelemetry.io/otel/attribute" "github.com/onflow/flow-go/cmd/util/ledger/util" + migrationSnapshot "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/ledger" ) @@ -378,7 +379,7 @@ func newReadonlyStorageRuntime(payloads []*ledger.Payload) ( *readonlyStorageRuntime, error, ) { - snapshot, err := util.NewMapBasedPayloadSnapshot(payloads) + snapshot, err := migrationSnapshot.NewPayloadSnapshot(payloads, migrationSnapshot.MapBased) if err != nil { return nil, fmt.Errorf("failed to create payload snapshot: %w", err) } diff --git a/cmd/util/ledger/migrations/cadence_value_validation_test.go b/cmd/util/ledger/migrations/cadence_value_validation_test.go index d2cd063af75..637c0f74d59 100644 --- a/cmd/util/ledger/migrations/cadence_value_validation_test.go +++ b/cmd/util/ledger/migrations/cadence_value_validation_test.go @@ -10,6 +10,7 @@ import ( "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" @@ -57,6 +58,7 @@ func TestValidateCadenceValues(t *testing.T) { address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) @@ -149,6 +151,7 @@ func createTestPayloads(t *testing.T, address common.Address, domain string) []* address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) diff --git a/cmd/util/ledger/migrations/cadence_values_migration.go b/cmd/util/ledger/migrations/cadence_values_migration.go index 92062b2f7bb..ebf74c7b679 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration.go +++ b/cmd/util/ledger/migrations/cadence_values_migration.go @@ -22,6 +22,7 @@ import ( "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/ledger" @@ -98,6 +99,7 @@ func (m *CadenceBaseMigrator) MigrateAccount( address, oldPayloads, m.runtimeInterfaceConfig, + snapshot.IndexMapBased, ) if err != nil { return nil, fmt.Errorf("failed to create migrator runtime: %w", err) diff --git a/cmd/util/ledger/migrations/cadence_values_migration_test.go b/cmd/util/ledger/migrations/cadence_values_migration_test.go index 812b14343b1..0ab80625cfc 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration_test.go +++ b/cmd/util/ledger/migrations/cadence_values_migration_test.go @@ -19,6 +19,7 @@ import ( "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/ledger" @@ -124,6 +125,7 @@ func checkMigratedPayloads( address, newPayloads, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) @@ -381,7 +383,7 @@ func checkMigratedPayloads( } } -func checkAccountID(t *testing.T, mr *migratorRuntime, address common.Address) { +func checkAccountID(t *testing.T, mr *MigratorRuntime, address common.Address) { id := flow.AccountStatusRegisterID(flow.Address(address)) statusBytes, err := mr.Accounts.GetValue(id) require.NoError(t, err) @@ -741,6 +743,7 @@ func TestProgramParsingError(t *testing.T) { testAddress, payloads, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) @@ -882,6 +885,7 @@ func TestCoreContractUsage(t *testing.T) { testAddress, payloads, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) @@ -969,6 +973,7 @@ func TestCoreContractUsage(t *testing.T) { testAddress, payloads, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) diff --git a/cmd/util/ledger/migrations/migrator_runtime.go b/cmd/util/ledger/migrations/migrator_runtime.go index 65d07e9d653..3e19ecb8e74 100644 --- a/cmd/util/ledger/migrations/migrator_runtime.go +++ b/cmd/util/ledger/migrations/migrator_runtime.go @@ -9,6 +9,7 @@ import ( "github.com/onflow/cadence/runtime/stdlib" "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" @@ -25,16 +26,18 @@ type migrationTransactionPreparer struct { var _ storage.TransactionPreparer = migrationTransactionPreparer{} -// migratorRuntime is a runtime that can be used to run a migration on a single account +// NewMigratorRuntime is a runtime that can be used to run a migration on a single account func NewMigratorRuntime( address common.Address, payloads []*ledger.Payload, config util.RuntimeInterfaceConfig, + snapshotType snapshot.MigrationSnapshotType, ) ( - *migratorRuntime, + *MigratorRuntime, error, ) { - snapshot, err := util.NewMapBasedPayloadSnapshot(payloads) + // TODO: the snapshot type should be a parameter + snapshot, err := snapshot.NewPayloadSnapshot(payloads, snapshotType) if err != nil { return nil, fmt.Errorf("failed to create payload snapshot: %w", err) } @@ -92,7 +95,7 @@ func NewMigratorRuntime( return nil, err } - return &migratorRuntime{ + return &MigratorRuntime{ Address: address, Payloads: payloads, Snapshot: snapshot, @@ -106,8 +109,8 @@ func NewMigratorRuntime( }, nil } -type migratorRuntime struct { - Snapshot util.MigrationStorageSnapshot +type MigratorRuntime struct { + Snapshot snapshot.MigrationSnapshot TransactionState state.NestedTransactionPreparer Interpreter *interpreter.Interpreter Storage *runtime.Storage @@ -119,13 +122,13 @@ type migratorRuntime struct { ContractNamesProvider stdlib.AccountContractNamesProvider } -var _ stdlib.AccountContractNamesProvider = &migratorRuntime{} +var _ stdlib.AccountContractNamesProvider = &MigratorRuntime{} -func (mr *migratorRuntime) GetReadOnlyStorage() *runtime.Storage { +func (mr *MigratorRuntime) GetReadOnlyStorage() *runtime.Storage { readonlyLedger := util.NewPayloadsReadonlyLedger(mr.Snapshot) return runtime.NewStorage(readonlyLedger, nil) } -func (mr *migratorRuntime) GetAccountContractNames(address common.Address) ([]string, error) { +func (mr *MigratorRuntime) GetAccountContractNames(address common.Address) ([]string, error) { return mr.Accounts.GetContractNames(flow.Address(address)) } diff --git a/cmd/util/ledger/migrations/staged_contracts_migration.go b/cmd/util/ledger/migrations/staged_contracts_migration.go index dad890389d7..7c402d7c842 100644 --- a/cmd/util/ledger/migrations/staged_contracts_migration.go +++ b/cmd/util/ledger/migrations/staged_contracts_migration.go @@ -18,6 +18,7 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" "github.com/onflow/flow-go/model/flow" @@ -131,7 +132,11 @@ func (m *StagedContractsMigration) InitMigration( } // Pass empty address. We are only interested in the created `env` object. - mr, err := NewMigratorRuntime(common.Address{}, allPayloads, config) + mr, err := NewMigratorRuntime( + common.Address{}, + allPayloads, + config, + snapshot.IndexMapBased) if err != nil { return err } diff --git a/cmd/util/ledger/migrations/transaction_migration.go b/cmd/util/ledger/migrations/transaction_migration.go index 59e8d023e2c..a54f7af3838 100644 --- a/cmd/util/ledger/migrations/transaction_migration.go +++ b/cmd/util/ledger/migrations/transaction_migration.go @@ -3,7 +3,7 @@ package migrations import ( "github.com/rs/zerolog" - "github.com/onflow/flow-go/cmd/util/ledger/util" + migrationSnapshot "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/engine/execution/computation" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/ledger" @@ -27,7 +27,7 @@ func NewTransactionBasedMigration( fvm.WithTransactionFeesEnabled(false)) ctx := fvm.NewContext(options...) - snapshot, err := util.NewMapBasedPayloadSnapshot(payloads) + snapshot, err := migrationSnapshot.NewPayloadSnapshot(payloads, migrationSnapshot.IndexMapBased) if err != nil { return nil, err } diff --git a/cmd/util/ledger/util/payload_snapshot.go b/cmd/util/ledger/util/snapshot/payload_snapshot.go similarity index 75% rename from cmd/util/ledger/util/payload_snapshot.go rename to cmd/util/ledger/util/snapshot/payload_snapshot.go index 59e3eca47f7..20a6079892f 100644 --- a/cmd/util/ledger/util/payload_snapshot.go +++ b/cmd/util/ledger/util/snapshot/payload_snapshot.go @@ -1,4 +1,4 @@ -package util +package snapshot import ( "sort" @@ -11,7 +11,18 @@ import ( "github.com/onflow/flow-go/model/flow" ) -type MigrationStorageSnapshot interface { +type MigrationSnapshotType int + +const ( + // MapBased is more efficient when the number of payloads that are going to change is either 0 + // or large compared to the total number of payloads (more than 30% of the current number of payloads). + MapBased MigrationSnapshotType = iota + // IndexMapBased is more efficient when the number of payloads that are going to change is small + // compared to the total number of payloads (less than 30% of the current number of payloads). + IndexMapBased +) + +type MigrationSnapshot interface { snapshot.StorageSnapshot Exists(id flow.RegisterID) bool @@ -25,14 +36,30 @@ type MigrationStorageSnapshot interface { PayloadMap() map[flow.RegisterID]*ledger.Payload } -type PayloadSnapshot struct { +func NewPayloadSnapshot( + payloads []*ledger.Payload, + snapshotType MigrationSnapshotType, +) (MigrationSnapshot, error) { + switch snapshotType { + case MapBased: + return newIndexMapSnapshot(payloads) + case IndexMapBased: + return newMapSnapshot(payloads) + default: + // should never happen + panic("unknown snapshot type") + } + +} + +type mapSnapshot struct { Payloads map[flow.RegisterID]*ledger.Payload } -var _ MigrationStorageSnapshot = PayloadSnapshot{} +var _ MigrationSnapshot = mapSnapshot{} -func NewPayloadSnapshot(payloads []*ledger.Payload) (*PayloadSnapshot, error) { - l := &PayloadSnapshot{ +func newMapSnapshot(payloads []*ledger.Payload) (*mapSnapshot, error) { + l := &mapSnapshot{ Payloads: make(map[flow.RegisterID]*ledger.Payload, len(payloads)), } for _, payload := range payloads { @@ -49,7 +76,7 @@ func NewPayloadSnapshot(payloads []*ledger.Payload) (*PayloadSnapshot, error) { return l, nil } -func (p PayloadSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { +func (p mapSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { value, exists := p.Payloads[id] if !exists { return nil, nil @@ -57,22 +84,22 @@ func (p PayloadSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { return value.Value(), nil } -func (p PayloadSnapshot) Exists(id flow.RegisterID) bool { +func (p mapSnapshot) Exists(id flow.RegisterID) bool { _, exists := p.Payloads[id] return exists } -func (p PayloadSnapshot) Len() int { +func (p mapSnapshot) Len() int { return len(p.Payloads) } -func (p PayloadSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { +func (p mapSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { return p.Payloads } // ApplyChangesAndGetNewPayloads applies the given changes to the snapshot and returns the new payloads. // the snapshot is destroyed. -func (p PayloadSnapshot) ApplyChangesAndGetNewPayloads( +func (p mapSnapshot) ApplyChangesAndGetNewPayloads( changes map[flow.RegisterID]flow.RegisterValue, expectedChangeAddresses map[flow.Address]struct{}, logger zerolog.Logger, @@ -120,17 +147,17 @@ func (p PayloadSnapshot) ApplyChangesAndGetNewPayloads( return newPayloads, nil } -type MapBasedPayloadSnapshot struct { +type IndexMapSnapshot struct { reverseMap map[flow.RegisterID]int payloads []*ledger.Payload } -var _ MigrationStorageSnapshot = (*MapBasedPayloadSnapshot)(nil) +var _ MigrationSnapshot = (*IndexMapSnapshot)(nil) -func NewMapBasedPayloadSnapshot(payloads []*ledger.Payload) (*MapBasedPayloadSnapshot, error) { +func newIndexMapSnapshot(payloads []*ledger.Payload) (*IndexMapSnapshot, error) { payloadsCopy := make([]*ledger.Payload, len(payloads)) copy(payloadsCopy, payloads) - l := &MapBasedPayloadSnapshot{ + l := &IndexMapSnapshot{ reverseMap: make(map[flow.RegisterID]int, len(payloads)), payloads: payloadsCopy, } @@ -148,7 +175,7 @@ func NewMapBasedPayloadSnapshot(payloads []*ledger.Payload) (*MapBasedPayloadSna return l, nil } -func (p *MapBasedPayloadSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { +func (p *IndexMapSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { index, exists := p.reverseMap[id] if !exists { return nil, nil @@ -156,16 +183,16 @@ func (p *MapBasedPayloadSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, e return p.payloads[index].Value(), nil } -func (p *MapBasedPayloadSnapshot) Exists(id flow.RegisterID) bool { +func (p *IndexMapSnapshot) Exists(id flow.RegisterID) bool { _, exists := p.reverseMap[id] return exists } -func (p *MapBasedPayloadSnapshot) Len() int { +func (p *IndexMapSnapshot) Len() int { return len(p.payloads) } -func (p *MapBasedPayloadSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { +func (p *IndexMapSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { result := make(map[flow.RegisterID]*ledger.Payload, len(p.payloads)) for id, index := range p.reverseMap { result[id] = p.payloads[index] @@ -175,7 +202,7 @@ func (p *MapBasedPayloadSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Paylo // ApplyChangesAndGetNewPayloads applies the given changes to the snapshot and returns the new payloads. // the snapshot is destroyed. -func (p *MapBasedPayloadSnapshot) ApplyChangesAndGetNewPayloads( +func (p *IndexMapSnapshot) ApplyChangesAndGetNewPayloads( changes map[flow.RegisterID]flow.RegisterValue, expectedChangeAddresses map[flow.Address]struct{}, logger zerolog.Logger, diff --git a/cmd/util/ledger/util/payload_snapshot_test.go b/cmd/util/ledger/util/snapshot/payload_snapshot_test.go similarity index 85% rename from cmd/util/ledger/util/payload_snapshot_test.go rename to cmd/util/ledger/util/snapshot/payload_snapshot_test.go index 0b909e2af42..e5982b46958 100644 --- a/cmd/util/ledger/util/payload_snapshot_test.go +++ b/cmd/util/ledger/util/snapshot/payload_snapshot_test.go @@ -1,4 +1,4 @@ -package util_test +package snapshot_test import ( "crypto/rand" @@ -9,7 +9,7 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" "github.com/onflow/flow-go/model/flow" @@ -130,14 +130,14 @@ func benchmarkMerge(b *testing.B, payloadsNum int, changes []int) { // third of the changes should be new payloads // third of the changes should be existing payloads // third of the changes should be removals - change, remove, add := changesNum/3, changesNum/3, changesNum-2*(changesNum/3) + change, remove, add := changesNum/2, changesNum/2, changesNum/2 changes := creatChanges(len(payloads), change, remove, add) - b.Run("MapBasedPayloadSnapshot", func(b *testing.B) { + b.Run("IndexMapSnapshot", func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - snapshot, err := util.NewMapBasedPayloadSnapshot(payloads) + snapshot, err := snapshot.NewPayloadSnapshot(payloads, snapshot.IndexMapBased) require.NoError(b, err) b.StartTimer() @@ -146,10 +146,10 @@ func benchmarkMerge(b *testing.B, payloadsNum int, changes []int) { } }) - b.Run("PayloadSnapshot", func(b *testing.B) { + b.Run("mapSnapshot", func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - snapshot, err := util.NewPayloadSnapshot(payloads) + snapshot, err := snapshot.NewPayloadSnapshot(payloads, snapshot.MapBased) require.NoError(b, err) b.StartTimer() @@ -168,16 +168,16 @@ func benchmarkCreate( payloads := createPayloads(payloadsNum) - b.Run("MapBasedPayloadSnapshot", func(b *testing.B) { + b.Run("IndexMapSnapshot", func(b *testing.B) { for i := 0; i < b.N; i++ { - _, err := util.NewMapBasedPayloadSnapshot(payloads) + _, err := snapshot.NewPayloadSnapshot(payloads, snapshot.IndexMapBased) require.NoError(b, err) } }) - b.Run("PayloadSnapshot", func(b *testing.B) { + b.Run("mapSnapshot", func(b *testing.B) { for i := 0; i < b.N; i++ { - _, err := util.NewPayloadSnapshot(payloads) + _, err := snapshot.NewPayloadSnapshot(payloads, snapshot.MapBased) require.NoError(b, err) } }) diff --git a/cmd/util/ledger/util/util.go b/cmd/util/ledger/util/util.go index 92a9c8e40c8..a804efb1974 100644 --- a/cmd/util/ledger/util/util.go +++ b/cmd/util/ledger/util/util.go @@ -11,6 +11,7 @@ import ( "github.com/onflow/atree" "github.com/onflow/cadence/runtime/common" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" @@ -110,7 +111,7 @@ func (n NopMemoryGauge) MeterMemory(common.MemoryUsage) error { var _ common.MemoryGauge = (*NopMemoryGauge)(nil) type PayloadsReadonlyLedger struct { - Snapshot MigrationStorageSnapshot + Snapshot snapshot.MigrationSnapshot AllocateStorageIndexFunc func(owner []byte) (atree.StorageIndex, error) SetValueFunc func(owner, key, value []byte) (err error) @@ -145,7 +146,7 @@ func (p *PayloadsReadonlyLedger) AllocateStorageIndex(owner []byte) (atree.Stora panic("AllocateStorageIndex not expected to be called") } -func NewPayloadsReadonlyLedger(snapshot MigrationStorageSnapshot) *PayloadsReadonlyLedger { +func NewPayloadsReadonlyLedger(snapshot snapshot.MigrationSnapshot) *PayloadsReadonlyLedger { return &PayloadsReadonlyLedger{Snapshot: snapshot} } From de493cbdcd3d146abf21d2402b43a250e4a0f646 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Mon, 15 Apr 2024 20:04:02 +0200 Subject: [PATCH 12/14] fix lint --- cmd/util/ledger/migrations/staged_contracts_migration.go | 1 + cmd/util/ledger/migrations/staged_contracts_migration_test.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/cmd/util/ledger/migrations/staged_contracts_migration.go b/cmd/util/ledger/migrations/staged_contracts_migration.go index 5f2091860e5..4dbfb6778ea 100644 --- a/cmd/util/ledger/migrations/staged_contracts_migration.go +++ b/cmd/util/ledger/migrations/staged_contracts_migration.go @@ -224,6 +224,7 @@ func (m *StagedContractsMigration) collectAndRegisterStagedContractsFromPayloads stagingAccountAddress, stagingAccountPayloads, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) if err != nil { return err diff --git a/cmd/util/ledger/migrations/staged_contracts_migration_test.go b/cmd/util/ledger/migrations/staged_contracts_migration_test.go index 6921ece5152..362013f20fb 100644 --- a/cmd/util/ledger/migrations/staged_contracts_migration_test.go +++ b/cmd/util/ledger/migrations/staged_contracts_migration_test.go @@ -8,6 +8,8 @@ import ( "strings" "testing" + "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -531,6 +533,7 @@ func TestStagedContractsMigration(t *testing.T) { stagingAccountAddress, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, + snapshot.IndexMapBased, ) require.NoError(t, err) From 7295f3f1ededfdd1b225c8baac8aa12d24cae702 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Wed, 17 Apr 2024 20:42:21 +0200 Subject: [PATCH 13/14] apply review comments --- .../migrations/account_storage_migration.go | 2 +- .../migrations/atree_register_migration.go | 2 +- .../migrations/cadence_value_diff_test.go | 10 ++-- .../migrations/cadence_value_validation.go | 2 +- .../cadence_value_validation_test.go | 4 +- .../migrations/cadence_values_migration.go | 2 +- .../cadence_values_migration_test.go | 8 ++-- .../ledger/migrations/migrator_runtime.go | 1 - .../migrations/staged_contracts_migration.go | 4 +- .../staged_contracts_migration_test.go | 2 +- .../migrations/transaction_migration.go | 2 +- .../ledger/util/snapshot/payload_snapshot.go | 46 ++++++++++--------- 12 files changed, 43 insertions(+), 42 deletions(-) diff --git a/cmd/util/ledger/migrations/account_storage_migration.go b/cmd/util/ledger/migrations/account_storage_migration.go index fb503929a80..972ccd6fd11 100644 --- a/cmd/util/ledger/migrations/account_storage_migration.go +++ b/cmd/util/ledger/migrations/account_storage_migration.go @@ -25,7 +25,7 @@ func NewAccountStorageMigration( address, payloads, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.SmallChangeSetSnapshot, ) if err != nil { return nil, fmt.Errorf("failed to create migrator runtime: %w", err) diff --git a/cmd/util/ledger/migrations/atree_register_migration.go b/cmd/util/ledger/migrations/atree_register_migration.go index d78d0563660..9d3c3f70acd 100644 --- a/cmd/util/ledger/migrations/atree_register_migration.go +++ b/cmd/util/ledger/migrations/atree_register_migration.go @@ -91,7 +91,7 @@ func (m *AtreeRegisterMigrator) MigrateAccount( oldPayloads []*ledger.Payload, ) ([]*ledger.Payload, error) { // create all the runtime components we need for the migration - mr, err := NewMigratorRuntime(address, oldPayloads, util.RuntimeInterfaceConfig{}, snapshot.MapBased) + mr, err := NewMigratorRuntime(address, oldPayloads, util.RuntimeInterfaceConfig{}, snapshot.LargeChangeSetOrReadonlySnapshot) if err != nil { return nil, fmt.Errorf("failed to create migrator runtime: %w", err) } diff --git a/cmd/util/ledger/migrations/cadence_value_diff_test.go b/cmd/util/ledger/migrations/cadence_value_diff_test.go index 730f8dd7141..fdd3460aaf3 100644 --- a/cmd/util/ledger/migrations/cadence_value_diff_test.go +++ b/cmd/util/ledger/migrations/cadence_value_diff_test.go @@ -175,7 +175,7 @@ func TestDiffCadenceValues(t *testing.T) { address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.LargeChangeSetOrReadonlySnapshot, ) require.NoError(t, err) @@ -276,7 +276,7 @@ func TestDiffCadenceValues(t *testing.T) { address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.LargeChangeSetOrReadonlySnapshot, ) require.NoError(t, err) @@ -384,7 +384,7 @@ func TestDiffCadenceValues(t *testing.T) { address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.LargeChangeSetOrReadonlySnapshot, ) require.NoError(t, err) @@ -503,7 +503,7 @@ func TestDiffCadenceValues(t *testing.T) { address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.LargeChangeSetOrReadonlySnapshot, ) require.NoError(t, err) @@ -629,7 +629,7 @@ func createStorageMapPayloads(t *testing.T, address common.Address, domain strin address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.LargeChangeSetOrReadonlySnapshot, ) require.NoError(t, err) diff --git a/cmd/util/ledger/migrations/cadence_value_validation.go b/cmd/util/ledger/migrations/cadence_value_validation.go index 35ade236305..e0103b03a66 100644 --- a/cmd/util/ledger/migrations/cadence_value_validation.go +++ b/cmd/util/ledger/migrations/cadence_value_validation.go @@ -389,7 +389,7 @@ func newReadonlyStorageRuntime(payloads []*ledger.Payload) ( *readonlyStorageRuntime, error, ) { - snapshot, err := migrationSnapshot.NewPayloadSnapshot(payloads, migrationSnapshot.MapBased) + snapshot, err := migrationSnapshot.NewPayloadSnapshot(payloads, migrationSnapshot.LargeChangeSetOrReadonlySnapshot) if err != nil { return nil, fmt.Errorf("failed to create payload snapshot: %w", err) } diff --git a/cmd/util/ledger/migrations/cadence_value_validation_test.go b/cmd/util/ledger/migrations/cadence_value_validation_test.go index 77f9e1f85df..7077b9cd57c 100644 --- a/cmd/util/ledger/migrations/cadence_value_validation_test.go +++ b/cmd/util/ledger/migrations/cadence_value_validation_test.go @@ -58,7 +58,7 @@ func TestValidateCadenceValues(t *testing.T) { address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.LargeChangeSetOrReadonlySnapshot, ) require.NoError(t, err) @@ -151,7 +151,7 @@ func createTestPayloads(t *testing.T, address common.Address, domain string) []* address, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.LargeChangeSetOrReadonlySnapshot, ) require.NoError(t, err) diff --git a/cmd/util/ledger/migrations/cadence_values_migration.go b/cmd/util/ledger/migrations/cadence_values_migration.go index 01cd8b90858..9d78554e639 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration.go +++ b/cmd/util/ledger/migrations/cadence_values_migration.go @@ -100,7 +100,7 @@ func (m *CadenceBaseMigrator) MigrateAccount( address, oldPayloads, m.runtimeInterfaceConfig, - snapshot.IndexMapBased, + snapshot.SmallChangeSetSnapshot, ) if err != nil { return nil, fmt.Errorf("failed to create migrator runtime: %w", err) diff --git a/cmd/util/ledger/migrations/cadence_values_migration_test.go b/cmd/util/ledger/migrations/cadence_values_migration_test.go index ca756c501fc..ea02f8c605f 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration_test.go +++ b/cmd/util/ledger/migrations/cadence_values_migration_test.go @@ -124,7 +124,7 @@ func checkMigratedPayloads( address, newPayloads, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.SmallChangeSetSnapshot, ) require.NoError(t, err) @@ -742,7 +742,7 @@ func TestProgramParsingError(t *testing.T) { testAddress, payloads, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.SmallChangeSetSnapshot, ) require.NoError(t, err) @@ -885,7 +885,7 @@ func TestCoreContractUsage(t *testing.T) { testAddress, payloads, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.SmallChangeSetSnapshot, ) require.NoError(t, err) @@ -973,7 +973,7 @@ func TestCoreContractUsage(t *testing.T) { testAddress, payloads, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.SmallChangeSetSnapshot, ) require.NoError(t, err) diff --git a/cmd/util/ledger/migrations/migrator_runtime.go b/cmd/util/ledger/migrations/migrator_runtime.go index 3e19ecb8e74..0c8ffd8819c 100644 --- a/cmd/util/ledger/migrations/migrator_runtime.go +++ b/cmd/util/ledger/migrations/migrator_runtime.go @@ -36,7 +36,6 @@ func NewMigratorRuntime( *MigratorRuntime, error, ) { - // TODO: the snapshot type should be a parameter snapshot, err := snapshot.NewPayloadSnapshot(payloads, snapshotType) if err != nil { return nil, fmt.Errorf("failed to create payload snapshot: %w", err) diff --git a/cmd/util/ledger/migrations/staged_contracts_migration.go b/cmd/util/ledger/migrations/staged_contracts_migration.go index 4dbfb6778ea..39a8e50d05f 100644 --- a/cmd/util/ledger/migrations/staged_contracts_migration.go +++ b/cmd/util/ledger/migrations/staged_contracts_migration.go @@ -165,7 +165,7 @@ func (m *StagedContractsMigration) InitMigration( common.Address{}, allPayloads, config, - snapshot.IndexMapBased) + snapshot.SmallChangeSetSnapshot) if err != nil { return err } @@ -224,7 +224,7 @@ func (m *StagedContractsMigration) collectAndRegisterStagedContractsFromPayloads stagingAccountAddress, stagingAccountPayloads, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.SmallChangeSetSnapshot, ) if err != nil { return err diff --git a/cmd/util/ledger/migrations/staged_contracts_migration_test.go b/cmd/util/ledger/migrations/staged_contracts_migration_test.go index 362013f20fb..eb1fb719369 100644 --- a/cmd/util/ledger/migrations/staged_contracts_migration_test.go +++ b/cmd/util/ledger/migrations/staged_contracts_migration_test.go @@ -533,7 +533,7 @@ func TestStagedContractsMigration(t *testing.T) { stagingAccountAddress, []*ledger.Payload{accountStatusPayload}, util.RuntimeInterfaceConfig{}, - snapshot.IndexMapBased, + snapshot.SmallChangeSetSnapshot, ) require.NoError(t, err) diff --git a/cmd/util/ledger/migrations/transaction_migration.go b/cmd/util/ledger/migrations/transaction_migration.go index a54f7af3838..8752d5ffd94 100644 --- a/cmd/util/ledger/migrations/transaction_migration.go +++ b/cmd/util/ledger/migrations/transaction_migration.go @@ -27,7 +27,7 @@ func NewTransactionBasedMigration( fvm.WithTransactionFeesEnabled(false)) ctx := fvm.NewContext(options...) - snapshot, err := migrationSnapshot.NewPayloadSnapshot(payloads, migrationSnapshot.IndexMapBased) + snapshot, err := migrationSnapshot.NewPayloadSnapshot(payloads, migrationSnapshot.SmallChangeSetSnapshot) if err != nil { return nil, err } diff --git a/cmd/util/ledger/util/snapshot/payload_snapshot.go b/cmd/util/ledger/util/snapshot/payload_snapshot.go index 20a6079892f..ec25f57553d 100644 --- a/cmd/util/ledger/util/snapshot/payload_snapshot.go +++ b/cmd/util/ledger/util/snapshot/payload_snapshot.go @@ -14,12 +14,14 @@ import ( type MigrationSnapshotType int const ( - // MapBased is more efficient when the number of payloads that are going to change is either 0 + // LargeChangeSetOrReadonlySnapshot creates a `map[flow.RegisterID]*ledger.Payload` from the payloads + // It is more efficient when the number of payloads that are going to change is either 0 // or large compared to the total number of payloads (more than 30% of the current number of payloads). - MapBased MigrationSnapshotType = iota - // IndexMapBased is more efficient when the number of payloads that are going to change is small + LargeChangeSetOrReadonlySnapshot MigrationSnapshotType = iota + // SmallChangeSetSnapshot creates a map of indexes `map[flow.RegisterID]int` from the payloads array + // It is more efficient when the number of payloads that are going to change is small // compared to the total number of payloads (less than 30% of the current number of payloads). - IndexMapBased + SmallChangeSetSnapshot ) type MigrationSnapshot interface { @@ -41,10 +43,10 @@ func NewPayloadSnapshot( snapshotType MigrationSnapshotType, ) (MigrationSnapshot, error) { switch snapshotType { - case MapBased: - return newIndexMapSnapshot(payloads) - case IndexMapBased: + case LargeChangeSetOrReadonlySnapshot: return newMapSnapshot(payloads) + case SmallChangeSetSnapshot: + return newIndexMapSnapshot(payloads) default: // should never happen panic("unknown snapshot type") @@ -56,7 +58,7 @@ type mapSnapshot struct { Payloads map[flow.RegisterID]*ledger.Payload } -var _ MigrationSnapshot = mapSnapshot{} +var _ MigrationSnapshot = (*mapSnapshot)(nil) func newMapSnapshot(payloads []*ledger.Payload) (*mapSnapshot, error) { l := &mapSnapshot{ @@ -76,7 +78,7 @@ func newMapSnapshot(payloads []*ledger.Payload) (*mapSnapshot, error) { return l, nil } -func (p mapSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { +func (p *mapSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { value, exists := p.Payloads[id] if !exists { return nil, nil @@ -84,22 +86,22 @@ func (p mapSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { return value.Value(), nil } -func (p mapSnapshot) Exists(id flow.RegisterID) bool { +func (p *mapSnapshot) Exists(id flow.RegisterID) bool { _, exists := p.Payloads[id] return exists } -func (p mapSnapshot) Len() int { +func (p *mapSnapshot) Len() int { return len(p.Payloads) } -func (p mapSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { +func (p *mapSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { return p.Payloads } // ApplyChangesAndGetNewPayloads applies the given changes to the snapshot and returns the new payloads. // the snapshot is destroyed. -func (p mapSnapshot) ApplyChangesAndGetNewPayloads( +func (p *mapSnapshot) ApplyChangesAndGetNewPayloads( changes map[flow.RegisterID]flow.RegisterValue, expectedChangeAddresses map[flow.Address]struct{}, logger zerolog.Logger, @@ -147,17 +149,17 @@ func (p mapSnapshot) ApplyChangesAndGetNewPayloads( return newPayloads, nil } -type IndexMapSnapshot struct { +type indexMapSnapshot struct { reverseMap map[flow.RegisterID]int payloads []*ledger.Payload } -var _ MigrationSnapshot = (*IndexMapSnapshot)(nil) +var _ MigrationSnapshot = (*indexMapSnapshot)(nil) -func newIndexMapSnapshot(payloads []*ledger.Payload) (*IndexMapSnapshot, error) { +func newIndexMapSnapshot(payloads []*ledger.Payload) (*indexMapSnapshot, error) { payloadsCopy := make([]*ledger.Payload, len(payloads)) copy(payloadsCopy, payloads) - l := &IndexMapSnapshot{ + l := &indexMapSnapshot{ reverseMap: make(map[flow.RegisterID]int, len(payloads)), payloads: payloadsCopy, } @@ -175,7 +177,7 @@ func newIndexMapSnapshot(payloads []*ledger.Payload) (*IndexMapSnapshot, error) return l, nil } -func (p *IndexMapSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { +func (p *indexMapSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { index, exists := p.reverseMap[id] if !exists { return nil, nil @@ -183,16 +185,16 @@ func (p *IndexMapSnapshot) Get(id flow.RegisterID) (flow.RegisterValue, error) { return p.payloads[index].Value(), nil } -func (p *IndexMapSnapshot) Exists(id flow.RegisterID) bool { +func (p *indexMapSnapshot) Exists(id flow.RegisterID) bool { _, exists := p.reverseMap[id] return exists } -func (p *IndexMapSnapshot) Len() int { +func (p *indexMapSnapshot) Len() int { return len(p.payloads) } -func (p *IndexMapSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { +func (p *indexMapSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { result := make(map[flow.RegisterID]*ledger.Payload, len(p.payloads)) for id, index := range p.reverseMap { result[id] = p.payloads[index] @@ -202,7 +204,7 @@ func (p *IndexMapSnapshot) PayloadMap() map[flow.RegisterID]*ledger.Payload { // ApplyChangesAndGetNewPayloads applies the given changes to the snapshot and returns the new payloads. // the snapshot is destroyed. -func (p *IndexMapSnapshot) ApplyChangesAndGetNewPayloads( +func (p *indexMapSnapshot) ApplyChangesAndGetNewPayloads( changes map[flow.RegisterID]flow.RegisterValue, expectedChangeAddresses map[flow.Address]struct{}, logger zerolog.Logger, From 3f3792dd16c7aae23a154f21b57916c8d676aa63 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Wed, 17 Apr 2024 21:12:06 +0200 Subject: [PATCH 14/14] merge fix --- .../ledger/migrations/atree_register_migration.go | 2 -- .../migrations/atree_register_migrator_runtime.go | 5 +++-- .../migrations/cadence_value_validation_test.go | 1 - .../migrations/staged_contracts_migration_test.go | 6 ++++-- .../ledger/util/snapshot/payload_snapshot_test.go | 12 ++++++------ 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/util/ledger/migrations/atree_register_migration.go b/cmd/util/ledger/migrations/atree_register_migration.go index e4e9773819a..c066bbfdc66 100644 --- a/cmd/util/ledger/migrations/atree_register_migration.go +++ b/cmd/util/ledger/migrations/atree_register_migration.go @@ -15,8 +15,6 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/cmd/util/ledger/reporters" - "github.com/onflow/flow-go/cmd/util/ledger/util" - "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" diff --git a/cmd/util/ledger/migrations/atree_register_migrator_runtime.go b/cmd/util/ledger/migrations/atree_register_migrator_runtime.go index 77f52d9198f..e8946c2ee00 100644 --- a/cmd/util/ledger/migrations/atree_register_migrator_runtime.go +++ b/cmd/util/ledger/migrations/atree_register_migrator_runtime.go @@ -8,6 +8,7 @@ import ( "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/flow-go/cmd/util/ledger/util" + migrationSnapshot "github.com/onflow/flow-go/cmd/util/ledger/util/snapshot" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/ledger" @@ -21,7 +22,7 @@ func NewAtreeRegisterMigratorRuntime( *AtreeRegisterMigratorRuntime, error, ) { - snapshot, err := util.NewPayloadSnapshot(payloads) + snapshot, err := migrationSnapshot.NewPayloadSnapshot(payloads, migrationSnapshot.LargeChangeSetOrReadonlySnapshot) if err != nil { return nil, fmt.Errorf("failed to create payload snapshot: %w", err) } @@ -54,7 +55,7 @@ func NewAtreeRegisterMigratorRuntime( } type AtreeRegisterMigratorRuntime struct { - Snapshot *util.PayloadSnapshot + Snapshot migrationSnapshot.MigrationSnapshot TransactionState state.NestedTransactionPreparer Interpreter *interpreter.Interpreter Storage *runtime.Storage diff --git a/cmd/util/ledger/migrations/cadence_value_validation_test.go b/cmd/util/ledger/migrations/cadence_value_validation_test.go index 04d48cae2c9..c0780c03909 100644 --- a/cmd/util/ledger/migrations/cadence_value_validation_test.go +++ b/cmd/util/ledger/migrations/cadence_value_validation_test.go @@ -9,7 +9,6 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" - "github.com/onflow/flow-go/cmd/util/ledger/util" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" diff --git a/cmd/util/ledger/migrations/staged_contracts_migration_test.go b/cmd/util/ledger/migrations/staged_contracts_migration_test.go index 52568396220..8a6a467f477 100644 --- a/cmd/util/ledger/migrations/staged_contracts_migration_test.go +++ b/cmd/util/ledger/migrations/staged_contracts_migration_test.go @@ -1899,6 +1899,7 @@ func TestStagedContractsUpdateValidationErrors(t *testing.T) { ) require.NoError(t, err) + var err error err = migration.Close() require.NoError(t, err) @@ -1906,7 +1907,7 @@ func TestStagedContractsUpdateValidationErrors(t *testing.T) { var jsonObject map[string]any - err := json.Unmarshal([]byte(logWriter.logs[0]), &jsonObject) + err = json.Unmarshal([]byte(logWriter.logs[0]), &jsonObject) require.NoError(t, err) assert.Equal( @@ -1994,6 +1995,7 @@ func TestStagedContractsUpdateValidationErrors(t *testing.T) { accountPayloads := []*ledger.Payload{contractACode} allPayloads := []*ledger.Payload{contractACode, nftCode} + var err error err = migration.InitMigration(log, allPayloads, 0) require.NoError(t, err) @@ -2007,7 +2009,7 @@ func TestStagedContractsUpdateValidationErrors(t *testing.T) { require.Len(t, logWriter.logs, 1) var jsonObject map[string]any - err := json.Unmarshal([]byte(logWriter.logs[0]), &jsonObject) + err = json.Unmarshal([]byte(logWriter.logs[0]), &jsonObject) require.NoError(t, err) assert.Equal( diff --git a/cmd/util/ledger/util/snapshot/payload_snapshot_test.go b/cmd/util/ledger/util/snapshot/payload_snapshot_test.go index e5982b46958..aaeddcb9b07 100644 --- a/cmd/util/ledger/util/snapshot/payload_snapshot_test.go +++ b/cmd/util/ledger/util/snapshot/payload_snapshot_test.go @@ -137,11 +137,11 @@ func benchmarkMerge(b *testing.B, payloadsNum int, changes []int) { b.Run("IndexMapSnapshot", func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - snapshot, err := snapshot.NewPayloadSnapshot(payloads, snapshot.IndexMapBased) + snap, err := snapshot.NewPayloadSnapshot(payloads, snapshot.SmallChangeSetSnapshot) require.NoError(b, err) b.StartTimer() - _, err = snapshot.ApplyChangesAndGetNewPayloads(changes, nil, zerolog.Nop()) + _, err = snap.ApplyChangesAndGetNewPayloads(changes, nil, zerolog.Nop()) require.NoError(b, err) } }) @@ -149,11 +149,11 @@ func benchmarkMerge(b *testing.B, payloadsNum int, changes []int) { b.Run("mapSnapshot", func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - snapshot, err := snapshot.NewPayloadSnapshot(payloads, snapshot.MapBased) + snap, err := snapshot.NewPayloadSnapshot(payloads, snapshot.LargeChangeSetOrReadonlySnapshot) require.NoError(b, err) b.StartTimer() - _, err = snapshot.ApplyChangesAndGetNewPayloads(changes, nil, zerolog.Nop()) + _, err = snap.ApplyChangesAndGetNewPayloads(changes, nil, zerolog.Nop()) require.NoError(b, err) } }) @@ -170,14 +170,14 @@ func benchmarkCreate( b.Run("IndexMapSnapshot", func(b *testing.B) { for i := 0; i < b.N; i++ { - _, err := snapshot.NewPayloadSnapshot(payloads, snapshot.IndexMapBased) + _, err := snapshot.NewPayloadSnapshot(payloads, snapshot.SmallChangeSetSnapshot) require.NoError(b, err) } }) b.Run("mapSnapshot", func(b *testing.B) { for i := 0; i < b.N; i++ { - _, err := snapshot.NewPayloadSnapshot(payloads, snapshot.MapBased) + _, err := snapshot.NewPayloadSnapshot(payloads, snapshot.LargeChangeSetOrReadonlySnapshot) require.NoError(b, err) } })