diff --git a/cmd/util/cmd/execution-state-extract/cmd.go b/cmd/util/cmd/execution-state-extract/cmd.go index 1b19896e224..a6eba3f1f53 100644 --- a/cmd/util/cmd/execution-state-extract/cmd.go +++ b/cmd/util/cmd/execution-state-extract/cmd.go @@ -46,6 +46,7 @@ var ( flagOutputPayloadFileName string flagOutputPayloadByAddresses string flagMaxAccountSize uint64 + flagFixSlabsWithBrokenReferences bool flagFilterUnreferencedSlabs bool ) @@ -153,6 +154,9 @@ func init() { Cmd.Flags().Uint64Var(&flagMaxAccountSize, "max-account-size", 0, "max account size") + Cmd.Flags().BoolVar(&flagFixSlabsWithBrokenReferences, "fix-testnet-slabs-with-broken-references", false, + "fix slabs with broken references in testnet") + Cmd.Flags().BoolVar(&flagFilterUnreferencedSlabs, "filter-unreferenced-slabs", false, "filter unreferenced slabs") } @@ -375,6 +379,7 @@ func run(*cobra.Command, []string) { Prune: flagPrune, MaxAccountSize: flagMaxAccountSize, VerboseErrorOutput: flagVerboseErrorOutput, + FixSlabsWithBrokenReferences: chainID == flow.Testnet && flagFixSlabsWithBrokenReferences, FilterUnreferencedSlabs: flagFilterUnreferencedSlabs, } diff --git a/cmd/util/ledger/migrations/account_based_migration.go b/cmd/util/ledger/migrations/account_based_migration.go index c183154d37a..d109fd47369 100644 --- a/cmd/util/ledger/migrations/account_based_migration.go +++ b/cmd/util/ledger/migrations/account_based_migration.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/cmd/util/ledger/util" "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/model/flow" moduleUtil "github.com/onflow/flow-go/module/util" ) @@ -186,22 +187,6 @@ func MigrateGroupConcurrently( continue } - if _, ok := knownProblematicAccounts[job.Address]; ok { - log.Info(). - Hex("address", job.Address[:]). - Int("payload_count", len(job.Payloads)). - Msg("skipping problematic account") - resultCh <- &migrationResult{ - migrationDuration: migrationDuration{ - Address: job.Address, - Duration: time.Since(start), - PayloadCount: len(job.Payloads), - }, - Migrated: job.Payloads, - } - continue - } - var err error accountMigrated := job.Payloads for m, migrator := range migrations { @@ -315,7 +300,35 @@ func MigrateGroupConcurrently( return migrated, nil } -var knownProblematicAccounts = map[common.Address]string{} +var testnetAccountsWithBrokenSlabReferences = func() map[common.Address]struct{} { + testnetAddresses := map[common.Address]struct{}{ + mustHexToAddress("434a1f199a7ae3ba"): {}, + mustHexToAddress("454c9991c2b8d947"): {}, + mustHexToAddress("48602d8056ff9d93"): {}, + mustHexToAddress("5d63c34d7f05e5a4"): {}, + mustHexToAddress("5e3448b3cffb97f2"): {}, + mustHexToAddress("7d8c7e050c694eaa"): {}, + mustHexToAddress("ba53f16ede01972d"): {}, + mustHexToAddress("c843c1f5a4805c3a"): {}, + mustHexToAddress("48d3be92e6e4a973"): {}, + } + + for address := range testnetAddresses { + if !flow.Testnet.Chain().IsValid(flow.Address(address)) { + panic(fmt.Sprintf("invalid testnet address: %s", address.Hex())) + } + } + + return testnetAddresses +}() + +func mustHexToAddress(hex string) common.Address { + address, err := common.HexToAddress(hex) + if err != nil { + panic(err) + } + return address +} type jobMigrateAccountGroup struct { Address common.Address diff --git a/cmd/util/ledger/migrations/cadence.go b/cmd/util/ledger/migrations/cadence.go index ce425c5b7d7..1b12600412f 100644 --- a/cmd/util/ledger/migrations/cadence.go +++ b/cmd/util/ledger/migrations/cadence.go @@ -384,6 +384,7 @@ type Options struct { StagedContracts []StagedContract Prune bool MaxAccountSize uint64 + FixSlabsWithBrokenReferences bool FilterUnreferencedSlabs bool } @@ -414,15 +415,32 @@ func NewCadence1Migrations( ) } - if opts.FilterUnreferencedSlabs { + if opts.FixSlabsWithBrokenReferences || opts.FilterUnreferencedSlabs { + + var accountBasedMigrations []AccountBasedMigration + + if opts.FixSlabsWithBrokenReferences { + accountBasedMigrations = append( + accountBasedMigrations, + NewFixBrokenReferencesInSlabsMigration(rwf, testnetAccountsWithBrokenSlabReferences), + ) + } + + if opts.FilterUnreferencedSlabs { + accountBasedMigrations = append( + accountBasedMigrations, + // NOTE: migration to filter unreferenced slabs should happen + // after migration to fix slabs with references to nonexistent slabs. + NewFilterUnreferencedSlabsMigration(outputDir, rwf), + ) + } + migrations = append(migrations, NamedMigration{ - Name: "filter-unreferenced-slabs-migration", + Name: "fix-slabs-migration", Migrate: NewAccountBasedMigration( log, opts.NWorker, - []AccountBasedMigration{ - NewFilterUnreferencedSlabsMigration(outputDir, rwf), - }, + accountBasedMigrations, ), }) } diff --git a/cmd/util/ledger/migrations/fix_broken_data_migration.go b/cmd/util/ledger/migrations/fix_broken_data_migration.go new file mode 100644 index 00000000000..ed9192ce83e --- /dev/null +++ b/cmd/util/ledger/migrations/fix_broken_data_migration.go @@ -0,0 +1,172 @@ +package migrations + +import ( + "context" + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/atree" + + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/cmd/util/ledger/reporters" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" +) + +type FixSlabsWithBrokenReferencesMigration struct { + log zerolog.Logger + rw reporters.ReportWriter + accountsToFix map[common.Address]struct{} + nWorkers int +} + +var _ AccountBasedMigration = &FixSlabsWithBrokenReferencesMigration{} + +const fixSlabsWithBrokenReferencesName = "fix-slabs-with-broken-references" + +func NewFixBrokenReferencesInSlabsMigration( + rwf reporters.ReportWriterFactory, + accountsToFix map[common.Address]struct{}, +) *FixSlabsWithBrokenReferencesMigration { + return &FixSlabsWithBrokenReferencesMigration{ + rw: rwf.ReportWriter(fixSlabsWithBrokenReferencesName), + accountsToFix: accountsToFix, + } +} + +func (m *FixSlabsWithBrokenReferencesMigration) InitMigration( + log zerolog.Logger, + _ []*ledger.Payload, + nWorkers int, +) error { + m.log = log. + With(). + Str("migration", fixSlabsWithBrokenReferencesName). + Logger() + m.nWorkers = nWorkers + + return nil +} + +func (m *FixSlabsWithBrokenReferencesMigration) MigrateAccount( + _ context.Context, + address common.Address, + oldPayloads []*ledger.Payload, +) ( + newPayloads []*ledger.Payload, + err error, +) { + + if _, exist := m.accountsToFix[address]; !exist { + return oldPayloads, nil + } + + migrationRuntime, err := NewAtreeRegisterMigratorRuntime(address, oldPayloads) + if err != nil { + return nil, fmt.Errorf("failed to create cadence runtime: %w", err) + } + + storage := migrationRuntime.Storage + + // Load all atree registers in storage + err = loadAtreeSlabsInStorge(storage, oldPayloads) + if err != nil { + return nil, err + } + + // Fix broken references + fixedStorageIDs, skippedStorageIDs, err := storage.FixLoadedBrokenReferences(func(old atree.Value) bool { + // TODO: Cadence may need to export functions to check type info, etc. + return true + }) + if err != nil { + return nil, err + } + + if len(skippedStorageIDs) > 0 { + m.log.Warn(). + Str("account", address.Hex()). + Msgf("skipped slabs with broken references: %v", skippedStorageIDs) + } + + if len(fixedStorageIDs) == 0 { + m.log.Warn(). + Str("account", address.Hex()). + Msgf("did not fix any slabs with broken references") + + return oldPayloads, nil + } + + m.log.Log(). + Str("account", address.Hex()). + Msgf("fixed slabs with broken references: %v", fixedStorageIDs) + + err = storage.FastCommit(m.nWorkers) + if err != nil { + return nil, err + } + + // Finalize the transaction + result, err := migrationRuntime.TransactionState.FinalizeMainTransaction() + if err != nil { + return nil, fmt.Errorf("failed to finalize main transaction: %w", err) + } + + // Merge the changes to the original payloads. + expectedAddresses := map[flow.Address]struct{}{ + flow.Address(address): {}, + } + + newPayloads, err = migrationRuntime.Snapshot.ApplyChangesAndGetNewPayloads( + result.WriteSet, + expectedAddresses, + m.log, + ) + if err != nil { + return nil, err + } + + // Log fixed payloads + fixedPayloads := make([]*ledger.Payload, 0, len(fixedStorageIDs)) + for _, payload := range newPayloads { + registerID, _, err := convert.PayloadToRegister(payload) + if err != nil { + return nil, fmt.Errorf("failed to convert payload to register: %w", err) + } + + if !registerID.IsSlabIndex() { + continue + } + + storageID := atree.NewStorageID( + atree.Address([]byte(registerID.Owner)), + atree.StorageIndex([]byte(registerID.Key[1:])), + ) + + if _, ok := fixedStorageIDs[storageID]; ok { + fixedPayloads = append(fixedPayloads, payload) + } + } + + m.rw.Write(fixedSlabsWithBrokenReferences{ + Account: address, + Payloads: fixedPayloads, + }) + + return newPayloads, nil +} + +func (m *FixSlabsWithBrokenReferencesMigration) Close() error { + // close the report writer so it flushes to file + m.rw.Close() + + return nil +} + +type fixedSlabsWithBrokenReferences struct { + Account common.Address `json:"account"` + Payloads []*ledger.Payload `json:"payloads"` +} diff --git a/cmd/util/ledger/migrations/fix_broken_data_migration_test.go b/cmd/util/ledger/migrations/fix_broken_data_migration_test.go new file mode 100644 index 00000000000..8fd5a74eaef --- /dev/null +++ b/cmd/util/ledger/migrations/fix_broken_data_migration_test.go @@ -0,0 +1,187 @@ +package migrations + +import ( + "bytes" + "context" + "encoding/hex" + "runtime" + "testing" + + "github.com/rs/zerolog" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/ledger" +) + +func TestFixSlabsWithBrokenReferences(t *testing.T) { + + rawAddress := mustDecodeHex("5e3448b3cffb97f2") + + address := common.MustBytesToAddress(rawAddress) + + ownerKey := ledger.KeyPart{Type: 0, Value: rawAddress} + + oldPayloads := []*ledger.Payload{ + // account status "a.s" register + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("612e73")}}), + ledger.Value(mustDecodeHex("00000000000000083900000000000000090000000000000001")), + ), + + // storage domain register + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("73746f72616765")}}), + ledger.Value(mustDecodeHex("0000000000000008")), + ), + + // public domain register + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("7075626c6963")}}), + ledger.Value(mustDecodeHex("0000000000000007")), + ), + + // MapDataSlab [balance:1000.00089000 uuid:13797744] + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("240000000000000001")}}), + ledger.Value(mustDecodeHex("008883d88483d8c082487e60df042a9c086869466c6f77546f6b656e6f466c6f77546f6b656e2e5661756c7402021b146e6a6a4c5eee08008883005b00000000000000100887f9d0544c60cbefe0afc51d7f46609b0000000000000002826762616c616e6365d8bc1b00000017487843a8826475756964d8a41a00d28970")), + ), + + // MapDataSlab [uuid:13799884 roles:StorageIDStorable({[94 52 72 179 207 251 151 242] [0 0 0 0 0 0 0 3]}) recipient:0x5e3448b3cffb97f2] + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("240000000000000002")}}), + ledger.Value(mustDecodeHex("00c883d88483d8c0824848602d8056ff9d937046616e546f705065726d697373696f6e7746616e546f705065726d697373696f6e2e486f6c64657202031bb9d0e9f36650574100c883005b000000000000001820d6c23f2e85e694b0070dbc21a9822de5725916c4a005e99b0000000000000003826475756964d8a41a00d291cc8265726f6c6573d8ff505e3448b3cffb97f200000000000000038269726563697069656e74d883485e3448b3cffb97f2")), + ), + + // This slab contains broken references. + // MapDataSlab [StorageIDStorable({[0 0 0 0 0 0 0 0] [0 0 0 0 0 0 0 45]}):Capability<&A.48602d8056ff9d93.FanTopPermission.Admin>(address: 0x48602d8056ff9d93, path: /private/FanTopAdmin)] + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("240000000000000003")}}), + ledger.Value(mustDecodeHex("00c883d8d982d8d582d8c0824848602d8056ff9d937046616e546f705065726d697373696f6e7546616e546f705065726d697373696f6e2e526f6c65d8ddf6011b535c9de83a38cab000c883005b000000000000000856c1dcdf34d761b79b000000000000000182d8ff500000000000000000000000000000002dd8c983d8834848602d8056ff9d93d8c882026b46616e546f7041646d696ed8db82f4d8d582d8c0824848602d8056ff9d937046616e546f705065726d697373696f6e7646616e546f705065726d697373696f6e2e41646d696e")), + ), + + // MapDataSlab [resources:StorageIDStorable({[94 52 72 179 207 251 151 242] [0 0 0 0 0 0 0 5]}) uuid:15735719 address:0x5e3448b3cffb97f2] + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("240000000000000004")}}), + ledger.Value(mustDecodeHex("00c883d88483d8c0824848602d8056ff9d937346616e546f705065726d697373696f6e563261781a46616e546f705065726d697373696f6e5632612e486f6c64657202031b5a99ef3adb06d40600c883005b00000000000000185c9fead93697b692967de568f789d3c2d5e974502c8b12e99b000000000000000382697265736f7572636573d8ff505e3448b3cffb97f20000000000000005826475756964d8a41a00f01ba7826761646472657373d883485e3448b3cffb97f2")), + ), + + // MapDataSlab ["admin":StorageIDStorable({[94 52 72 179 207 251 151 242] [0 0 0 0 0 0 0 6]})] + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("240000000000000005")}}), + ledger.Value(mustDecodeHex("00c883d8d982d8d408d8dc82d8d40581d8d682d8c0824848602d8056ff9d937346616e546f705065726d697373696f6e563261781846616e546f705065726d697373696f6e5632612e526f6c65011b8059ccce9aa48cfb00c883005b00000000000000087a89c005baa53d9a9b000000000000000182d8876561646d696ed8ff505e3448b3cffb97f20000000000000006")), + ), + + // MapDataSlab [role:"admin" uuid:15735727] + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("240000000000000006")}}), + ledger.Value(mustDecodeHex("008883d88483d8c0824848602d8056ff9d937346616e546f705065726d697373696f6e563261781946616e546f705065726d697373696f6e5632612e41646d696e02021b4fc212cd0f233183008883005b0000000000000010858862f5e3e45e48d2bf75097a8aaf819b00000000000000028264726f6c65d8876561646d696e826475756964d8a41a00f01baf")), + ), + + // MapDataSlab [ + // FanTopPermissionV2a:PathLink<&{A.48602d8056ff9d93.FanTopPermissionV2a.Receiver}>(/storage/FanTopPermissionV2a) + // flowTokenReceiver:PathLink<&{A.9a0766d93b6608b7.FungibleToken.Receiver}>(/storage/flowTokenVault) + // flowTokenBalance:PathLink<&{A.9a0766d93b6608b7.FungibleToken.Balance}>(/storage/flowTokenVault) + // FanTopPermission:PathLink<&{A.48602d8056ff9d93.FanTopPermission.Receiver}>(/storage/FanTopPermission)] + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("240000000000000007")}}), + ledger.Value(mustDecodeHex("008883f6041bc576c5f201b94974008883005b00000000000000207971082fb163397089dbafb546246f429beff1dc622768dcb916d25455dc0be39b0000000000000004827346616e546f705065726d697373696f6e563261d8cb82d8c882017346616e546f705065726d697373696f6e563261d8db82f4d8dc82d8d40581d8d682d8c0824848602d8056ff9d937346616e546f705065726d697373696f6e563261781c46616e546f705065726d697373696f6e5632612e52656365697665728271666c6f77546f6b656e5265636569766572d8cb82d8c882016e666c6f77546f6b656e5661756c74d8db82f4d8dc82d8d582d8c082487e60df042a9c086869466c6f77546f6b656e6f466c6f77546f6b656e2e5661756c7481d8d682d8c082489a0766d93b6608b76d46756e6769626c65546f6b656e7646756e6769626c65546f6b656e2e52656365697665728270666c6f77546f6b656e42616c616e6365d8cb82d8c882016e666c6f77546f6b656e5661756c74d8db82f4d8dc82d8d582d8c082487e60df042a9c086869466c6f77546f6b656e6f466c6f77546f6b656e2e5661756c7481d8d682d8c082489a0766d93b6608b76d46756e6769626c65546f6b656e7546756e6769626c65546f6b656e2e42616c616e6365827046616e546f705065726d697373696f6ed8cb82d8c882017046616e546f705065726d697373696f6ed8db82f4d8dc82d8d40581d8d682d8c0824848602d8056ff9d937046616e546f705065726d697373696f6e781946616e546f705065726d697373696f6e2e5265636569766572")), + ), + + // MapDataSlab [ + // FanTopPermission:StorageIDStorable({[94 52 72 179 207 251 151 242] [0 0 0 0 0 0 0 2]}) + // FanTopPermissionV2a:StorageIDStorable({[94 52 72 179 207 251 151 242] [0 0 0 0 0 0 0 4]}) + // flowTokenVault:StorageIDStorable({[94 52 72 179 207 251 151 242] [0 0 0 0 0 0 0 1]})] + ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: mustDecodeHex("240000000000000008")}}), + ledger.Value(mustDecodeHex("008883f6031b7d303e276f3b803f008883005b00000000000000180a613a86f5856a480b3a715aa29b9876e5d7742a5a1df8e09b0000000000000003827046616e546f705065726d697373696f6ed8ff505e3448b3cffb97f20000000000000002827346616e546f705065726d697373696f6e563261d8ff505e3448b3cffb97f20000000000000004826e666c6f77546f6b656e5661756c74d8ff505e3448b3cffb97f20000000000000001")), + ), + } + + slabIndexWithBrokenReferences := mustDecodeHex("240000000000000003") + fixedSlabWithBrokenReferences := ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: slabIndexWithBrokenReferences}}), + ledger.Value(mustDecodeHex("008883d8d982d8d582d8c0824848602d8056ff9d937046616e546f705065726d697373696f6e7546616e546f705065726d697373696f6e2e526f6c65d8ddf6001b535c9de83a38cab0008883005b00000000000000009b0000000000000000")), + ) + + // Account status register is updated to include address ID counter and new storage used. + accountStatusRegisterID := mustDecodeHex("612e73") + updatedAccountStatusRegister := ledger.NewPayload( + ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: accountStatusRegisterID}}), + ledger.Value([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0xcc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + ) + + expectedNewPayloads := make([]*ledger.Payload, len(oldPayloads)) + copy(expectedNewPayloads, oldPayloads) + + for i, payload := range expectedNewPayloads { + key, err := payload.Key() + require.NoError(t, err) + + if bytes.Equal(key.KeyParts[1].Value, slabIndexWithBrokenReferences) { + expectedNewPayloads[i] = fixedSlabWithBrokenReferences + } else if bytes.Equal(key.KeyParts[1].Value, accountStatusRegisterID) { + expectedNewPayloads[i] = updatedAccountStatusRegister + } + } + + rwf := &testReportWriterFactory{} + + log := zerolog.New(zerolog.NewTestWriter(t)) + + accountsToFix := map[common.Address]struct{}{ + address: struct{}{}, + } + + migration := NewFixBrokenReferencesInSlabsMigration(rwf, accountsToFix) + + err := migration.InitMigration(log, nil, runtime.NumCPU()) + require.NoError(t, err) + + defer migration.Close() + + newPayloads, err := migration.MigrateAccount( + context.Background(), + address, + oldPayloads, + ) + require.NoError(t, err) + require.Equal(t, len(expectedNewPayloads), len(newPayloads)) + + for _, expected := range expectedNewPayloads { + k, _ := expected.Key() + rawExpectedKey := expected.EncodedKey() + + var found bool + for _, p := range newPayloads { + if bytes.Equal(rawExpectedKey, p.EncodedKey()) { + found = true + require.Equal(t, expected.Value(), p.Value(), k.String()) + break + } + } + require.True(t, found) + } + + writer := rwf.reportWriters[fixSlabsWithBrokenReferencesName] + assert.Equal(t, + []any{ + fixedSlabsWithBrokenReferences{ + Account: address, + Payloads: []*ledger.Payload{fixedSlabWithBrokenReferences}, + }, + }, + writer.entries, + ) +} + +func mustDecodeHex(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} diff --git a/cmd/util/ledger/migrations/utils.go b/cmd/util/ledger/migrations/utils.go index f9ce19b84e8..6104b244d73 100644 --- a/cmd/util/ledger/migrations/utils.go +++ b/cmd/util/ledger/migrations/utils.go @@ -12,12 +12,7 @@ import ( "github.com/onflow/flow-go/ledger/common/convert" ) -func checkStorageHealth( - address common.Address, - storage *runtime.Storage, - payloads []*ledger.Payload, -) error { - +func loadAtreeSlabsInStorge(storage *runtime.Storage, payloads []*ledger.Payload) error { for _, payload := range payloads { registerID, _, err := convert.PayloadToRegister(payload) if err != nil { @@ -40,6 +35,20 @@ func checkStorageHealth( } } + return nil +} + +func checkStorageHealth( + address common.Address, + storage *runtime.Storage, + payloads []*ledger.Payload, +) error { + + err := loadAtreeSlabsInStorge(storage, payloads) + if err != nil { + return err + } + for _, domain := range allStorageMapDomains { _ = storage.GetStorageMap(address, domain, false) }