Merge pull request #5192 from onflow/supun/cadence-migrations

cmd/util/cmd/execution-state-extract/execution_state_extract.go

@@ -21,6 +21,8 @@ import (
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/module/metrics"
 	"github.com/onflow/flow-go/storage"
+
+	"github.com/onflow/cadence/runtime/interpreter"
 )
 
 func getStateCommitment(commits storage.Commits, blockHash flow.Identifier) (flow.StateCommitment, error) {
@@ -89,18 +91,15 @@ func extractExecutionState(
 	if runMigrations {
 		rwf := reporters.NewReportFileWriterFactory(dir, log)
 
+		capabilityIDs := map[interpreter.AddressPath]interpreter.UInt64Value{}
+
 		migrations = []ledger.Migration{
 			migrators.CreateAccountBasedMigration(
 				log,
 				nWorker,
 				[]migrators.AccountBasedMigration{
-					migrators.NewAtreeRegisterMigrator(
-						rwf,
-						flagValidateMigration,
-						flagLogVerboseValidationError,
-					),
-
-					&migrators.DeduplicateContractNamesMigration{},
+					migrators.NewCadenceLinkValueMigrator(rwf, capabilityIDs),
+					migrators.NewCadenceValueMigrator(rwf, capabilityIDs),
 
 					// This will fix storage used discrepancies caused by the
 					// DeduplicateContractNamesMigration.

cmd/util/ledger/migrations/cadence_values_migration.go

@@ -0,0 +1,312 @@
+package migrations
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/rs/zerolog"
+
+	"github.com/onflow/flow-go/cmd/util/ledger/reporters"
+	"github.com/onflow/flow-go/cmd/util/ledger/util"
+	"github.com/onflow/flow-go/fvm/environment"
+	"github.com/onflow/flow-go/fvm/tracing"
+	"github.com/onflow/flow-go/ledger"
+	"github.com/onflow/flow-go/ledger/common/convert"
+	"github.com/onflow/flow-go/model/flow"
+
+	"github.com/onflow/cadence/migrations"
+	"github.com/onflow/cadence/migrations/capcons"
+	"github.com/onflow/cadence/migrations/entitlements"
+	"github.com/onflow/cadence/migrations/statictypes"
+	"github.com/onflow/cadence/migrations/string_normalization"
+	"github.com/onflow/cadence/runtime/common"
+	"github.com/onflow/cadence/runtime/interpreter"
+)
+
+type CadenceBaseMigrator struct {
+	name            string
+	log             zerolog.Logger
+	reporter        reporters.ReportWriter
+	valueMigrations func(
+		inter *interpreter.Interpreter,
+		accounts environment.Accounts,
+		reporter *cadenceValueMigrationReporter,
+	) []migrations.ValueMigration
+}
+
+var _ AccountBasedMigration = (*CadenceBaseMigrator)(nil)
+var _ io.Closer = (*CadenceBaseMigrator)(nil)
+
+func (m *CadenceBaseMigrator) Close() error {
+	// Close the report writer so it flushes to file.
+	m.reporter.Close()
+	return nil
+}
+
+func (m *CadenceBaseMigrator) InitMigration(
+	log zerolog.Logger,
+	_ []*ledger.Payload,
+	_ int,
+) error {
+	m.log = log.With().Str("migration", m.name).Logger()
+	return nil
+}
+
+func (m *CadenceBaseMigrator) MigrateAccount(
+	_ context.Context,
+	address common.Address,
+	oldPayloads []*ledger.Payload,
+) ([]*ledger.Payload, error) {
+
+	// Create all the runtime components we need for the migration
+	migrationRuntime, err := newMigratorRuntime(address, oldPayloads)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create migrator runtime: %w", err)
+	}
+
+	migration := migrations.NewStorageMigration(
+		migrationRuntime.Interpreter,
+		migrationRuntime.Storage,
+	)
+
+	reporter := newValueMigrationReporter(m.reporter, m.log)
+
+	m.log.Info().Msg("Migrating cadence values")
+
+	migration.Migrate(
+		&migrations.AddressSliceIterator{
+			Addresses: []common.Address{
+				address,
+			},
+		},
+		migration.NewValueMigrationsPathMigrator(
+			reporter,
+			m.valueMigrations(migrationRuntime.Interpreter, migrationRuntime.Accounts, reporter)...,
+		),
+	)
+
+	m.log.Info().Msg("Committing changes")
+	err = migration.Commit()
+	if err != nil {
+		return nil, fmt.Errorf("failed to commit changes: %w", err)
+	}
+
+	// finalize the transaction
+	result, err := migrationRuntime.TransactionState.FinalizeMainTransaction()
+	if err != nil {
+		return nil, fmt.Errorf("failed to finalize main transaction: %w", err)
+	}
+
+	// Merge the changes to the original payloads.
+	return m.mergeRegisterChanges(migrationRuntime, result.WriteSet)
+}
+
+func (m *CadenceBaseMigrator) mergeRegisterChanges(
+	mr *migratorRuntime,
+	changes map[flow.RegisterID]flow.RegisterValue,
+) ([]*ledger.Payload, error) {
+
+	originalPayloads := mr.Snapshot.Payloads
+	newPayloads := make([]*ledger.Payload, 0, len(originalPayloads))
+
+	// Add all new payloads.
+	for id, value := range changes {
+		key := convert.RegisterIDToLedgerKey(id)
+		newPayloads = append(newPayloads, ledger.NewPayload(key, value))
+	}
+
+	// Add any old payload that wasn't updated.
+	for id, value := range originalPayloads {
+		if len(value.Value()) == 0 {
+			// This is strange, but we don't want to add empty values. Log it.
+			m.log.Warn().Msgf("empty value for key %s", id)
+			continue
+		}
+
+		// If the payload had changed, then it has been added earlier.
+		// So skip old payload.
+		if _, contains := changes[id]; contains {
+			continue
+		}
+
+		newPayloads = append(newPayloads, value)
+	}
+
+	return newPayloads, nil
+}
+
+func NewCadenceValueMigrator(
+	rwf reporters.ReportWriterFactory,
+	capabilityIDs map[interpreter.AddressPath]interpreter.UInt64Value,
+) *CadenceBaseMigrator {
+	return &CadenceBaseMigrator{
+		name:     "cadence-value-migration",
+		reporter: rwf.ReportWriter("cadence-value-migrator"),
+		valueMigrations: func(
+			inter *interpreter.Interpreter,
+			_ environment.Accounts,
+			reporter *cadenceValueMigrationReporter,
+		) []migrations.ValueMigration {
+			// All cadence migrations except the `capcons.LinkValueMigration`.
+			return []migrations.ValueMigration{
+				&capcons.CapabilityValueMigration{
+					CapabilityIDs: capabilityIDs,
+					Reporter:      reporter,
+				},
+				entitlements.NewEntitlementsMigration(inter),
+				string_normalization.NewStringNormalizingMigration(),
+				statictypes.NewStaticTypeMigration().
+					WithCompositeTypeConverter(func(staticType *interpreter.CompositeStaticType) interpreter.StaticType {
+						// Returning `nil` indicates the type wasn't converted.
+						return nil
+					}).
+					WithInterfaceTypeConverter(func(staticType *interpreter.InterfaceStaticType) interpreter.StaticType {
+						// Returning `nil` indicates the type wasn't converted.
+						return nil
+					}),
+			}
+		},
+	}
+}
+
+func NewCadenceLinkValueMigrator(
+	rwf reporters.ReportWriterFactory,
+	capabilityIDs map[interpreter.AddressPath]interpreter.UInt64Value,
+) *CadenceBaseMigrator {
+	return &CadenceBaseMigrator{
+		name:     "cadence-link-value-migration",
+		reporter: rwf.ReportWriter("cadence-link-value-migrator"),
+		valueMigrations: func(
+			_ *interpreter.Interpreter,
+			accounts environment.Accounts,
+			reporter *cadenceValueMigrationReporter,
+		) []migrations.ValueMigration {
+			idGenerator := environment.NewAccountLocalIDGenerator(
+				tracing.NewMockTracerSpan(),
+				util.NopMeter{},
+				accounts,
+			)
+			return []migrations.ValueMigration{
+				&capcons.LinkValueMigration{
+					CapabilityIDs:      capabilityIDs,
+					AccountIDGenerator: idGenerator,
+					Reporter:           reporter,
+				},
+			}
+		},
+	}
+}
+
+// cadenceValueMigrationReporter is the reporter for cadence value migrations
+type cadenceValueMigrationReporter struct {
+	rw  reporters.ReportWriter
+	log zerolog.Logger
+}
+
+var _ capcons.LinkMigrationReporter = &cadenceValueMigrationReporter{}
+var _ capcons.CapabilityMigrationReporter = &cadenceValueMigrationReporter{}
+var _ migrations.Reporter = &cadenceValueMigrationReporter{}
+
+func newValueMigrationReporter(rw reporters.ReportWriter, log zerolog.Logger) *cadenceValueMigrationReporter {
+	return &cadenceValueMigrationReporter{
+		rw:  rw,
+		log: log,
+	}
+}
+
+func (t *cadenceValueMigrationReporter) Migrated(
+	storageKey interpreter.StorageKey,
+	storageMapKey interpreter.StorageMapKey,
+	migration string,
+) {
+	t.rw.Write(cadenceValueMigrationReportEntry{
+		StorageKey:    storageKey,
+		StorageMapKey: storageMapKey,
+		Migration:     migration,
+	})
+}
+
+func (t *cadenceValueMigrationReporter) Error(
+	storageKey interpreter.StorageKey,
+	storageMapKey interpreter.StorageMapKey,
+	migration string,
+	err error,
+) {
+	t.log.Error().Msgf(
+		"failed to run %s for path /%s/%s in account %s: %s",
+		migration,
+		storageKey.Key,
+		storageMapKey,
+		storageKey.Address,
+		err,
+	)
+}
+
+func (t *cadenceValueMigrationReporter) MigratedPathCapability(
+	accountAddress common.Address,
+	addressPath interpreter.AddressPath,
+	borrowType *interpreter.ReferenceStaticType,
+) {
+	t.rw.Write(capConsPathCapabilityMigration{
+		AccountAddress: accountAddress,
+		AddressPath:    addressPath,
+		BorrowType:     borrowType,
+	})
+}
+
+func (t *cadenceValueMigrationReporter) MissingCapabilityID(
+	accountAddress common.Address,
+	addressPath interpreter.AddressPath,
+) {
+	t.rw.Write(capConsMissingCapabilityID{
+		AccountAddress: accountAddress,
+		AddressPath:    addressPath,
+	})
+}
+
+func (t *cadenceValueMigrationReporter) MigratedLink(
+	accountAddressPath interpreter.AddressPath,
+	capabilityID interpreter.UInt64Value,
+) {
+	t.rw.Write(capConsLinkMigration{
+		AccountAddressPath: accountAddressPath,
+		CapabilityID:       capabilityID,
+	})
+}
+
+func (t *cadenceValueMigrationReporter) CyclicLink(err capcons.CyclicLinkError) {
+	t.rw.Write(err)
+}
+
+func (t *cadenceValueMigrationReporter) MissingTarget(accountAddressPath interpreter.AddressPath) {
+	t.rw.Write(capConsMissingTarget{
+		AddressPath: accountAddressPath,
+	})
+}
+
+type cadenceValueMigrationReportEntry struct {
+	StorageKey    interpreter.StorageKey    `json:"storageKey"`
+	StorageMapKey interpreter.StorageMapKey `json:"storageMapKey"`
+	Migration     string                    `json:"migration"`
+}
+
+type capConsLinkMigration struct {
+	AccountAddressPath interpreter.AddressPath `json:"address"`
+	CapabilityID       interpreter.UInt64Value `json:"capabilityID"`
+}
+
+type capConsPathCapabilityMigration struct {
+	AccountAddress common.Address                   `json:"address"`
+	AddressPath    interpreter.AddressPath          `json:"addressPath"`
+	BorrowType     *interpreter.ReferenceStaticType `json:"borrowType"`
+}
+
+type capConsMissingCapabilityID struct {
+	AccountAddress common.Address          `json:"address"`
+	AddressPath    interpreter.AddressPath `json:"addressPath"`
+}
+
+type capConsMissingTarget struct {
+	AddressPath interpreter.AddressPath `json:"addressPath"`
+}