diff --git a/builtin/v8/miner/deadline_state.go b/builtin/v8/miner/deadline_state.go index 1315c8a4..4af82796 100644 --- a/builtin/v8/miner/deadline_state.go +++ b/builtin/v8/miner/deadline_state.go @@ -95,10 +95,57 @@ const DeadlineExpirationAmtBitwidth = 5 // only exceed the partition AMT's height at ~0.75EiB of storage. const DeadlineOptimisticPoStSubmissionsAmtBitwidth = 2 +// +// Deadline (singular) +// + +func ConstructDeadline(store adt.Store) (*Deadline, error) { + emptyPartitionsArrayCid, err := adt.StoreEmptyArray(store, DeadlinePartitionsAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty partitions array: %w", err) + } + emptyDeadlineExpirationArrayCid, err := adt.StoreEmptyArray(store, DeadlineExpirationAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty deadline expiration array: %w", err) + } + + emptySectorsSnapshotArrayCid, err := adt.StoreEmptyArray(store, SectorsAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty sectors snapshot array: %w", err) + } + + emptyPoStSubmissionsArrayCid, err := adt.StoreEmptyArray(store, DeadlineOptimisticPoStSubmissionsAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty proofs array: %w", err) + } + + return &Deadline{ + Partitions: emptyPartitionsArrayCid, + ExpirationsEpochs: emptyDeadlineExpirationArrayCid, + EarlyTerminations: bitfield.New(), + LiveSectors: 0, + TotalSectors: 0, + FaultyPower: NewPowerPairZero(), + PartitionsPoSted: bitfield.New(), + OptimisticPoStSubmissions: emptyPoStSubmissionsArrayCid, + PartitionsSnapshot: emptyPartitionsArrayCid, + SectorsSnapshot: emptySectorsSnapshotArrayCid, + OptimisticPoStSubmissionsSnapshot: emptyPoStSubmissionsArrayCid, + }, nil +} + // // Deadlines (plural) // +func ConstructDeadlines(emptyDeadlineCid cid.Cid) *Deadlines { + d := new(Deadlines) + for i := range d.Due { + d.Due[i] = emptyDeadlineCid + } + return d +} + func (d *Deadlines) LoadDeadline(store adt.Store, dlIdx uint64) (*Deadline, error) { if dlIdx >= uint64(len(d.Due)) { return nil, xc.ErrIllegalArgument.Wrapf("invalid deadline %d", dlIdx) diff --git a/builtin/v9/datacap/datacap_state.go b/builtin/v9/datacap/datacap_state.go index 05cd663f..afc16305 100644 --- a/builtin/v9/datacap/datacap_state.go +++ b/builtin/v9/datacap/datacap_state.go @@ -4,14 +4,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/builtin" "github.com/filecoin-project/go-state-types/builtin/v9/util/adt" "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) -var DatacapGranularity = builtin.TokenPrecision - type State struct { Governor address.Address Token TokenState diff --git a/builtin/v9/datacap/datacap_types.go b/builtin/v9/datacap/datacap_types.go index d644ad2a..7691d5eb 100644 --- a/builtin/v9/datacap/datacap_types.go +++ b/builtin/v9/datacap/datacap_types.go @@ -3,8 +3,12 @@ package datacap import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/builtin" ) +var InfiniteAllowance = big.Mul(big.MustFromString("1000000000000000000000"), builtin.TokenPrecision) + type MintParams struct { To address.Address Amount abi.TokenAmount diff --git a/builtin/v9/market/market_state.go b/builtin/v9/market/market_state.go index fd0827f9..05fef2ec 100644 --- a/builtin/v9/market/market_state.go +++ b/builtin/v9/market/market_state.go @@ -52,7 +52,7 @@ type State struct { TotalClientStorageFee abi.TokenAmount // Verified registry allocation IDs for deals that are not yet activated. - PendingDealAllocationIds cid.Cid + PendingDealAllocationIds cid.Cid // HAMT[DealID]AllocationID } func ConstructState(store adt.Store) (*State, error) { diff --git a/builtin/v9/migration/miner.go b/builtin/v9/migration/miner.go index 8e445531..37ad84ac 100644 --- a/builtin/v9/migration/miner.go +++ b/builtin/v9/migration/miner.go @@ -3,13 +3,17 @@ package migration import ( "context" + adt9 "github.com/filecoin-project/go-state-types/builtin/v9/util/adt" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-amt-ipld/v4" "golang.org/x/xerrors" commp "github.com/filecoin-project/go-commp-utils/nonffi" "github.com/filecoin-project/go-state-types/builtin" "github.com/filecoin-project/go-state-types/builtin/v8/market" miner8 "github.com/filecoin-project/go-state-types/builtin/v8/miner" - "github.com/filecoin-project/go-state-types/builtin/v8/util/adt" + adt8 "github.com/filecoin-project/go-state-types/builtin/v8/util/adt" miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/ipfs/go-cid" @@ -18,9 +22,72 @@ import ( "github.com/filecoin-project/go-state-types/abi" ) +// The minerMigrator performs the following migrations: +// FIP-0029: Sets the Beneficary to the Owner, and sets empty values for BeneficiaryTerm and PendingBeneficiaryTerm +// FIP-0034: For each SectorPreCommitOnChainInfo in PreCommitedSectors, calculates the unsealed CID (assuming there are deals) +// FIP-0045: For each SectorOnChainInfo in Sectors, set SimpleQAPower = (DealWeight == 0 && VerifiedDealWeight == 0) +// FIP-0045: For each Deadline in Deadlines: for each SectorOnChainInfo in SectorsSnapshot, set SimpleQAPower = (DealWeight == 0 && VerifiedDealWeight == 0) + type minerMigrator struct { - proposals *market.DealArray - OutCodeCID cid.Cid + emptyPrecommitOnChainInfosV9 cid.Cid + emptyDeadlineV8 cid.Cid + emptyDeadlinesV8 cid.Cid + emptyDeadlineV9 cid.Cid + emptyDeadlinesV9 cid.Cid + proposals *market.DealArray + OutCodeCID cid.Cid +} + +func newMinerMigrator(ctx context.Context, store cbor.IpldStore, marketProposals *market.DealArray, outCode cid.Cid) (*minerMigrator, error) { + ctxStore := adt8.WrapStore(ctx, store) + + emptyPrecommitMapCidV9, err := adt9.StoreEmptyMap(ctxStore, builtin.DefaultHamtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty precommit map v9: %w", err) + } + + edv8, err := miner8.ConstructDeadline(ctxStore) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty deadline v8: %w", err) + } + + edv8cid, err := store.Put(ctx, edv8) + if err != nil { + return nil, xerrors.Errorf("failed to put empty deadline v8: %w", err) + } + + edsv8 := miner8.ConstructDeadlines(edv8cid) + edsv8cid, err := store.Put(ctx, edsv8) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty deadlines v8: %w", err) + } + + edv9, err := miner9.ConstructDeadline(ctxStore) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty deadline v9: %w", err) + } + + edv9cid, err := store.Put(ctx, edv9) + if err != nil { + return nil, xerrors.Errorf("failed to put empty deadline v9: %w", err) + } + + edsv9 := miner9.ConstructDeadlines(edv9cid) + edsv9cid, err := store.Put(ctx, edsv9) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty deadlines v9: %w", err) + + } + + return &minerMigrator{ + emptyPrecommitOnChainInfosV9: emptyPrecommitMapCidV9, + emptyDeadlineV8: edv8cid, + emptyDeadlinesV8: edsv8cid, + emptyDeadlineV9: edv9cid, + emptyDeadlinesV9: edsv9cid, + proposals: marketProposals, + OutCodeCID: outCode, + }, nil } func (m minerMigrator) migratedCodeCID() cid.Cid { @@ -36,21 +103,90 @@ func (m minerMigrator) migrateState(ctx context.Context, store cbor.IpldStore, i if err := store.Get(ctx, inState.Info, &inInfo); err != nil { return nil, err } - wrappedStore := adt.WrapStore(ctx, store) + wrappedStore := adt8.WrapStore(ctx, store) - oldPrecommitOnChainInfos, err := adt.AsMap(wrappedStore, inState.PreCommittedSectors, builtin.DefaultHamtBitwidth) + newPrecommits, err := m.migratePrecommits(ctx, wrappedStore, inState.PreCommittedSectors) if err != nil { - return nil, xerrors.Errorf("failed to load old precommit onchain infos for miner %s: %w", in.address, err) + return nil, xerrors.Errorf("failed to migrate precommits for miner: %s: %w", in.address, err) } - emptyMap, err := adt.StoreEmptyMap(wrappedStore, builtin.DefaultHamtBitwidth) + newSectors, err := migrateSectorsWithCache(ctx, wrappedStore, in.cache, in.address, inState.Sectors) if err != nil { - return nil, xerrors.Errorf("failed to make empty map: %w", err) + return nil, xerrors.Errorf("failed to migrate sectors for miner: %s: %w", in.address, err) } - newPrecommitOnChainInfos, err := adt.AsMap(wrappedStore, emptyMap, builtin.DefaultHamtBitwidth) + deadlinesOut, err := m.migrateDeadlines(ctx, wrappedStore, in.cache, inState.Deadlines) if err != nil { - return nil, xerrors.Errorf("failed to load empty map: %w", err) + return nil, xerrors.Errorf("failed to migrate deadlines: %w", err) + } + + var newPendingWorkerKey *miner9.WorkerKeyChange + if inInfo.PendingWorkerKey != nil { + newPendingWorkerKey = &miner9.WorkerKeyChange{ + NewWorker: inInfo.PendingWorkerKey.NewWorker, + EffectiveAt: inInfo.PendingWorkerKey.EffectiveAt, + } + } + + outInfo := miner9.MinerInfo{ + Owner: inInfo.Owner, + Worker: inInfo.Worker, + Beneficiary: inInfo.Owner, + BeneficiaryTerm: miner9.BeneficiaryTerm{ + Quota: abi.NewTokenAmount(0), + UsedQuota: abi.NewTokenAmount(0), + Expiration: 0, + }, + PendingBeneficiaryTerm: nil, + ControlAddresses: inInfo.ControlAddresses, + PendingWorkerKey: newPendingWorkerKey, + PeerId: inInfo.PeerId, + Multiaddrs: inInfo.Multiaddrs, + WindowPoStProofType: inInfo.WindowPoStProofType, + SectorSize: inInfo.SectorSize, + WindowPoStPartitionSectors: inInfo.WindowPoStPartitionSectors, + ConsensusFaultElapsed: inInfo.ConsensusFaultElapsed, + PendingOwnerAddress: inInfo.PendingOwnerAddress, + } + newInfoCid, err := store.Put(ctx, &outInfo) + if err != nil { + return nil, xerrors.Errorf("failed to flush new miner info: %w", err) + } + + outState := miner9.State{ + Info: newInfoCid, + PreCommitDeposits: inState.PreCommitDeposits, + LockedFunds: inState.LockedFunds, + VestingFunds: inState.VestingFunds, + FeeDebt: inState.FeeDebt, + InitialPledge: inState.InitialPledge, + PreCommittedSectors: newPrecommits, + PreCommittedSectorsCleanUp: inState.PreCommittedSectorsCleanUp, + AllocatedSectors: inState.AllocatedSectors, + Sectors: newSectors, + ProvingPeriodStart: inState.ProvingPeriodStart, + CurrentDeadline: inState.CurrentDeadline, + Deadlines: deadlinesOut, + EarlyTerminations: inState.EarlyTerminations, + DeadlineCronActive: inState.DeadlineCronActive, + } + + newHead, err := store.Put(ctx, &outState) + return &actorMigrationResult{ + newCodeCID: m.migratedCodeCID(), + newHead: newHead, + }, err +} + +func (m minerMigrator) migratePrecommits(ctx context.Context, wrappedStore adt8.Store, inRoot cid.Cid) (cid.Cid, error) { + oldPrecommitOnChainInfos, err := adt8.AsMap(wrappedStore, inRoot, builtin.DefaultHamtBitwidth) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to load old precommit onchain infos: %w", err) + } + + newPrecommitOnChainInfos, err := adt9.AsMap(wrappedStore, m.emptyPrecommitOnChainInfosV9, builtin.DefaultHamtBitwidth) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to load empty map: %w", err) } var info miner8.SectorPreCommitOnChainInfo @@ -100,68 +236,197 @@ func (m minerMigrator) migrateState(ctx context.Context, store cbor.IpldStore, i }) if err != nil { - return nil, xerrors.Errorf("failed to iterate over precommitinfos: %w", err) + return cid.Undef, xerrors.Errorf("failed to iterate over precommitinfos: %w", err) } newPrecommits, err := newPrecommitOnChainInfos.Root() if err != nil { - return nil, xerrors.Errorf("failed to flush new precommits: %w", err) + return cid.Undef, xerrors.Errorf("failed to flush new precommits: %w", err) } - var newPendingWorkerKey *miner9.WorkerKeyChange - if inInfo.PendingWorkerKey != nil { - newPendingWorkerKey = &miner9.WorkerKeyChange{ - NewWorker: inInfo.PendingWorkerKey.NewWorker, - EffectiveAt: inInfo.PendingWorkerKey.EffectiveAt, + return newPrecommits, nil +} + +func migrateSectorsWithCache(ctx context.Context, store adt8.Store, cache MigrationCache, minerAddr address.Address, inRoot cid.Cid) (cid.Cid, error) { + return cache.Load(SectorsAmtKey(inRoot), func() (cid.Cid, error) { + inArray, err := adt8.AsArray(store, inRoot, miner8.SectorsAmtBitwidth) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to read sectors array: %w", err) + } + + okIn, prevInRoot, err := cache.Read(MinerPrevSectorsInKey(minerAddr)) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to get previous inRoot from cache: %w", err) + } + + okOut, prevOutRoot, err := cache.Read(MinerPrevSectorsOutKey(minerAddr)) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) + } + + var outArray *adt9.Array + if okIn && okOut { + // we have previous work, but the AMT has changed -- diff them + diffs, err := amt.Diff(ctx, store, store, prevInRoot, inRoot, amt.UseTreeBitWidth(miner9.SectorsAmtBitwidth)) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to diff old and new Sector AMTs: %w", err) + } + + inSectors, err := miner8.LoadSectors(store, inRoot) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to load inSectors: %w", err) + } + + prevOutSectors, err := miner9.LoadSectors(store, prevOutRoot) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to load prevOutSectors: %w", err) + } + + for _, change := range diffs { + switch change.Type { + case amt.Remove: + if err := prevOutSectors.Delete(change.Key); err != nil { + return cid.Undef, xerrors.Errorf("failed to delete sector from prevOutSectors: %w", err) + } + case amt.Add: + fallthrough + case amt.Modify: + sectorNo := abi.SectorNumber(change.Key) + info, found, err := inSectors.Get(sectorNo) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to get sector %d in inSectors: %w", sectorNo, err) + } + + if !found { + return cid.Undef, xerrors.Errorf("didn't find sector %d in inSectors", sectorNo) + } + + if err := prevOutSectors.Set(change.Key, migrateSectorInfo(*info)); err != nil { + return cid.Undef, xerrors.Errorf("failed to set migrated sector %d in prevOutSectors", sectorNo) + } + } + } + + outArray = prevOutSectors.Array + } else { + // first time we're doing this, do all the work + outArray, err = migrateSectorsFromScratch(ctx, store, inArray) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to migrate sectors from scratch: %w", err) + } + + } + + outRoot, err := outArray.Root() + if err != nil { + return cid.Undef, xerrors.Errorf("error writing new sectors AMT: %w", err) } + + _ = cache.Write(MinerPrevSectorsInKey(minerAddr), inRoot) + + _ = cache.Write(MinerPrevSectorsOutKey(minerAddr), outRoot) + return outRoot, nil + }) +} + +func migrateSectorsFromScratch(ctx context.Context, store adt8.Store, inArray *adt8.Array) (*adt9.Array, error) { + outArray, err := adt9.MakeEmptyArray(store, miner9.SectorsAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct new sectors array: %w", err) } - outInfo := miner9.MinerInfo{ - Owner: inInfo.Owner, - Worker: inInfo.Worker, - Beneficiary: inInfo.Owner, - BeneficiaryTerm: miner9.BeneficiaryTerm{ - Quota: abi.NewTokenAmount(0), - UsedQuota: abi.NewTokenAmount(0), - Expiration: 0, - }, - PendingBeneficiaryTerm: nil, - ControlAddresses: inInfo.ControlAddresses, - PendingWorkerKey: newPendingWorkerKey, - PeerId: inInfo.PeerId, - Multiaddrs: inInfo.Multiaddrs, - WindowPoStProofType: inInfo.WindowPoStProofType, - SectorSize: inInfo.SectorSize, - WindowPoStPartitionSectors: inInfo.WindowPoStPartitionSectors, - ConsensusFaultElapsed: inInfo.ConsensusFaultElapsed, - PendingOwnerAddress: inInfo.PendingOwnerAddress, + var sectorInfo miner8.SectorOnChainInfo + if err = inArray.ForEach(§orInfo, func(k int64) error { + return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo)) + }); err != nil { + return nil, err } - newInfoCid, err := store.Put(ctx, &outInfo) + + return outArray, err +} + +// Need to introduce caching here too +func (m minerMigrator) migrateDeadlines(ctx context.Context, store adt8.Store, cache MigrationCache, deadlines cid.Cid) (cid.Cid, error) { + if deadlines == m.emptyDeadlinesV8 { + return m.emptyDeadlinesV9, nil + } + + var inDeadlines miner8.Deadlines + err := store.Get(store.Context(), deadlines, &inDeadlines) if err != nil { - return nil, xerrors.Errorf("failed to flush new miner info: %w", err) + return cid.Undef, err } - outState := miner9.State{ - Info: newInfoCid, - PreCommitDeposits: inState.PreCommitDeposits, - LockedFunds: inState.LockedFunds, - VestingFunds: inState.VestingFunds, - FeeDebt: inState.FeeDebt, - InitialPledge: inState.InitialPledge, - PreCommittedSectors: newPrecommits, - PreCommittedSectorsCleanUp: inState.PreCommittedSectorsCleanUp, - AllocatedSectors: inState.AllocatedSectors, - Sectors: inState.Sectors, - ProvingPeriodStart: inState.ProvingPeriodStart, - CurrentDeadline: inState.CurrentDeadline, - Deadlines: inState.Deadlines, - EarlyTerminations: inState.EarlyTerminations, - DeadlineCronActive: inState.DeadlineCronActive, + var outDeadlines miner9.Deadlines + for i, c := range inDeadlines.Due { + if c == m.emptyDeadlineV8 { + outDeadlines.Due[i] = m.emptyDeadlineV9 + } else { + var inDeadline miner8.Deadline + if err = store.Get(ctx, c, &inDeadline); err != nil { + return cid.Undef, err + } + + outSectorsSnapshotCid, err := cache.Load(SectorsAmtKey(inDeadline.SectorsSnapshot), func() (cid.Cid, error) { + inSectorsSnapshot, err := adt8.AsArray(store, inDeadline.SectorsSnapshot, miner8.SectorsAmtBitwidth) + if err != nil { + return cid.Undef, err + } + + outSectorsSnapshot, err := migrateSectorsFromScratch(ctx, store, inSectorsSnapshot) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to migrate sectors: %w", err) + } + + return store.Put(ctx, outSectorsSnapshot) + }) + + if err != nil { + return cid.Undef, xerrors.Errorf("failed to migrate sectors snapshot: %w", err) + } + + outDeadline := miner9.Deadline{ + Partitions: inDeadline.Partitions, + ExpirationsEpochs: inDeadline.ExpirationsEpochs, + PartitionsPoSted: inDeadline.PartitionsPoSted, + EarlyTerminations: inDeadline.EarlyTerminations, + LiveSectors: inDeadline.LiveSectors, + TotalSectors: inDeadline.TotalSectors, + FaultyPower: miner9.PowerPair(inDeadline.FaultyPower), + OptimisticPoStSubmissions: inDeadline.OptimisticPoStSubmissions, + SectorsSnapshot: outSectorsSnapshotCid, + PartitionsSnapshot: inDeadline.PartitionsSnapshot, + OptimisticPoStSubmissionsSnapshot: inDeadline.OptimisticPoStSubmissionsSnapshot, + } + + outDlCid, err := store.Put(ctx, &outDeadline) + if err != nil { + return cid.Undef, err + } + + outDeadlines.Due[i] = outDlCid + } } - newHead, err := store.Put(ctx, &outState) - return &actorMigrationResult{ - newCodeCID: m.migratedCodeCID(), - newHead: newHead, - }, err + return store.Put(ctx, &outDeadlines) +} + +func migrateSectorInfo(sectorInfo miner8.SectorOnChainInfo) *miner9.SectorOnChainInfo { + return &miner9.SectorOnChainInfo{ + SectorNumber: sectorInfo.SectorNumber, + SealProof: sectorInfo.SealProof, + SealedCID: sectorInfo.SealedCID, + DealIDs: sectorInfo.DealIDs, + Activation: sectorInfo.Activation, + Expiration: sectorInfo.Expiration, + DealWeight: sectorInfo.DealWeight, + VerifiedDealWeight: sectorInfo.VerifiedDealWeight, + InitialPledge: sectorInfo.InitialPledge, + ExpectedDayReward: sectorInfo.ExpectedDayReward, + ExpectedStoragePledge: sectorInfo.ExpectedStoragePledge, + ReplacedSectorAge: sectorInfo.ReplacedSectorAge, + ReplacedDayReward: sectorInfo.ReplacedDayReward, + SectorKeyCID: sectorInfo.SectorKeyCID, + SimpleQAPower: sectorInfo.DealWeight.IsZero() && sectorInfo.VerifiedDealWeight.IsZero(), + } } diff --git a/builtin/v9/migration/system.go b/builtin/v9/migration/system.go index 35713570..f0bac8de 100644 --- a/builtin/v9/migration/system.go +++ b/builtin/v9/migration/system.go @@ -15,6 +15,10 @@ type systemActorMigrator struct { ManifestData cid.Cid } +func (m systemActorMigrator) migratedCodeCID() cid.Cid { + return m.OutCodeCID +} + func (m systemActorMigrator) migrateState(ctx context.Context, store cbor.IpldStore, in actorMigrationInput) (*actorMigrationResult, error) { // The ManifestData itself is already in the blockstore state := system.State{BuiltinActors: m.ManifestData} diff --git a/builtin/v9/migration/test/migration_test.go b/builtin/v9/migration/test/migration_test.go index 1bd98bb9..1397cd17 100644 --- a/builtin/v9/migration/test/migration_test.go +++ b/builtin/v9/migration/test/migration_test.go @@ -87,7 +87,7 @@ func TestMigration(t *testing.T) { _ = oldStateTree.ForEach(func(addr address.Address, oldActor *migration.Actor) error { newActor, ok, err := newStateTree.GetActor(addr) require.NoError(t, err, "failed to get actor") - require.True(t, ok, "didn't find actor") + require.True(t, ok, "didn't find actor: %s", addr) expectedCid, ok := cidsMap[oldActor.Code] require.True(t, ok, "didn't find code in cidsmap") require.Equal(t, expectedCid, newActor.Code) diff --git a/builtin/v9/migration/test/util.go b/builtin/v9/migration/test/util.go index ed86881d..3db1cde1 100644 --- a/builtin/v9/migration/test/util.go +++ b/builtin/v9/migration/test/util.go @@ -38,7 +38,7 @@ func makeTestManifest(t *testing.T, store adt.Store, prefix string) (cid.Cid, ci builder := cid.V1Builder{Codec: cid.Raw, MhType: mh.IDENTITY} newManifestData := manifest.ManifestData{} - for _, name := range []string{"system", "init", "cron", "account", "storagepower", "storageminer", "storagemarket", "paymentchannel", "multisig", "reward", "verifiedregistry"} { + for _, name := range manifest.GetBuiltinActorsKeys() { codeCid, err := builder.Sum([]byte(fmt.Sprintf("%s%s", prefix, name))) if err != nil { t.Fatal(err) diff --git a/builtin/v9/migration/top.go b/builtin/v9/migration/top.go index df339f83..e5a68fd5 100644 --- a/builtin/v9/migration/top.go +++ b/builtin/v9/migration/top.go @@ -6,6 +6,15 @@ import ( "sync/atomic" "time" + "github.com/filecoin-project/go-state-types/builtin/v9/datacap" + + "github.com/filecoin-project/go-state-types/builtin/v9/verifreg" + + "github.com/filecoin-project/go-state-types/big" + adt9 "github.com/filecoin-project/go-state-types/builtin/v9/util/adt" + + verifreg8 "github.com/filecoin-project/go-state-types/builtin/v8/verifreg" + market8 "github.com/filecoin-project/go-state-types/builtin/v8/market" "github.com/filecoin-project/go-state-types/builtin/v8/system" @@ -63,6 +72,23 @@ func ActorHeadKey(addr address.Address, head cid.Cid) string { return addr.String() + "-head-" + headKey } +func SectorsAmtKey(sectorsAmt cid.Cid) string { + sectorsAmtKey, err := sectorsAmt.StringOfBase(multibase.Base32) + if err != nil { + panic(err) + } + + return "sectorsAmt-" + sectorsAmtKey +} + +func MinerPrevSectorsInKey(m address.Address) string { + return "prevSectorsIn-" + m.String() +} + +func MinerPrevSectorsOutKey(m address.Address) string { + return "prevSectorsOut-" + m.String() +} + // Migrates the filecoin state tree starting from the global state tree and upgrading all actor state. // The store must support concurrent writes (even if the configured worker count is 1). func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID cid.Cid, actorsRootIn cid.Cid, priorEpoch abi.ChainEpoch, cfg Config, log Logger, cache MigrationCache) (cid.Cid, error) { @@ -114,19 +140,16 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID // Maps prior version code CIDs to migration functions. migrations := make(map[cid.Cid]actorMigration) - // Set of prior version code CIDs for actors to defer during iteration, for explicit migration afterwards. - var deferredCodeIDs = map[cid.Cid]struct{}{ - // None - } + deferredCodeIDs := make(map[cid.Cid]struct{}) - // simple code migrations - simpleMigrations := make(map[string]cid.Cid, len(oldManifestData.Entries)) + // Populated from oldManifestData + oldCodeIDMap := make(map[string]cid.Cid, len(oldManifestData.Entries)) miner8Cid := cid.Undef for _, entry := range oldManifestData.Entries { - simpleMigrations[entry.Name] = entry.Code - if entry.Name == "storageminer" { + oldCodeIDMap[entry.Name] = entry.Code + if entry.Name == manifest.MinerKey { miner8Cid = entry.Code } } @@ -135,30 +158,34 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("didn't find miner in old manifest entries") } - for name, oldCodeCID := range simpleMigrations { //nolint:nomaprange - newCodeCID, ok := newManifest.Get(name) - if !ok { - return cid.Undef, xerrors.Errorf("code cid for %s actor not found in new manifest", name) - } + for name, oldCodeCID := range oldCodeIDMap { //nolint:nomaprange + if name == manifest.MarketKey || name == manifest.VerifregKey { + deferredCodeIDs[oldCodeCID] = struct{}{} + } else { + newCodeCID, ok := newManifest.Get(name) + if !ok { + return cid.Undef, xerrors.Errorf("code cid for %s actor not found in new manifest", name) + } - migrations[oldCodeCID] = codeMigrator{newCodeCID} + migrations[oldCodeCID] = codeMigrator{newCodeCID} + } } - // migrations that migrate both code and state + // migrations that migrate both code and state, override entries in `migrations` + + // The System Actor + newSystemCodeCID, ok := newManifest.Get("system") if !ok { return cid.Undef, xerrors.Errorf("code cid for system actor not found in manifest") } - miner9Cid, ok := newManifest.Get("storageminer") - if !ok { - return cid.Undef, xerrors.Errorf("code cid for miner actor not found in new manifest") - } - migrations[systemActor.Code] = systemActorMigrator{newSystemCodeCID, newManifest.Data} + // The Miner Actor + // load market proposals - marketActor, ok, err := actorsIn.GetActor(builtin.StorageMarketActorAddr) + marketActorV8, ok, err := actorsIn.GetActor(builtin.StorageMarketActorAddr) if err != nil { return cid.Undef, xerrors.Errorf("failed to get market actor: %w", err) } @@ -167,17 +194,27 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.New("didn't find market actor") } - var marketState market8.State - if err := store.Get(ctx, marketActor.Head, &marketState); err != nil { - return cid.Undef, xerrors.Errorf("failed to get system actor state: %w", err) + var marketStateV8 market8.State + if err := store.Get(ctx, marketActorV8.Head, &marketStateV8); err != nil { + return cid.Undef, xerrors.Errorf("failed to get market actor state: %w", err) } - proposals, err := market8.AsDealProposalArray(adtStore, marketState.Proposals) + proposals, err := market8.AsDealProposalArray(adtStore, marketStateV8.Proposals) if err != nil { return cid.Undef, xerrors.Errorf("failed to get proposals: %w", err) } - migrations[miner8Cid] = minerMigrator{proposals, miner9Cid} + miner9Cid, ok := newManifest.Get(manifest.MinerKey) + if !ok { + return cid.Undef, xerrors.Errorf("code cid for miner actor not found in new manifest") + } + + mm, err := newMinerMigrator(ctx, store, proposals, miner9Cid) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to create miner migrator: %w", err) + } + + migrations[miner8Cid] = cachedMigration(cache, *mm) if len(migrations)+len(deferredCodeIDs) != len(oldManifestData.Entries) { return cid.Undef, xerrors.Errorf("incomplete migration specification with %d code CIDs, need %d", len(migrations), len(oldManifestData.Entries)) @@ -307,7 +344,150 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID elapsed := time.Since(startTime) rate := float64(doneCount) / elapsed.Seconds() - log.Log(rt.INFO, "All %d done after %v (%.0f/s). Flushing state tree root.", doneCount, elapsed, rate) + log.Log(rt.INFO, "All %d done after %v (%.0f/s). Starting deferred migrations.", doneCount, elapsed, rate) + + // Create the Datacap actor + + verifregActorV8, ok, err := actorsIn.GetActor(builtin.VerifiedRegistryActorAddr) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to get verifreg actor: %w", err) + } + + if !ok { + return cid.Undef, xerrors.New("didn't find verifreg actor") + } + + var verifregStateV8 verifreg8.State + if err = store.Get(ctx, verifregActorV8.Head, &verifregStateV8); err != nil { + return cid.Undef, xerrors.Errorf("failed to get verifreg actor state: %w", err) + } + + verifiedClients, err := adt8.AsMap(adtStore, verifregStateV8.VerifiedClients, builtin.DefaultHamtBitwidth) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to get verified clients: %w", err) + } + + tokenSupply := big.Zero() + + emptyMapCid, err := adt9.StoreEmptyMap(adtStore, builtin.DefaultHamtBitwidth) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to create empty map: %w", err) + } + + balancesMap, err := adt9.AsMap(adtStore, emptyMapCid, builtin.DefaultHamtBitwidth) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to load empty map: %w", err) + } + + allowancesMap, err := adt9.AsMap(adtStore, emptyMapCid, builtin.DefaultHamtBitwidth) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to load empty map: %w", err) + } + + var dcap abi.StoragePower + if err = verifiedClients.ForEach(&dcap, func(key string) error { + a, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + + tokenAmount := verifreg.DataCapToTokens(dcap) + tokenSupply = big.Add(tokenSupply, tokenAmount) + if err = balancesMap.Put(abi.IdAddrKey(a), &tokenAmount); err != nil { + return xerrors.Errorf("failed to put new balancesMap entry: %w", err) + } + + allowancesMapEntry, err := adt9.AsMap(adtStore, emptyMapCid, builtin.DefaultHamtBitwidth) + if err != nil { + return xerrors.Errorf("failed to load empty map: %w", err) + } + + if err = allowancesMapEntry.Put(abi.IdAddrKey(builtin.StorageMarketActorAddr), &datacap.InfiniteAllowance); err != nil { + return xerrors.Errorf("failed to populate allowance map: %w", err) + } + + return allowancesMap.Put(abi.IdAddrKey(a), allowancesMapEntry) + }); err != nil { + return cid.Undef, xerrors.Errorf("failed to loop over verified clients: %w", err) + } + + balancesMapRoot, err := balancesMap.Root() + if err != nil { + return cid.Undef, xerrors.Errorf("failed to flush balances map: %w", err) + } + + allowancesMapRoot, err := allowancesMap.Root() + if err != nil { + return cid.Undef, xerrors.Errorf("failed to flush allowances map: %w", err) + } + + dataCapState := datacap.State{ + Governor: builtin.VerifiedRegistryActorAddr, + Token: datacap.TokenState{ + Supply: tokenSupply, + Balances: balancesMapRoot, + Allowances: allowancesMapRoot, + HamtBitWidth: builtin.DefaultHamtBitwidth, + }, + } + + dataCapHead, err := adtStore.Put(ctx, &dataCapState) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to put data cap state: %w", err) + } + + dataCapCode, ok := newManifest.Get(manifest.DataCapKey) + if !ok { + return cid.Undef, xerrors.Errorf("failed to find datacap code ID: %w", err) + } + + if err = actorsOut.SetActor(builtin.DatacapActorAddr, &Actor{ + Code: dataCapCode, + Head: dataCapHead, + CallSeqNum: 0, + Balance: big.Zero(), + }); err != nil { + return cid.Undef, xerrors.Errorf("failed to set datacap actor: %w", err) + } + + // Migrate the Verified Registry Actor + + verifregCode, ok := newManifest.Get(manifest.VerifregKey) + if !ok { + return cid.Undef, xerrors.Errorf("failed to find verifreg code ID: %w", err) + } + + if err = actorsOut.SetActor(builtin.VerifiedRegistryActorAddr, &Actor{ + Code: verifregCode, + // TODO: Actual head after migrate + Head: dataCapHead, + // TODO: Actual nonce from old state (should just be 0) + CallSeqNum: 0, + // TODO: Actual balance from old state + Balance: big.Zero(), + }); err != nil { + return cid.Undef, xerrors.Errorf("failed to set datacap actor: %w", err) + } + + // Migrate the Market Actor + + marketCode, ok := newManifest.Get(manifest.MarketKey) + if !ok { + return cid.Undef, xerrors.Errorf("failed to find market code ID: %w", err) + } + + if err = actorsOut.SetActor(builtin.StorageMarketActorAddr, &Actor{ + Code: marketCode, + // TODO: Actual head after migrate + Head: dataCapHead, + // TODO: Actual nonce from old state (should just be 0) + CallSeqNum: 0, + // TODO: Actual balance from old state + Balance: big.Zero(), + }); err != nil { + return cid.Undef, xerrors.Errorf("failed to set market actor: %w", err) + } + return actorsOut.Flush() } @@ -327,6 +507,7 @@ type actorMigration interface { // Loads an actor's state from an input store and writes new state to an output store. // Returns the new state head CID. migrateState(ctx context.Context, store cbor.IpldStore, input actorMigrationInput) (result *actorMigrationResult, err error) + migratedCodeCID() cid.Cid } type migrationJob struct { @@ -376,3 +557,37 @@ func (n codeMigrator) migrateState(_ context.Context, _ cbor.IpldStore, in actor newHead: in.head, }, nil } + +func (n codeMigrator) migratedCodeCID() cid.Cid { + return n.OutCodeCID +} + +// Migrator that uses cached transformation if it exists +type cachedMigrator struct { + cache MigrationCache + actorMigration +} + +func (c cachedMigrator) migrateState(ctx context.Context, store cbor.IpldStore, in actorMigrationInput) (*actorMigrationResult, error) { + newHead, err := c.cache.Load(ActorHeadKey(in.address, in.head), func() (cid.Cid, error) { + result, err := c.actorMigration.migrateState(ctx, store, in) + if err != nil { + return cid.Undef, err + } + return result.newHead, nil + }) + if err != nil { + return nil, err + } + return &actorMigrationResult{ + newCodeCID: c.migratedCodeCID(), + newHead: newHead, + }, nil +} + +func cachedMigration(cache MigrationCache, m actorMigration) actorMigration { + return cachedMigrator{ + actorMigration: m, + cache: cache, + } +} diff --git a/builtin/v9/miner/deadline_state.go b/builtin/v9/miner/deadline_state.go index b189817f..24c00797 100644 --- a/builtin/v9/miner/deadline_state.go +++ b/builtin/v9/miner/deadline_state.go @@ -63,7 +63,7 @@ type Deadline struct { // Snapshot of the miner's sectors AMT at the end of the previous challenge // window for this deadline. - SectorsSnapshot cid.Cid + SectorsSnapshot cid.Cid // Array, AMT[SectorNumber]SectorOnChainInfo (sparse) // Snapshot of partition state at the end of the previous challenge // window for this deadline. @@ -95,10 +95,57 @@ const DeadlineExpirationAmtBitwidth = 5 // only exceed the partition AMT's height at ~0.75EiB of storage. const DeadlineOptimisticPoStSubmissionsAmtBitwidth = 2 +// +// Deadline (singular) +// + +func ConstructDeadline(store adt.Store) (*Deadline, error) { + emptyPartitionsArrayCid, err := adt.StoreEmptyArray(store, DeadlinePartitionsAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty partitions array: %w", err) + } + emptyDeadlineExpirationArrayCid, err := adt.StoreEmptyArray(store, DeadlineExpirationAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty deadline expiration array: %w", err) + } + + emptySectorsSnapshotArrayCid, err := adt.StoreEmptyArray(store, SectorsAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty sectors snapshot array: %w", err) + } + + emptyPoStSubmissionsArrayCid, err := adt.StoreEmptyArray(store, DeadlineOptimisticPoStSubmissionsAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty proofs array: %w", err) + } + + return &Deadline{ + Partitions: emptyPartitionsArrayCid, + ExpirationsEpochs: emptyDeadlineExpirationArrayCid, + EarlyTerminations: bitfield.New(), + LiveSectors: 0, + TotalSectors: 0, + FaultyPower: NewPowerPairZero(), + PartitionsPoSted: bitfield.New(), + OptimisticPoStSubmissions: emptyPoStSubmissionsArrayCid, + PartitionsSnapshot: emptyPartitionsArrayCid, + SectorsSnapshot: emptySectorsSnapshotArrayCid, + OptimisticPoStSubmissionsSnapshot: emptyPoStSubmissionsArrayCid, + }, nil +} + // // Deadlines (plural) // +func ConstructDeadlines(emptyDeadlineCid cid.Cid) *Deadlines { + d := new(Deadlines) + for i := range d.Due { + d.Due[i] = emptyDeadlineCid + } + return d +} + func (d *Deadlines) LoadDeadline(store adt.Store, dlIdx uint64) (*Deadline, error) { if dlIdx >= uint64(len(d.Due)) { return nil, xc.ErrIllegalArgument.Wrapf("invalid deadline %d", dlIdx) diff --git a/builtin/v9/util/adt/map.go b/builtin/v9/util/adt/map.go index de8bcb7d..afa70eab 100644 --- a/builtin/v9/util/adt/map.go +++ b/builtin/v9/util/adt/map.go @@ -3,6 +3,7 @@ package adt import ( "bytes" "crypto/sha256" + "io" hamt "github.com/filecoin-project/go-hamt-ipld/v3" "github.com/filecoin-project/go-state-types/abi" @@ -28,6 +29,16 @@ type Map struct { store Store } +func (m *Map) MarshalCBOR(w io.Writer) error { + rootCid, err := m.Root() + if err != nil { + return xerrors.Errorf("failed to flush map: %w", err) + } + + scratch := make([]byte, 9) + return cbg.WriteCidBuf(scratch, w, rootCid) +} + // AsMap interprets a store as a HAMT-based map with root `r`. // The HAMT is interpreted with branching factor 2^bitwidth. // We could drop this parameter if https://github.com/filecoin-project/go-hamt-ipld/issues/79 is implemented. diff --git a/builtin/v9/verifreg/verified_registry_state.go b/builtin/v9/verifreg/verified_registry_state.go index d57e806a..38fd4212 100644 --- a/builtin/v9/verifreg/verified_registry_state.go +++ b/builtin/v9/verifreg/verified_registry_state.go @@ -1,6 +1,7 @@ package verifreg import ( + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/builtin" "github.com/filecoin-project/go-state-types/builtin/v9/util/adt" @@ -14,6 +15,16 @@ import ( // We can introduce policy changes and replace this in the future. type DataCap = abi.StoragePower +var DatacapGranularity = builtin.TokenPrecision + +func DataCapToTokens(d DataCap) abi.TokenAmount { + return big.Mul(d, DatacapGranularity) +} + +func TokensToDatacap(t abi.TokenAmount) DataCap { + return big.Div(t, DatacapGranularity) +} + const SignatureDomainSeparation_RemoveDataCap = "fil_removedatacap:" type RmDcProposalID struct { diff --git a/manifest/manifest.go b/manifest/manifest.go index 16f35e02..59340fdf 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -12,6 +12,39 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" ) +const ( + AccountKey = "account" + CronKey = "cron" + DataCapKey = "datacap" + InitKey = "init" + MarketKey = "storagemarket" + MinerKey = "storageminer" + MultisigKey = "multisig" + PaychKey = "paymentchannel" + PowerKey = "storagepower" + RewardKey = "reward" + SystemKey = "system" + VerifregKey = "verifiedregistry" +) + +func GetBuiltinActorsKeys() []string { + keys := []string{ + AccountKey, + CronKey, + DataCapKey, + InitKey, + MarketKey, + MinerKey, + MultisigKey, + PaychKey, + PowerKey, + RewardKey, + SystemKey, + VerifregKey, + } + return keys +} + type Manifest struct { Version uint64 // this is really u32, but cbor-gen can't deal with it Data cid.Cid