Merge remote-tracking branch 'origin/master' into feat/async-restarta…

…ble-workers

CHANGELOG.md

@@ -1,5 +1,58 @@
 # Lotus changelog
 
+# 1.1.2 / 2020-10-24
+
+This is a patch release of Lotus that builds on the fixes involving worker keys that was introduced in v1.1.1. Miners and node operators should update to this release as soon as possible in order to ensure their blocks are propagated and validated.
+
+## Changes 
+
+- Handle worker key changes correctly in runtime (https://github.com/filecoin-project/lotus/pull/4579)
+
+# 1.1.1 / 2020-10-24
+
+This is a patch release of Lotus that addresses some issues caused by when miners change their worker keys. Miners and node operators should update to this release as soon as possible, especially any miner who has changed their worker key recently.
+
+## Changes
+
+- Miner finder for interactive client deal CLI (https://github.com/filecoin-project/lotus/pull/4504)
+- Disable blockstore bloom filter (https://github.com/filecoin-project/lotus/pull/4512)
+- Add api for getting status given a code (https://github.com/filecoin-project/lotus/pull/4210)
+- add batch api for push messages (https://github.com/filecoin-project/lotus/pull/4236)
+- add measure datastore wrapper around bench chain datastore (https://github.com/filecoin-project/lotus/pull/4302)
+- Look at block base fee for PCR (https://github.com/filecoin-project/lotus/pull/4313)
+- Add a shed util to determine % of power that has won a block (https://github.com/filecoin-project/lotus/pull/4318)
+- Shed/borked cmd (https://github.com/filecoin-project/lotus/pull/4339)
+- optimize mining code (https://github.com/filecoin-project/lotus/pull/4379)
+- heaviestTipSet reurning nil is a ok (https://github.com/filecoin-project/lotus/pull/4523)
+- Remove most v0 actor imports (https://github.com/filecoin-project/lotus/pull/4383)
+- Small chain export optimization (https://github.com/filecoin-project/lotus/pull/4536)
+- Add block list to pcr (https://github.com/filecoin-project/lotus/pull/4314)
+- Fix circ supply default in conformance (https://github.com/filecoin-project/lotus/pull/4449)
+- miner: fix init --create-worker-key (https://github.com/filecoin-project/lotus/pull/4475)
+- make push and addLocal atomic (https://github.com/filecoin-project/lotus/pull/4500)
+- add some methods that oni needs (https://github.com/filecoin-project/lotus/pull/4501)
+- MinerGetBaseInfo: if miner is not found in lookback, check current (https://github.com/filecoin-project/lotus/pull/4508)
+- Delete wallet from local wallet cache (https://github.com/filecoin-project/lotus/pull/4526)
+- Fix lotus-shed ledger list (https://github.com/filecoin-project/lotus/pull/4521)
+- Manage sectors by size instead of proof type (https://github.com/filecoin-project/lotus/pull/4511)
+- Feat/api request metrics wrapper (https://github.com/filecoin-project/lotus/pull/4516)
+- Fix chain sync stopping to sync (https://github.com/filecoin-project/lotus/pull/4541)
+- Use the correct lookback for the worker key when creating blocks (https://github.com/filecoin-project/lotus/pull/4539)
+- Cleanup test initialization and always validate VRFs in tests (https://github.com/filecoin-project/lotus/pull/4538)
+- Add a market WithdrawBalance CLI (https://github.com/filecoin-project/lotus/pull/4524)
+- wallet list: Add market balance and ID address flags (https://github.com/filecoin-project/lotus/pull/4555)
+- tvx simulate command; tvx extract --ignore-sanity-checks (https://github.com/filecoin-project/lotus/pull/4554)
+- lotus-lite: CLI tests for `lotus client` commands (https://github.com/filecoin-project/lotus/pull/4497)
+- lite-mode - market storage and retrieval clients (https://github.com/filecoin-project/lotus/pull/4263)
+- Chore: update drand to v1.2.0 (https://github.com/filecoin-project/lotus/pull/4420)
+- Fix random test failures (https://github.com/filecoin-project/lotus/pull/4559)
+- Fix flaky TestTimedBSSimple (https://github.com/filecoin-project/lotus/pull/4561)
+- Make wallet market withdraw usable with miner addresses (https://github.com/filecoin-project/lotus/pull/4556)
+- Fix flaky TestChainExportImportFull (https://github.com/filecoin-project/lotus/pull/4564)
+- Use older randomness for the PoSt commit on specs-actors version 2 (https://github.com/filecoin-project/lotus/pull/4563)
+- shed: Commad to decode messages (https://github.com/filecoin-project/lotus/pull/4565)
+- Fetch worker key from correct block on sync (https://github.com/filecoin-project/lotus/pull/4573)
+
 # 1.1.0 / 2020-10-20
 
 This is a mandatory release that introduces the first post-liftoff upgrade to the Filecoin network. The changes that break consensus are an upgrade to specs-actors v2.2.0 at epoch 170000.

api/api_full.go

@@ -312,6 +312,8 @@ type FullNode interface {
 	ClientDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
 	// ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
 	ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
+	// ClientCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
+	ClientCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
 	// ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel
 	// which are stuck due to insufficient funds
 	ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error

api/api_storage.go

@@ -5,8 +5,10 @@ import (
 	"context"
 	"time"
 
+	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-fil-markets/piecestore"
@@ -83,6 +85,10 @@ type StorageMiner interface {
 	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)
 	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)
 	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
+	// MinerRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
+	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
+	// ClientCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
+	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
 
 	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
 	DealsList(ctx context.Context) ([]MarketDeal, error)

api/apistruct/struct.go

@@ -173,6 +173,7 @@ type FullNodeStruct struct {
 		ClientListDataTransfers                   func(ctx context.Context) ([]api.DataTransferChannel, error)                                                      `perm:"write"`
 		ClientDataTransferUpdates                 func(ctx context.Context) (<-chan api.DataTransferChannel, error)                                                 `perm:"write"`
 		ClientRestartDataTransfer                 func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error          `perm:"write"`
+		ClientCancelDataTransfer                  func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error          `perm:"write"`
 		ClientRetrieveTryRestartInsufficientFunds func(ctx context.Context, paymentChannel address.Address) error                                                   `perm:"write"`
 
 		StateNetworkName                   func(context.Context) (dtypes.NetworkName, error)                                                                   `perm:"read"`
@@ -286,6 +287,8 @@ type StorageMinerStruct struct {
 		MarketGetRetrievalAsk     func(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                                      `perm:"read"`
 		MarketListDataTransfers   func(ctx context.Context) ([]api.DataTransferChannel, error)                                                                                                                 `perm:"write"`
 		MarketDataTransferUpdates func(ctx context.Context) (<-chan api.DataTransferChannel, error)                                                                                                            `perm:"write"`
+		MarketRestartDataTransfer func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error                                                                     `perm:"read"`
+		MarketCancelDataTransfer  func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error                                                                     `perm:"read"`
 
 		PledgeSector func(context.Context) error `perm:"write"`
 
@@ -597,6 +600,10 @@ func (c *FullNodeStruct) ClientRestartDataTransfer(ctx context.Context, transfer
 	return c.Internal.ClientRestartDataTransfer(ctx, transferID, otherPeer, isInitiator)
 }
 
+func (c *FullNodeStruct) ClientCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error {
+	return c.Internal.ClientCancelDataTransfer(ctx, transferID, otherPeer, isInitiator)
+}
+
 func (c *FullNodeStruct) ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error {
 	return c.Internal.ClientRetrieveTryRestartInsufficientFunds(ctx, paymentChannel)
 }
@@ -1381,6 +1388,14 @@ func (c *StorageMinerStruct) MarketDataTransferUpdates(ctx context.Context) (<-c
 	return c.Internal.MarketDataTransferUpdates(ctx)
 }
 
+func (c *StorageMinerStruct) MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error {
+	return c.Internal.MarketRestartDataTransfer(ctx, transferID, otherPeer, isInitiator)
+}
+
+func (c *StorageMinerStruct) MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error {
+	return c.Internal.MarketCancelDataTransfer(ctx, transferID, otherPeer, isInitiator)
+}
+
 func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error {
 	return c.Internal.DealsImportData(ctx, dealPropCid, file)
 }

build/version.go

@@ -29,7 +29,7 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "1.1.0"
+const BuildVersion = "1.1.2"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit

chain/actors/builtin/miner/miner.go

@@ -77,6 +77,7 @@ type State interface {
 	DeadlinesChanged(State) (bool, error)
 
 	Info() (MinerInfo, error)
+	MinerInfoChanged(State) (bool, error)
 
 	DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error)
 

chain/actors/builtin/miner/v0.go

@@ -277,6 +277,15 @@ func (s *state0) DeadlinesChanged(other State) (bool, error) {
 	return !s.State.Deadlines.Equals(other0.Deadlines), nil
 }
 
+func (s *state0) MinerInfoChanged(other State) (bool, error) {
+	other0, ok := other.(*state0)
+	if !ok {
+		// treat an upgrade as a change, always
+		return true, nil
+	}
+	return !s.State.Info.Equals(other0.State.Info), nil
+}
+
 func (s *state0) Info() (MinerInfo, error) {
 	info, err := s.State.GetInfo(s.store)
 	if err != nil {

chain/actors/builtin/miner/v2.go

@@ -276,6 +276,15 @@ func (s *state2) DeadlinesChanged(other State) (bool, error) {
 	return !s.State.Deadlines.Equals(other2.Deadlines), nil
 }
 
+func (s *state2) MinerInfoChanged(other State) (bool, error) {
+	other0, ok := other.(*state2)
+	if !ok {
+		// treat an upgrade as a change, always
+		return true, nil
+	}
+	return !s.State.Info.Equals(other0.State.Info), nil
+}
+
 func (s *state2) Info() (MinerInfo, error) {
 	info, err := s.State.GetInfo(s.store)
 	if err != nil {

chain/gen/genesis/miners.go

@@ -56,9 +56,9 @@ func (fss *fakedSigSyscalls) VerifySignature(signature crypto.Signature, signer
 }
 
 func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder {
-	return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime2.Syscalls {
+	return func(ctx context.Context, rt *vm.Runtime) runtime2.Syscalls {
 		return &fakedSigSyscalls{
-			base(ctx, cstate, cst),
+			base(ctx, rt),
 		}
 	}
 }

chain/gen/mining.go

@@ -27,16 +27,11 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletA
 		return nil, xerrors.Errorf("failed to load tipset state: %w", err)
 	}
 
-	lbts, err := stmgr.GetLookbackTipSetForRound(ctx, sm, pts, bt.Epoch)
+	_, lbst, err := stmgr.GetLookbackTipSetForRound(ctx, sm, pts, bt.Epoch)
 	if err != nil {
 		return nil, xerrors.Errorf("getting lookback miner actor state: %w", err)
 	}
 
-	lbst, _, err := sm.TipSetState(ctx, lbts)
-	if err != nil {
-		return nil, err
-	}
-
 	worker, err := stmgr.GetMinerWorkerRaw(ctx, sm, lbst, bt.Miner)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to get miner worker: %w", err)

chain/stmgr/call.go

@@ -64,6 +64,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NtwkVersion:    sm.GetNtwkVersion,
 		BaseFee:        types.NewInt(0),
+		LookbackState:  LookbackStateGetterForTipset(sm, ts),
 	}
 
 	vmi, err := sm.newVM(ctx, vmopt)
@@ -178,6 +179,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NtwkVersion:    sm.GetNtwkVersion,
 		BaseFee:        ts.Blocks()[0].ParentBaseFee,
+		LookbackState:  LookbackStateGetterForTipset(sm, ts),
 	}
 	vmi, err := sm.newVM(ctx, vmopt)
 	if err != nil {

chain/stmgr/stmgr.go

@@ -241,6 +241,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 			CircSupplyCalc: sm.GetVMCirculatingSupply,
 			NtwkVersion:    sm.GetNtwkVersion,
 			BaseFee:        baseFee,
+			LookbackState:  LookbackStateGetterForTipset(sm, ts),
 		}
 
 		return sm.newVM(ctx, vmopt)

chain/stmgr/utils.go

@@ -366,6 +366,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NtwkVersion:    sm.GetNtwkVersion,
 		BaseFee:        ts.Blocks()[0].ParentBaseFee,
+		LookbackState:  LookbackStateGetterForTipset(sm, ts),
 	}
 	vmi, err := sm.newVM(ctx, vmopt)
 	if err != nil {
@@ -391,24 +392,51 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 	return root, trace, nil
 }
 
-func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.TipSet, round abi.ChainEpoch) (*types.TipSet, error) {
+func LookbackStateGetterForTipset(sm *StateManager, ts *types.TipSet) vm.LookbackStateGetter {
+	return func(ctx context.Context, round abi.ChainEpoch) (*state.StateTree, error) {
+		_, st, err := GetLookbackTipSetForRound(ctx, sm, ts, round)
+		if err != nil {
+			return nil, err
+		}
+		return sm.StateTree(st)
+	}
+}
+
+func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.TipSet, round abi.ChainEpoch) (*types.TipSet, cid.Cid, error) {
 	var lbr abi.ChainEpoch
 	lb := policy.GetWinningPoStSectorSetLookback(sm.GetNtwkVersion(ctx, round))
 	if round > lb {
 		lbr = round - lb
 	}
 
 	// more null blocks than our lookback
-	if lbr > ts.Height() {
-		return ts, nil
+	if lbr >= ts.Height() {
+		// This should never happen at this point, but may happen before
+		// network version 3 (where the lookback was only 10 blocks).
+		st, _, err := sm.TipSetState(ctx, ts)
+		if err != nil {
+			return nil, cid.Undef, err
+		}
+		return ts, st, nil
 	}
 
-	lbts, err := sm.ChainStore().GetTipsetByHeight(ctx, lbr, ts, true)
+	// Get the tipset after the lookback tipset, or the next non-null one.
+	nextTs, err := sm.ChainStore().GetTipsetByHeight(ctx, lbr+1, ts, false)
 	if err != nil {
-		return nil, xerrors.Errorf("failed to get lookback tipset: %w", err)
+		return nil, cid.Undef, xerrors.Errorf("failed to get lookback tipset+1: %w", err)
 	}
 
-	return lbts, nil
+	if lbr > nextTs.Height() {
+		return nil, cid.Undef, xerrors.Errorf("failed to find non-null tipset %s (%d) which is known to exist, found %s (%d)", ts.Key(), ts.Height(), nextTs.Key(), nextTs.Height())
+
+	}
+
+	lbts, err := sm.ChainStore().GetTipSetFromKey(nextTs.Parents())
+	if err != nil {
+		return nil, cid.Undef, xerrors.Errorf("failed to resolve lookback tipset: %w", err)
+	}
+
+	return lbts, nextTs.ParentState(), nil
 }
 
 func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule, tsk types.TipSetKey, round abi.ChainEpoch, maddr address.Address, pv ffiwrapper.Verifier) (*api.MiningBaseInfo, error) {
@@ -436,17 +464,11 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule
 		rbase = entries[len(entries)-1]
 	}
 
-	lbts, err := GetLookbackTipSetForRound(ctx, sm, ts, round)
+	lbts, lbst, err := GetLookbackTipSetForRound(ctx, sm, ts, round)
 	if err != nil {
 		return nil, xerrors.Errorf("getting lookback miner actor state: %w", err)
 	}
 
-	// TODO: load the state instead of computing it?
-	lbst, _, err := sm.TipSetState(ctx, lbts)
-	if err != nil {
-		return nil, err
-	}
-
 	act, err := sm.LoadActorRaw(ctx, maddr, lbst)
 	if xerrors.Is(err, types.ErrActorNotFound) {
 		_, err := sm.LoadActor(ctx, maddr, ts)