diff --git a/CHANGELOG.md b/CHANGELOG.md index fa6330907f9..11f7d300b9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,143 @@ # Lotus changelog +# 1.10.0 / 2021-06-23 + +This is a mandatory release of Lotus that introduces Filecoin network v13, codenamed the HyperDrive upgrade. The +Filecoin mainnet will upgrade, which is epoch 892800, on 2021-06-30T22:00:00Z. The network upgrade introduces the +following FIPs: + +- [FIP-0008](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0008.md): Add miner batched sector pre-commit method +- [FIP-0011](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0011.md): Remove reward auction from reporting consensus faults +- [FIP-0012](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0012.md): DataCap Top up for FIL+ Client Addresses +- [FIP-0013](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0013.md): Add ProveCommitSectorAggregated method to reduce on-chain congestion +- [FIP-0015](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0015.md): Revert FIP-0009(Exempt Window PoSts from BaseFee burn) + +Note that this release is built on top of Lotus v1.9.0. Enterprising users can use the `master` branch of Lotus to get the latest functionality, including all changes in this release candidate. + +## Proof batching and aggregation + +FIPs [0008](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0008.md) and [0013](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0013.md) combine to allow for a significant increase in the rate of onboarding storage on the Filecoin network. This aims to lead to more useful data being stored on the network, reduced network congestion, and lower network base fee. + +**Check out the documentation [here](https://docs.filecoin.io/mine/lotus/miner-configuration/#precommitsectorsbatch) for details on the new Lotus miner sealing config options, [here](https://docs.filecoin.io/mine/lotus/miner-configuration/#fees-section) for fee config options, and explanations of the new features.** + +Note: + - We recommend to keep `PreCommitSectorsBatch` as 1. + - We recommend miners to set `PreCommitBatchWait` lower than 30 hours. + - We recommend miners to set a longer `CommitBatchSlack` and `PreCommitBatchSlack` to prevent message failures + due to expirations. + +### Projected state tree growth + +In order to validate the Hyperdrive changes, we wrote a simulation to seal as many sectors as quickly as possible, assuming the same number and mix of 32GiB and 64GiB miners as the current network. + +Given these assumptions: + +- We'd expect a network storage growth rate of around 530PiB per day. 😳 πŸŽ‰ πŸ₯³ πŸ˜… +- We'd expect network bandwidth dedicated to `SubmitWindowedPoSt` to grow by about 0.02% per day. +- We'd expect the [state-tree](https://spec.filecoin.io/#section-systems.filecoin_vm.state_tree) (and therefore [snapshot](https://docs.filecoin.io/get-started/lotus/chain/#lightweight-snapshot)) size to grow by 1.16GiB per day. + - Nearly all of the state-tree growth is expected to come from new sector metadata. +- We'd expect the daily lotus datastore growth rate to increase by about 10-15% (from current ~21GiB/day). + - Most "growth" of the lotus datastore is due to "churn", historical data that's no longer referenced by the latest state-tree. + +### Future improvements + +Various Lotus improvements are planned moving forward to mitigate the effects of the growing state tree size. The primary improvement is the [Lotus splitstore](https://github.com/filecoin-project/lotus/discussions/5788), which will soon be enabled by default. The feature allows for [online garbage collection](https://github.com/filecoin-project/lotus/issues/6577) for nodes that do not seek to maintain full chain and state history, thus eliminating the need for users to delete their datastores and sync from snapshots. + +Other improvements including better compressed snapshots, faster pre-migrations, and improved chain exports are in the roadmap. + +## WindowPost base fee burn + +Included in the HyperDrive upgrade is [FIP-0015](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0015.md) which eliminates the special-case gas treatment of `SubmitWindowedPoSt` messages that was introduced in [FIP-0009](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0009.md). Although `SubmitWindowedPoSt` messages will be relatively cheap, thanks to the introduction of optimistic acceptance of these proofs in [FIP-0010](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0010.md), storage providers should pay attention to their `MaxWindowPoStGasFee` config option: too low and PoSts may not land on chain; too high and they may cost an exorbitant amount! + +## Changelog + +### New Features + +- Implement FIP-0015 ([filecoin-project/lotus#6361](https://github.com/filecoin-project/lotus/pull/6361)) +- Integrate FIP0013 and FIP0008 ([filecoin-project/lotus#6235](https://github.com/filecoin-project/lotus/pull/6235)) + - [Configuration docs and cli examples](https://docs.filecoin.io/mine/lotus/miner-configuration/#precommitsectorsbatch) + - [cli docs](https://github.com/filecoin-project/lotus/blob/master/documentation/en/cli-lotus-miner.md#lotus-miner-sectors-batching) + - Introduce gas prices for aggregate verifications ([filecoin-project/lotus#6347](https://github.com/filecoin-project/lotus/pull/6347)) +- Introduce v5 actors ([filecoin-project/lotus#6195](https://github.com/filecoin-project/lotus/pull/6195)) +- Robustify commit batcher ([filecoin-project/lotus#6367](https://github.com/filecoin-project/lotus/pull/6367)) +- Always flush when timer goes off ([filecoin-project/lotus#6563](https://github.com/filecoin-project/lotus/pull/6563)) +- Update default fees for aggregates ([filecoin-project/lotus#6548](https://github.com/filecoin-project/lotus/pull/6548)) +- sealing: Early finalization option ([filecoin-project/lotus#6452](https://github.com/filecoin-project/lotus/pull/6452)) + - `./lotus-miner/config.toml/[Sealing.FinalizeEarly]`: default to false. Enable if you want to FinalizeSector before commiting +- Add filplus utils to CLI ([filecoin-project/lotus#6351](https://github.com/filecoin-project/lotus/pull/6351)) + - cli doc can be found [here](https://github.com/filecoin-project/lotus/blob/master/documentation/en/cli-lotus.md#lotus-filplus) +- Add miner-side MaxDealStartDelay config ([filecoin-project/lotus#6576](https://github.com/filecoin-project/lotus/pull/6576)) + + +### Bug Fixes +- chainstore: Don't take heaviestLk with backlogged reorgCh ([filecoin-project/lotus#6526](https://github.com/filecoin-project/lotus/pull/6526)) +- Backport #6041 - storagefsm: Fix batch deal packing behavior ([filecoin-project/lotus#6519](https://github.com/filecoin-project/lotus/pull/6519)) +- backport: pick the correct partitions-per-post limit ([filecoin-project/lotus#6503](https://github.com/filecoin-project/lotus/pull/6503)) +- failed sectors should be added into res correctly ([filecoin-project/lotus#6472](https://github.com/filecoin-project/lotus/pull/6472)) +- sealing: Fix restartSectors race ([filecoin-project/lotus#6491](https://github.com/filecoin-project/lotus/pull/6491)) +- Fund miners with the aggregate fee when ProveCommitting ([filecoin-project/lotus#6428](https://github.com/filecoin-project/lotus/pull/6428)) +- Commit and Precommit batcher cannot share a getSectorDeadline method ([filecoin-project/lotus#6416](https://github.com/filecoin-project/lotus/pull/6416)) +- Fix supported proof type manipulations for v5 actors ([filecoin-project/lotus#6366](https://github.com/filecoin-project/lotus/pull/6366)) +- events: Fix handling of multiple matched events per epoch ([filecoin-project/lotus#6362](https://github.com/filecoin-project/lotus/pull/6362)) +- Fix randomness fetching around null blocks ([filecoin-project/lotus#6240](https://github.com/filecoin-project/lotus/pull/6240)) + +### Improvements +- Appimage v1.10.0 rc3 ([filecoin-project/lotus#6492](https://github.com/filecoin-project/lotus/pull/6492)) +- Expand on Drand change testing ([filecoin-project/lotus#6500](https://github.com/filecoin-project/lotus/pull/6500)) +- Backport Fix logging around mineOne ([filecoin-project/lotus#6499](https://github.com/filecoin-project/lotus/pull/6499)) +- mpool: Add more metrics ([filecoin-project/lotus#6453](https://github.com/filecoin-project/lotus/pull/6453)) +- Merge backported PRs into v1.10 release branch ([filecoin-project/lotus#6436](https://github.com/filecoin-project/lotus/pull/6436)) +- Fix tests ([filecoin-project/lotus#6371](https://github.com/filecoin-project/lotus/pull/6371)) +- Extend the default deal start epoch delay ([filecoin-project/lotus#6350](https://github.com/filecoin-project/lotus/pull/6350)) +- sealing: Wire up context to batchers ([filecoin-project/lotus#6497](https://github.com/filecoin-project/lotus/pull/6497)) +- Improve address resolution for messages ([filecoin-project/lotus#6364](https://github.com/filecoin-project/lotus/pull/6364)) + +### Dependency Updates +- Proofs v8.0.2 ([filecoin-project/lotus#6524](https://github.com/filecoin-project/lotus/pull/6524)) +- Update to fixed Bellperson ([filecoin-project/lotus#6480](https://github.com/filecoin-project/lotus/pull/6480)) +- Update to go-praamfetch with fslocks ([filecoin-project/lotus#6473](https://github.com/filecoin-project/lotus/pull/6473)) +- Update ffi with fixed multicore sdr support ([filecoin-project/lotus#6471](https://github.com/filecoin-project/lotus/pull/6471)) +- github.com/filecoin-project/go-paramfetch (v0.0.2-0.20200701152213-3e0f0afdc261 -> v0.0.2-0.20210614165157-25a6c7769498) +- github.com/filecoin-project/specs-actors/v5 (v5.0.0-20210512015452-4fe3889fff57 -> v5.0.0) +- github.com/filecoin-project/go-hamt-ipld/v3 (v3.0.1 -> v3.1.0) +- github.com/ipfs/go-log/v2 (v2.1.2-0.20200626104915-0016c0b4b3e4 -> v2.1.3) +- github.com/filecoin-project/go-amt-ipld/v3 (v3.0.0 -> v3.1.0) + +### Network Version v13 HyperDrive Upgrade +- Set HyperDrive upgrade epoch ([filecoin-project/lotus#6565](https://github.com/filecoin-project/lotus/pull/6565)) +- version bump to lotus v1.10.0-rc6 ([filecoin-project/lotus#6529](https://github.com/filecoin-project/lotus/pull/6529)) +- Upgrade epochs for calibration reset ([filecoin-project/lotus#6528](https://github.com/filecoin-project/lotus/pull/6528)) +- Lotus version 1.10.0-rc5 ([filecoin-project/lotus#6504](https://github.com/filecoin-project/lotus/pull/6504)) +- Merge releases into v1.10 release ([filecoin-project/lotus#6494](https://github.com/filecoin-project/lotus/pull/6494)) +- update lotus to v1.10.0-rc3 ([filecoin-project/lotus#6481](https://github.com/filecoin-project/lotus/pull/6481)) +- updated configuration comments for docs +- Lotus version 1.10.0-rc2 ([filecoin-project/lotus#6443](https://github.com/filecoin-project/lotus/pull/6443)) +- Set ntwk v13 HyperDrive Calibration upgrade epoch ([filecoin-project/lotus#6442](https://github.com/filecoin-project/lotus/pull/6442)) + + +## Contributors + +πŸ’™Thank you to all the contributors! + +| Contributor | Commits | Lines Β± | Files Changed | +|--------------------|---------|-------------|---------------| +| @magik6k | 81 | +9606/-1536 | 361 | +| @arajasek | 41 | +6543/-679 | 189 | +| @ZenGround0 | 11 | +4074/-727 | 110 | +| @anorth | 10 | +2035/-1177 | 55 | +| @iand | 1 | +779/-12 | 5 | +| @frrist | 2 | +722/-6 | 6 | +| @Stebalien | 6 | +368/-24 | 15 | +| @jennijuju | 11 | +204/-111 | 19 | +| @vyzo | 6 | +155/-66 | 13 | +| @coryschwartz | 10 | +171/-27 | 14 | +| @Kubuxu | 4 | +177/-13 | 7 | +| @ribasushi | 4 | +65/-42 | 5 | +| @travisperson | 2 | +11/-11 | 4 | +| @kirk-baird | 1 | +1/-5 | 1 | +| @wangchao | 2 | +3/-2 | 2 | + + # 1.9.0 / 2021-05-17 This is an optional Lotus release that introduces various improvements to the sealing, mining, and deal-making processes. diff --git a/extern/storage-sealing/commit_batch.go b/extern/storage-sealing/commit_batch.go index 6d2196f3276..097d55a1ad9 100644 --- a/extern/storage-sealing/commit_batch.go +++ b/extern/storage-sealing/commit_batch.go @@ -110,7 +110,8 @@ func (b *CommitBatcher) run() { } lastMsg = nil - var sendAboveMax, sendAboveMin bool + // indicates whether we should only start a batch if we have reached or exceeded cfg.MaxCommitBatch + var sendAboveMax bool select { case <-b.stop: close(b.stopped) @@ -118,13 +119,13 @@ func (b *CommitBatcher) run() { case <-b.notify: sendAboveMax = true case <-b.batchWait(cfg.CommitBatchWait, cfg.CommitBatchSlack): - sendAboveMin = true + // do nothing case fr := <-b.force: // user triggered forceRes = fr } var err error - lastMsg, err = b.maybeStartBatch(sendAboveMax, sendAboveMin) + lastMsg, err = b.maybeStartBatch(sendAboveMax) if err != nil { log.Warnw("CommitBatcher processBatch error", "error", err) } @@ -172,7 +173,7 @@ func (b *CommitBatcher) batchWait(maxWait, slack time.Duration) <-chan time.Time return time.After(wait) } -func (b *CommitBatcher) maybeStartBatch(notif, after bool) ([]sealiface.CommitBatchRes, error) { +func (b *CommitBatcher) maybeStartBatch(notif bool) ([]sealiface.CommitBatchRes, error) { b.lk.Lock() defer b.lk.Unlock() @@ -190,10 +191,6 @@ func (b *CommitBatcher) maybeStartBatch(notif, after bool) ([]sealiface.CommitBa return nil, nil } - if after && total < cfg.MinCommitBatch { - return nil, nil - } - var res []sealiface.CommitBatchRes if total < cfg.MinCommitBatch || total < miner5.MinAggregatedSectors { diff --git a/extern/storage-sealing/commit_batch_test.go b/extern/storage-sealing/commit_batch_test.go index bbf09766fce..ad2bc8f6f70 100644 --- a/extern/storage-sealing/commit_batch_test.go +++ b/extern/storage-sealing/commit_batch_test.go @@ -48,7 +48,6 @@ func TestCommitBatcher(t *testing.T) { AlwaysKeepUnsealedCopy: true, BatchPreCommits: true, - MinPreCommitBatch: 1, MaxPreCommitBatch: miner5.PreCommitSectorBatchMaxSize, PreCommitBatchWait: 24 * time.Hour, PreCommitBatchSlack: 3 * time.Hour, diff --git a/extern/storage-sealing/precommit_batch.go b/extern/storage-sealing/precommit_batch.go index d85a60bc68b..3dc3510c282 100644 --- a/extern/storage-sealing/precommit_batch.go +++ b/extern/storage-sealing/precommit_batch.go @@ -95,7 +95,7 @@ func (b *PreCommitBatcher) run() { } lastRes = nil - var sendAboveMax, sendAboveMin bool + var sendAboveMax bool select { case <-b.stop: close(b.stopped) @@ -103,13 +103,13 @@ func (b *PreCommitBatcher) run() { case <-b.notify: sendAboveMax = true case <-b.batchWait(cfg.PreCommitBatchWait, cfg.PreCommitBatchSlack): - sendAboveMin = true + // do nothing case fr := <-b.force: // user triggered forceRes = fr } var err error - lastRes, err = b.maybeStartBatch(sendAboveMax, sendAboveMin) + lastRes, err = b.maybeStartBatch(sendAboveMax) if err != nil { log.Warnw("PreCommitBatcher processBatch error", "error", err) } @@ -157,7 +157,7 @@ func (b *PreCommitBatcher) batchWait(maxWait, slack time.Duration) <-chan time.T return time.After(wait) } -func (b *PreCommitBatcher) maybeStartBatch(notif, after bool) ([]sealiface.PreCommitBatchRes, error) { +func (b *PreCommitBatcher) maybeStartBatch(notif bool) ([]sealiface.PreCommitBatchRes, error) { b.lk.Lock() defer b.lk.Unlock() @@ -175,10 +175,6 @@ func (b *PreCommitBatcher) maybeStartBatch(notif, after bool) ([]sealiface.PreCo return nil, nil } - if after && total < cfg.MinPreCommitBatch { - return nil, nil - } - // todo support multiple batches res, err := b.processBatch(cfg) if err != nil && len(res) == 0 { diff --git a/extern/storage-sealing/precommit_batch_test.go b/extern/storage-sealing/precommit_batch_test.go index 141742c14ab..b6c35362e02 100644 --- a/extern/storage-sealing/precommit_batch_test.go +++ b/extern/storage-sealing/precommit_batch_test.go @@ -55,7 +55,6 @@ func TestPrecommitBatcher(t *testing.T) { AlwaysKeepUnsealedCopy: true, BatchPreCommits: true, - MinPreCommitBatch: 1, MaxPreCommitBatch: maxBatch, PreCommitBatchWait: 24 * time.Hour, PreCommitBatchSlack: 3 * time.Hour, diff --git a/extern/storage-sealing/sealiface/config.go b/extern/storage-sealing/sealiface/config.go index 499a2befa0e..b237072d304 100644 --- a/extern/storage-sealing/sealiface/config.go +++ b/extern/storage-sealing/sealiface/config.go @@ -22,7 +22,6 @@ type Config struct { BatchPreCommits bool MaxPreCommitBatch int - MinPreCommitBatch int PreCommitBatchWait time.Duration PreCommitBatchSlack time.Duration diff --git a/go.mod b/go.mod index 411522a367a..94e87b05158 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/filecoin-project/specs-actors/v2 v2.3.5 github.com/filecoin-project/specs-actors/v3 v3.1.1 github.com/filecoin-project/specs-actors/v4 v4.0.1 - github.com/filecoin-project/specs-actors/v5 v5.0.0-20210609212542-73e0409ac77c + github.com/filecoin-project/specs-actors/v5 v5.0.1 github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506 github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 5573587fd33..c127451713c 100644 --- a/go.sum +++ b/go.sum @@ -331,8 +331,8 @@ github.com/filecoin-project/specs-actors/v4 v4.0.0/go.mod h1:TkHXf/l7Wyw4ZejyXIP github.com/filecoin-project/specs-actors/v4 v4.0.1 h1:AiWrtvJZ63MHGe6rn7tPu4nSUY8bA1KDNszqJaD5+Fg= github.com/filecoin-project/specs-actors/v4 v4.0.1/go.mod h1:TkHXf/l7Wyw4ZejyXIPS2rK8bBO0rdwhTZyQQgaglng= github.com/filecoin-project/specs-actors/v5 v5.0.0-20210512015452-4fe3889fff57/go.mod h1:283yBMMUSDB2abcjP/hhrwTkhb9h3sfM6KGrep/ZlBI= -github.com/filecoin-project/specs-actors/v5 v5.0.0-20210609212542-73e0409ac77c h1:GnDJ6q3QEm2ytTKjPFQSvczAltgCSb3j9F1FeynwvPA= -github.com/filecoin-project/specs-actors/v5 v5.0.0-20210609212542-73e0409ac77c/go.mod h1:b/btpRl84Q9SeDKlyIoORBQwe2OTmq14POrYrVvBWCM= +github.com/filecoin-project/specs-actors/v5 v5.0.1 h1:PrYm5AKdMlJ/55eRW5laWcnaX66gyyDYBWvH38kNAMo= +github.com/filecoin-project/specs-actors/v5 v5.0.1/go.mod h1:74euMDIXorusOBs/QL/LNkYsXZdDpLJwojWw6T03pdE= github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506 h1:Ur/l2+6qN+lQiqjozWWc5p9UDaAMDZKTlDS98oRnlIw= github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg= @@ -710,6 +710,7 @@ github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Ax github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= github.com/ipfs/go-merkledag v0.3.2 h1:MRqj40QkrWkvPswXs4EfSslhZ4RVPRbxwX11js0t1xY= github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= @@ -728,6 +729,7 @@ github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUn github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.2.1/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= +github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= @@ -738,13 +740,16 @@ github.com/ipfs/iptb v1.4.0 h1:YFYTrCkLMRwk/35IMyC6+yjoQSHTEcNcefBStLJzgvo= github.com/ipfs/iptb v1.4.0/go.mod h1:1rzHpCYtNp87/+hTxG5TfCVn/yMY3dKnLn8tBiMfdmg= github.com/ipfs/iptb-plugins v0.2.1 h1:au4HWn9/pRPbkxA08pDx2oRAs4cnbgQWgV0teYXuuGA= github.com/ipfs/iptb-plugins v0.2.1/go.mod h1:QXMbtIWZ+jRsW8a4h13qAKU7jcM7qaittO8wOsTP0Rs= +github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g= github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D1vEwl+YAXsg/JfA98XGpXDwnkIL4Aimqw= github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U= github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ= +github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8= github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018 h1:RbRHv8epkmvBYA5cGfz68GUSbOgx5j/7ObLIl4Rsif0= github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= +github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= github.com/ipld/go-ipld-prime-proto v0.0.0-20200428191222-c1ffdadc01e1/go.mod h1:OAV6xBmuTLsPZ+epzKkPB1e25FHk/vCtyatkdHcArLs= github.com/ipld/go-ipld-prime-proto v0.0.0-20200922192210-9a2bfd4440a6/go.mod h1:3pHYooM9Ea65jewRwrb2u5uHZCNkNTe9ABsVB+SrkH0= github.com/ipld/go-ipld-prime-proto v0.1.0 h1:j7gjqrfwbT4+gXpHwEx5iMssma3mnctC7YaCimsFP70= diff --git a/node/builder.go b/node/builder.go index 7740e8e977c..c737e85b82c 100644 --- a/node/builder.go +++ b/node/builder.go @@ -450,6 +450,8 @@ var MinerNode = Options( Override(new(dtypes.GetSealingConfigFunc), modules.NewGetSealConfigFunc), Override(new(dtypes.SetExpectedSealDurationFunc), modules.NewSetExpectedSealDurationFunc), Override(new(dtypes.GetExpectedSealDurationFunc), modules.NewGetExpectedSealDurationFunc), + Override(new(dtypes.SetMaxDealStartDelayFunc), modules.NewSetMaxDealStartDelayFunc), + Override(new(dtypes.GetMaxDealStartDelayFunc), modules.NewGetMaxDealStartDelayFunc), ) // Online sets up basic libp2p node diff --git a/node/config/def.go b/node/config/def.go index 5a672e10964..b2ed43ac657 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -66,6 +66,8 @@ type DealmakingConfig struct { ConsiderUnverifiedStorageDeals bool PieceCidBlocklist []cid.Cid ExpectedSealDuration Duration + // Maximum amount of time proposed deal StartEpoch can be in future + MaxDealStartDelay Duration // The amount of time to wait for more deals to arrive before // publishing PublishMsgPeriod Duration @@ -124,7 +126,6 @@ type SealingConfig struct { BatchPreCommits bool // maximum precommit batch size - batches will be sent immediately above this size MaxPreCommitBatch int - MinPreCommitBatch int // how long to wait before submitting a batch after crossing the minimum batch size PreCommitBatchWait Duration // time buffer for forceful batch submission before sectors/deal in batch would start expiring @@ -316,7 +317,6 @@ func DefaultStorageMiner() *StorageMiner { FinalizeEarly: false, BatchPreCommits: true, - MinPreCommitBatch: 1, // we must have at least one precommit to batch MaxPreCommitBatch: miner5.PreCommitSectorBatchMaxSize, // up to 256 sectors PreCommitBatchWait: Duration(24 * time.Hour), // this should be less than 31.5 hours, which is the expiration of a precommit ticket PreCommitBatchSlack: Duration(3 * time.Hour), // time buffer for forceful batch submission before sectors/deals in batch would start expiring, higher value will lower the chances for message fail due to expiration @@ -356,6 +356,7 @@ func DefaultStorageMiner() *StorageMiner { ConsiderUnverifiedStorageDeals: true, PieceCidBlocklist: []cid.Cid{}, // TODO: It'd be nice to set this based on sector size + MaxDealStartDelay: Duration(time.Hour * 24 * 14), ExpectedSealDuration: Duration(time.Hour * 24), PublishMsgPeriod: Duration(time.Hour), MaxDealsPerPublishMsg: 8, diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index 3c0c14383f4..9a391223dc9 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -90,6 +90,9 @@ type SetExpectedSealDurationFunc func(time.Duration) error // too determine how long sealing is expected to take type GetExpectedSealDurationFunc func() (time.Duration, error) +type SetMaxDealStartDelayFunc func(time.Duration) error +type GetMaxDealStartDelayFunc func() (time.Duration, error) + type StorageDealFilter func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) type RetrievalDealFilter func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 3b346a80fc5..5ed092d3c42 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -60,7 +60,6 @@ import ( "github.com/filecoin-project/lotus/api/v1api" "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" @@ -487,6 +486,7 @@ func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.Conside unverifiedOk dtypes.ConsiderUnverifiedStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc, expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc, + startDelay dtypes.GetMaxDealStartDelayFunc, spn storagemarket.StorageProviderNode) dtypes.StorageDealFilter { return func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc, @@ -494,6 +494,7 @@ func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.Conside unverifiedOk dtypes.ConsiderUnverifiedStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc, expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc, + startDelay dtypes.GetMaxDealStartDelayFunc, spn storagemarket.StorageProviderNode) dtypes.StorageDealFilter { return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { @@ -565,9 +566,14 @@ func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.Conside return false, fmt.Sprintf("cannot seal a sector before %s", deal.Proposal.StartEpoch), nil } + sd, err := startDelay() + if err != nil { + return false, "miner error", err + } + // Reject if it's more than 7 days in the future // TODO: read from cfg - maxStartEpoch := earliest + abi.ChainEpoch(7*builtin.SecondsInDay/build.BlockDelaySecs) + maxStartEpoch := earliest + abi.ChainEpoch(uint64(sd.Seconds())/build.BlockDelaySecs) if deal.Proposal.StartEpoch > maxStartEpoch { return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil } @@ -854,7 +860,6 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error FinalizeEarly: cfg.FinalizeEarly, BatchPreCommits: cfg.BatchPreCommits, - MinPreCommitBatch: cfg.MinPreCommitBatch, MaxPreCommitBatch: cfg.MaxPreCommitBatch, PreCommitBatchWait: config.Duration(cfg.PreCommitBatchWait), PreCommitBatchSlack: config.Duration(cfg.PreCommitBatchSlack), @@ -886,7 +891,6 @@ func NewGetSealConfigFunc(r repo.LockedRepo) (dtypes.GetSealingConfigFunc, error FinalizeEarly: cfg.Sealing.FinalizeEarly, BatchPreCommits: cfg.Sealing.BatchPreCommits, - MinPreCommitBatch: cfg.Sealing.MinPreCommitBatch, MaxPreCommitBatch: cfg.Sealing.MaxPreCommitBatch, PreCommitBatchWait: time.Duration(cfg.Sealing.PreCommitBatchWait), PreCommitBatchSlack: time.Duration(cfg.Sealing.PreCommitBatchSlack), @@ -924,6 +928,24 @@ func NewGetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.GetExpectedSealDu }, nil } +func NewSetMaxDealStartDelayFunc(r repo.LockedRepo) (dtypes.SetMaxDealStartDelayFunc, error) { + return func(delay time.Duration) (err error) { + err = mutateCfg(r, func(cfg *config.StorageMiner) { + cfg.Dealmaking.MaxDealStartDelay = config.Duration(delay) + }) + return + }, nil +} + +func NewGetMaxDealStartDelayFunc(r repo.LockedRepo) (dtypes.GetMaxDealStartDelayFunc, error) { + return func() (out time.Duration, err error) { + err = readCfg(r, func(cfg *config.StorageMiner) { + out = time.Duration(cfg.Dealmaking.MaxDealStartDelay) + }) + return + }, nil +} + func readCfg(r repo.LockedRepo, accessor func(*config.StorageMiner)) error { raw, err := r.Config() if err != nil { diff --git a/scripts/mkreleaselog b/scripts/mkreleaselog index ffc3d935eb3..c9eaef4fb47 100755 --- a/scripts/mkreleaselog +++ b/scripts/mkreleaselog @@ -1,31 +1,11 @@ #!/bin/zsh -set -x set -euo pipefail export GO111MODULE=on export GOPATH="$(go env GOPATH)" alias jq="jq --unbuffered" -AUTHORS=( - # orgs - ipfs - ipld - libp2p - multiformats - filecoin-project - ipfs-shipyard - - # Authors of personal repos used by go-ipfs that should be mentioned in the - # release notes. - whyrusleeping - Kubuxu - jbenet - Stebalien - marten-seemann - hsanjuan - lucas-clemente - warpfork -) +AUTHORS=(filecoin-project) [[ -n "${REPO_FILTER+x}" ]] || REPO_FILTER="github.com/(${$(printf "|%s" "${AUTHORS[@]}"):1})" @@ -49,13 +29,15 @@ statlog() { mailmap_file="$ROOT_DIR/.mailmap" fi - git -C "$rpath" -c mailmap.file="$mailmap_file" log --use-mailmap --shortstat --no-merges --pretty="tformat:%H%n%aN%n%aE" "$start..$end" | while - read hash - read name - read email - read _ # empty line - read changes - do + local stack=() + git -C "$rpath" -c mailmap.file="$mailmap_file" log --use-mailmap --shortstat --no-merges --pretty="tformat:%H%x09%aN%x09%aE" "$start..$end" | while read -r line; do + if [[ -n "$line" ]]; then + stack+=("$line") + continue + fi + + read -r changes + changed=0 insertions=0 deletions=0 @@ -72,14 +54,18 @@ statlog() { fi done<<<"${changes//,/$NL}" - jq -n \ - --arg "hash" "$hash" \ - --arg "name" "$name" \ - --arg "email" "$email" \ - --argjson "changed" "$changed" \ - --argjson "insertions" "$insertions" \ - --argjson "deletions" "$deletions" \ - '{Commit: $hash, Author: $name, Email: $email, Files: $changed, Insertions: $insertions, Deletions: $deletions}' + for author in "${stack[@]}"; do + IFS=$'\t' read -r hash name email <<<"$author" + jq -n \ + --arg "hash" "$hash" \ + --arg "name" "$name" \ + --arg "email" "$email" \ + --argjson "changed" "$changed" \ + --argjson "insertions" "$insertions" \ + --argjson "deletions" "$deletions" \ + '{Commit: $hash, Author: $name, Email: $email, Files: $changed, Insertions: $insertions, Deletions: $deletions}' + done + stack=() done }