diff --git a/.circleci/config.yml b/.circleci/config.yml index c2ce75f9c98..52a37a59fa3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -992,7 +992,7 @@ workflows: - test: go-test-flags: "-run=TestMulticoreSDR" suite: multicore-sdr-check - target: "./extern/sector-storage/ffiwrapper" + target: "./storage/sealer/ffiwrapper" proofs-log-test: "1" - test-conformance: suite: conformance diff --git a/.circleci/template.yml b/.circleci/template.yml index 437eec9d396..482eb72bf93 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -782,7 +782,7 @@ workflows: - test: go-test-flags: "-run=TestMulticoreSDR" suite: multicore-sdr-check - target: "./extern/sector-storage/ffiwrapper" + target: "./storage/sealer/ffiwrapper" proofs-log-test: "1" - test-conformance: suite: conformance diff --git a/api/api_net.go b/api/api_net.go index ae53e4c0d43..32c280828b9 100644 --- a/api/api_net.go +++ b/api/api_net.go @@ -4,7 +4,7 @@ import ( "context" "time" - metrics "github.com/libp2p/go-libp2p-core/metrics" + "github.com/libp2p/go-libp2p-core/metrics" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" diff --git a/api/api_storage.go b/api/api_storage.go index 69d631a08f7..0584f85fdf1 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -22,9 +22,9 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // MODIFYING THE API INTERFACE diff --git a/api/api_worker.go b/api/api_worker.go index 485cd0b011c..688151a8cde 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -10,8 +10,8 @@ import ( "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // MODIFYING THE API INTERFACE diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index 8d8f64c01a9..569d366df62 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -27,7 +27,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" datatransfer "github.com/filecoin-project/go-data-transfer" - filestore "github.com/filecoin-project/go-fil-markets/filestore" + "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/go-state-types/abi" @@ -39,11 +39,11 @@ import ( "github.com/filecoin-project/lotus/api/v0api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo/imports" + sealing "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var ExampleValues = map[reflect.Type]interface{}{ diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 09a655b73dd..64fe24c53df 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -10,11 +10,11 @@ import ( "github.com/google/uuid" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - metrics "github.com/libp2p/go-libp2p-core/metrics" + "github.com/libp2p/go-libp2p-core/metrics" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" @@ -36,13 +36,13 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/journal/alerting" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo/imports" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var ErrNotSupported = xerrors.New("method not supported") diff --git a/api/version.go b/api/version.go index cc0c7b270c1..b47a6ef33a4 100644 --- a/api/version.go +++ b/api/version.go @@ -3,7 +3,7 @@ package api import ( "fmt" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" ) type Version uint32 diff --git a/blockstore/badger/blockstore_test.go b/blockstore/badger/blockstore_test.go index 82ea7368c6b..8d1465a7955 100644 --- a/blockstore/badger/blockstore_test.go +++ b/blockstore/badger/blockstore_test.go @@ -11,7 +11,7 @@ import ( "testing" blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" diff --git a/blockstore/blockstore.go b/blockstore/blockstore.go index 7e2e0b5c6a6..7c3deadb869 100644 --- a/blockstore/blockstore.go +++ b/blockstore/blockstore.go @@ -3,7 +3,7 @@ package blockstore import ( "context" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" blockstore "github.com/ipfs/go-ipfs-blockstore" logging "github.com/ipfs/go-log/v2" diff --git a/blockstore/discard.go b/blockstore/discard.go index 575c752d40c..a8e6cfec7a9 100644 --- a/blockstore/discard.go +++ b/blockstore/discard.go @@ -5,7 +5,7 @@ import ( "io" blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" ) var _ Blockstore = (*discardstore)(nil) diff --git a/blockstore/idstore.go b/blockstore/idstore.go index ef4cb76e11f..cfe4dbb9f5b 100644 --- a/blockstore/idstore.go +++ b/blockstore/idstore.go @@ -5,7 +5,7 @@ import ( "io" blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" "golang.org/x/xerrors" ) diff --git a/blockstore/splitstore/checkpoint.go b/blockstore/splitstore/checkpoint.go index dd9e84aef69..a8b41b6052f 100644 --- a/blockstore/splitstore/checkpoint.go +++ b/blockstore/splitstore/checkpoint.go @@ -5,7 +5,7 @@ import ( "io" "os" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" "golang.org/x/xerrors" ) diff --git a/blockstore/splitstore/coldset.go b/blockstore/splitstore/coldset.go index 76d7c81177f..27dd30db41d 100644 --- a/blockstore/splitstore/coldset.go +++ b/blockstore/splitstore/coldset.go @@ -5,7 +5,7 @@ import ( "io" "os" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) diff --git a/blockstore/splitstore/debug.go b/blockstore/splitstore/debug.go index d3607ea6986..f059ae4ced9 100644 --- a/blockstore/splitstore/debug.go +++ b/blockstore/splitstore/debug.go @@ -13,7 +13,7 @@ import ( "time" blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "go.uber.org/multierr" "golang.org/x/xerrors" ) diff --git a/blockstore/splitstore/markset.go b/blockstore/splitstore/markset.go index 498034016de..7a584d027ca 100644 --- a/blockstore/splitstore/markset.go +++ b/blockstore/splitstore/markset.go @@ -3,7 +3,7 @@ package splitstore import ( "errors" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go index ce3fc57866a..e98046862f0 100644 --- a/blockstore/splitstore/markset_badger.go +++ b/blockstore/splitstore/markset_badger.go @@ -8,7 +8,7 @@ import ( "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/badger/v2/options" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "go.uber.org/zap" "golang.org/x/xerrors" ) diff --git a/blockstore/splitstore/markset_map.go b/blockstore/splitstore/markset_map.go index dd22e987e41..52efcb8a484 100644 --- a/blockstore/splitstore/markset_map.go +++ b/blockstore/splitstore/markset_map.go @@ -7,7 +7,7 @@ import ( "path/filepath" "sync" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) diff --git a/blockstore/splitstore/markset_test.go b/blockstore/splitstore/markset_test.go index 4d67909b03c..bcfb9454ba1 100644 --- a/blockstore/splitstore/markset_test.go +++ b/blockstore/splitstore/markset_test.go @@ -4,7 +4,7 @@ package splitstore import ( "testing" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" ) diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index 91df77c6f8d..45d7720e0a5 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -9,7 +9,7 @@ import ( "time" blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" dstore "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" "go.opencensus.io/stats" diff --git a/blockstore/splitstore/splitstore_check.go b/blockstore/splitstore/splitstore_check.go index be4f9f5902a..dc8de7b96c6 100644 --- a/blockstore/splitstore/splitstore_check.go +++ b/blockstore/splitstore/splitstore_check.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "time" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" bstore "github.com/filecoin-project/lotus/blockstore" diff --git a/blockstore/splitstore/splitstore_compact.go b/blockstore/splitstore/splitstore_compact.go index dda356e9d4d..25dbd65f4f8 100644 --- a/blockstore/splitstore/splitstore_compact.go +++ b/blockstore/splitstore/splitstore_compact.go @@ -11,7 +11,7 @@ import ( "time" blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "go.opencensus.io/stats" "golang.org/x/sync/errgroup" diff --git a/blockstore/splitstore/splitstore_expose.go b/blockstore/splitstore/splitstore_expose.go index 6f838229d68..7cef66b11de 100644 --- a/blockstore/splitstore/splitstore_expose.go +++ b/blockstore/splitstore/splitstore_expose.go @@ -5,7 +5,7 @@ import ( "errors" blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" bstore "github.com/filecoin-project/lotus/blockstore" ) diff --git a/blockstore/splitstore/splitstore_reify.go b/blockstore/splitstore/splitstore_reify.go index f36098a75ec..eb24ac20904 100644 --- a/blockstore/splitstore/splitstore_reify.go +++ b/blockstore/splitstore/splitstore_reify.go @@ -6,7 +6,7 @@ import ( "sync/atomic" blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) diff --git a/blockstore/splitstore/splitstore_test.go b/blockstore/splitstore/splitstore_test.go index d64c1798759..8d210f9ed18 100644 --- a/blockstore/splitstore/splitstore_test.go +++ b/blockstore/splitstore/splitstore_test.go @@ -12,8 +12,8 @@ import ( "time" blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" - datastore "github.com/ipfs/go-datastore" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" logging "github.com/ipfs/go-log/v2" mh "github.com/multiformats/go-multihash" diff --git a/blockstore/splitstore/splitstore_util.go b/blockstore/splitstore/splitstore_util.go index 6ab60b7604a..6e317f4d4bf 100644 --- a/blockstore/splitstore/splitstore_util.go +++ b/blockstore/splitstore/splitstore_util.go @@ -3,7 +3,7 @@ package splitstore import ( "encoding/binary" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" "golang.org/x/xerrors" diff --git a/blockstore/splitstore/splitstore_warmup.go b/blockstore/splitstore/splitstore_warmup.go index b4ba7b03a5a..a496f335fae 100644 --- a/blockstore/splitstore/splitstore_warmup.go +++ b/blockstore/splitstore/splitstore_warmup.go @@ -6,7 +6,7 @@ import ( "time" blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" diff --git a/blockstore/splitstore/visitor.go b/blockstore/splitstore/visitor.go index 4a78f1db1b5..f43bef67518 100644 --- a/blockstore/splitstore/visitor.go +++ b/blockstore/splitstore/visitor.go @@ -3,7 +3,7 @@ package splitstore import ( "sync" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" ) // ObjectVisitor is an interface for deduplicating objects during walks diff --git a/chain/actors/adt/store.go b/chain/actors/adt/store.go index b4116bb4356..71a7df7cb69 100644 --- a/chain/actors/adt/store.go +++ b/chain/actors/adt/store.go @@ -5,7 +5,7 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" - adt "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/specs-actors/actors/util/adt" ) type Store interface { diff --git a/chain/actors/builtin/paych/actor.go.template b/chain/actors/builtin/paych/actor.go.template index 8730e61775f..a815c880e2f 100644 --- a/chain/actors/builtin/paych/actor.go.template +++ b/chain/actors/builtin/paych/actor.go.template @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - big "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/cbor" ipldcbor "github.com/ipfs/go-ipld-cbor" diff --git a/chain/actors/builtin/paych/paych.go b/chain/actors/builtin/paych/paych.go index 75ea6f33442..20d9d255215 100644 --- a/chain/actors/builtin/paych/paych.go +++ b/chain/actors/builtin/paych/paych.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - big "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/big" paychtypes "github.com/filecoin-project/go-state-types/builtin/v8/paych" "github.com/filecoin-project/go-state-types/cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" diff --git a/chain/actors/manifest.go b/chain/actors/manifest.go index c59af7c5549..555a7e1e835 100644 --- a/chain/actors/manifest.go +++ b/chain/actors/manifest.go @@ -5,7 +5,7 @@ import ( "strings" "sync" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" diff --git a/chain/consensus/filcns/filecoin.go b/chain/consensus/filcns/filecoin.go index b33dc1ace38..b4fb40d5213 100644 --- a/chain/consensus/filcns/filecoin.go +++ b/chain/consensus/filcns/filecoin.go @@ -39,10 +39,10 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/lib/async" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) var log = logging.Logger("fil-consensus") diff --git a/chain/exchange/peer_tracker.go b/chain/exchange/peer_tracker.go index 835a5b8a479..489408624d6 100644 --- a/chain/exchange/peer_tracker.go +++ b/chain/exchange/peer_tracker.go @@ -8,7 +8,7 @@ import ( "sync" "time" - host "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" "go.uber.org/fx" diff --git a/chain/gen/gen.go b/chain/gen/gen.go index bfaacefc811..768da71e824 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -40,10 +40,10 @@ import ( "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/genesis" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) const msgsPerBlock = 20 diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index ab01fe80e1c..a4f43e7faec 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -47,7 +47,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/genesis" "github.com/filecoin-project/lotus/lib/sigs" ) @@ -240,7 +240,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal return cid.Undef, fmt.Errorf("failed to marshal proposal: %w", err) } - sig, err := sigs.Sign(wallet.ActSigType(preseal.DealClientKey.Type), preseal.DealClientKey.PrivateKey, buf) + sig, err := sigs.Sign(key.ActSigType(preseal.DealClientKey.Type), preseal.DealClientKey.PrivateKey, buf) if err != nil { return cid.Undef, fmt.Errorf("failed to sign proposal: %w", err) } diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index ce772a80b85..981842255bd 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -9,7 +9,7 @@ import ( "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" - address "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/network" builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" diff --git a/chain/stmgr/actors.go b/chain/stmgr/actors.go index c950f8d02b8..dca02ce3497 100644 --- a/chain/stmgr/actors.go +++ b/chain/stmgr/actors.go @@ -5,7 +5,7 @@ import ( "context" "os" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -26,7 +26,7 @@ import ( "github.com/filecoin-project/lotus/chain/rand" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (address.Address, error) { diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 24d5b66b398..86aa84cb84a 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -6,7 +6,7 @@ import ( "context" "testing" - datastore "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore" syncds "github.com/ipfs/go-datastore/sync" "github.com/stretchr/testify/assert" diff --git a/chain/store/store_test.go b/chain/store/store_test.go index cf0d6e16c7d..95de8401b9e 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -7,7 +7,7 @@ import ( "io" "testing" - datastore "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 7c4df3c9a9a..41a10042ff2 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -12,14 +12,14 @@ import ( bserv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - connmgr "github.com/libp2p/go-libp2p-core/connmgr" + "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/peer" pubsub "github.com/libp2p/go-libp2p-pubsub" "go.opencensus.io/stats" "go.opencensus.io/tag" "golang.org/x/xerrors" - address "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-legs/dtsync" "github.com/filecoin-project/lotus/build" diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 82de35add45..a472bafb5e3 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -9,7 +9,7 @@ import ( "sync" "time" - peer "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peer" "github.com/filecoin-project/go-address" diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index b700c96991a..e0b9e6b30bb 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -7,7 +7,7 @@ import ( block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/minio/blake2b-simd" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" diff --git a/chain/types/blockheader_test.go b/chain/types/blockheader_test.go index 80c97482c71..248016b2870 100644 --- a/chain/types/blockheader_test.go +++ b/chain/types/blockheader_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-address" diff --git a/chain/types/message.go b/chain/types/message.go index dc9accc4346..5b2aad47e90 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -7,7 +7,7 @@ import ( block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" diff --git a/chain/types/vmcontext.go b/chain/types/vmcontext.go index a8fd907a203..2702153b6a7 100644 --- a/chain/types/vmcontext.go +++ b/chain/types/vmcontext.go @@ -1,7 +1,7 @@ package types import ( - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/go-address" diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 8533e29ebb1..7120393194c 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -26,8 +26,8 @@ import ( "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/lib/sigs" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) func init() { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index d016914dd00..00007f29bdb 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -8,7 +8,7 @@ import ( "time" block "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" mh "github.com/multiformats/go-multihash" diff --git a/chain/wallet/key.go b/chain/wallet/key/key.go similarity index 99% rename from chain/wallet/key.go rename to chain/wallet/key/key.go index 1b191cc4b3a..66053525b55 100644 --- a/chain/wallet/key.go +++ b/chain/wallet/key/key.go @@ -1,4 +1,4 @@ -package wallet +package key import ( "golang.org/x/xerrors" diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 91d7e3d54ae..b88d776b503 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/lib/sigs" _ "github.com/filecoin-project/lotus/lib/sigs/bls" // enable bls signatures _ "github.com/filecoin-project/lotus/lib/sigs/secp" // enable secp signatures @@ -28,7 +29,7 @@ const ( ) type LocalWallet struct { - keys map[address.Address]*Key + keys map[address.Address]*key.Key keystore types.KeyStore lk sync.Mutex @@ -41,15 +42,15 @@ type Default interface { func NewWallet(keystore types.KeyStore) (*LocalWallet, error) { w := &LocalWallet{ - keys: make(map[address.Address]*Key), + keys: make(map[address.Address]*key.Key), keystore: keystore, } return w, nil } -func KeyWallet(keys ...*Key) *LocalWallet { - m := make(map[address.Address]*Key) +func KeyWallet(keys ...*key.Key) *LocalWallet { + m := make(map[address.Address]*key.Key) for _, key := range keys { m[key.Address] = key } @@ -68,10 +69,10 @@ func (w *LocalWallet) WalletSign(ctx context.Context, addr address.Address, msg return nil, xerrors.Errorf("signing using key '%s': %w", addr.String(), types.ErrKeyInfoNotFound) } - return sigs.Sign(ActSigType(ki.Type), ki.PrivateKey, msg) + return sigs.Sign(key.ActSigType(ki.Type), ki.PrivateKey, msg) } -func (w *LocalWallet) findKey(addr address.Address) (*Key, error) { +func (w *LocalWallet) findKey(addr address.Address) (*key.Key, error) { w.lk.Lock() defer w.lk.Unlock() @@ -91,7 +92,7 @@ func (w *LocalWallet) findKey(addr address.Address) (*Key, error) { } return nil, xerrors.Errorf("getting from keystore: %w", err) } - k, err = NewKey(ki) + k, err = key.NewKey(ki) if err != nil { return nil, xerrors.Errorf("decoding from keystore: %w", err) } @@ -149,7 +150,7 @@ func (w *LocalWallet) WalletImport(ctx context.Context, ki *types.KeyInfo) (addr w.lk.Lock() defer w.lk.Unlock() - k, err := NewKey(*ki) + k, err := key.NewKey(*ki) if err != nil { return address.Undef, xerrors.Errorf("failed to make key: %w", err) } @@ -203,7 +204,7 @@ func (w *LocalWallet) GetDefault() (address.Address, error) { return address.Undef, xerrors.Errorf("failed to get default key: %w", err) } - k, err := NewKey(ki) + k, err := key.NewKey(ki) if err != nil { return address.Undef, xerrors.Errorf("failed to read default key from keystore: %w", err) } @@ -237,7 +238,7 @@ func (w *LocalWallet) WalletNew(ctx context.Context, typ types.KeyType) (address w.lk.Lock() defer w.lk.Unlock() - k, err := GenerateKey(typ) + k, err := key.GenerateKey(typ) if err != nil { return address.Undef, err } diff --git a/cli/chain.go b/cli/chain.go index 856751ed779..be9279542b2 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -16,7 +16,7 @@ import ( "strings" "time" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -38,7 +38,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/consensus/filcns" - types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/types" ) var ChainCmd = &cli.Command{ diff --git a/cli/chain_test.go b/cli/chain_test.go index 41f0525b2c1..dfef27b115f 100644 --- a/cli/chain_test.go +++ b/cli/chain_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/golang/mock/gomock" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/stretchr/testify/assert" "github.com/filecoin-project/go-state-types/abi" @@ -19,7 +19,7 @@ import ( "github.com/filecoin-project/specs-actors/v7/actors/builtin" "github.com/filecoin-project/lotus/api" - types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" ) diff --git a/cli/mpool.go b/cli/mpool.go index 0224b15d2b9..1410814b53a 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -7,7 +7,7 @@ import ( "sort" "strconv" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" "golang.org/x/xerrors" diff --git a/cli/mpool_manage.go b/cli/mpool_manage.go index d344557305a..67724c77332 100644 --- a/cli/mpool_manage.go +++ b/cli/mpool_manage.go @@ -7,7 +7,7 @@ import ( "github.com/Kubuxu/imtui" "github.com/gdamore/tcell/v2" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/messagepool" - types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/types" ) var mpoolManage = &cli.Command{ diff --git a/cli/multisig.go b/cli/multisig.go index 84803183e61..d88f5d963dc 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -10,7 +10,7 @@ import ( "strconv" "text/tabwriter" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "github.com/urfave/cli/v2" cbg "github.com/whyrusleeping/cbor-gen" diff --git a/cli/net.go b/cli/net.go index 5c9578616ba..23691974fe3 100644 --- a/cli/net.go +++ b/cli/net.go @@ -15,7 +15,7 @@ import ( "github.com/fatih/color" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - protocol "github.com/libp2p/go-libp2p-core/protocol" + "github.com/libp2p/go-libp2p-core/protocol" "github.com/multiformats/go-multiaddr" "github.com/urfave/cli/v2" "golang.org/x/xerrors" diff --git a/cli/params.go b/cli/params.go index d391c1b8573..4dcbe67e2a0 100644 --- a/cli/params.go +++ b/cli/params.go @@ -5,7 +5,7 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" - paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/lotus/build" ) diff --git a/cli/send_test.go b/cli/send_test.go index 571cb42b673..3ee6ad992d5 100644 --- a/cli/send_test.go +++ b/cli/send_test.go @@ -6,7 +6,7 @@ import ( "bytes" "testing" - gomock "github.com/golang/mock/gomock" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" ucli "github.com/urfave/cli/v2" @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" - types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/types" ) func mustAddr(a address.Address, err error) address.Address { diff --git a/cli/sending_ui.go b/cli/sending_ui.go index 5cd763fc2db..d2d2ed3c110 100644 --- a/cli/sending_ui.go +++ b/cli/sending_ui.go @@ -9,7 +9,7 @@ import ( "github.com/Kubuxu/imtui" "github.com/gdamore/tcell/v2" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -18,7 +18,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/types" ) func InteractiveSend(ctx context.Context, cctx *cli.Context, srv ServicesAPI, diff --git a/cli/services.go b/cli/services.go index 287deb5127c..d90da419c67 100644 --- a/cli/services.go +++ b/cli/services.go @@ -7,7 +7,7 @@ import ( "fmt" "reflect" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -18,7 +18,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/consensus/filcns" - types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/types" ) //go:generate go run github.com/golang/mock/mockgen -destination=servicesmock_test.go -package=cli -self_package github.com/filecoin-project/lotus/cli . ServicesAPI diff --git a/cli/services_send_test.go b/cli/services_send_test.go index a73fabe3cef..fe9bb0e898d 100644 --- a/cli/services_send_test.go +++ b/cli/services_send_test.go @@ -7,7 +7,7 @@ import ( "fmt" "testing" - gomock "github.com/golang/mock/gomock" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/filecoin-project/go-address" @@ -15,8 +15,8 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/api" - mocks "github.com/filecoin-project/lotus/api/mocks" - types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/api/mocks" + "github.com/filecoin-project/lotus/chain/types" ) type markerKeyType struct{} diff --git a/cli/sync.go b/cli/sync.go index 2dddbab89fc..02e4e381ff7 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" "github.com/filecoin-project/go-state-types/abi" diff --git a/cli/wallet_test.go b/cli/wallet_test.go index 9a806266b35..01aa0dd7c32 100644 --- a/cli/wallet_test.go +++ b/cli/wallet_test.go @@ -20,7 +20,7 @@ import ( "github.com/filecoin-project/lotus/api" apitypes "github.com/filecoin-project/lotus/api/types" - types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/types" ) func TestWalletNew(t *testing.T) { diff --git a/cmd/lotus-bench/caching_verifier.go b/cmd/lotus-bench/caching_verifier.go index 733b060c7c0..f300a657754 100644 --- a/cmd/lotus-bench/caching_verifier.go +++ b/cmd/lotus-bench/caching_verifier.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" prooftypes "github.com/filecoin-project/go-state-types/proof" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) type cachingVerifier struct { diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index ef3c8312f86..31ee0d51eaa 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -40,10 +40,10 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) type TipSetExec struct { diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 10a3c6dc17f..6803fd90216 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -19,7 +19,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-state-types/abi" prooftypes "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/specs-storage/storage" @@ -29,10 +29,10 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper/basicfs" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper/basicfs" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("lotus-bench") diff --git a/cmd/lotus-bench/simple.go b/cmd/lotus-bench/simple.go index 7454a370134..7ddf161aa77 100644 --- a/cmd/lotus-bench/simple.go +++ b/cmd/lotus-bench/simple.go @@ -24,8 +24,8 @@ import ( "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var simpleCmd = &cli.Command{ diff --git a/cmd/lotus-health/main.go b/cmd/lotus-health/main.go index c1bf7a45945..a7052f21480 100644 --- a/cmd/lotus-health/main.go +++ b/cmd/lotus-health/main.go @@ -8,7 +8,7 @@ import ( "syscall" "time" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/urfave/cli/v2" diff --git a/cmd/lotus-health/main_test.go b/cmd/lotus-health/main_test.go index f2264227911..40ca9d40929 100644 --- a/cmd/lotus-health/main_test.go +++ b/cmd/lotus-health/main_test.go @@ -4,7 +4,7 @@ package main import ( "testing" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" "github.com/stretchr/testify/assert" ) diff --git a/cmd/lotus-miner/info.go b/cmd/lotus-miner/info.go index 8b68099423a..2c3e29b2959 100644 --- a/cmd/lotus-miner/info.go +++ b/cmd/lotus-miner/info.go @@ -33,9 +33,9 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/journal/alerting" + sealing "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" ) var infoCmd = &cli.Command{ diff --git a/cmd/lotus-miner/init.go b/cmd/lotus-miner/init.go index 1e7b07a5911..2daced42e32 100644 --- a/cmd/lotus-miner/init.go +++ b/cmd/lotus-miner/init.go @@ -25,7 +25,7 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" - paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/builtin" @@ -46,11 +46,6 @@ import ( "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/genesis" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/journal/fsjournal" @@ -59,6 +54,11 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/paths" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var initCmd = &cli.Command{ @@ -213,7 +213,7 @@ var initCmd = &cli.Command{ return err } - var localPaths []stores.LocalPath + var localPaths []paths.LocalPath if pssb := cctx.StringSlice("pre-sealed-sectors"); len(pssb) != 0 { log.Infof("Setting up storage config with presealed sectors: %v", pssb) @@ -223,14 +223,14 @@ var initCmd = &cli.Command{ if err != nil { return err } - localPaths = append(localPaths, stores.LocalPath{ + localPaths = append(localPaths, paths.LocalPath{ Path: psp, }) } } if !cctx.Bool("no-local-storage") { - b, err := json.MarshalIndent(&stores.LocalStorageMeta{ + b, err := json.MarshalIndent(&paths.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), Weight: 10, CanSeal: true, @@ -244,12 +244,12 @@ var initCmd = &cli.Command{ return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) } - localPaths = append(localPaths, stores.LocalPath{ + localPaths = append(localPaths, paths.LocalPath{ Path: lr.Path(), }) } - if err := lr.SetStorage(func(sc *stores.StorageConfig) { + if err := lr.SetStorage(func(sc *paths.StorageConfig) { sc.StoragePaths = append(sc.StoragePaths, localPaths...) }); err != nil { return xerrors.Errorf("set storage config: %w", err) @@ -303,7 +303,7 @@ func migratePreSealMeta(ctx context.Context, api v1api.FullNode, metadata string maxSectorID := abi.SectorNumber(0) for _, sector := range meta.Sectors { - sectorKey := datastore.NewKey(sealing.SectorStorePrefix).ChildString(fmt.Sprint(sector.SectorID)) + sectorKey := datastore.NewKey(pipeline.SectorStorePrefix).ChildString(fmt.Sprint(sector.SectorID)) dealID, err := findMarketDealID(ctx, api, sector.Deal) if err != nil { @@ -312,10 +312,10 @@ func migratePreSealMeta(ctx context.Context, api v1api.FullNode, metadata string commD := sector.CommD commR := sector.CommR - info := &sealing.SectorInfo{ - State: sealing.Proving, + info := &pipeline.SectorInfo{ + State: pipeline.Proving, SectorNumber: sector.SectorID, - Pieces: []sealing.Piece{ + Pieces: []pipeline.Piece{ { Piece: abi.PieceInfo{ Size: abi.PaddedPieceSize(meta.SectorSize), @@ -458,15 +458,15 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode wsts := statestore.New(namespace.Wrap(mds, modules.WorkerCallsPrefix)) smsts := statestore.New(namespace.Wrap(mds, modules.ManagerWorkPrefix)) - si := stores.NewIndex() + si := paths.NewIndex() - lstor, err := stores.NewLocal(ctx, lr, si, nil) + lstor, err := paths.NewLocal(ctx, lr, si, nil) if err != nil { return err } - stor := stores.NewRemote(lstor, si, http.Header(sa), 10, &stores.DefaultPartialFileHandler{}) + stor := paths.NewRemote(lstor, si, http.Header(sa), 10, &paths.DefaultPartialFileHandler{}) - smgr, err := sectorstorage.New(ctx, lstor, stor, lr, si, sectorstorage.Config{ + smgr, err := sealer.New(ctx, lstor, stor, lr, si, sealer.Config{ ParallelFetchLimit: 10, AllowAddPiece: true, AllowPreCommit1: true, diff --git a/cmd/lotus-miner/init_restore.go b/cmd/lotus-miner/init_restore.go index fc55f845cba..483c081c3d7 100644 --- a/cmd/lotus-miner/init_restore.go +++ b/cmd/lotus-miner/init_restore.go @@ -24,10 +24,10 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/paths" ) var restoreCmd = &cli.Command{ @@ -52,7 +52,7 @@ var restoreCmd = &cli.Command{ ctx := lcli.ReqContext(cctx) log.Info("Initializing lotus miner using a backup") - var storageCfg *stores.StorageConfig + var storageCfg *paths.StorageConfig if cctx.IsSet("storage-config") { cf, err := homedir.Expand(cctx.String("storage-config")) if err != nil { @@ -64,7 +64,7 @@ var restoreCmd = &cli.Command{ return xerrors.Errorf("reading storage config: %w", err) } - storageCfg = &stores.StorageConfig{} + storageCfg = &paths.StorageConfig{} err = json.Unmarshal(cfb, storageCfg) if err != nil { return xerrors.Errorf("cannot unmarshal json for storage config: %w", err) @@ -95,7 +95,7 @@ var restoreCmd = &cli.Command{ }, } -func restore(ctx context.Context, cctx *cli.Context, targetPath string, strConfig *stores.StorageConfig, manageConfig func(*config.StorageMiner) error, after func(api lapi.FullNode, addr address.Address, peerid peer.ID, mi api.MinerInfo) error) error { +func restore(ctx context.Context, cctx *cli.Context, targetPath string, strConfig *paths.StorageConfig, manageConfig func(*config.StorageMiner) error, after func(api lapi.FullNode, addr address.Address, peerid peer.ID, mi api.MinerInfo) error) error { if cctx.Args().Len() != 1 { return xerrors.Errorf("expected 1 argument") } @@ -214,7 +214,7 @@ func restore(ctx context.Context, cctx *cli.Context, targetPath string, strConfi if strConfig != nil { log.Info("Restoring storage path config") - err = lr.SetStorage(func(scfg *stores.StorageConfig) { + err = lr.SetStorage(func(scfg *paths.StorageConfig) { *scfg = *strConfig }) if err != nil { @@ -223,8 +223,8 @@ func restore(ctx context.Context, cctx *cli.Context, targetPath string, strConfi } else { log.Warn("--storage-config NOT SET. NO SECTOR PATHS WILL BE CONFIGURED") // setting empty config to allow miner to be started - if err := lr.SetStorage(func(sc *stores.StorageConfig) { - sc.StoragePaths = append(sc.StoragePaths, stores.LocalPath{}) + if err := lr.SetStorage(func(sc *paths.StorageConfig) { + sc.StoragePaths = append(sc.StoragePaths, paths.LocalPath{}) }); err != nil { return xerrors.Errorf("set storage config: %w", err) } diff --git a/cmd/lotus-miner/init_service.go b/cmd/lotus-miner/init_service.go index 0c2d19cfd3d..9bbfd18ed1a 100644 --- a/cmd/lotus-miner/init_service.go +++ b/cmd/lotus-miner/init_service.go @@ -16,8 +16,8 @@ import ( "github.com/filecoin-project/lotus/api/client" lcli "github.com/filecoin-project/lotus/cli" cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/storage/paths" ) const ( @@ -78,7 +78,7 @@ var serviceCmd = &cli.Command{ return xerrors.Errorf("please provide Lotus markets repo path via flag %s", FlagMarketsRepo) } - if err := restore(ctx, cctx, repoPath, &stores.StorageConfig{}, func(cfg *config.StorageMiner) error { + if err := restore(ctx, cctx, repoPath, &paths.StorageConfig{}, func(cfg *config.StorageMiner) error { cfg.Subsystems.EnableMarkets = es.Contains(MarketsService) cfg.Subsystems.EnableMining = false cfg.Subsystems.EnableSealing = false diff --git a/cmd/lotus-miner/proving.go b/cmd/lotus-miner/proving.go index 76d3edc60b9..00531462a3a 100644 --- a/cmd/lotus-miner/proving.go +++ b/cmd/lotus-miner/proving.go @@ -21,7 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var provingCmd = &cli.Command{ diff --git a/cmd/lotus-miner/sealing.go b/cmd/lotus-miner/sealing.go index 975bec7d108..d334f657f2d 100644 --- a/cmd/lotus-miner/sealing.go +++ b/cmd/lotus-miner/sealing.go @@ -24,9 +24,9 @@ import ( "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/lib/httpreader" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var sealingCmd = &cli.Command{ diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index edbb62b1f26..be5623c2735 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -32,8 +32,8 @@ import ( "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/lib/tablewriter" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) var sectorsCmd = &cli.Command{ diff --git a/cmd/lotus-miner/storage.go b/cmd/lotus-miner/storage.go index 80a722f629d..7823f484d54 100644 --- a/cmd/lotus-miner/storage.go +++ b/cmd/lotus-miner/storage.go @@ -27,11 +27,11 @@ import ( "github.com/filecoin-project/lotus/api/v0api" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/lib/tablewriter" + "github.com/filecoin-project/lotus/storage/paths" + sealing "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const metaFile = "sectorstore.json" @@ -145,7 +145,7 @@ over time } } - cfg := &stores.LocalStorageMeta{ + cfg := &paths.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), Weight: cctx.Uint64("weight"), CanSeal: cctx.Bool("seal"), diff --git a/cmd/lotus-seed/genesis.go b/cmd/lotus-seed/genesis.go index a3cc890174e..5c3ac5ff14d 100644 --- a/cmd/lotus-seed/genesis.go +++ b/cmd/lotus-seed/genesis.go @@ -25,10 +25,10 @@ import ( genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/genesis" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/modules/testing" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) var genesisCmd = &cli.Command{ diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 2cb629beccd..5af234e5c83 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -26,17 +26,17 @@ import ( "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper/basicfs" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper/basicfs" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("preseal") -func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.SectorNumber, sectors int, sbroot string, preimage []byte, key *types.KeyInfo, fakeSectors bool) (*genesis.Miner, *types.KeyInfo, error) { +func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.SectorNumber, sectors int, sbroot string, preimage []byte, ki *types.KeyInfo, fakeSectors bool) (*genesis.Miner, *types.KeyInfo, error) { mid, err := address.IDFromAddress(maddr) if err != nil { return nil, nil, err @@ -84,14 +84,14 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect sealedSectors = append(sealedSectors, preseal) } - var minerAddr *wallet.Key - if key != nil { - minerAddr, err = wallet.NewKey(*key) + var minerAddr *key.Key + if ki != nil { + minerAddr, err = key.NewKey(*ki) if err != nil { return nil, nil, err } } else { - minerAddr, err = wallet.GenerateKey(types.KTBLS) + minerAddr, err = key.GenerateKey(types.KTBLS) if err != nil { return nil, nil, err } @@ -127,7 +127,7 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect } { - b, err := json.MarshalIndent(&stores.LocalStorageMeta{ + b, err := json.MarshalIndent(&paths.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), Weight: 0, // read-only CanSeal: false, @@ -249,7 +249,7 @@ func WriteGenesisMiner(maddr address.Address, sbroot string, gm *genesis.Miner, return nil } -func createDeals(m *genesis.Miner, k *wallet.Key, maddr address.Address, ssize abi.SectorSize) error { +func createDeals(m *genesis.Miner, k *key.Key, maddr address.Address, ssize abi.SectorSize) error { for i, sector := range m.Sectors { label, err := market8.NewLabelFromString(fmt.Sprintf("%d", i)) if err != nil { diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 1318b8c6a13..9ce4faf723c 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -40,8 +40,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) type accountInfo struct { diff --git a/cmd/lotus-shed/blockmsgid.go b/cmd/lotus-shed/blockmsgid.go index 4a0250d1e5f..c52c87c36c0 100644 --- a/cmd/lotus-shed/blockmsgid.go +++ b/cmd/lotus-shed/blockmsgid.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/ipfs/go-cid" - blake2b "github.com/minio/blake2b-simd" + "github.com/minio/blake2b-simd" "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/chain/types" diff --git a/cmd/lotus-shed/fr32.go b/cmd/lotus-shed/fr32.go index f3f6d06696c..1bdca198cf0 100644 --- a/cmd/lotus-shed/fr32.go +++ b/cmd/lotus-shed/fr32.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/fr32" + "github.com/filecoin-project/lotus/storage/sealer/fr32" ) var fr32Cmd = &cli.Command{ diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index a8c5e1484ba..6830847110c 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/node/modules" @@ -317,7 +318,7 @@ var keyinfoInfoCmd = &cli.Command{ case types.KTSecp256k1, types.KTBLS: kio.Type = keyInfo.Type - key, err := wallet.NewKey(keyInfo) + key, err := key.NewKey(keyInfo) if err != nil { return err } @@ -402,7 +403,7 @@ var keyinfoNewCmd = &cli.Command{ break case types.KTSecp256k1, types.KTBLS: - key, err := wallet.GenerateKey(keyType) + key, err := key.GenerateKey(keyType) if err != nil { return err } diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index 55625c8eb24..7e3daf1f23e 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -16,8 +16,8 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) var migrationsCmd = &cli.Command{ diff --git a/cmd/lotus-shed/params.go b/cmd/lotus-shed/params.go index 107a04ac209..08ce5ba83f1 100644 --- a/cmd/lotus-shed/params.go +++ b/cmd/lotus-shed/params.go @@ -5,7 +5,7 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" - paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" diff --git a/cmd/lotus-sim/simulation/mock/mock.go b/cmd/lotus-sim/simulation/mock/mock.go index c172907b3b6..a37845daafd 100644 --- a/cmd/lotus-sim/simulation/mock/mock.go +++ b/cmd/lotus-sim/simulation/mock/mock.go @@ -15,10 +15,10 @@ import ( miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" tutils "github.com/filecoin-project/specs-actors/v5/support/testing" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) -// Ideally, we'd use extern/sector-storage/mock. Unfortunately, those mocks are a bit _too_ accurate +// Ideally, we'd use extern/sealer/mock. Unfortunately, those mocks are a bit _too_ accurate // and would force us to load sector info for window post proofs. const ( diff --git a/cmd/lotus-stats/main.go b/cmd/lotus-stats/main.go index 2c3b80b3cc3..20b2ee45c81 100644 --- a/cmd/lotus-stats/main.go +++ b/cmd/lotus-stats/main.go @@ -10,7 +10,7 @@ import ( "contrib.go.opencensus.io/exporter/prometheus" logging "github.com/ipfs/go-log/v2" "github.com/urfave/cli/v2" - stats "go.opencensus.io/stats" + "go.opencensus.io/stats" "go.opencensus.io/stats/view" "github.com/filecoin-project/go-state-types/abi" diff --git a/cmd/lotus-worker/info.go b/cmd/lotus-worker/info.go index 057e1303c8b..338345c2857 100644 --- a/cmd/lotus-worker/info.go +++ b/cmd/lotus-worker/info.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" ) var infoCmd = &cli.Command{ diff --git a/cmd/lotus-worker/main.go b/cmd/lotus-worker/main.go index ba4c4caf8c0..41a59f6b458 100644 --- a/cmd/lotus-worker/main.go +++ b/cmd/lotus-worker/main.go @@ -22,7 +22,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-jsonrpc/auth" - paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" @@ -30,15 +30,15 @@ import ( lcli "github.com/filecoin-project/lotus/cli" cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/lib/ulimit" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("main") @@ -384,10 +384,10 @@ var runCmd = &cli.Command{ return err } - var localPaths []stores.LocalPath + var localPaths []paths.LocalPath if !cctx.Bool("no-local-storage") { - b, err := json.MarshalIndent(&stores.LocalStorageMeta{ + b, err := json.MarshalIndent(&paths.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), Weight: 10, CanSeal: true, @@ -401,12 +401,12 @@ var runCmd = &cli.Command{ return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) } - localPaths = append(localPaths, stores.LocalPath{ + localPaths = append(localPaths, paths.LocalPath{ Path: lr.Path(), }) } - if err := lr.SetStorage(func(sc *stores.StorageConfig) { + if err := lr.SetStorage(func(sc *paths.StorageConfig) { sc.StoragePaths = append(sc.StoragePaths, localPaths...) }); err != nil { return xerrors.Errorf("set storage config: %w", err) @@ -456,7 +456,7 @@ var runCmd = &cli.Command{ } } - localStore, err := stores.NewLocal(ctx, lr, nodeApi, []string{"http://" + address + "/remote"}) + localStore, err := paths.NewLocal(ctx, lr, nodeApi, []string{"http://" + address + "/remote"}) if err != nil { return err } @@ -467,10 +467,10 @@ var runCmd = &cli.Command{ return xerrors.Errorf("could not get api info: %w", err) } - remote := stores.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"), - &stores.DefaultPartialFileHandler{}) + remote := paths.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"), + &paths.DefaultPartialFileHandler{}) - fh := &stores.FetchHandler{Local: localStore, PfHandler: &stores.DefaultPartialFileHandler{}} + fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}} remoteHandler := func(w http.ResponseWriter, r *http.Request) { if !auth.HasPerm(r.Context(), nil, api.PermAdmin) { w.WriteHeader(401) @@ -486,7 +486,7 @@ var runCmd = &cli.Command{ wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix)) workerApi := &sealworker.Worker{ - LocalWorker: sectorstorage.NewLocalWorker(sectorstorage.WorkerConfig{ + LocalWorker: sealer.NewLocalWorker(sealer.WorkerConfig{ TaskTypes: taskTypes, NoSwap: cctx.Bool("no-swap"), MaxParallelChallengeReads: cctx.Int("post-parallel-reads"), @@ -561,7 +561,7 @@ var runCmd = &cli.Command{ } go func() { - heartbeats := time.NewTicker(stores.HeartbeatInterval) + heartbeats := time.NewTicker(paths.HeartbeatInterval) defer heartbeats.Stop() var redeclareStorage bool diff --git a/cmd/lotus-worker/resources.go b/cmd/lotus-worker/resources.go index 539f141b9f7..45fd01e702b 100644 --- a/cmd/lotus-worker/resources.go +++ b/cmd/lotus-worker/resources.go @@ -7,7 +7,7 @@ import ( "github.com/urfave/cli/v2" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var resourcesCmd = &cli.Command{ diff --git a/cmd/lotus-worker/sealworker/rpc.go b/cmd/lotus-worker/sealworker/rpc.go index 91f96d9a80b..120d6c04e87 100644 --- a/cmd/lotus-worker/sealworker/rpc.go +++ b/cmd/lotus-worker/sealworker/rpc.go @@ -16,11 +16,11 @@ import ( "github.com/filecoin-project/lotus/api" apitypes "github.com/filecoin-project/lotus/api/types" "github.com/filecoin-project/lotus/build" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/lib/rpcenc" "github.com/filecoin-project/lotus/metrics/proxy" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func WorkerHandler(authv func(ctx context.Context, token string) ([]auth.Permission, error), remote http.HandlerFunc, a api.Worker, permissioned bool) http.Handler { @@ -53,10 +53,10 @@ func WorkerHandler(authv func(ctx context.Context, token string) ([]auth.Permiss } type Worker struct { - *sectorstorage.LocalWorker + *sealer.LocalWorker - LocalStore *stores.Local - Storage stores.LocalStorage + LocalStore *paths.Local + Storage paths.LocalStorage disabled int64 } @@ -75,8 +75,8 @@ func (w *Worker) StorageAddLocal(ctx context.Context, path string) error { return xerrors.Errorf("opening local path: %w", err) } - if err := w.Storage.SetStorage(func(sc *stores.StorageConfig) { - sc.StoragePaths = append(sc.StoragePaths, stores.LocalPath{Path: path}) + if err := w.Storage.SetStorage(func(sc *paths.StorageConfig) { + sc.StoragePaths = append(sc.StoragePaths, paths.LocalPath{Path: path}) }); err != nil { return xerrors.Errorf("get storage config: %w", err) } diff --git a/cmd/lotus-worker/storage.go b/cmd/lotus-worker/storage.go index 0855ddf6a83..2f1cd2f711c 100644 --- a/cmd/lotus-worker/storage.go +++ b/cmd/lotus-worker/storage.go @@ -13,8 +13,8 @@ import ( "golang.org/x/xerrors" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const metaFile = "sectorstore.json" @@ -101,7 +101,7 @@ var storageAttachCmd = &cli.Command{ } } - cfg := &stores.LocalStorageMeta{ + cfg := &paths.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), Weight: cctx.Uint64("weight"), CanSeal: cctx.Bool("seal"), diff --git a/cmd/lotus-worker/tasks.go b/cmd/lotus-worker/tasks.go index 880381fd259..7b446207318 100644 --- a/cmd/lotus-worker/tasks.go +++ b/cmd/lotus-worker/tasks.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/lotus/api" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" ) var tasksCmd = &cli.Command{ diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index b77cb0bae18..33c579e3a95 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -28,7 +28,7 @@ import ( "gopkg.in/cheggaaa/pb.v1" "github.com/filecoin-project/go-jsonrpc" - paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -38,7 +38,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/journal/fsjournal" "github.com/filecoin-project/lotus/lib/peermgr" @@ -49,6 +48,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) const ( diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index c0caf173e52..0764463fc88 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -11,14 +11,14 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - mock2 "github.com/filecoin-project/specs-actors/v2/support/mock" + "github.com/filecoin-project/specs-actors/v2/support/mock" atesting2 "github.com/filecoin-project/specs-actors/v2/support/testing" ) func TestSingleton(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_BUILDER_001 receiver := atesting2.NewIDAddr(t, 100) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -33,7 +33,7 @@ func TestSingleton(t *testing.T) { func TestCallerValidationNone(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_CALLER_VALIDATION_001 receiver := atesting2.NewIDAddr(t, 100) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -46,7 +46,7 @@ func TestCallerValidationIs(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_CALLER_VALIDATION_001 caller := atesting2.NewIDAddr(t, 100) receiver := atesting2.NewIDAddr(t, 101) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) rt.SetCaller(caller, builtin2.AccountActorCodeID) @@ -76,7 +76,7 @@ func TestCallerValidationType(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_CALLER_VALIDATION_001 caller := atesting2.NewIDAddr(t, 100) receiver := atesting2.NewIDAddr(t, 101) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) rt.SetCaller(caller, builtin2.AccountActorCodeID) @@ -102,7 +102,7 @@ func TestCallerValidationType(t *testing.T) { func TestCallerValidationInvalidBranch(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_CALLER_VALIDATION_001 receiver := atesting2.NewIDAddr(t, 100) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -117,7 +117,7 @@ func TestDeleteActor(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_CREATE_ACTOR_001 receiver := atesting2.NewIDAddr(t, 100) beneficiary := atesting2.NewIDAddr(t, 101) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -131,7 +131,7 @@ func TestDeleteActor(t *testing.T) { func TestMutateStateInTransaction(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_CREATE_STATE_001, @CHAIN_ACTOR_CHAOS_MUTATE_STATE_001 receiver := atesting2.NewIDAddr(t, 100) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -159,7 +159,7 @@ func TestMutateStateInTransaction(t *testing.T) { func TestMutateStateAfterTransaction(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_CREATE_STATE_001, @CHAIN_ACTOR_CHAOS_MUTATE_STATE_001 receiver := atesting2.NewIDAddr(t, 100) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -194,7 +194,7 @@ func TestMutateStateAfterTransaction(t *testing.T) { func TestMutateStateReadonly(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_CREATE_STATE_001, @CHAIN_ACTOR_CHAOS_MUTATE_STATE_001 receiver := atesting2.NewIDAddr(t, 100) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -229,7 +229,7 @@ func TestMutateStateReadonly(t *testing.T) { func TestMutateStateInvalidBranch(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_MUTATE_STATE_001 receiver := atesting2.NewIDAddr(t, 100) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -244,7 +244,7 @@ func TestMutateStateInvalidBranch(t *testing.T) { func TestAbortWith(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_ABORT_WITH_001 receiver := atesting2.NewIDAddr(t, 100) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -263,7 +263,7 @@ func TestAbortWith(t *testing.T) { func TestAbortWithUncontrolled(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_ABORT_WITH_001 receiver := atesting2.NewIDAddr(t, 100) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -282,7 +282,7 @@ func TestInspectRuntime(t *testing.T) { //stm: @CHAIN_ACTOR_CHAOS_INSPECT_RUNTIME_001, @CHAIN_ACTOR_CHAOS_CREATE_STATE_001 caller := atesting2.NewIDAddr(t, 100) receiver := atesting2.NewIDAddr(t, 101) - builder := mock2.NewBuilder(context.Background(), receiver) + builder := mock.NewBuilder(context.Background(), receiver) var a Actor diff --git a/conformance/driver.go b/conformance/driver.go index f63fcaaf148..982ce6daeb3 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -25,9 +25,9 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/conformance/chaos" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" _ "github.com/filecoin-project/lotus/lib/sigs/bls" // enable bls signatures _ "github.com/filecoin-project/lotus/lib/sigs/secp" // enable secp signatures + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) var ( diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index 282d3f3c6a1..44a9117e3e6 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -562,7 +562,7 @@ # to use when evaluating tasks against this worker. An empty value defaults # to "hardware". # - # type: sectorstorage.ResourceFilteringStrategy + # type: sealer.ResourceFilteringStrategy # env var: LOTUS_STORAGE_RESOURCEFILTERING #ResourceFiltering = "hardware" diff --git a/extern/sector-storage/mock/util.go b/extern/sector-storage/mock/util.go deleted file mode 100644 index 2d2ebbfe23f..00000000000 --- a/extern/sector-storage/mock/util.go +++ /dev/null @@ -1,9 +0,0 @@ -package mock - -func CommDR(in []byte) (out [32]byte) { - for i, b := range in { - out[i] = ^b - } - - return out -} diff --git a/gen/main.go b/gen/main.go index 9cb68574067..fcb1d6dfa98 100644 --- a/gen/main.go +++ b/gen/main.go @@ -12,10 +12,10 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/cmd/lotus-shed/shedgen" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/paychmgr" + sectorstorage "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func main() { @@ -99,7 +99,7 @@ func main() { os.Exit(1) } - err = gen.WriteMapEncodersToFile("./extern/sector-storage/storiface/cbor_gen.go", "storiface", + err = gen.WriteMapEncodersToFile("./storage/sealer/storiface/cbor_gen.go", "storiface", storiface.CallID{}, ) if err != nil { @@ -107,7 +107,7 @@ func main() { os.Exit(1) } - err = gen.WriteMapEncodersToFile("./extern/sector-storage/cbor_gen.go", "sectorstorage", + err = gen.WriteMapEncodersToFile("./storage/sealer/cbor_gen.go", "sealer", sectorstorage.Call{}, sectorstorage.WorkState{}, sectorstorage.WorkID{}, diff --git a/genesis/types.go b/genesis/types.go index 327a164f57f..021cdd28703 100644 --- a/genesis/types.go +++ b/genesis/types.go @@ -11,7 +11,7 @@ import ( market8 "github.com/filecoin-project/go-state-types/builtin/v8/market" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" ) type ActorType string @@ -26,7 +26,7 @@ type PreSeal struct { CommD cid.Cid SectorID abi.SectorNumber Deal market8.DealProposal - DealClientKey *wallet.Key + DealClientKey *key.Key ProofType abi.RegisteredSealProof } diff --git a/itests/batch_deal_test.go b/itests/batch_deal_test.go index d1a32241200..64326c82264 100644 --- a/itests/batch_deal_test.go +++ b/itests/batch_deal_test.go @@ -13,13 +13,13 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" ) func TestBatchDealInput(t *testing.T) { diff --git a/itests/deadlines_test.go b/itests/deadlines_test.go index 1a0f94711da..edcabba2152 100644 --- a/itests/deadlines_test.go +++ b/itests/deadlines_test.go @@ -28,9 +28,9 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/mock" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/storage/sealer/mock" ) // TestDeadlineToggling: diff --git a/itests/deals_pricing_test.go b/itests/deals_pricing_test.go index f544a58efe6..ad3c15d69bd 100644 --- a/itests/deals_pricing_test.go +++ b/itests/deals_pricing_test.go @@ -10,8 +10,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/itests/kit" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func TestQuotePriceForUnsealedRetrieval(t *testing.T) { diff --git a/itests/deals_publish_test.go b/itests/deals_publish_test.go index bd4a3d1c0f7..b12fe31b70f 100644 --- a/itests/deals_publish_test.go +++ b/itests/deals_publish_test.go @@ -16,13 +16,13 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules" - "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/ctladdr" ) func TestPublishDealsBatching(t *testing.T) { @@ -41,7 +41,7 @@ func TestPublishDealsBatching(t *testing.T) { kit.QuietMiningLogs() - publisherKey, err := wallet.GenerateKey(types.KTSecp256k1) + publisherKey, err := key.GenerateKey(types.KTSecp256k1) require.NoError(t, err) opts := node.Options( @@ -51,7 +51,7 @@ func TestPublishDealsBatching(t *testing.T) { MaxDealsPerMsg: maxDealsPerMsg, }), ), - node.Override(new(*storage.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ + node.Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ DealPublishControl: []string{ publisherKey.Address.String(), }, diff --git a/itests/deals_retry_deal_no_funds_test.go b/itests/deals_retry_deal_no_funds_test.go index f0582a3636f..317538cf132 100644 --- a/itests/deals_retry_deal_no_funds_test.go +++ b/itests/deals_retry_deal_no_funds_test.go @@ -11,13 +11,13 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules" - "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/ctladdr" ) var ( @@ -42,7 +42,7 @@ func TestDealsRetryLackOfFunds(t *testing.T) { // Allow 8MB sectors eightMBSectorsOpt := kit.SectorSize(8 << 20) - publishStorageDealKey, err := wallet.GenerateKey(types.KTSecp256k1) + publishStorageDealKey, err := key.GenerateKey(types.KTSecp256k1) require.NoError(t, err) opts := node.Options( @@ -52,7 +52,7 @@ func TestDealsRetryLackOfFunds(t *testing.T) { MaxDealsPerMsg: maxDealsPerMsg, }), ), - node.Override(new(*storage.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ + node.Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ DealPublishControl: []string{ publishStorageDealKey.Address.String(), }, @@ -118,7 +118,7 @@ func TestDealsRetryLackOfFunds_blockInPublishDeal(t *testing.T) { // Allow 8MB sectors eightMBSectorsOpt := kit.SectorSize(8 << 20) - publishStorageDealKey, err := wallet.GenerateKey(types.KTSecp256k1) + publishStorageDealKey, err := key.GenerateKey(types.KTSecp256k1) require.NoError(t, err) opts := node.Options( @@ -128,7 +128,7 @@ func TestDealsRetryLackOfFunds_blockInPublishDeal(t *testing.T) { MaxDealsPerMsg: maxDealsPerMsg, }), ), - node.Override(new(*storage.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ + node.Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ DealPublishControl: []string{ publishStorageDealKey.Address.String(), }, @@ -191,7 +191,7 @@ func TestDealsRetryLackOfFunds_belowLimit(t *testing.T) { // Allow 8MB sectors eightMBSectorsOpt := kit.SectorSize(8 << 20) - publishStorageDealKey, err := wallet.GenerateKey(types.KTSecp256k1) + publishStorageDealKey, err := key.GenerateKey(types.KTSecp256k1) require.NoError(t, err) opts := node.Options( @@ -201,7 +201,7 @@ func TestDealsRetryLackOfFunds_belowLimit(t *testing.T) { MaxDealsPerMsg: maxDealsPerMsg, }), ), - node.Override(new(*storage.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ + node.Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ DealPublishControl: []string{ publishStorageDealKey.Address.String(), }, diff --git a/itests/kit/deals.go b/itests/kit/deals.go index e256438d088..9c48242c790 100644 --- a/itests/kit/deals.go +++ b/itests/kit/deals.go @@ -28,7 +28,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) type DealHarness struct { diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index 75b38776a9d..4c71c238b23 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -42,16 +42,12 @@ import ( "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/mock" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/genesis" "github.com/filecoin-project/lotus/markets/idxprov" - idxprov_test "github.com/filecoin-project/lotus/markets/idxprov/idxprov_test" + "github.com/filecoin-project/lotus/markets/idxprov/idxprov_test" lotusminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/config" @@ -59,7 +55,10 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" testing2 "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/mockstorage" + "github.com/filecoin-project/lotus/storage/paths" + sectorstorage "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/mock" ) func init() { @@ -183,7 +182,7 @@ func (n *Ensemble) FullNode(full *TestFullNode, opts ...NodeOpt) *Ensemble { require.NoError(n.t, err) } - key, err := wallet.GenerateKey(types.KTBLS) + key, err := key.GenerateKey(types.KTBLS) require.NoError(n.t, err) if !n.bootstrapped && !options.balance.IsZero() { @@ -247,7 +246,7 @@ func (n *Ensemble) Miner(minerNode *TestMiner, full *TestFullNode, opts ...NodeO // Create the preseal commitment. if n.options.mockProofs { - genm, k, err = mockstorage.PreSeal(proofType, actorAddr, sectors) + genm, k, err = mock.PreSeal(proofType, actorAddr, sectors) } else { genm, k, err = seed.PreSeal(actorAddr, proofType, 0, sectors, tdir, []byte("make genesis mem random"), nil, true) } @@ -256,7 +255,7 @@ func (n *Ensemble) Miner(minerNode *TestMiner, full *TestFullNode, opts ...NodeO genm.PeerId = peerId // create an owner key, and assign it some FIL. - ownerKey, err = wallet.NewKey(*k) + ownerKey, err = key.NewKey(*k) require.NoError(n.t, err) genacc := genesis.Actor{ @@ -547,8 +546,8 @@ func (n *Ensemble) Start() *Ensemble { // using real proofs, therefore need real sectors. if !n.bootstrapped && !n.options.mockProofs { psd := m.PresealDir - err := lr.SetStorage(func(sc *stores.StorageConfig) { - sc.StoragePaths = append(sc.StoragePaths, stores.LocalPath{Path: psd}) + err := lr.SetStorage(func(sc *paths.StorageConfig) { + sc.StoragePaths = append(sc.StoragePaths, paths.LocalPath{Path: psd}) }) require.NoError(n.t, err) @@ -699,15 +698,15 @@ func (n *Ensemble) Start() *Ensemble { addr := m.RemoteListener.Addr().String() - localStore, err := stores.NewLocal(ctx, lr, m.MinerNode, []string{"http://" + addr + "/remote"}) + localStore, err := paths.NewLocal(ctx, lr, m.MinerNode, []string{"http://" + addr + "/remote"}) require.NoError(n.t, err) auth := http.Header(nil) - remote := stores.NewRemote(localStore, m.MinerNode, auth, 20, &stores.DefaultPartialFileHandler{}) + remote := paths.NewRemote(localStore, m.MinerNode, auth, 20, &paths.DefaultPartialFileHandler{}) store := m.options.workerStorageOpt(remote) - fh := &stores.FetchHandler{Local: localStore, PfHandler: &stores.DefaultPartialFileHandler{}} + fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}} m.FetchHandler = fh.ServeHTTP wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix)) diff --git a/itests/kit/ensemble_opts.go b/itests/kit/ensemble_opts.go index f0a984e4ea4..61a54550ee4 100644 --- a/itests/kit/ensemble_opts.go +++ b/itests/kit/ensemble_opts.go @@ -7,13 +7,13 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" ) type EnsembleOpt func(opts *ensembleOpts) error type genesisAccount struct { - key *wallet.Key + key *key.Key initialBalance abi.TokenAmount } @@ -47,7 +47,7 @@ func MockProofs() EnsembleOpt { // RootVerifier specifies the key to be enlisted as the verified registry root, // as well as the initial balance to be attributed during genesis. -func RootVerifier(key *wallet.Key, balance abi.TokenAmount) EnsembleOpt { +func RootVerifier(key *key.Key, balance abi.TokenAmount) EnsembleOpt { return func(opts *ensembleOpts) error { opts.verifiedRoot.key = key opts.verifiedRoot.initialBalance = balance @@ -56,7 +56,7 @@ func RootVerifier(key *wallet.Key, balance abi.TokenAmount) EnsembleOpt { } // Account sets up an account at genesis with the specified key and balance. -func Account(key *wallet.Key, balance abi.TokenAmount) EnsembleOpt { +func Account(key *key.Key, balance abi.TokenAmount) EnsembleOpt { return func(opts *ensembleOpts) error { opts.accounts = append(opts.accounts, genesisAccount{ key: key, diff --git a/itests/kit/files.go b/itests/kit/files.go index ca76ad36755..3fa8a5f4d11 100644 --- a/itests/kit/files.go +++ b/itests/kit/files.go @@ -11,7 +11,7 @@ import ( "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" - cidutil "github.com/ipfs/go-cidutil" + "github.com/ipfs/go-cidutil" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" bstore "github.com/ipfs/go-ipfs-blockstore" diff --git a/itests/kit/node_full.go b/itests/kit/node_full.go index a4cdd0c7a6d..710962e6a03 100644 --- a/itests/kit/node_full.go +++ b/itests/kit/node_full.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/v1api" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" ) // TestFullNode represents a full node enrolled in an Ensemble. @@ -27,7 +27,7 @@ type TestFullNode struct { // ListenAddr is the address on which an API server is listening, if an // API server is created for this Node. ListenAddr multiaddr.Multiaddr - DefaultKey *wallet.Key + DefaultKey *key.Key options nodeOpts } diff --git a/itests/kit/node_miner.go b/itests/kit/node_miner.go index 6eea1b43075..cedbb92041f 100644 --- a/itests/kit/node_miner.go +++ b/itests/kit/node_miner.go @@ -23,11 +23,11 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/miner" + "github.com/filecoin-project/lotus/storage/paths" + sealing "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type MinerSubsystem int @@ -72,7 +72,7 @@ type TestMiner struct { ListenAddr multiaddr.Multiaddr ActorAddr address.Address - OwnerKey *wallet.Key + OwnerKey *key.Key MineOne func(context.Context, miner.MineReq) error Stop func(context.Context) error @@ -182,7 +182,7 @@ func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, weight uint64 require.NoError(t, err) } - cfg := &stores.LocalStorageMeta{ + cfg := &paths.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), Weight: weight, CanSeal: seal, diff --git a/itests/kit/node_opts.go b/itests/kit/node_opts.go index 030b94b194d..69e6acbb278 100644 --- a/itests/kit/node_opts.go +++ b/itests/kit/node_opts.go @@ -6,15 +6,15 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" ) // DefaultPresealsPerBootstrapMiner is the number of preseals that every @@ -31,7 +31,7 @@ type nodeOpts struct { lite bool sectors int rpc bool - ownerKey *wallet.Key + ownerKey *key.Key extraNodeOpts []node.Option subsystems MinerSubsystem @@ -45,7 +45,7 @@ type nodeOpts struct { disallowRemoteFinalize bool workerTasks []sealtasks.TaskType - workerStorageOpt func(stores.Store) stores.Store + workerStorageOpt func(paths.Store) paths.Store } // DefaultNodeOpts are the default options that will be applied to test nodes. @@ -55,7 +55,7 @@ var DefaultNodeOpts = nodeOpts{ sectorSize: abi.SectorSize(2 << 10), // 2KiB. workerTasks: []sealtasks.TaskType{sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTFinalize}, - workerStorageOpt: func(store stores.Store) stores.Store { return store }, + workerStorageOpt: func(store paths.Store) paths.Store { return store }, } // OptBuilder is used to create an option after some other node is already @@ -165,7 +165,7 @@ func ThroughRPC() NodeOpt { // OwnerAddr sets the owner address of a miner. Only relevant when creating // a miner. -func OwnerAddr(wk *wallet.Key) NodeOpt { +func OwnerAddr(wk *key.Key) NodeOpt { return func(opts *nodeOpts) error { opts.ownerKey = wk return nil @@ -210,7 +210,7 @@ func WithTaskTypes(tt []sealtasks.TaskType) NodeOpt { } } -func WithWorkerStorage(transform func(stores.Store) stores.Store) NodeOpt { +func WithWorkerStorage(transform func(paths.Store) paths.Store) NodeOpt { return func(opts *nodeOpts) error { opts.workerStorageOpt = transform return nil diff --git a/itests/sector_make_cc_avail_test.go b/itests/sector_make_cc_avail_test.go index 094367e964f..437e803b053 100644 --- a/itests/sector_make_cc_avail_test.go +++ b/itests/sector_make_cc_avail_test.go @@ -13,9 +13,9 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node/config" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) func TestMakeAvailable(t *testing.T) { diff --git a/itests/sector_miner_collateral_test.go b/itests/sector_miner_collateral_test.go index 22c04b5eaef..07dad8edd74 100644 --- a/itests/sector_miner_collateral_test.go +++ b/itests/sector_miner_collateral_test.go @@ -13,14 +13,14 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" + sealing "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" ) func TestMinerBalanceCollateral(t *testing.T) { diff --git a/itests/sector_pledge_test.go b/itests/sector_pledge_test.go index 0c76e9076bf..6f966430d26 100644 --- a/itests/sector_pledge_test.go +++ b/itests/sector_pledge_test.go @@ -18,9 +18,9 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node/impl" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) func TestPledgeSectors(t *testing.T) { diff --git a/itests/sector_prefer_no_upgrade_test.go b/itests/sector_prefer_no_upgrade_test.go index 11fd2c1de0a..30674b36e38 100644 --- a/itests/sector_prefer_no_upgrade_test.go +++ b/itests/sector_prefer_no_upgrade_test.go @@ -13,9 +13,9 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node/config" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) func TestPreferNoUpgrade(t *testing.T) { diff --git a/itests/sector_revert_available_test.go b/itests/sector_revert_available_test.go index 6827a85fac9..559e8e8b050 100644 --- a/itests/sector_revert_available_test.go +++ b/itests/sector_revert_available_test.go @@ -13,8 +13,8 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/itests/kit" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) func TestAbortUpgradeAvailable(t *testing.T) { diff --git a/itests/sector_terminate_test.go b/itests/sector_terminate_test.go index fd685dc53c3..b7004b3129c 100644 --- a/itests/sector_terminate_test.go +++ b/itests/sector_terminate_test.go @@ -12,8 +12,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/itests/kit" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) func TestTerminate(t *testing.T) { diff --git a/itests/tape_test.go b/itests/tape_test.go index 3cb1e83397c..d179951ed8f 100644 --- a/itests/tape_test.go +++ b/itests/tape_test.go @@ -12,8 +12,8 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/itests/kit" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) func TestTapeFix(t *testing.T) { diff --git a/itests/verifreg_test.go b/itests/verifreg_test.go index be80043c174..b235c45da2b 100644 --- a/itests/verifreg_test.go +++ b/itests/verifreg_test.go @@ -19,7 +19,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node/impl" ) @@ -35,13 +35,13 @@ func TestVerifiedClientTopUp(t *testing.T) { test := func(nv network.Version, shouldWork bool) func(*testing.T) { return func(t *testing.T) { - rootKey, err := wallet.GenerateKey(types.KTSecp256k1) + rootKey, err := key.GenerateKey(types.KTSecp256k1) require.NoError(t, err) - verifierKey, err := wallet.GenerateKey(types.KTSecp256k1) + verifierKey, err := key.GenerateKey(types.KTSecp256k1) require.NoError(t, err) - verifiedClientKey, err := wallet.GenerateKey(types.KTBLS) + verifiedClientKey, err := key.GenerateKey(types.KTBLS) require.NoError(t, err) bal, err := types.ParseFIL("100fil") diff --git a/itests/wdpost_test.go b/itests/wdpost_test.go index bbeedb8d80b..66857401c0c 100644 --- a/itests/wdpost_test.go +++ b/itests/wdpost_test.go @@ -18,9 +18,9 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/mock" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/storage/sealer/mock" ) func TestWindowedPost(t *testing.T) { diff --git a/itests/worker_test.go b/itests/worker_test.go index fda8f9901e6..3f8a3e628a4 100644 --- a/itests/worker_test.go +++ b/itests/worker_test.go @@ -17,14 +17,14 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/filecoin-project/lotus/storage/wdpost" ) func TestWorkerPledge(t *testing.T) { @@ -145,7 +145,7 @@ func TestWindowPostWorker(t *testing.T) { di = di.NextNotElapsed() t.Log("Running one proving period") - waitUntil := di.Open + di.WPoStChallengeWindow*2 + storage.SubmitConfidence + waitUntil := di.Open + di.WPoStChallengeWindow*2 + wdpost.SubmitConfidence client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) t.Log("Waiting for post message") @@ -227,7 +227,7 @@ func TestWindowPostWorker(t *testing.T) { } type badWorkerStorage struct { - stores.Store + paths.Store badsector *uint64 notBadCount int @@ -258,14 +258,14 @@ func TestWindowPostWorkerSkipBadSector(t *testing.T) { kit.LatestActorsAt(-1), kit.ThroughRPC(), kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWindowPoSt}), - kit.WithWorkerStorage(func(store stores.Store) stores.Store { + kit.WithWorkerStorage(func(store paths.Store) paths.Store { return &badWorkerStorage{ Store: store, badsector: &badsector, } }), kit.ConstructorOpts(node.ApplyIf(node.IsType(repo.StorageMiner), - node.Override(new(stores.Store), func(store *stores.Remote) stores.Store { + node.Override(new(paths.Store), func(store *paths.Remote) paths.Store { return &badWorkerStorage{ Store: store, badsector: &badsector, @@ -284,7 +284,7 @@ func TestWindowPostWorkerSkipBadSector(t *testing.T) { di = di.NextNotElapsed() t.Log("Running one proving period") - waitUntil := di.Open + di.WPoStChallengeWindow*2 + storage.SubmitConfidence + waitUntil := di.Open + di.WPoStChallengeWindow*2 + wdpost.SubmitConfidence client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) t.Log("Waiting for post message") diff --git a/lib/oldpath/oldresolver/resolver.go b/lib/oldpath/oldresolver/resolver.go index f92aaee2e1d..7c598c66e63 100644 --- a/lib/oldpath/oldresolver/resolver.go +++ b/lib/oldpath/oldresolver/resolver.go @@ -6,7 +6,7 @@ import ( "fmt" "time" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" dag "github.com/ipfs/go-merkledag" diff --git a/lib/oldpath/path.go b/lib/oldpath/path.go index fe4bd67a24b..9cc99ce5ce7 100644 --- a/lib/oldpath/path.go +++ b/lib/oldpath/path.go @@ -5,7 +5,7 @@ import ( "path" "strings" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" ) // helper type so path parsing errors include the path diff --git a/lib/peermgr/peermgr.go b/lib/peermgr/peermgr.go index 6f4cf88d73d..1ab81aff292 100644 --- a/lib/peermgr/peermgr.go +++ b/lib/peermgr/peermgr.go @@ -7,9 +7,9 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/event" - host "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/host" net "github.com/libp2p/go-libp2p-core/network" - peer "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peer" dht "github.com/libp2p/go-libp2p-kad-dht" "go.opencensus.io/stats" "go.uber.org/fx" diff --git a/lib/rpcenc/reader.go b/lib/rpcenc/reader.go index a4f16982e05..99b0faf05ef 100644 --- a/lib/rpcenc/reader.go +++ b/lib/rpcenc/reader.go @@ -22,8 +22,8 @@ import ( "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" "github.com/filecoin-project/lotus/lib/httpreader" + "github.com/filecoin-project/lotus/storage/pipeline/lib/nullreader" ) var log = logging.Logger("rpcenc") diff --git a/lib/rpcenc/reader_test.go b/lib/rpcenc/reader_test.go index 881cbeaae2c..ac78d1109ee 100644 --- a/lib/rpcenc/reader_test.go +++ b/lib/rpcenc/reader_test.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/go-jsonrpc" - "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" + "github.com/filecoin-project/lotus/storage/pipeline/lib/nullreader" ) type ReaderHandler struct { diff --git a/lib/sigs/bls/bls_test.go b/lib/sigs/bls/bls_test.go index d6956a383d1..3187fea6705 100644 --- a/lib/sigs/bls/bls_test.go +++ b/lib/sigs/bls/bls_test.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/lib/sigs" _ "github.com/filecoin-project/lotus/lib/sigs/bls" ) @@ -24,7 +24,7 @@ func TestRoundtrip(t *testing.T) { Type: types.KTBLS, PrivateKey: pk, } - k, err := wallet.NewKey(ki) + k, err := key.NewKey(ki) require.NoError(t, err) p := []byte("potato") diff --git a/lib/ulimit/ulimit_unix.go b/lib/ulimit/ulimit_unix.go index ac9f4ca908c..f36d7a58b66 100644 --- a/lib/ulimit/ulimit_unix.go +++ b/lib/ulimit/ulimit_unix.go @@ -4,7 +4,7 @@ package ulimit import ( - unix "golang.org/x/sys/unix" + "golang.org/x/sys/unix" ) func init() { diff --git a/markets/sectoraccessor/sectoraccessor.go b/markets/sectoraccessor/sectoraccessor.go index 4320e3fb1e9..a01895bb8c9 100644 --- a/markets/sectoraccessor/sectoraccessor.go +++ b/markets/sectoraccessor/sectoraccessor.go @@ -17,10 +17,10 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/v1api" "github.com/filecoin-project/lotus/chain/types" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/storiface" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -29,13 +29,13 @@ var log = logging.Logger("sectoraccessor") type sectorAccessor struct { maddr address.Address secb sectorblocks.SectorBuilder - pp sectorstorage.PieceProvider + pp sealer.PieceProvider full v1api.FullNode } var _ retrievalmarket.SectorAccessor = (*sectorAccessor)(nil) -func NewSectorAccessor(maddr dtypes.MinerAddress, secb sectorblocks.SectorBuilder, pp sectorstorage.PieceProvider, full v1api.FullNode) dagstore.SectorAccessor { +func NewSectorAccessor(maddr dtypes.MinerAddress, secb sectorblocks.SectorBuilder, pp sealer.PieceProvider, full v1api.FullNode) dagstore.SectorAccessor { return §orAccessor{address.Address(maddr), secb, pp, full} } diff --git a/markets/storageadapter/dealpublisher.go b/markets/storageadapter/dealpublisher.go index a98da7140ec..c2864bdf949 100644 --- a/markets/storageadapter/dealpublisher.go +++ b/markets/storageadapter/dealpublisher.go @@ -23,7 +23,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/ctladdr" ) type dealPublisherAPI interface { @@ -48,7 +48,7 @@ type dealPublisherAPI interface { // publish message with all deals in the queue. type DealPublisher struct { api dealPublisherAPI - as *storage.AddressSelector + as *ctladdr.AddressSelector ctx context.Context Shutdown context.CancelFunc @@ -99,8 +99,8 @@ type PublishMsgConfig struct { func NewDealPublisher( feeConfig *config.MinerFeeConfig, publishMsgCfg PublishMsgConfig, -) func(lc fx.Lifecycle, full api.FullNode, as *storage.AddressSelector) *DealPublisher { - return func(lc fx.Lifecycle, full api.FullNode, as *storage.AddressSelector) *DealPublisher { +) func(lc fx.Lifecycle, full api.FullNode, as *ctladdr.AddressSelector) *DealPublisher { + return func(lc fx.Lifecycle, full api.FullNode, as *ctladdr.AddressSelector) *DealPublisher { maxFee := abi.NewTokenAmount(0) if feeConfig != nil { maxFee = abi.TokenAmount(feeConfig.MaxPublishDealsFee) @@ -119,7 +119,7 @@ func NewDealPublisher( func newDealPublisher( dpapi dealPublisherAPI, - as *storage.AddressSelector, + as *ctladdr.AddressSelector, publishMsgCfg PublishMsgConfig, publishSpec *api.MessageSendSpec, ) *DealPublisher { diff --git a/markets/storageadapter/ondealsectorcommitted.go b/markets/storageadapter/ondealsectorcommitted.go index 0567a45145c..426f018b9d4 100644 --- a/markets/storageadapter/ondealsectorcommitted.go +++ b/markets/storageadapter/ondealsectorcommitted.go @@ -20,7 +20,7 @@ import ( lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" ) type eventsCalledAPI interface { @@ -28,7 +28,7 @@ type eventsCalledAPI interface { } type dealInfoAPI interface { - GetCurrentDealInfo(ctx context.Context, tok sealing.TipSetToken, proposal *market.DealProposal, publishCid cid.Cid) (sealing.CurrentDealInfo, error) + GetCurrentDealInfo(ctx context.Context, tok pipeline.TipSetToken, proposal *market.DealProposal, publishCid cid.Cid) (pipeline.CurrentDealInfo, error) } type diffPreCommitsAPI interface { @@ -41,9 +41,9 @@ type SectorCommittedManager struct { dpc diffPreCommitsAPI } -func NewSectorCommittedManager(ev eventsCalledAPI, tskAPI sealing.CurrentDealInfoTskAPI, dpcAPI diffPreCommitsAPI) *SectorCommittedManager { - dim := &sealing.CurrentDealInfoManager{ - CDAPI: &sealing.CurrentDealInfoAPIAdapter{CurrentDealInfoTskAPI: tskAPI}, +func NewSectorCommittedManager(ev eventsCalledAPI, tskAPI pipeline.CurrentDealInfoTskAPI, dpcAPI diffPreCommitsAPI) *SectorCommittedManager { + dim := &pipeline.CurrentDealInfoManager{ + CDAPI: &pipeline.CurrentDealInfoAPIAdapter{CurrentDealInfoTskAPI: tskAPI}, } return newSectorCommittedManager(ev, dim, dpcAPI) } @@ -280,7 +280,7 @@ func (mgr *SectorCommittedManager) OnDealSectorCommitted(ctx context.Context, pr return nil } -func dealSectorInReplicaUpdateSuccess(msg *types.Message, rec *types.MessageReceipt, res sealing.CurrentDealInfo) (*abi.SectorNumber, error) { +func dealSectorInReplicaUpdateSuccess(msg *types.Message, rec *types.MessageReceipt, res pipeline.CurrentDealInfo) (*abi.SectorNumber, error) { var params miner.ProveReplicaUpdatesParams if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return nil, xerrors.Errorf("unmarshal prove replica update: %w", err) @@ -317,7 +317,7 @@ func dealSectorInReplicaUpdateSuccess(msg *types.Message, rec *types.MessageRece } // dealSectorInPreCommitMsg tries to find a sector containing the specified deal -func dealSectorInPreCommitMsg(msg *types.Message, res sealing.CurrentDealInfo) (*abi.SectorNumber, error) { +func dealSectorInPreCommitMsg(msg *types.Message, res pipeline.CurrentDealInfo) (*abi.SectorNumber, error) { switch msg.Method { case builtin.MethodsMiner.PreCommitSector: var params miner.SectorPreCommitInfo @@ -383,7 +383,7 @@ func sectorInCommitMsg(msg *types.Message, sectorNumber abi.SectorNumber) (bool, } } -func (mgr *SectorCommittedManager) checkIfDealAlreadyActive(ctx context.Context, ts *types.TipSet, proposal *market.DealProposal, publishCid cid.Cid) (sealing.CurrentDealInfo, bool, error) { +func (mgr *SectorCommittedManager) checkIfDealAlreadyActive(ctx context.Context, ts *types.TipSet, proposal *market.DealProposal, publishCid cid.Cid) (pipeline.CurrentDealInfo, bool, error) { res, err := mgr.dealInfo.GetCurrentDealInfo(ctx, ts.Key().Bytes(), proposal, publishCid) if err != nil { // TODO: This may be fine for some errors diff --git a/markets/storageadapter/ondealsectorcommitted_test.go b/markets/storageadapter/ondealsectorcommitted_test.go index 458d2209eff..097a2fc1c90 100644 --- a/markets/storageadapter/ondealsectorcommitted_test.go +++ b/markets/storageadapter/ondealsectorcommitted_test.go @@ -29,7 +29,7 @@ import ( "github.com/filecoin-project/lotus/chain/events" test "github.com/filecoin-project/lotus/chain/events/state/mock" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" ) func TestOnDealSectorPreCommitted(t *testing.T) { @@ -76,7 +76,7 @@ func TestOnDealSectorPreCommitted(t *testing.T) { }, } type testCase struct { - currentDealInfo sealing.CurrentDealInfo + currentDealInfo pipeline.CurrentDealInfo currentDealInfoErr error currentDealInfoErr2 error preCommitDiff *miner.PreCommitChanges @@ -90,7 +90,7 @@ func TestOnDealSectorPreCommitted(t *testing.T) { } testCases := map[string]testCase{ "normal sequence": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: unfinishedDeal, }, @@ -108,7 +108,7 @@ func TestOnDealSectorPreCommitted(t *testing.T) { expectedCBSectorNumber: sectorNumber, }, "ignores unsuccessful pre-commit message": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: unfinishedDeal, }, @@ -126,7 +126,7 @@ func TestOnDealSectorPreCommitted(t *testing.T) { expectedCBCallCount: 0, }, "deal already pre-committed": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: unfinishedDeal, }, @@ -148,7 +148,7 @@ func TestOnDealSectorPreCommitted(t *testing.T) { expectedError: xerrors.Errorf("failed to set up called handler: failed to look up deal on chain: something went wrong"), }, "sector already active": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: activeDeal, }, @@ -156,7 +156,7 @@ func TestOnDealSectorPreCommitted(t *testing.T) { expectedCBIsActive: true, }, "sector was slashed": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: slashedDeal, PublishMsgTipSet: nil, @@ -165,7 +165,7 @@ func TestOnDealSectorPreCommitted(t *testing.T) { expectedError: xerrors.Errorf("failed to set up called handler: deal %d was slashed at epoch %d", dealID, slashedDeal.State.SlashEpoch), }, "error getting current deal info in called func": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: unfinishedDeal, }, @@ -183,7 +183,7 @@ func TestOnDealSectorPreCommitted(t *testing.T) { expectedCBError: errors.New("handling applied event: something went wrong"), }, "proposed deal epoch timeout": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: activeDeal, }, @@ -298,9 +298,9 @@ func TestOnDealSectorCommitted(t *testing.T) { }, } type testCase struct { - currentDealInfo sealing.CurrentDealInfo + currentDealInfo pipeline.CurrentDealInfo currentDealInfoErr error - currentDealInfo2 sealing.CurrentDealInfo + currentDealInfo2 pipeline.CurrentDealInfo currentDealInfoErr2 error matchStates []matchState dealStartEpochTimeout bool @@ -310,11 +310,11 @@ func TestOnDealSectorCommitted(t *testing.T) { } testCases := map[string]testCase{ "normal sequence": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: unfinishedDeal, }, - currentDealInfo2: sealing.CurrentDealInfo{ + currentDealInfo2: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: activeDeal, }, @@ -328,11 +328,11 @@ func TestOnDealSectorCommitted(t *testing.T) { expectedCBCallCount: 1, }, "ignores unsuccessful prove-commit message": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: unfinishedDeal, }, - currentDealInfo2: sealing.CurrentDealInfo{ + currentDealInfo2: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: activeDeal, }, @@ -353,14 +353,14 @@ func TestOnDealSectorCommitted(t *testing.T) { expectedError: xerrors.Errorf("failed to set up called handler: failed to look up deal on chain: something went wrong"), }, "sector already active": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: activeDeal, }, expectedCBCallCount: 1, }, "sector was slashed": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: slashedDeal, }, @@ -368,7 +368,7 @@ func TestOnDealSectorCommitted(t *testing.T) { expectedError: xerrors.Errorf("failed to set up called handler: deal %d was slashed at epoch %d", dealID, slashedDeal.State.SlashEpoch), }, "error getting current deal info in called func": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: unfinishedDeal, }, @@ -384,7 +384,7 @@ func TestOnDealSectorCommitted(t *testing.T) { expectedCBError: xerrors.Errorf("handling applied event: failed to look up deal on chain: something went wrong"), }, "proposed deal epoch timeout": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: unfinishedDeal, }, @@ -393,11 +393,11 @@ func TestOnDealSectorCommitted(t *testing.T) { expectedCBError: xerrors.Errorf("handling applied event: deal with piece CID %s was not activated by proposed deal start epoch 0", unfinishedDeal.Proposal.PieceCID), }, "got prove-commit but deal not active": { - currentDealInfo: sealing.CurrentDealInfo{ + currentDealInfo: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: unfinishedDeal, }, - currentDealInfo2: sealing.CurrentDealInfo{ + currentDealInfo2: pipeline.CurrentDealInfo{ DealID: dealID, MarketDeal: unfinishedDeal, }, @@ -568,13 +568,13 @@ func (m *mockPreCommitsAPI) diffPreCommits(ctx context.Context, actor address.Ad type mockDealInfoAPI struct { count int - CurrentDealInfo sealing.CurrentDealInfo + CurrentDealInfo pipeline.CurrentDealInfo Err error - CurrentDealInfo2 sealing.CurrentDealInfo + CurrentDealInfo2 pipeline.CurrentDealInfo Err2 error } -func (m *mockDealInfoAPI) GetCurrentDealInfo(ctx context.Context, tok sealing.TipSetToken, proposal *market.DealProposal, publishCid cid.Cid) (sealing.CurrentDealInfo, error) { +func (m *mockDealInfoAPI) GetCurrentDealInfo(ctx context.Context, tok pipeline.TipSetToken, proposal *market.DealProposal, publishCid cid.Cid) (pipeline.CurrentDealInfo, error) { m.count++ if m.count == 2 { return m.CurrentDealInfo2, m.Err2 diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 0de8de59500..c09e4030254 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -27,11 +27,11 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/helpers" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -108,7 +108,7 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema curTime := build.Clock.Now() for build.Clock.Since(curTime) < addPieceRetryTimeout { // Check if there was an error because of too many sectors being sealed - if !xerrors.Is(err, sealing.ErrTooManySectorsSealing) { + if !xerrors.Is(err, pipeline.ErrTooManySectorsSealing) { if err != nil { log.Errorf("failed to addPiece for deal %d, err: %v", deal.DealID, err) } @@ -258,13 +258,13 @@ func (n *ProviderNodeAdapter) LocatePieceForDealWithinSector(ctx context.Context if err != nil { return 0, 0, 0, xerrors.Errorf("getting sector info: %w", err) } - if si.State == api.SectorState(sealing.Proving) { + if si.State == api.SectorState(pipeline.Proving) { best = r bestSi = si break } } - if bestSi.State == api.SectorState(sealing.UndefinedSectorState) { + if bestSi.State == api.SectorState(pipeline.UndefinedSectorState) { return 0, 0, 0, xerrors.New("no sealed sector found") } return best.SectorID, best.Offset, best.Size.Padded(), nil diff --git a/markets/utils/converters.go b/markets/utils/converters.go index 9a715b40fdb..f77297070d0 100644 --- a/markets/utils/converters.go +++ b/markets/utils/converters.go @@ -1,7 +1,7 @@ package utils import ( - peer "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/go-address" diff --git a/node/builder.go b/node/builder.go index 4b0b5b745e2..a8b082008b9 100644 --- a/node/builder.go +++ b/node/builder.go @@ -25,7 +25,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/journal/alerting" "github.com/filecoin-project/lotus/lib/lotuslog" @@ -42,6 +41,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/lp2p" "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/paths" "github.com/filecoin-project/lotus/system" ) @@ -262,10 +262,10 @@ func ConfigCommon(cfg *config.Common, enableLibp2pNode bool) Option { Override(SetApiEndpointKey, func(lr repo.LockedRepo, e dtypes.APIEndpoint) error { return lr.SetAPIEndpoint(e) }), - Override(new(stores.URLs), func(e dtypes.APIEndpoint) (stores.URLs, error) { + Override(new(paths.URLs), func(e dtypes.APIEndpoint) (paths.URLs, error) { ip := cfg.API.RemoteListenAddress - var urls stores.URLs + var urls paths.URLs urls = append(urls, "http://"+ip+"/remote") // TODO: This makes no assumptions, and probably could... return urls, nil }), diff --git a/node/builder_chain.go b/node/builder_chain.go index d2c116b9627..21727459187 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -28,7 +28,6 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" "github.com/filecoin-project/lotus/chain/wallet/remotewallet" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/lib/peermgr" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/node/config" @@ -40,6 +39,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/paychmgr" "github.com/filecoin-project/lotus/paychmgr/settler" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) // Chain node provides access to the Filecoin blockchain, by setting up a full diff --git a/node/builder_miner.go b/node/builder_miner.go index 2223d14ce7f..17cd5d293e0 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -19,11 +19,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/dealfilter" "github.com/filecoin-project/lotus/markets/idxprov" @@ -37,7 +32,14 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/ctladdr" + "github.com/filecoin-project/lotus/storage/paths" + sealing "github.com/filecoin-project/lotus/storage/pipeline" + sectorstorage "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/storiface" "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/filecoin-project/lotus/storage/wdpost" ) var MinerNode = Options( @@ -50,7 +52,7 @@ var MinerNode = Options( Override(new(dtypes.NetworkName), modules.StorageNetworkName), // Mining / proving - Override(new(*storage.AddressSelector), modules.AddressSelector(nil)), + Override(new(*ctladdr.AddressSelector), modules.AddressSelector(nil)), ) func ConfigStorageMiner(c interface{}) Option { @@ -85,10 +87,10 @@ func ConfigStorageMiner(c interface{}) Option { Override(CheckFDLimit, modules.CheckFdLimit(build.MinerFDLimit)), // recommend at least 100k FD limit to miners Override(new(api.MinerSubsystems), modules.ExtractEnabledMinerSubsystems(cfg.Subsystems)), - Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), - Override(new(*stores.Local), modules.LocalStorage), - Override(new(*stores.Remote), modules.RemoteStorage), - Override(new(stores.Store), From(new(*stores.Remote))), + Override(new(paths.LocalStorage), From(new(repo.LockedRepo))), + Override(new(*paths.Local), modules.LocalStorage), + Override(new(*paths.Remote), modules.RemoteStorage), + Override(new(paths.Store), From(new(*paths.Remote))), Override(new(dtypes.RetrievalPricingFunc), modules.RetrievalPricingFunc(cfg.Dealmaking)), If(!cfg.Subsystems.EnableMining, @@ -116,14 +118,14 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(*miner.Miner), modules.SetupBlockProducer), Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), Override(new(*storage.Miner), modules.StorageMiner(cfg.Fees)), - Override(new(*storage.WindowPoStScheduler), modules.WindowPostScheduler(cfg.Fees)), + Override(new(*wdpost.WindowPoStScheduler), modules.WindowPostScheduler(cfg.Fees)), Override(new(sectorblocks.SectorBuilder), From(new(*storage.Miner))), ), If(cfg.Subsystems.EnableSectorStorage, // Sector storage - Override(new(*stores.Index), stores.NewIndex), - Override(new(stores.SectorIndex), From(new(*stores.Index))), + Override(new(*paths.Index), paths.NewIndex), + Override(new(paths.SectorIndex), From(new(*paths.Index))), Override(new(*sectorstorage.Manager), modules.SectorStorage), Override(new(sectorstorage.Unsealer), From(new(*sectorstorage.Manager))), Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))), @@ -138,7 +140,7 @@ func ConfigStorageMiner(c interface{}) Option { ), If(!cfg.Subsystems.EnableSealing, Override(new(modules.MinerSealingService), modules.ConnectSealingService(cfg.Subsystems.SealerApiInfo)), - Override(new(stores.SectorIndex), From(new(modules.MinerSealingService))), + Override(new(paths.SectorIndex), From(new(modules.MinerSealingService))), ), If(cfg.Subsystems.EnableMarkets, @@ -220,7 +222,7 @@ func ConfigStorageMiner(c interface{}) Option { ), Override(new(sectorstorage.Config), cfg.StorageManager()), - Override(new(*storage.AddressSelector), modules.AddressSelector(&cfg.Addresses)), + Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)), ) } diff --git a/node/bundle/bundle.go b/node/bundle/bundle.go index 2863cab54ce..cb97838f5c3 100644 --- a/node/bundle/bundle.go +++ b/node/bundle/bundle.go @@ -6,7 +6,7 @@ import ( "io" "os" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/ipld/go-car" "golang.org/x/xerrors" diff --git a/node/config/def.go b/node/config/def.go index c6d87788f2a..452a31b4c21 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" + "github.com/filecoin-project/lotus/storage/sealer" ) const ( @@ -161,7 +161,7 @@ func DefaultStorageMiner() *StorageMiner { Assigner: "utilization", // By default use the hardware resource filtering strategy. - ResourceFiltering: sectorstorage.ResourceFilteringHardware, + ResourceFiltering: sealer.ResourceFilteringHardware, }, Dealmaking: DealmakingConfig{ diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index c3fed5197b0..68d81f80273 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -782,7 +782,7 @@ If you see stuck Finalize tasks after enabling this setting, check }, { Name: "ResourceFiltering", - Type: "sectorstorage.ResourceFilteringStrategy", + Type: "sealer.ResourceFilteringStrategy", Comment: `ResourceFiltering instructs the system which resource filtering strategy to use when evaluating tasks against this worker. An empty value defaults diff --git a/node/config/storage.go b/node/config/storage.go index de65dc60a41..16f688075ee 100644 --- a/node/config/storage.go +++ b/node/config/storage.go @@ -8,11 +8,11 @@ import ( "golang.org/x/xerrors" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer" ) -func StorageFromFile(path string, def *stores.StorageConfig) (*stores.StorageConfig, error) { +func StorageFromFile(path string, def *paths.StorageConfig) (*paths.StorageConfig, error) { file, err := os.Open(path) switch { case os.IsNotExist(err): @@ -28,8 +28,8 @@ func StorageFromFile(path string, def *stores.StorageConfig) (*stores.StorageCon return StorageFromReader(file) } -func StorageFromReader(reader io.Reader) (*stores.StorageConfig, error) { - var cfg stores.StorageConfig +func StorageFromReader(reader io.Reader) (*paths.StorageConfig, error) { + var cfg paths.StorageConfig err := json.NewDecoder(reader).Decode(&cfg) if err != nil { return nil, err @@ -38,7 +38,7 @@ func StorageFromReader(reader io.Reader) (*stores.StorageConfig, error) { return &cfg, nil } -func WriteStorageFile(path string, config stores.StorageConfig) error { +func WriteStorageFile(path string, config paths.StorageConfig) error { b, err := json.MarshalIndent(config, "", " ") if err != nil { return xerrors.Errorf("marshaling storage config: %w", err) @@ -51,8 +51,8 @@ func WriteStorageFile(path string, config stores.StorageConfig) error { return nil } -func (c *StorageMiner) StorageManager() sectorstorage.Config { - return sectorstorage.Config{ +func (c *StorageMiner) StorageManager() sealer.Config { + return sealer.Config{ ParallelFetchLimit: c.Storage.ParallelFetchLimit, AllowAddPiece: c.Storage.AllowAddPiece, AllowPreCommit1: c.Storage.AllowPreCommit1, diff --git a/node/config/types.go b/node/config/types.go index 6d1939f9494..cd05c8ce5e7 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -4,7 +4,7 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/lotus/chain/types" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" + "github.com/filecoin-project/lotus/storage/sealer" ) // // NOTE: ONLY PUT STRUCT DEFINITIONS IN THIS FILE @@ -352,7 +352,7 @@ type SealerConfig struct { // ResourceFiltering instructs the system which resource filtering strategy // to use when evaluating tasks against this worker. An empty value defaults // to "hardware". - ResourceFiltering sectorstorage.ResourceFilteringStrategy + ResourceFiltering sealer.ResourceFilteringStrategy } type BatchFeeConfig struct { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index e2dfe56a181..90f531ea7d6 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -7,7 +7,7 @@ import ( "fmt" "strconv" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" "go.uber.org/fx" "golang.org/x/xerrors" @@ -40,8 +40,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) type StateModuleAPI interface { diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index bd7de65b150..2809d41dcf2 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -4,7 +4,7 @@ import ( "context" "sync/atomic" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/fx" "golang.org/x/xerrors" diff --git a/node/impl/remoteworker.go b/node/impl/remoteworker.go index d27b3baff42..b3e68869d94 100644 --- a/node/impl/remoteworker.go +++ b/node/impl/remoteworker.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" + "github.com/filecoin-project/lotus/storage/sealer" ) type remoteWorker struct { @@ -56,4 +56,4 @@ func (r *remoteWorker) Close() error { return nil } -var _ sectorstorage.Worker = &remoteWorker{} +var _ sealer.Worker = &remoteWorker{} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 69249e1c63c..80410ac9a98 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -42,18 +42,20 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/ctladdr" + "github.com/filecoin-project/lotus/storage/paths" + sealing "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/storiface" "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/filecoin-project/lotus/storage/wdpost" ) type StorageMinerAPI struct { @@ -65,8 +67,8 @@ type StorageMinerAPI struct { EnabledSubsystems api.MinerSubsystems Full api.FullNode - LocalStore *stores.Local - RemoteStore *stores.Remote + LocalStore *paths.Local + RemoteStore *paths.Remote // Markets PieceStore dtypes.ProviderPieceStore `optional:"true"` @@ -83,15 +85,15 @@ type StorageMinerAPI struct { DAGStoreWrapper *mktsdagstore.Wrapper `optional:"true"` // Miner / storage - Miner *storage.Miner `optional:"true"` - BlockMiner *miner.Miner `optional:"true"` - StorageMgr *sectorstorage.Manager `optional:"true"` - IStorageMgr sectorstorage.SectorManager `optional:"true"` - stores.SectorIndex + Miner *storage.Miner `optional:"true"` + BlockMiner *miner.Miner `optional:"true"` + StorageMgr *sealer.Manager `optional:"true"` + IStorageMgr sealer.SectorManager `optional:"true"` + paths.SectorIndex storiface.WorkerReturn `optional:"true"` - AddrSel *storage.AddressSelector + AddrSel *ctladdr.AddressSelector - WdPoSt *storage.WindowPoStScheduler `optional:"true"` + WdPoSt *wdpost.WindowPoStScheduler `optional:"true"` Epp gen.WinningPoStProver `optional:"true"` DS dtypes.MetadataDS diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index 9a391223dc9..24bcc714c17 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" ) type MinerAddress address.Address diff --git a/node/modules/dtypes/protector.go b/node/modules/dtypes/protector.go index 0d9625fc1cd..cb7069dc8e7 100644 --- a/node/modules/dtypes/protector.go +++ b/node/modules/dtypes/protector.go @@ -1,7 +1,7 @@ package dtypes import ( - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" ) type GCReferenceProtector interface { diff --git a/node/modules/dtypes/scorekeeper.go b/node/modules/dtypes/scorekeeper.go index 7999d19e56c..7154bbe8b34 100644 --- a/node/modules/dtypes/scorekeeper.go +++ b/node/modules/dtypes/scorekeeper.go @@ -3,7 +3,7 @@ package dtypes import ( "sync" - peer "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peer" pubsub "github.com/libp2p/go-libp2p-pubsub" ) diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index a59ff94dbcb..33f17a5c1dd 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -6,11 +6,11 @@ import ( "net" "time" - host "github.com/libp2p/go-libp2p-core/host" - peer "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/peer" pubsub "github.com/libp2p/go-libp2p-pubsub" pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" - blake2b "github.com/minio/blake2b-simd" + "github.com/minio/blake2b-simd" ma "github.com/multiformats/go-multiaddr" "go.opencensus.io/stats" "go.uber.org/fx" diff --git a/node/modules/lp2p/relay.go b/node/modules/lp2p/relay.go index a6bedb12b37..78d04e33fa9 100644 --- a/node/modules/lp2p/relay.go +++ b/node/modules/lp2p/relay.go @@ -5,7 +5,7 @@ import ( "github.com/libp2p/go-libp2p" coredisc "github.com/libp2p/go-libp2p-core/discovery" - routing "github.com/libp2p/go-libp2p-core/routing" + "github.com/libp2p/go-libp2p-core/routing" discovery "github.com/libp2p/go-libp2p-discovery" ) diff --git a/node/modules/lp2p/routing.go b/node/modules/lp2p/routing.go index fe7f9d3f37c..8cc06a6f695 100644 --- a/node/modules/lp2p/routing.go +++ b/node/modules/lp2p/routing.go @@ -4,7 +4,7 @@ import ( "context" "sort" - routing "github.com/libp2p/go-libp2p-core/routing" + "github.com/libp2p/go-libp2p-core/routing" dht "github.com/libp2p/go-libp2p-kad-dht" record "github.com/libp2p/go-libp2p-record" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" diff --git a/node/modules/lp2p/transport.go b/node/modules/lp2p/transport.go index 5ea2d1c837c..4349fcd87c5 100644 --- a/node/modules/lp2p/transport.go +++ b/node/modules/lp2p/transport.go @@ -2,7 +2,7 @@ package lp2p import ( "github.com/libp2p/go-libp2p" - metrics "github.com/libp2p/go-libp2p-core/metrics" + "github.com/libp2p/go-libp2p-core/metrics" noise "github.com/libp2p/go-libp2p-noise" libp2pquic "github.com/libp2p/go-libp2p-quic-transport" tls "github.com/libp2p/go-libp2p-tls" diff --git a/node/modules/services.go b/node/modules/services.go index 2a0303fa86b..15015443538 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -8,8 +8,8 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - eventbus "github.com/libp2p/go-eventbus" - event "github.com/libp2p/go-libp2p-core/event" + "github.com/libp2p/go-eventbus" + "github.com/libp2p/go-libp2p-core/event" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" pubsub "github.com/libp2p/go-libp2p-pubsub" diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 72ae4dfbfa2..44d00bd4d18 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -53,11 +53,6 @@ import ( "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/types" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/markets" "github.com/filecoin-project/lotus/markets/dagstore" @@ -70,6 +65,13 @@ import ( "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/ctladdr" + "github.com/filecoin-project/lotus/storage/paths" + sealing "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/wdpost" ) var ( @@ -150,9 +152,9 @@ func SectorIDCounter(ds dtypes.MetadataDS) sealing.SectorIDCounter { return &sidsc{sc} } -func AddressSelector(addrConf *config.MinerAddressConfig) func() (*storage.AddressSelector, error) { - return func() (*storage.AddressSelector, error) { - as := &storage.AddressSelector{} +func AddressSelector(addrConf *config.MinerAddressConfig) func() (*ctladdr.AddressSelector, error) { + return func() (*ctladdr.AddressSelector, error) { + as := &ctladdr.AddressSelector{} if addrConf == nil { return as, nil } @@ -207,13 +209,13 @@ type StorageMinerParams struct { MetricsCtx helpers.MetricsCtx API v1api.FullNode MetadataDS dtypes.MetadataDS - Sealer sectorstorage.SectorManager + Sealer sealer.SectorManager SectorIDCounter sealing.SectorIDCounter Verifier ffiwrapper.Verifier Prover ffiwrapper.Prover GetSealingConfigFn dtypes.GetSealingConfigFunc Journal journal.Journal - AddrSel *storage.AddressSelector + AddrSel *ctladdr.AddressSelector Maddr dtypes.MinerAddress } @@ -252,8 +254,8 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st } } -func WindowPostScheduler(fc config.MinerFeeConfig) func(params StorageMinerParams) (*storage.WindowPoStScheduler, error) { - return func(params StorageMinerParams) (*storage.WindowPoStScheduler, error) { +func WindowPostScheduler(fc config.MinerFeeConfig) func(params StorageMinerParams) (*wdpost.WindowPoStScheduler, error) { + return func(params StorageMinerParams) (*wdpost.WindowPoStScheduler, error) { var ( mctx = params.MetricsCtx lc = params.Lifecycle @@ -267,7 +269,7 @@ func WindowPostScheduler(fc config.MinerFeeConfig) func(params StorageMinerParam ctx := helpers.LifecycleCtx(mctx, lc) - fps, err := storage.NewWindowedPoStScheduler(api, fc, as, sealer, verif, sealer, j, maddr) + fps, err := wdpost.NewWindowedPoStScheduler(api, fc, as, sealer, verif, sealer, j, maddr) if err != nil { return nil, err } @@ -733,22 +735,22 @@ func RetrievalProvider( var WorkerCallsPrefix = datastore.NewKey("/worker/calls") var ManagerWorkPrefix = datastore.NewKey("/stmgr/calls") -func LocalStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, urls stores.URLs) (*stores.Local, error) { +func LocalStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls paths.LocalStorage, si paths.SectorIndex, urls paths.URLs) (*paths.Local, error) { ctx := helpers.LifecycleCtx(mctx, lc) - return stores.NewLocal(ctx, ls, si, urls) + return paths.NewLocal(ctx, ls, si, urls) } -func RemoteStorage(lstor *stores.Local, si stores.SectorIndex, sa sectorstorage.StorageAuth, sc sectorstorage.Config) *stores.Remote { - return stores.NewRemote(lstor, si, http.Header(sa), sc.ParallelFetchLimit, &stores.DefaultPartialFileHandler{}) +func RemoteStorage(lstor *paths.Local, si paths.SectorIndex, sa sealer.StorageAuth, sc sealer.Config) *paths.Remote { + return paths.NewRemote(lstor, si, http.Header(sa), sc.ParallelFetchLimit, &paths.DefaultPartialFileHandler{}) } -func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *stores.Local, stor stores.Store, ls stores.LocalStorage, si stores.SectorIndex, sc sectorstorage.Config, ds dtypes.MetadataDS) (*sectorstorage.Manager, error) { +func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *paths.Local, stor paths.Store, ls paths.LocalStorage, si paths.SectorIndex, sc sealer.Config, ds dtypes.MetadataDS) (*sealer.Manager, error) { ctx := helpers.LifecycleCtx(mctx, lc) wsts := statestore.New(namespace.Wrap(ds, WorkerCallsPrefix)) smsts := statestore.New(namespace.Wrap(ds, ManagerWorkPrefix)) - sst, err := sectorstorage.New(ctx, lstor, stor, ls, si, sc, wsts, smsts) + sst, err := sealer.New(ctx, lstor, stor, ls, si, sc, wsts, smsts) if err != nil { return nil, err } @@ -760,7 +762,7 @@ func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *stores.Local return sst, nil } -func StorageAuth(ctx helpers.MetricsCtx, ca v0api.Common) (sectorstorage.StorageAuth, error) { +func StorageAuth(ctx helpers.MetricsCtx, ca v0api.Common) (sealer.StorageAuth, error) { token, err := ca.AuthNew(ctx, []auth.Permission{"admin"}) if err != nil { return nil, xerrors.Errorf("creating storage auth header: %w", err) @@ -768,18 +770,18 @@ func StorageAuth(ctx helpers.MetricsCtx, ca v0api.Common) (sectorstorage.Storage headers := http.Header{} headers.Add("Authorization", "Bearer "+string(token)) - return sectorstorage.StorageAuth(headers), nil + return sealer.StorageAuth(headers), nil } -func StorageAuthWithURL(apiInfo string) func(ctx helpers.MetricsCtx, ca v0api.Common) (sectorstorage.StorageAuth, error) { - return func(ctx helpers.MetricsCtx, ca v0api.Common) (sectorstorage.StorageAuth, error) { +func StorageAuthWithURL(apiInfo string) func(ctx helpers.MetricsCtx, ca v0api.Common) (sealer.StorageAuth, error) { + return func(ctx helpers.MetricsCtx, ca v0api.Common) (sealer.StorageAuth, error) { s := strings.Split(apiInfo, ":") if len(s) != 2 { return nil, errors.New("unexpected format of `apiInfo`") } headers := http.Header{} headers.Add("Authorization", "Bearer "+s[0]) - return sectorstorage.StorageAuth(headers), nil + return sealer.StorageAuth(headers), nil } } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 1f90a1fea42..9327575dd5b 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -24,9 +24,9 @@ import ( "github.com/filecoin-project/lotus/blockstore" badgerbs "github.com/filecoin-project/lotus/blockstore/badger" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" ) const ( @@ -572,26 +572,26 @@ func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error { return nil } -func (fsr *fsLockedRepo) GetStorage() (stores.StorageConfig, error) { +func (fsr *fsLockedRepo) GetStorage() (paths.StorageConfig, error) { fsr.storageLk.Lock() defer fsr.storageLk.Unlock() return fsr.getStorage(nil) } -func (fsr *fsLockedRepo) getStorage(def *stores.StorageConfig) (stores.StorageConfig, error) { +func (fsr *fsLockedRepo) getStorage(def *paths.StorageConfig) (paths.StorageConfig, error) { c, err := config.StorageFromFile(fsr.join(fsStorageConfig), def) if err != nil { - return stores.StorageConfig{}, err + return paths.StorageConfig{}, err } return *c, nil } -func (fsr *fsLockedRepo) SetStorage(c func(*stores.StorageConfig)) error { +func (fsr *fsLockedRepo) SetStorage(c func(*paths.StorageConfig)) error { fsr.storageLk.Lock() defer fsr.storageLk.Unlock() - sc, err := fsr.getStorage(&stores.StorageConfig{}) + sc, err := fsr.getStorage(&paths.StorageConfig{}) if err != nil { return xerrors.Errorf("get storage: %w", err) } diff --git a/node/repo/interface.go b/node/repo/interface.go index 88c2f566602..4f029471375 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -9,8 +9,8 @@ import ( "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" ) // BlockstoreDomain represents the domain of a blockstore. @@ -73,8 +73,8 @@ type LockedRepo interface { Config() (interface{}, error) SetConfig(func(interface{})) error - GetStorage() (stores.StorageConfig, error) - SetStorage(func(*stores.StorageConfig)) error + GetStorage() (paths.StorageConfig, error) + SetStorage(func(*paths.StorageConfig)) error Stat(path string) (fsutil.FsStat, error) DiskUsage(path string) (int64, error) diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 1b2e59c66e0..534487417c2 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -17,10 +17,10 @@ import ( "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type MemRepo struct { @@ -37,7 +37,7 @@ type MemRepo struct { keystore map[string]types.KeyInfo blockstore blockstore.Blockstore - sc *stores.StorageConfig + sc *paths.StorageConfig tempDir string // holds the current config value @@ -59,13 +59,13 @@ func (lmem *lockedMemRepo) RepoType() RepoType { return lmem.t } -func (lmem *lockedMemRepo) GetStorage() (stores.StorageConfig, error) { +func (lmem *lockedMemRepo) GetStorage() (paths.StorageConfig, error) { if err := lmem.checkToken(); err != nil { - return stores.StorageConfig{}, err + return paths.StorageConfig{}, err } if lmem.mem.sc == nil { - lmem.mem.sc = &stores.StorageConfig{StoragePaths: []stores.LocalPath{ + lmem.mem.sc = &paths.StorageConfig{StoragePaths: []paths.LocalPath{ {Path: lmem.Path()}, }} } @@ -73,7 +73,7 @@ func (lmem *lockedMemRepo) GetStorage() (stores.StorageConfig, error) { return *lmem.mem.sc, nil } -func (lmem *lockedMemRepo) SetStorage(c func(*stores.StorageConfig)) error { +func (lmem *lockedMemRepo) SetStorage(c func(*paths.StorageConfig)) error { if err := lmem.checkToken(); err != nil { return err } @@ -126,14 +126,14 @@ func (lmem *lockedMemRepo) Path() string { } func (lmem *lockedMemRepo) initSectorStore(t string) { - if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), stores.StorageConfig{ - StoragePaths: []stores.LocalPath{ + if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), paths.StorageConfig{ + StoragePaths: []paths.LocalPath{ {Path: t}, }}); err != nil { panic(err) } - b, err := json.MarshalIndent(&stores.LocalStorageMeta{ + b, err := json.MarshalIndent(&paths.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), Weight: 10, CanSeal: true, diff --git a/paychmgr/manager.go b/paychmgr/manager.go index 38cb8b8e6d0..fda9b101f2c 100644 --- a/paychmgr/manager.go +++ b/paychmgr/manager.go @@ -8,7 +8,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" diff --git a/paychmgr/paychget_test.go b/paychmgr/paychget_test.go index cdb37065069..5f18766644e 100644 --- a/paychmgr/paychget_test.go +++ b/paychmgr/paychget_test.go @@ -515,18 +515,21 @@ func TestPaychGetRestartAfterCreateChannelMsg(t *testing.T) { from := tutils.NewIDAddr(t, 101) to := tutils.NewIDAddr(t, 102) - mock := newMockManagerAPI() + var createMsgCid cid.Cid + { + mock := newMockManagerAPI() - mgr, err := newManager(store, mock) - require.NoError(t, err) + mgr, err := newManager(store, mock) + require.NoError(t, err) - // Send create message for a channel with value 10 - amt := big.NewInt(10) - _, createMsgCid, err := mgr.GetPaych(ctx, from, to, amt, onChainReserve) - require.NoError(t, err) + // Send create message for a channel with value 10 + amt := big.NewInt(10) + _, createMsgCid, err = mgr.GetPaych(ctx, from, to, amt, onChainReserve) + require.NoError(t, err) - // Simulate shutting down system - mock.close() + // Simulate shutting down system + mock.close() + } // Create a new manager with the same datastore mock2 := newMockManagerAPI() diff --git a/storage/adapter_events.go b/storage/adapter_events.go index 0f9c62039f8..fc84bca1d4e 100644 --- a/storage/adapter_events.go +++ b/storage/adapter_events.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) var _ sealing.Events = new(EventsAdapter) diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index d270b7b2093..a28a837677a 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -27,10 +27,10 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" ) -var _ sealing.SealingAPI = new(SealingAPIAdapter) +var _ pipeline.SealingAPI = new(SealingAPIAdapter) type SealingAPIAdapter struct { delegate fullNodeFilteredAPI @@ -40,7 +40,7 @@ func NewSealingAPIAdapter(api fullNodeFilteredAPI) SealingAPIAdapter { return SealingAPIAdapter{delegate: api} } -func (s SealingAPIAdapter) StateMinerSectorSize(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) (abi.SectorSize, error) { +func (s SealingAPIAdapter) StateMinerSectorSize(ctx context.Context, maddr address.Address, tok pipeline.TipSetToken) (abi.SectorSize, error) { // TODO: update storage-fsm to just StateMinerInfo mi, err := s.StateMinerInfo(ctx, maddr, tok) if err != nil { @@ -49,7 +49,7 @@ func (s SealingAPIAdapter) StateMinerSectorSize(ctx context.Context, maddr addre return mi.SectorSize, nil } -func (s SealingAPIAdapter) StateMinerPreCommitDepositForPower(ctx context.Context, a address.Address, pci minertypes.SectorPreCommitInfo, tok sealing.TipSetToken) (big.Int, error) { +func (s SealingAPIAdapter) StateMinerPreCommitDepositForPower(ctx context.Context, a address.Address, pci minertypes.SectorPreCommitInfo, tok pipeline.TipSetToken) (big.Int, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return big.Zero(), xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -58,7 +58,7 @@ func (s SealingAPIAdapter) StateMinerPreCommitDepositForPower(ctx context.Contex return s.delegate.StateMinerPreCommitDepositForPower(ctx, a, pci, tsk) } -func (s SealingAPIAdapter) StateMinerInitialPledgeCollateral(ctx context.Context, a address.Address, pci minertypes.SectorPreCommitInfo, tok sealing.TipSetToken) (big.Int, error) { +func (s SealingAPIAdapter) StateMinerInitialPledgeCollateral(ctx context.Context, a address.Address, pci minertypes.SectorPreCommitInfo, tok pipeline.TipSetToken) (big.Int, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return big.Zero(), xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -67,7 +67,7 @@ func (s SealingAPIAdapter) StateMinerInitialPledgeCollateral(ctx context.Context return s.delegate.StateMinerInitialPledgeCollateral(ctx, a, pci, tsk) } -func (s SealingAPIAdapter) StateMinerInfo(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) (api.MinerInfo, error) { +func (s SealingAPIAdapter) StateMinerInfo(ctx context.Context, maddr address.Address, tok pipeline.TipSetToken) (api.MinerInfo, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return api.MinerInfo{}, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -77,7 +77,7 @@ func (s SealingAPIAdapter) StateMinerInfo(ctx context.Context, maddr address.Add return s.delegate.StateMinerInfo(ctx, maddr, tsk) } -func (s SealingAPIAdapter) StateMinerAvailableBalance(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) (big.Int, error) { +func (s SealingAPIAdapter) StateMinerAvailableBalance(ctx context.Context, maddr address.Address, tok pipeline.TipSetToken) (big.Int, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return big.Zero(), xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -86,7 +86,7 @@ func (s SealingAPIAdapter) StateMinerAvailableBalance(ctx context.Context, maddr return s.delegate.StateMinerAvailableBalance(ctx, maddr, tsk) } -func (s SealingAPIAdapter) StateMinerWorkerAddress(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) (address.Address, error) { +func (s SealingAPIAdapter) StateMinerWorkerAddress(ctx context.Context, maddr address.Address, tok pipeline.TipSetToken) (address.Address, error) { // TODO: update storage-fsm to just StateMinerInfo mi, err := s.StateMinerInfo(ctx, maddr, tok) if err != nil { @@ -95,7 +95,7 @@ func (s SealingAPIAdapter) StateMinerWorkerAddress(ctx context.Context, maddr ad return mi.Worker, nil } -func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) ([]api.Deadline, error) { +func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok pipeline.TipSetToken) ([]api.Deadline, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -104,7 +104,7 @@ func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr addres return s.delegate.StateMinerDeadlines(ctx, maddr, tsk) } -func (s SealingAPIAdapter) StateMinerSectorAllocated(ctx context.Context, maddr address.Address, sid abi.SectorNumber, tok sealing.TipSetToken) (bool, error) { +func (s SealingAPIAdapter) StateMinerSectorAllocated(ctx context.Context, maddr address.Address, sid abi.SectorNumber, tok pipeline.TipSetToken) (bool, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return false, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -113,7 +113,7 @@ func (s SealingAPIAdapter) StateMinerSectorAllocated(ctx context.Context, maddr return s.delegate.StateMinerSectorAllocated(ctx, maddr, sid, tsk) } -func (s SealingAPIAdapter) StateMinerActiveSectors(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) (bitfield.BitField, error) { +func (s SealingAPIAdapter) StateMinerActiveSectors(ctx context.Context, maddr address.Address, tok pipeline.TipSetToken) (bitfield.BitField, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return bitfield.BitField{}, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -134,14 +134,14 @@ func (s SealingAPIAdapter) StateMinerActiveSectors(ctx context.Context, maddr ad return miner.AllPartSectors(state, miner.Partition.ActiveSectors) } -func (s SealingAPIAdapter) StateWaitMsg(ctx context.Context, mcid cid.Cid) (sealing.MsgLookup, error) { +func (s SealingAPIAdapter) StateWaitMsg(ctx context.Context, mcid cid.Cid) (pipeline.MsgLookup, error) { wmsg, err := s.delegate.StateWaitMsg(ctx, mcid, build.MessageConfidence, api.LookbackNoLimit, true) if err != nil { - return sealing.MsgLookup{}, err + return pipeline.MsgLookup{}, err } - return sealing.MsgLookup{ - Receipt: sealing.MessageReceipt{ + return pipeline.MsgLookup{ + Receipt: pipeline.MessageReceipt{ ExitCode: wmsg.Receipt.ExitCode, Return: wmsg.Receipt.Return, GasUsed: wmsg.Receipt.GasUsed, @@ -151,7 +151,7 @@ func (s SealingAPIAdapter) StateWaitMsg(ctx context.Context, mcid cid.Cid) (seal }, nil } -func (s SealingAPIAdapter) StateSearchMsg(ctx context.Context, c cid.Cid) (*sealing.MsgLookup, error) { +func (s SealingAPIAdapter) StateSearchMsg(ctx context.Context, c cid.Cid) (*pipeline.MsgLookup, error) { wmsg, err := s.delegate.StateSearchMsg(ctx, types.EmptyTSK, c, api.LookbackNoLimit, true) if err != nil { return nil, err @@ -161,8 +161,8 @@ func (s SealingAPIAdapter) StateSearchMsg(ctx context.Context, c cid.Cid) (*seal return nil, nil } - return &sealing.MsgLookup{ - Receipt: sealing.MessageReceipt{ + return &pipeline.MsgLookup{ + Receipt: pipeline.MessageReceipt{ ExitCode: wmsg.Receipt.ExitCode, Return: wmsg.Receipt.Return, GasUsed: wmsg.Receipt.GasUsed, @@ -172,7 +172,7 @@ func (s SealingAPIAdapter) StateSearchMsg(ctx context.Context, c cid.Cid) (*seal }, nil } -func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tok sealing.TipSetToken) (cid.Cid, error) { +func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tok pipeline.TipSetToken) (cid.Cid, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return cid.Undef, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -240,7 +240,7 @@ func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr return cid.Cid(cr.CommDs[0]), nil } -func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok sealing.TipSetToken) (*minertypes.SectorPreCommitOnChainInfo, error) { +func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok pipeline.TipSetToken) (*minertypes.SectorPreCommitOnChainInfo, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -268,7 +268,7 @@ func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr a return nil, xerrors.Errorf("checking if sector is allocated: %w", err) } if set { - return nil, sealing.ErrSectorAllocated + return nil, pipeline.ErrSectorAllocated } return nil, nil @@ -277,7 +277,7 @@ func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr a return pci, nil } -func (s SealingAPIAdapter) StateSectorGetInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok sealing.TipSetToken) (*miner.SectorOnChainInfo, error) { +func (s SealingAPIAdapter) StateSectorGetInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok pipeline.TipSetToken) (*miner.SectorOnChainInfo, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -286,7 +286,7 @@ func (s SealingAPIAdapter) StateSectorGetInfo(ctx context.Context, maddr address return s.delegate.StateSectorGetInfo(ctx, maddr, sectorNumber, tsk) } -func (s SealingAPIAdapter) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok sealing.TipSetToken) (*sealing.SectorLocation, error) { +func (s SealingAPIAdapter) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok pipeline.TipSetToken) (*pipeline.SectorLocation, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -297,7 +297,7 @@ func (s SealingAPIAdapter) StateSectorPartition(ctx context.Context, maddr addre return nil, err } if l != nil { - return &sealing.SectorLocation{ + return &pipeline.SectorLocation{ Deadline: l.Deadline, Partition: l.Partition, }, nil @@ -306,7 +306,7 @@ func (s SealingAPIAdapter) StateSectorPartition(ctx context.Context, maddr addre return nil, nil // not found } -func (s SealingAPIAdapter) StateMinerPartitions(ctx context.Context, maddr address.Address, dlIdx uint64, tok sealing.TipSetToken) ([]api.Partition, error) { +func (s SealingAPIAdapter) StateMinerPartitions(ctx context.Context, maddr address.Address, dlIdx uint64, tok pipeline.TipSetToken) ([]api.Partition, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -315,7 +315,7 @@ func (s SealingAPIAdapter) StateMinerPartitions(ctx context.Context, maddr addre return s.delegate.StateMinerPartitions(ctx, maddr, dlIdx, tsk) } -func (s SealingAPIAdapter) StateLookupID(ctx context.Context, addr address.Address, tok sealing.TipSetToken) (address.Address, error) { +func (s SealingAPIAdapter) StateLookupID(ctx context.Context, addr address.Address, tok pipeline.TipSetToken) (address.Address, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return address.Undef, err @@ -324,7 +324,7 @@ func (s SealingAPIAdapter) StateLookupID(ctx context.Context, addr address.Addre return s.delegate.StateLookupID(ctx, addr, tsk) } -func (s SealingAPIAdapter) StateMarketStorageDeal(ctx context.Context, dealID abi.DealID, tok sealing.TipSetToken) (*api.MarketDeal, error) { +func (s SealingAPIAdapter) StateMarketStorageDeal(ctx context.Context, dealID abi.DealID, tok pipeline.TipSetToken) (*api.MarketDeal, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, err @@ -333,7 +333,7 @@ func (s SealingAPIAdapter) StateMarketStorageDeal(ctx context.Context, dealID ab return s.delegate.StateMarketStorageDeal(ctx, dealID, tsk) } -func (s SealingAPIAdapter) StateMarketStorageDealProposal(ctx context.Context, dealID abi.DealID, tok sealing.TipSetToken) (market.DealProposal, error) { +func (s SealingAPIAdapter) StateMarketStorageDealProposal(ctx context.Context, dealID abi.DealID, tok pipeline.TipSetToken) (market.DealProposal, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return market.DealProposal{}, err @@ -347,7 +347,7 @@ func (s SealingAPIAdapter) StateMarketStorageDealProposal(ctx context.Context, d return deal.Proposal, nil } -func (s SealingAPIAdapter) StateNetworkVersion(ctx context.Context, tok sealing.TipSetToken) (network.Version, error) { +func (s SealingAPIAdapter) StateNetworkVersion(ctx context.Context, tok pipeline.TipSetToken) (network.Version, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return network.VersionMax, err @@ -356,7 +356,7 @@ func (s SealingAPIAdapter) StateNetworkVersion(ctx context.Context, tok sealing. return s.delegate.StateNetworkVersion(ctx, tsk) } -func (s SealingAPIAdapter) StateMinerProvingDeadline(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) (*dline.Info, error) { +func (s SealingAPIAdapter) StateMinerProvingDeadline(ctx context.Context, maddr address.Address, tok pipeline.TipSetToken) (*dline.Info, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, err @@ -382,7 +382,7 @@ func (s SealingAPIAdapter) SendMsg(ctx context.Context, from, to address.Address return smsg.Cid(), nil } -func (s SealingAPIAdapter) ChainHead(ctx context.Context) (sealing.TipSetToken, abi.ChainEpoch, error) { +func (s SealingAPIAdapter) ChainHead(ctx context.Context) (pipeline.TipSetToken, abi.ChainEpoch, error) { head, err := s.delegate.ChainHead(ctx) if err != nil { return nil, 0, err @@ -391,7 +391,7 @@ func (s SealingAPIAdapter) ChainHead(ctx context.Context) (sealing.TipSetToken, return head.Key().Bytes(), head.Height(), nil } -func (s SealingAPIAdapter) ChainBaseFee(ctx context.Context, tok sealing.TipSetToken) (abi.TokenAmount, error) { +func (s SealingAPIAdapter) ChainBaseFee(ctx context.Context, tok pipeline.TipSetToken) (abi.TokenAmount, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return big.Zero(), err @@ -409,7 +409,7 @@ func (s SealingAPIAdapter) ChainGetMessage(ctx context.Context, mc cid.Cid) (*ty return s.delegate.ChainGetMessage(ctx, mc) } -func (s SealingAPIAdapter) StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tok sealing.TipSetToken) (abi.Randomness, error) { +func (s SealingAPIAdapter) StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tok pipeline.TipSetToken) (abi.Randomness, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, err @@ -418,7 +418,7 @@ func (s SealingAPIAdapter) StateGetRandomnessFromBeacon(ctx context.Context, per return s.delegate.StateGetRandomnessFromBeacon(ctx, personalization, randEpoch, entropy, tsk) } -func (s SealingAPIAdapter) StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tok sealing.TipSetToken) (abi.Randomness, error) { +func (s SealingAPIAdapter) StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tok pipeline.TipSetToken) (abi.Randomness, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, err diff --git a/storage/addresses.go b/storage/ctladdr/addresses.go similarity index 98% rename from storage/addresses.go rename to storage/ctladdr/addresses.go index 2c9b381cd19..701fea933bb 100644 --- a/storage/addresses.go +++ b/storage/ctladdr/addresses.go @@ -1,8 +1,10 @@ -package storage +package ctladdr import ( "context" + logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" @@ -11,6 +13,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +var log = logging.Logger("ctladdr") + type addrSelectApi interface { WalletBalance(context.Context, address.Address) (types.BigInt, error) WalletHas(context.Context, address.Address) (bool, error) diff --git a/storage/miner.go b/storage/miner.go index 8c15f544d15..6fd076d18a7 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -28,12 +28,13 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/storage/ctladdr" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) var log = logging.Logger("storageminer") @@ -42,24 +43,24 @@ var log = logging.Logger("storageminer") // instantiated in the node builder, along with the WindowPoStScheduler. // // This object is the owner of the sealing pipeline. Most of the actual logic -// lives in the storage-sealing module (sealing.Sealing), and the Miner object +// lives in the pipeline module (sealing.Sealing), and the Miner object // exposes it to the rest of the system by proxying calls. // // Miner#Run starts the sealing FSM. type Miner struct { api fullNodeFilteredAPI feeCfg config.MinerFeeConfig - sealer sectorstorage.SectorManager + sealer sealer.SectorManager ds datastore.Batching - sc sealing.SectorIDCounter + sc pipeline.SectorIDCounter verif ffiwrapper.Verifier prover ffiwrapper.Prover - addrSel *AddressSelector + addrSel *ctladdr.AddressSelector maddr address.Address getSealConfig dtypes.GetSealingConfigFunc - sealing *sealing.Sealing + sealing *pipeline.Sealing sealingEvtType journal.EventType @@ -70,8 +71,8 @@ type Miner struct { type SealingStateEvt struct { SectorNumber abi.SectorNumber SectorType abi.RegisteredSealProof - From sealing.SectorState - After sealing.SectorState + From pipeline.SectorState + After pipeline.SectorState Error string } @@ -132,14 +133,14 @@ type fullNodeFilteredAPI interface { func NewMiner(api fullNodeFilteredAPI, maddr address.Address, ds datastore.Batching, - sealer sectorstorage.SectorManager, - sc sealing.SectorIDCounter, + sealer sealer.SectorManager, + sc pipeline.SectorIDCounter, verif ffiwrapper.Verifier, prover ffiwrapper.Prover, gsd dtypes.GetSealingConfigFunc, feeCfg config.MinerFeeConfig, journal journal.Journal, - as *AddressSelector) (*Miner, error) { + as *ctladdr.AddressSelector) (*Miner, error) { m := &Miner{ api: api, feeCfg: feeCfg, @@ -184,10 +185,10 @@ func (m *Miner) Run(ctx context.Context) error { adaptedAPI := NewSealingAPIAdapter(m.api) // Instantiate a precommit policy. - cfg := sealing.GetSealingConfigFunc(m.getSealConfig) + cfg := pipeline.GetSealingConfigFunc(m.getSealConfig) provingBuffer := md.WPoStProvingPeriod * 2 - pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, cfg, provingBuffer) + pcp := pipeline.NewBasicPreCommitPolicy(adaptedAPI, cfg, provingBuffer) // address selector. as := func(ctx context.Context, mi api.MinerInfo, use api.AddrUse, goodFunds, minFunds abi.TokenAmount) (address.Address, abi.TokenAmount, error) { @@ -195,7 +196,7 @@ func (m *Miner) Run(ctx context.Context) error { } // Instantiate the sealing FSM. - m.sealing = sealing.New(ctx, adaptedAPI, m.feeCfg, evtsAdapter, m.maddr, m.ds, m.sealer, m.sc, m.verif, m.prover, &pcp, cfg, m.handleSealingNotifications, as) + m.sealing = pipeline.New(ctx, adaptedAPI, m.feeCfg, evtsAdapter, m.maddr, m.ds, m.sealer, m.sc, m.verif, m.prover, &pcp, cfg, m.handleSealingNotifications, as) // Run the sealing FSM. go m.sealing.Run(ctx) //nolint:errcheck // logged intside the function @@ -203,7 +204,7 @@ func (m *Miner) Run(ctx context.Context) error { return nil } -func (m *Miner) handleSealingNotifications(before, after sealing.SectorInfo) { +func (m *Miner) handleSealingNotifications(before, after pipeline.SectorInfo) { m.journal.RecordEvent(m.sealingEvtType, func() interface{} { return SealingStateEvt{ SectorNumber: before.SectorNumber, diff --git a/storage/miner_sealing.go b/storage/miner_sealing.go index 59a587fdcfd..04f11282310 100644 --- a/storage/miner_sealing.go +++ b/storage/miner_sealing.go @@ -12,8 +12,8 @@ import ( "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -27,7 +27,7 @@ func (m *Miner) StartPackingSector(sectorNum abi.SectorNumber) error { return m.sealing.StartPacking(sectorNum) } -func (m *Miner) ListSectors() ([]sealing.SectorInfo, error) { +func (m *Miner) ListSectors() ([]pipeline.SectorInfo, error) { return m.sealing.ListSectors() } @@ -35,7 +35,7 @@ func (m *Miner) PledgeSector(ctx context.Context) (storage.SectorRef, error) { return m.sealing.PledgeSector(ctx) } -func (m *Miner) ForceSectorState(ctx context.Context, id abi.SectorNumber, state sealing.SectorState) error { +func (m *Miner) ForceSectorState(ctx context.Context, id abi.SectorNumber, state pipeline.SectorState) error { return m.sealing.ForceSectorState(ctx, id, state) } diff --git a/extern/sector-storage/stores/http_handler.go b/storage/paths/http_handler.go similarity index 97% rename from extern/sector-storage/stores/http_handler.go rename to storage/paths/http_handler.go index 6667b166268..f15afbf3d9a 100644 --- a/extern/sector-storage/stores/http_handler.go +++ b/storage/paths/http_handler.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "bytes" @@ -15,9 +15,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" + "github.com/filecoin-project/lotus/storage/sealer/partialfile" + "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/filecoin-project/lotus/storage/sealer/tarutil" ) var log = logging.Logger("stores") diff --git a/extern/sector-storage/stores/http_handler_test.go b/storage/paths/http_handler_test.go similarity index 97% rename from extern/sector-storage/stores/http_handler_test.go rename to storage/paths/http_handler_test.go index 3533e4f4e86..624f264ff88 100644 --- a/extern/sector-storage/stores/http_handler_test.go +++ b/storage/paths/http_handler_test.go @@ -1,4 +1,4 @@ -package stores_test +package paths_test import ( "fmt" @@ -16,10 +16,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/stores/mocks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/paths/mocks" + "github.com/filecoin-project/lotus/storage/sealer/partialfile" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func TestRemoteGetAllocated(t *testing.T) { @@ -207,7 +207,7 @@ func TestRemoteGetAllocated(t *testing.T) { lstore := mocks.NewMockStore(mockCtrl) pfhandler := mocks.NewMockPartialFileHandler(mockCtrl) - handler := &stores.FetchHandler{ + handler := &paths.FetchHandler{ lstore, pfhandler, } @@ -406,7 +406,7 @@ func TestRemoteGetSector(t *testing.T) { path = tempDir } - handler := &stores.FetchHandler{ + handler := &paths.FetchHandler{ lstore, pfhandler, } diff --git a/extern/sector-storage/stores/index.go b/storage/paths/index.go similarity index 98% rename from extern/sector-storage/stores/index.go rename to storage/paths/index.go index 5a0d0f3a3fa..634c7b6237e 100644 --- a/extern/sector-storage/stores/index.go +++ b/storage/paths/index.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "context" @@ -17,9 +17,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var HeartbeatInterval = 10 * time.Second diff --git a/extern/sector-storage/stores/index_locks.go b/storage/paths/index_locks.go similarity index 97% rename from extern/sector-storage/stores/index_locks.go rename to storage/paths/index_locks.go index ade437c6b0f..ab95cb4a7ab 100644 --- a/extern/sector-storage/stores/index_locks.go +++ b/storage/paths/index_locks.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "context" @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type sectorLock struct { diff --git a/extern/sector-storage/stores/index_locks_test.go b/storage/paths/index_locks_test.go similarity index 98% rename from extern/sector-storage/stores/index_locks_test.go rename to storage/paths/index_locks_test.go index ec7f3492760..d3134055bcb 100644 --- a/extern/sector-storage/stores/index_locks_test.go +++ b/storage/paths/index_locks_test.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "context" @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var aSector = abi.SectorID{ diff --git a/extern/sector-storage/stores/index_locks_util.go b/storage/paths/index_locks_util.go similarity index 97% rename from extern/sector-storage/stores/index_locks_util.go rename to storage/paths/index_locks_util.go index a8327fdedae..c8a8ae9acbf 100644 --- a/extern/sector-storage/stores/index_locks_util.go +++ b/storage/paths/index_locks_util.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "context" diff --git a/extern/sector-storage/stores/index_test.go b/storage/paths/index_test.go similarity index 95% rename from extern/sector-storage/stores/index_test.go rename to storage/paths/index_test.go index 64a9271c415..3eb2dbec47f 100644 --- a/extern/sector-storage/stores/index_test.go +++ b/storage/paths/index_test.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "context" @@ -10,8 +10,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func init() { diff --git a/extern/sector-storage/stores/interface.go b/storage/paths/interface.go similarity index 91% rename from extern/sector-storage/stores/interface.go rename to storage/paths/interface.go index 980beee47ea..6b5519bf408 100644 --- a/extern/sector-storage/stores/interface.go +++ b/storage/paths/interface.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "context" @@ -7,9 +7,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/partialfile" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) //go:generate go run github.com/golang/mock/mockgen -destination=mocks/pf.go -package=mocks . PartialFileHandler diff --git a/extern/sector-storage/stores/local.go b/storage/paths/local.go similarity index 99% rename from extern/sector-storage/stores/local.go rename to storage/paths/local.go index 549a8d4ecfd..e8f46353390 100644 --- a/extern/sector-storage/stores/local.go +++ b/storage/paths/local.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "context" @@ -18,8 +18,8 @@ import ( "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // LocalStorageMeta [path]/sectorstore.json diff --git a/extern/sector-storage/stores/local_test.go b/storage/paths/local_test.go similarity index 91% rename from extern/sector-storage/stores/local_test.go rename to storage/paths/local_test.go index 913a73b7727..c4891811a33 100644 --- a/extern/sector-storage/stores/local_test.go +++ b/storage/paths/local_test.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "context" @@ -11,8 +11,8 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const pathSize = 16 << 20 diff --git a/extern/sector-storage/stores/localstorage_cached.go b/storage/paths/localstorage_cached.go similarity index 97% rename from extern/sector-storage/stores/localstorage_cached.go rename to storage/paths/localstorage_cached.go index 9ada06cbfd7..4ccabb15e99 100644 --- a/extern/sector-storage/stores/localstorage_cached.go +++ b/storage/paths/localstorage_cached.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "sync" @@ -6,7 +6,7 @@ import ( lru "github.com/hashicorp/golang-lru" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" ) var StatTimeout = 5 * time.Second diff --git a/extern/sector-storage/stores/mocks/index.go b/storage/paths/mocks/index.go similarity index 97% rename from extern/sector-storage/stores/mocks/index.go rename to storage/paths/mocks/index.go index b194d0d84e0..030692b8f5f 100644 --- a/extern/sector-storage/stores/mocks/index.go +++ b/storage/paths/mocks/index.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/filecoin-project/lotus/extern/sector-storage/stores (interfaces: SectorIndex) +// Source: github.com/filecoin-project/lotus/storage/paths (interfaces: SectorIndex) // Package mocks is a generated GoMock package. package mocks @@ -12,8 +12,8 @@ import ( abi "github.com/filecoin-project/go-state-types/abi" - fsutil "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - storiface "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + fsutil "github.com/filecoin-project/lotus/storage/sealer/fsutil" + storiface "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // MockSectorIndex is a mock of SectorIndex interface. diff --git a/extern/sector-storage/stores/mocks/pf.go b/storage/paths/mocks/pf.go similarity index 92% rename from extern/sector-storage/stores/mocks/pf.go rename to storage/paths/mocks/pf.go index 5e8fe62cce3..50b020aaa8f 100644 --- a/extern/sector-storage/stores/mocks/pf.go +++ b/storage/paths/mocks/pf.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/filecoin-project/lotus/extern/sector-storage/stores (interfaces: PartialFileHandler) +// Source: github.com/filecoin-project/lotus/storage/paths (interfaces: PartialFileHandler) // Package mocks is a generated GoMock package. package mocks @@ -12,8 +12,8 @@ import ( abi "github.com/filecoin-project/go-state-types/abi" - partialfile "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" - storiface "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + partialfile "github.com/filecoin-project/lotus/storage/sealer/partialfile" + storiface "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // MockPartialFileHandler is a mock of PartialFileHandler interface. diff --git a/extern/sector-storage/stores/mocks/store.go b/storage/paths/mocks/store.go similarity index 95% rename from extern/sector-storage/stores/mocks/store.go rename to storage/paths/mocks/store.go index 2377433ee04..b1238c479b1 100644 --- a/extern/sector-storage/stores/mocks/store.go +++ b/storage/paths/mocks/store.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/filecoin-project/lotus/extern/sector-storage/stores (interfaces: Store) +// Source: github.com/filecoin-project/lotus/storage/paths (interfaces: Store) // Package mocks is a generated GoMock package. package mocks @@ -13,8 +13,8 @@ import ( abi "github.com/filecoin-project/go-state-types/abi" storage "github.com/filecoin-project/specs-storage/storage" - fsutil "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - storiface "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + fsutil "github.com/filecoin-project/lotus/storage/sealer/fsutil" + storiface "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // MockStore is a mock of Store interface. diff --git a/extern/sector-storage/stores/remote.go b/storage/paths/remote.go similarity index 99% rename from extern/sector-storage/stores/remote.go rename to storage/paths/remote.go index 989e913ee48..d44c0b55e33 100644 --- a/extern/sector-storage/stores/remote.go +++ b/storage/paths/remote.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "context" @@ -23,9 +23,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/filecoin-project/lotus/storage/sealer/tarutil" ) var FetchTempSubdir = "fetching" diff --git a/extern/sector-storage/stores/remote_test.go b/storage/paths/remote_test.go similarity index 95% rename from extern/sector-storage/stores/remote_test.go rename to storage/paths/remote_test.go index 1b6160a8f37..237501b7a88 100644 --- a/extern/sector-storage/stores/remote_test.go +++ b/storage/paths/remote_test.go @@ -1,5 +1,5 @@ //stm: #unit -package stores_test +package paths_test import ( "context" @@ -23,23 +23,23 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/stores/mocks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/paths/mocks" + "github.com/filecoin-project/lotus/storage/sealer/partialfile" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const metaFile = "sectorstore.json" -func createTestStorage(t *testing.T, p string, seal bool, att ...*stores.Local) storiface.ID { +func createTestStorage(t *testing.T, p string, seal bool, att ...*paths.Local) storiface.ID { if err := os.MkdirAll(p, 0755); err != nil { if !os.IsExist(err) { require.NoError(t, err) } } - cfg := &stores.LocalStorageMeta{ + cfg := &paths.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), Weight: 10, CanSeal: seal, @@ -61,7 +61,7 @@ func createTestStorage(t *testing.T, p string, seal bool, att ...*stores.Local) func TestMoveShared(t *testing.T) { logging.SetAllLoggers(logging.LevelDebug) - index := stores.NewIndex() + index := paths.NewIndex() ctx := context.Background() @@ -78,8 +78,8 @@ func TestMoveShared(t *testing.T) { _ = lr.Close() }) - err = lr.SetStorage(func(config *stores.StorageConfig) { - *config = stores.StorageConfig{} + err = lr.SetStorage(func(config *paths.StorageConfig) { + *config = paths.StorageConfig{} }) require.NoError(t, err) @@ -98,9 +98,9 @@ func TestMoveShared(t *testing.T) { hs1 := httptest.NewServer(mux1) hs2 := httptest.NewServer(mux2) - ls1, err := stores.NewLocal(ctx, lr1, index, []string{hs1.URL + "/remote"}) + ls1, err := paths.NewLocal(ctx, lr1, index, []string{hs1.URL + "/remote"}) require.NoError(t, err) - ls2, err := stores.NewLocal(ctx, lr2, index, []string{hs2.URL + "/remote"}) + ls2, err := paths.NewLocal(ctx, lr2, index, []string{hs2.URL + "/remote"}) require.NoError(t, err) dirStor := filepath.Join(dir, "stor") @@ -109,11 +109,11 @@ func TestMoveShared(t *testing.T) { id1 := createTestStorage(t, dirStor, false, ls1, ls2) id2 := createTestStorage(t, dirSeal, true, ls1) - rs1 := stores.NewRemote(ls1, index, nil, 20, &stores.DefaultPartialFileHandler{}) - rs2 := stores.NewRemote(ls2, index, nil, 20, &stores.DefaultPartialFileHandler{}) + rs1 := paths.NewRemote(ls1, index, nil, 20, &paths.DefaultPartialFileHandler{}) + rs2 := paths.NewRemote(ls2, index, nil, 20, &paths.DefaultPartialFileHandler{}) _ = rs2 - mux1.PathPrefix("/").Handler(&stores.FetchHandler{Local: ls1, PfHandler: &stores.DefaultPartialFileHandler{}}) - mux2.PathPrefix("/").Handler(&stores.FetchHandler{Local: ls2, PfHandler: &stores.DefaultPartialFileHandler{}}) + mux1.PathPrefix("/").Handler(&paths.FetchHandler{Local: ls1, PfHandler: &paths.DefaultPartialFileHandler{}}) + mux2.PathPrefix("/").Handler(&paths.FetchHandler{Local: ls2, PfHandler: &paths.DefaultPartialFileHandler{}}) // add a sealed replica file to the sealing (non-shared) path @@ -468,7 +468,7 @@ func TestReader(t *testing.T) { tc.indexFnc(index, tc.serverUrl) } - remoteStore := stores.NewRemote(lstore, index, nil, 6000, pfhandler) + remoteStore := paths.NewRemote(lstore, index, nil, 6000, pfhandler) rdg, err := remoteStore.Reader(ctx, sectorRef, offset, size) var rd io.ReadCloser @@ -771,7 +771,7 @@ func TestCheckIsUnsealed(t *testing.T) { tc.indexFnc(index, tc.serverUrl) } - remoteStore := stores.NewRemote(lstore, index, nil, 6000, pfhandler) + remoteStore := paths.NewRemote(lstore, index, nil, 6000, pfhandler) isUnsealed, err := remoteStore.CheckIsUnsealed(ctx, sectorRef, offset, size) diff --git a/extern/sector-storage/stores/util_unix.go b/storage/paths/util_unix.go similarity index 98% rename from extern/sector-storage/stores/util_unix.go rename to storage/paths/util_unix.go index 943681b498c..f691bad091a 100644 --- a/extern/sector-storage/stores/util_unix.go +++ b/storage/paths/util_unix.go @@ -1,4 +1,4 @@ -package stores +package paths import ( "bytes" diff --git a/extern/storage-sealing/cbor_gen.go b/storage/pipeline/cbor_gen.go similarity index 100% rename from extern/storage-sealing/cbor_gen.go rename to storage/pipeline/cbor_gen.go diff --git a/extern/storage-sealing/checks.go b/storage/pipeline/checks.go similarity index 100% rename from extern/storage-sealing/checks.go rename to storage/pipeline/checks.go diff --git a/extern/storage-sealing/commit_batch.go b/storage/pipeline/commit_batch.go similarity index 99% rename from extern/storage-sealing/commit_batch.go rename to storage/pipeline/commit_batch.go index 09f122fe9dc..0cd55da3b06 100644 --- a/extern/storage-sealing/commit_batch.go +++ b/storage/pipeline/commit_batch.go @@ -24,9 +24,9 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) var aggFeeNum = big.NewInt(110) diff --git a/extern/storage-sealing/commit_batch_test.go b/storage/pipeline/commit_batch_test.go similarity index 93% rename from extern/storage-sealing/commit_batch_test.go rename to storage/pipeline/commit_batch_test.go index 59a79d459b3..abb44f4b419 100644 --- a/extern/storage-sealing/commit_batch_test.go +++ b/storage/pipeline/commit_batch_test.go @@ -22,10 +22,10 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/extern/storage-sealing/mocks" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/pipeline/mocks" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) func TestCommitBatcher(t *testing.T) { @@ -71,10 +71,10 @@ func TestCommitBatcher(t *testing.T) { } type promise func(t *testing.T) - type action func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *sealing.CommitBatcher) promise + type action func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *pipeline.CommitBatcher) promise actions := func(as ...action) action { - return func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *sealing.CommitBatcher) promise { + return func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *pipeline.CommitBatcher) promise { var ps []promise for _, a := range as { p := a(t, s, pcb) @@ -95,13 +95,13 @@ func TestCommitBatcher(t *testing.T) { } addSector := func(sn abi.SectorNumber) action { - return func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *sealing.CommitBatcher) promise { + return func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *pipeline.CommitBatcher) promise { var pcres sealiface.CommitBatchRes var pcerr error done := sync.Mutex{} done.Lock() - si := sealing.SectorInfo{ + si := pipeline.SectorInfo{ SectorNumber: sn, } @@ -113,7 +113,7 @@ func TestCommitBatcher(t *testing.T) { go func() { defer done.Unlock() - pcres, pcerr = pcb.AddCommit(ctx, si, sealing.AggregateInput{ + pcres, pcerr = pcb.AddCommit(ctx, si, pipeline.AggregateInput{ Info: prooftypes.AggregateSealVerifyInfo{ Number: sn, }, @@ -138,7 +138,7 @@ func TestCommitBatcher(t *testing.T) { } waitPending := func(n int) action { - return func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *sealing.CommitBatcher) promise { + return func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *pipeline.CommitBatcher) promise { require.Eventually(t, func() bool { p, err := pcb.Pending(ctx) require.NoError(t, err) @@ -151,7 +151,7 @@ func TestCommitBatcher(t *testing.T) { //stm: @CHAIN_STATE_MINER_INFO_001, @CHAIN_STATE_NETWORK_VERSION_001, @CHAIN_STATE_MINER_GET_COLLATERAL_001 expectSend := func(expect []abi.SectorNumber, aboveBalancer, failOnePCI bool) action { - return func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *sealing.CommitBatcher) promise { + return func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *pipeline.CommitBatcher) promise { s.EXPECT().StateMinerInfo(gomock.Any(), gomock.Any(), gomock.Any()).Return(api.MinerInfo{Owner: t0123, Worker: t0123}, nil) ti := len(expect) @@ -217,7 +217,7 @@ func TestCommitBatcher(t *testing.T) { } flush := func(expect []abi.SectorNumber, aboveBalancer, failOnePCI bool) action { - return func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *sealing.CommitBatcher) promise { + return func(t *testing.T, s *mocks.MockCommitBatcherApi, pcb *pipeline.CommitBatcher) promise { _ = expectSend(expect, aboveBalancer, failOnePCI)(t, s, pcb) batch := len(expect) >= minBatch && aboveBalancer @@ -357,7 +357,7 @@ func TestCommitBatcher(t *testing.T) { // create them mocks pcapi := mocks.NewMockCommitBatcherApi(mockCtrl) - pcb := sealing.NewCommitBatcher(ctx, t0123, pcapi, as, fc, cfg, &fakeProver{}) + pcb := pipeline.NewCommitBatcher(ctx, t0123, pcapi, as, fc, cfg, &fakeProver{}) var promises []promise diff --git a/extern/storage-sealing/constants.go b/storage/pipeline/constants.go similarity index 100% rename from extern/storage-sealing/constants.go rename to storage/pipeline/constants.go diff --git a/extern/storage-sealing/currentdealinfo.go b/storage/pipeline/currentdealinfo.go similarity index 100% rename from extern/storage-sealing/currentdealinfo.go rename to storage/pipeline/currentdealinfo.go diff --git a/extern/storage-sealing/currentdealinfo_test.go b/storage/pipeline/currentdealinfo_test.go similarity index 100% rename from extern/storage-sealing/currentdealinfo_test.go rename to storage/pipeline/currentdealinfo_test.go diff --git a/extern/storage-sealing/events.go b/storage/pipeline/events.go similarity index 100% rename from extern/storage-sealing/events.go rename to storage/pipeline/events.go diff --git a/extern/storage-sealing/fsm.go b/storage/pipeline/fsm.go similarity index 99% rename from extern/storage-sealing/fsm.go rename to storage/pipeline/fsm.go index 4e36ada6c57..3df57a501ad 100644 --- a/extern/storage-sealing/fsm.go +++ b/storage/pipeline/fsm.go @@ -13,7 +13,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" - statemachine "github.com/filecoin-project/go-statemachine" + "github.com/filecoin-project/go-statemachine" ) func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, uint64, error) { diff --git a/extern/storage-sealing/fsm_events.go b/storage/pipeline/fsm_events.go similarity index 100% rename from extern/storage-sealing/fsm_events.go rename to storage/pipeline/fsm_events.go diff --git a/extern/storage-sealing/fsm_test.go b/storage/pipeline/fsm_test.go similarity index 100% rename from extern/storage-sealing/fsm_test.go rename to storage/pipeline/fsm_test.go diff --git a/extern/storage-sealing/garbage.go b/storage/pipeline/garbage.go similarity index 100% rename from extern/storage-sealing/garbage.go rename to storage/pipeline/garbage.go diff --git a/extern/storage-sealing/gen/main.go b/storage/pipeline/gen/main.go similarity index 80% rename from extern/storage-sealing/gen/main.go rename to storage/pipeline/gen/main.go index 825ce8d284b..6da1d96b7e2 100644 --- a/extern/storage-sealing/gen/main.go +++ b/storage/pipeline/gen/main.go @@ -6,7 +6,7 @@ import ( gen "github.com/whyrusleeping/cbor-gen" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) func main() { diff --git a/extern/storage-sealing/input.go b/storage/pipeline/input.go similarity index 97% rename from extern/storage-sealing/input.go rename to storage/pipeline/input.go index f34277d9089..09661ea0e61 100644 --- a/extern/storage-sealing/input.go +++ b/storage/pipeline/input.go @@ -19,10 +19,10 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + "github.com/filecoin-project/lotus/storage/pipeline/lib/nullreader" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) func (m *Sealing) handleWaitDeals(ctx statemachine.Context, sector SectorInfo) error { @@ -216,7 +216,7 @@ func (m *Sealing) handleAddPiece(ctx statemachine.Context, sector SectorInfo) er for _, p := range pads { expectCid := zerocomm.ZeroPieceCommitment(p.Unpadded()) - ppi, err := m.sealer.AddPiece(sectorstorage.WithPriority(ctx.Context(), DealSectorPriority), + ppi, err := m.sealer.AddPiece(sealer.WithPriority(ctx.Context(), DealSectorPriority), m.minerSector(sector.SectorType, sector.SectorNumber), pieceSizes, p.Unpadded(), @@ -238,7 +238,7 @@ func (m *Sealing) handleAddPiece(ctx statemachine.Context, sector SectorInfo) er }) } - ppi, err := m.sealer.AddPiece(sectorstorage.WithPriority(ctx.Context(), DealSectorPriority), + ppi, err := m.sealer.AddPiece(sealer.WithPriority(ctx.Context(), DealSectorPriority), m.minerSector(sector.SectorType, sector.SectorNumber), pieceSizes, deal.size, diff --git a/extern/storage-sealing/lib/nullreader/nullreader.go b/storage/pipeline/lib/nullreader/nullreader.go similarity index 100% rename from extern/storage-sealing/lib/nullreader/nullreader.go rename to storage/pipeline/lib/nullreader/nullreader.go diff --git a/extern/storage-sealing/mocks/api.go b/storage/pipeline/mocks/api.go similarity index 99% rename from extern/storage-sealing/mocks/api.go rename to storage/pipeline/mocks/api.go index db7bad01f5c..0f04688e54c 100644 --- a/extern/storage-sealing/mocks/api.go +++ b/storage/pipeline/mocks/api.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/filecoin-project/lotus/extern/storage-sealing (interfaces: SealingAPI) +// Source: github.com/filecoin-project/lotus/storage/pipeline (interfaces: SealingAPI) // Package mocks is a generated GoMock package. package mocks @@ -23,7 +23,7 @@ import ( api "github.com/filecoin-project/lotus/api" types "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) // MockSealingAPI is a mock of SealingAPI interface. diff --git a/extern/storage-sealing/mocks/mock_commit_batcher.go b/storage/pipeline/mocks/mock_commit_batcher.go similarity index 97% rename from extern/storage-sealing/mocks/mock_commit_batcher.go rename to storage/pipeline/mocks/mock_commit_batcher.go index 80e6d0655a7..a5475c4c7f5 100644 --- a/extern/storage-sealing/mocks/mock_commit_batcher.go +++ b/storage/pipeline/mocks/mock_commit_batcher.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/filecoin-project/lotus/extern/storage-sealing (interfaces: CommitBatcherApi) +// Source: github.com/filecoin-project/lotus/storage/pipeline (interfaces: CommitBatcherApi) // Package mocks is a generated GoMock package. package mocks @@ -18,7 +18,7 @@ import ( network "github.com/filecoin-project/go-state-types/network" api "github.com/filecoin-project/lotus/api" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) // MockCommitBatcherApi is a mock of CommitBatcherApi interface. diff --git a/extern/storage-sealing/mocks/mock_precommit_batcher.go b/storage/pipeline/mocks/mock_precommit_batcher.go similarity index 96% rename from extern/storage-sealing/mocks/mock_precommit_batcher.go rename to storage/pipeline/mocks/mock_precommit_batcher.go index eae9c8339aa..b6d2964ebab 100644 --- a/extern/storage-sealing/mocks/mock_precommit_batcher.go +++ b/storage/pipeline/mocks/mock_precommit_batcher.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/filecoin-project/lotus/extern/storage-sealing (interfaces: PreCommitBatcherApi) +// Source: github.com/filecoin-project/lotus/storage/pipeline (interfaces: PreCommitBatcherApi) // Package mocks is a generated GoMock package. package mocks @@ -17,7 +17,7 @@ import ( network "github.com/filecoin-project/go-state-types/network" api "github.com/filecoin-project/lotus/api" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + sealing "github.com/filecoin-project/lotus/storage/pipeline" ) // MockPreCommitBatcherApi is a mock of PreCommitBatcherApi interface. diff --git a/extern/storage-sealing/mocks/statemachine.go b/storage/pipeline/mocks/statemachine.go similarity index 94% rename from extern/storage-sealing/mocks/statemachine.go rename to storage/pipeline/mocks/statemachine.go index 9fdabdc879b..8ed7cc625a0 100644 --- a/extern/storage-sealing/mocks/statemachine.go +++ b/storage/pipeline/mocks/statemachine.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/filecoin-project/lotus/extern/storage-sealing (interfaces: Context) +// Source: github.com/filecoin-project/lotus/storage/pipeline (interfaces: Context) // Package mocks is a generated GoMock package. package mocks diff --git a/extern/storage-sealing/precommit_batch.go b/storage/pipeline/precommit_batch.go similarity index 99% rename from extern/storage-sealing/precommit_batch.go rename to storage/pipeline/precommit_batch.go index dd93175ca08..fb77f21f727 100644 --- a/extern/storage-sealing/precommit_batch.go +++ b/storage/pipeline/precommit_batch.go @@ -21,8 +21,8 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" ) //go:generate go run github.com/golang/mock/mockgen -destination=mocks/mock_precommit_batcher.go -package=mocks . PreCommitBatcherApi diff --git a/extern/storage-sealing/precommit_batch_test.go b/storage/pipeline/precommit_batch_test.go similarity index 93% rename from extern/storage-sealing/precommit_batch_test.go rename to storage/pipeline/precommit_batch_test.go index 29e02650efe..3881da90ddd 100644 --- a/extern/storage-sealing/precommit_batch_test.go +++ b/storage/pipeline/precommit_batch_test.go @@ -21,10 +21,10 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/extern/storage-sealing/mocks" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/node/config" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/pipeline/mocks" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" ) var fc = config.MinerFeeConfig{ @@ -75,10 +75,10 @@ func TestPrecommitBatcher(t *testing.T) { } type promise func(t *testing.T) - type action func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *sealing.PreCommitBatcher) promise + type action func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *pipeline.PreCommitBatcher) promise actions := func(as ...action) action { - return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *sealing.PreCommitBatcher) promise { + return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *pipeline.PreCommitBatcher) promise { var ps []promise for _, a := range as { p := a(t, s, pcb) @@ -99,13 +99,13 @@ func TestPrecommitBatcher(t *testing.T) { } addSector := func(sn abi.SectorNumber) action { - return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *sealing.PreCommitBatcher) promise { + return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *pipeline.PreCommitBatcher) promise { var pcres sealiface.PreCommitBatchRes var pcerr error done := sync.Mutex{} done.Lock() - si := sealing.SectorInfo{ + si := pipeline.SectorInfo{ SectorNumber: sn, } @@ -139,7 +139,7 @@ func TestPrecommitBatcher(t *testing.T) { } waitPending := func(n int) action { - return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *sealing.PreCommitBatcher) promise { + return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *pipeline.PreCommitBatcher) promise { require.Eventually(t, func() bool { p, err := pcb.Pending(ctx) require.NoError(t, err) @@ -152,7 +152,7 @@ func TestPrecommitBatcher(t *testing.T) { //stm: @CHAIN_STATE_MINER_INFO_001, @CHAIN_STATE_NETWORK_VERSION_001 expectSend := func(expect []abi.SectorNumber) action { - return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *sealing.PreCommitBatcher) promise { + return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *pipeline.PreCommitBatcher) promise { s.EXPECT().ChainHead(gomock.Any()).Return(nil, abi.ChainEpoch(1), nil) s.EXPECT().ChainBaseFee(gomock.Any(), gomock.Any()).Return(big.NewInt(10001), nil) s.EXPECT().StateNetworkVersion(gomock.Any(), gomock.Any()).Return(network.Version14, nil) @@ -173,7 +173,7 @@ func TestPrecommitBatcher(t *testing.T) { //stm: @CHAIN_STATE_MINER_INFO_001, @CHAIN_STATE_NETWORK_VERSION_001 expectSendsSingle := func(expect []abi.SectorNumber) action { - return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *sealing.PreCommitBatcher) promise { + return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *pipeline.PreCommitBatcher) promise { s.EXPECT().ChainHead(gomock.Any()).Return(nil, abi.ChainEpoch(1), nil) s.EXPECT().ChainBaseFee(gomock.Any(), gomock.Any()).Return(big.NewInt(9999), nil) s.EXPECT().StateNetworkVersion(gomock.Any(), gomock.Any()).Return(network.Version14, nil) @@ -194,7 +194,7 @@ func TestPrecommitBatcher(t *testing.T) { } flush := func(expect []abi.SectorNumber) action { - return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *sealing.PreCommitBatcher) promise { + return func(t *testing.T, s *mocks.MockPreCommitBatcherApi, pcb *pipeline.PreCommitBatcher) promise { _ = expectSend(expect)(t, s, pcb) r, err := pcb.Flush(ctx) @@ -261,7 +261,7 @@ func TestPrecommitBatcher(t *testing.T) { // create them mocks pcapi := mocks.NewMockPreCommitBatcherApi(mockCtrl) - pcb := sealing.NewPreCommitBatcher(ctx, t0123, pcapi, as, fc, cfg) + pcb := pipeline.NewPreCommitBatcher(ctx, t0123, pcapi, as, fc, cfg) var promises []promise diff --git a/extern/storage-sealing/precommit_policy.go b/storage/pipeline/precommit_policy.go similarity index 100% rename from extern/storage-sealing/precommit_policy.go rename to storage/pipeline/precommit_policy.go diff --git a/extern/storage-sealing/precommit_policy_test.go b/storage/pipeline/precommit_policy_test.go similarity index 84% rename from extern/storage-sealing/precommit_policy_test.go rename to storage/pipeline/precommit_policy_test.go index 38553d461df..86e5c60e263 100644 --- a/extern/storage-sealing/precommit_policy_test.go +++ b/storage/pipeline/precommit_policy_test.go @@ -13,12 +13,12 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" - api "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/policy" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" ) type fakeChain struct { @@ -29,7 +29,7 @@ type fakeConfigStub struct { CCSectorLifetime time.Duration } -func fakeConfigGetter(stub *fakeConfigStub) sealing.GetSealingConfigFunc { +func fakeConfigGetter(stub *fakeConfigStub) pipeline.GetSealingConfigFunc { return func() (sealiface.Config, error) { if stub == nil { return sealiface.Config{}, nil @@ -41,11 +41,11 @@ func fakeConfigGetter(stub *fakeConfigStub) sealing.GetSealingConfigFunc { } } -func (f *fakeChain) StateNetworkVersion(ctx context.Context, tok sealing.TipSetToken) (network.Version, error) { +func (f *fakeChain) StateNetworkVersion(ctx context.Context, tok pipeline.TipSetToken) (network.Version, error) { return build.NewestNetworkVersion, nil } -func (f *fakeChain) ChainHead(ctx context.Context) (sealing.TipSetToken, abi.ChainEpoch, error) { +func (f *fakeChain) ChainHead(ctx context.Context) (pipeline.TipSetToken, abi.ChainEpoch, error) { return []byte{1, 2, 3}, f.h, nil } @@ -60,7 +60,7 @@ func TestBasicPolicyEmptySector(t *testing.T) { cfg := fakeConfigGetter(nil) h := abi.ChainEpoch(55) pBuffer := abi.ChainEpoch(2) - pcp := sealing.NewBasicPreCommitPolicy(&fakeChain{h: h}, cfg, pBuffer) + pcp := pipeline.NewBasicPreCommitPolicy(&fakeChain{h: h}, cfg, pBuffer) exp, err := pcp.Expiration(context.Background()) require.NoError(t, err) @@ -77,7 +77,7 @@ func TestCustomCCSectorConfig(t *testing.T) { cfg := fakeConfigGetter(&cfgStub) h := abi.ChainEpoch(55) pBuffer := abi.ChainEpoch(2) - pcp := sealing.NewBasicPreCommitPolicy(&fakeChain{h: h}, cfg, pBuffer) + pcp := pipeline.NewBasicPreCommitPolicy(&fakeChain{h: h}, cfg, pBuffer) exp, err := pcp.Expiration(context.Background()) require.NoError(t, err) @@ -89,11 +89,11 @@ func TestCustomCCSectorConfig(t *testing.T) { func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { cfg := fakeConfigGetter(nil) - policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + policy := pipeline.NewBasicPreCommitPolicy(&fakeChain{ h: abi.ChainEpoch(55), }, cfg, 2) longestDealEpochEnd := abi.ChainEpoch(547300) - pieces := []sealing.Piece{ + pieces := []pipeline.Piece{ { Piece: abi.PieceInfo{ Size: abi.PaddedPieceSize(1024), @@ -130,11 +130,11 @@ func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { func TestBasicPolicyIgnoresExistingScheduleIfExpired(t *testing.T) { cfg := fakeConfigGetter(nil) - policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + policy := pipeline.NewBasicPreCommitPolicy(&fakeChain{ h: abi.ChainEpoch(55), }, cfg, 0) - pieces := []sealing.Piece{ + pieces := []pipeline.Piece{ { Piece: abi.PieceInfo{ Size: abi.PaddedPieceSize(1024), @@ -159,11 +159,11 @@ func TestBasicPolicyIgnoresExistingScheduleIfExpired(t *testing.T) { func TestMissingDealIsIgnored(t *testing.T) { cfg := fakeConfigGetter(nil) - policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + policy := pipeline.NewBasicPreCommitPolicy(&fakeChain{ h: abi.ChainEpoch(55), }, cfg, 0) - pieces := []sealing.Piece{ + pieces := []pipeline.Piece{ { Piece: abi.PieceInfo{ Size: abi.PaddedPieceSize(1024), diff --git a/extern/storage-sealing/sealiface/batching.go b/storage/pipeline/sealiface/batching.go similarity index 100% rename from extern/storage-sealing/sealiface/batching.go rename to storage/pipeline/sealiface/batching.go diff --git a/extern/storage-sealing/sealiface/config.go b/storage/pipeline/sealiface/config.go similarity index 100% rename from extern/storage-sealing/sealiface/config.go rename to storage/pipeline/sealiface/config.go diff --git a/extern/storage-sealing/sealing.go b/storage/pipeline/sealing.go similarity index 95% rename from extern/storage-sealing/sealing.go rename to storage/pipeline/sealing.go index c7b156b1630..10dee5ac9fb 100644 --- a/extern/storage-sealing/sealing.go +++ b/storage/pipeline/sealing.go @@ -27,10 +27,10 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/market" lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) const SectorStorePrefix = "/sectors" @@ -95,7 +95,7 @@ type Sealing struct { maddr address.Address - sealer sectorstorage.SectorManager + sealer sealer.SectorManager sectors *statemachine.StateGroup sc SectorIDCounter verif ffiwrapper.Verifier @@ -160,7 +160,7 @@ type pendingPiece struct { accepted func(abi.SectorNumber, abi.UnpaddedPieceSize, error) } -func New(mctx context.Context, api SealingAPI, fc config.MinerFeeConfig, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, prov ffiwrapper.Prover, pcp PreCommitPolicy, gc GetSealingConfigFunc, notifee SectorStateNotifee, as AddrSel) *Sealing { +func New(mctx context.Context, api SealingAPI, fc config.MinerFeeConfig, events Events, maddr address.Address, ds datastore.Batching, sealer sealer.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, prov ffiwrapper.Prover, pcp PreCommitPolicy, gc GetSealingConfigFunc, notifee SectorStateNotifee, as AddrSel) *Sealing { s := &Sealing{ Api: api, DealInfo: &CurrentDealInfoManager{api}, diff --git a/extern/storage-sealing/sector_state.go b/storage/pipeline/sector_state.go similarity index 100% rename from extern/storage-sealing/sector_state.go rename to storage/pipeline/sector_state.go diff --git a/extern/storage-sealing/states_failed.go b/storage/pipeline/states_failed.go similarity index 100% rename from extern/storage-sealing/states_failed.go rename to storage/pipeline/states_failed.go diff --git a/extern/storage-sealing/states_failed_test.go b/storage/pipeline/states_failed_test.go similarity index 83% rename from extern/storage-sealing/states_failed_test.go rename to storage/pipeline/states_failed_test.go index 9aef97b28a4..78f55f1e77f 100644 --- a/extern/storage-sealing/states_failed_test.go +++ b/storage/pipeline/states_failed_test.go @@ -15,13 +15,13 @@ import ( "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-state-types/network" - statemachine "github.com/filecoin-project/go-statemachine" + "github.com/filecoin-project/go-statemachine" market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" api2 "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/market" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/extern/storage-sealing/mocks" + pipeline "github.com/filecoin-project/lotus/storage/pipeline" + "github.com/filecoin-project/lotus/storage/pipeline/mocks" ) func TestStateRecoverDealIDs(t *testing.T) { @@ -33,9 +33,9 @@ func TestStateRecoverDealIDs(t *testing.T) { api := mocks.NewMockSealingAPI(mockCtrl) - fakeSealing := &sealing.Sealing{ + fakeSealing := &pipeline.Sealing{ Api: api, - DealInfo: &sealing.CurrentDealInfoManager{CDAPI: api}, + DealInfo: &pipeline.CurrentDealInfoManager{CDAPI: api}, } sctx := mocks.NewMockContext(mockCtrl) @@ -55,8 +55,8 @@ func TestStateRecoverDealIDs(t *testing.T) { // expect GetCurrentDealInfo { - api.EXPECT().StateSearchMsg(ctx, pc).Return(&sealing.MsgLookup{ - Receipt: sealing.MessageReceipt{ + api.EXPECT().StateSearchMsg(ctx, pc).Return(&pipeline.MsgLookup{ + Receipt: pipeline.MessageReceipt{ ExitCode: exitcode.Ok, Return: cborRet(&market0.PublishStorageDealsReturn{ IDs: []abi.DealID{dealId}, @@ -70,12 +70,12 @@ func TestStateRecoverDealIDs(t *testing.T) { } - sctx.EXPECT().Send(sealing.SectorRemove{}).Return(nil) + sctx.EXPECT().Send(pipeline.SectorRemove{}).Return(nil) // TODO sctx should satisfy an interface so it can be useable for mocking. This will fail because we are passing in an empty context now to get this to build. // https://github.com/filecoin-project/lotus/issues/7867 - err := fakeSealing.HandleRecoverDealIDs(statemachine.Context{}, sealing.SectorInfo{ - Pieces: []sealing.Piece{ + err := fakeSealing.HandleRecoverDealIDs(statemachine.Context{}, pipeline.SectorInfo{ + Pieces: []pipeline.Piece{ { DealInfo: &api2.PieceDealInfo{ DealID: dealId, diff --git a/extern/storage-sealing/states_proving.go b/storage/pipeline/states_proving.go similarity index 98% rename from extern/storage-sealing/states_proving.go rename to storage/pipeline/states_proving.go index 8afa0871919..d73a61c62f5 100644 --- a/extern/storage-sealing/states_proving.go +++ b/storage/pipeline/states_proving.go @@ -6,7 +6,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/exitcode" - statemachine "github.com/filecoin-project/go-statemachine" + "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/policy" diff --git a/extern/storage-sealing/states_replica_update.go b/storage/pipeline/states_replica_update.go similarity index 98% rename from extern/storage-sealing/states_replica_update.go rename to storage/pipeline/states_replica_update.go index eaecea180dc..322ad1bbd96 100644 --- a/extern/storage-sealing/states_replica_update.go +++ b/storage/pipeline/states_replica_update.go @@ -12,9 +12,9 @@ import ( "github.com/filecoin-project/go-state-types/builtin" "github.com/filecoin-project/go-state-types/builtin/v8/miner" "github.com/filecoin-project/go-state-types/exitcode" - statemachine "github.com/filecoin-project/go-statemachine" + "github.com/filecoin-project/go-statemachine" - api "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/policy" ) diff --git a/extern/storage-sealing/states_sealing.go b/storage/pipeline/states_sealing.go similarity index 99% rename from extern/storage-sealing/states_sealing.go rename to storage/pipeline/states_sealing.go index 4ce82fa2b50..1b21e7c7357 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/storage/pipeline/states_sealing.go @@ -22,7 +22,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/policy" - "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" + "github.com/filecoin-project/lotus/storage/pipeline/lib/nullreader" ) var DealSectorPriority = 1024 diff --git a/extern/storage-sealing/stats.go b/storage/pipeline/stats.go similarity index 97% rename from extern/storage-sealing/stats.go rename to storage/pipeline/stats.go index 0502045199e..696cd56a560 100644 --- a/extern/storage-sealing/stats.go +++ b/storage/pipeline/stats.go @@ -9,8 +9,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" ) type statSectorState int diff --git a/extern/storage-sealing/terminate_batch.go b/storage/pipeline/terminate_batch.go similarity index 100% rename from extern/storage-sealing/terminate_batch.go rename to storage/pipeline/terminate_batch.go diff --git a/extern/storage-sealing/types.go b/storage/pipeline/types.go similarity index 95% rename from extern/storage-sealing/types.go rename to storage/pipeline/types.go index 28002e8cb81..3ca0cc603c8 100644 --- a/extern/storage-sealing/types.go +++ b/storage/pipeline/types.go @@ -13,8 +13,8 @@ import ( "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" + "github.com/filecoin-project/lotus/storage/sealer" ) //go:generate go run github.com/golang/mock/mockgen -destination=mocks/statemachine.go -package=mocks . Context @@ -157,7 +157,7 @@ func (t *SectorInfo) sealingCtx(ctx context.Context) context.Context { // we need sealed sooner if t.hasDeals() { - return sectorstorage.WithPriority(ctx, DealSectorPriority) + return sealer.WithPriority(ctx, DealSectorPriority) } return ctx diff --git a/extern/storage-sealing/types_test.go b/storage/pipeline/types_test.go similarity index 97% rename from extern/storage-sealing/types_test.go rename to storage/pipeline/types_test.go index e579009606b..69e102bf441 100644 --- a/extern/storage-sealing/types_test.go +++ b/storage/pipeline/types_test.go @@ -12,7 +12,7 @@ import ( markettypes "github.com/filecoin-project/go-state-types/builtin/v8/market" tutils "github.com/filecoin-project/specs-actors/v2/support/testing" - api "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api" ) func TestSectorInfoSerialization(t *testing.T) { diff --git a/extern/storage-sealing/upgrade_queue.go b/storage/pipeline/upgrade_queue.go similarity index 100% rename from extern/storage-sealing/upgrade_queue.go rename to storage/pipeline/upgrade_queue.go diff --git a/extern/storage-sealing/utils.go b/storage/pipeline/utils.go similarity index 97% rename from extern/storage-sealing/utils.go rename to storage/pipeline/utils.go index 3dc4c4d1ea3..24a67423e79 100644 --- a/extern/storage-sealing/utils.go +++ b/storage/pipeline/utils.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + "github.com/filecoin-project/lotus/storage/pipeline/sealiface" ) func fillersFromRem(in abi.UnpaddedPieceSize) ([]abi.UnpaddedPieceSize, error) { diff --git a/extern/storage-sealing/utils_test.go b/storage/pipeline/utils_test.go similarity index 100% rename from extern/storage-sealing/utils_test.go rename to storage/pipeline/utils_test.go diff --git a/extern/sector-storage/README.md b/storage/sealer/README.md similarity index 100% rename from extern/sector-storage/README.md rename to storage/sealer/README.md diff --git a/extern/sector-storage/cbor_gen.go b/storage/sealer/cbor_gen.go similarity index 95% rename from extern/sector-storage/cbor_gen.go rename to storage/sealer/cbor_gen.go index eeea2b4f70c..48bf0146a45 100644 --- a/extern/sector-storage/cbor_gen.go +++ b/storage/sealer/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package sectorstorage +package sealer import ( "fmt" @@ -12,7 +12,7 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" - sealtasks "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" + sealtasks "github.com/filecoin-project/lotus/storage/sealer/sealtasks" ) var _ = xerrors.Errorf @@ -48,7 +48,7 @@ func (t *Call) MarshalCBOR(w io.Writer) error { return err } - // t.RetType (sectorstorage.ReturnType) (string) + // t.RetType (sealer.ReturnType) (string) if len("RetType") > cbg.MaxLength { return xerrors.Errorf("Value in field \"RetType\" was too long") } @@ -71,7 +71,7 @@ func (t *Call) MarshalCBOR(w io.Writer) error { return err } - // t.State (sectorstorage.CallState) (uint64) + // t.State (sealer.CallState) (uint64) if len("State") > cbg.MaxLength { return xerrors.Errorf("Value in field \"State\" was too long") } @@ -87,7 +87,7 @@ func (t *Call) MarshalCBOR(w io.Writer) error { return err } - // t.Result (sectorstorage.ManyBytes) (struct) + // t.Result (sealer.ManyBytes) (struct) if len("Result") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Result\" was too long") } @@ -153,7 +153,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.RetType (sectorstorage.ReturnType) (string) + // t.RetType (sealer.ReturnType) (string) case "RetType": { @@ -164,7 +164,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) (err error) { t.RetType = ReturnType(sval) } - // t.State (sectorstorage.CallState) (uint64) + // t.State (sealer.CallState) (uint64) case "State": { @@ -179,7 +179,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) (err error) { t.State = CallState(extra) } - // t.Result (sectorstorage.ManyBytes) (struct) + // t.Result (sealer.ManyBytes) (struct) case "Result": { @@ -220,7 +220,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { return err } - // t.ID (sectorstorage.WorkID) (struct) + // t.ID (sealer.WorkID) (struct) if len("ID") > cbg.MaxLength { return xerrors.Errorf("Value in field \"ID\" was too long") } @@ -236,7 +236,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { return err } - // t.Status (sectorstorage.WorkStatus) (string) + // t.Status (sealer.WorkStatus) (string) if len("Status") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Status\" was too long") } @@ -383,7 +383,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) (err error) { } switch name { - // t.ID (sectorstorage.WorkID) (struct) + // t.ID (sealer.WorkID) (struct) case "ID": { @@ -393,7 +393,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.Status (sectorstorage.WorkStatus) (string) + // t.Status (sealer.WorkStatus) (string) case "Status": { diff --git a/extern/sector-storage/cgroups.go b/storage/sealer/cgroups.go similarity index 92% rename from extern/sector-storage/cgroups.go rename to storage/sealer/cgroups.go index e2ec0564ec7..5fce8ff6e45 100644 --- a/extern/sector-storage/cgroups.go +++ b/storage/sealer/cgroups.go @@ -1,7 +1,7 @@ //go:build !linux // +build !linux -package sectorstorage +package sealer func cgroupV1Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) { return 0, 0, 0, 0, nil diff --git a/extern/sector-storage/cgroups_linux.go b/storage/sealer/cgroups_linux.go similarity index 99% rename from extern/sector-storage/cgroups_linux.go rename to storage/sealer/cgroups_linux.go index 38fe88f19c2..8bee54f783d 100644 --- a/extern/sector-storage/cgroups_linux.go +++ b/storage/sealer/cgroups_linux.go @@ -1,7 +1,7 @@ //go:build linux // +build linux -package sectorstorage +package sealer import ( "bufio" diff --git a/extern/sector-storage/docs/sector-storage.svg b/storage/sealer/docs/sector-storage.svg similarity index 100% rename from extern/sector-storage/docs/sector-storage.svg rename to storage/sealer/docs/sector-storage.svg diff --git a/extern/sector-storage/faults.go b/storage/sealer/faults.go similarity index 97% rename from extern/sector-storage/faults.go rename to storage/sealer/faults.go index 04a94e58cdf..6100dbb876c 100644 --- a/extern/sector-storage/faults.go +++ b/storage/sealer/faults.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var PostCheckTimeout = 160 * time.Second diff --git a/extern/sector-storage/ffiwrapper/basicfs/fs.go b/storage/sealer/ffiwrapper/basicfs/fs.go similarity index 97% rename from extern/sector-storage/ffiwrapper/basicfs/fs.go rename to storage/sealer/ffiwrapper/basicfs/fs.go index 3d19e49ef82..5cd48bb8269 100644 --- a/extern/sector-storage/ffiwrapper/basicfs/fs.go +++ b/storage/sealer/ffiwrapper/basicfs/fs.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type sectorFile struct { diff --git a/extern/sector-storage/ffiwrapper/prover_cgo.go b/storage/sealer/ffiwrapper/prover_cgo.go similarity index 100% rename from extern/sector-storage/ffiwrapper/prover_cgo.go rename to storage/sealer/ffiwrapper/prover_cgo.go diff --git a/extern/sector-storage/ffiwrapper/sealer.go b/storage/sealer/ffiwrapper/sealer.go similarity index 100% rename from extern/sector-storage/ffiwrapper/sealer.go rename to storage/sealer/ffiwrapper/sealer.go diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/storage/sealer/ffiwrapper/sealer_cgo.go similarity index 99% rename from extern/sector-storage/ffiwrapper/sealer_cgo.go rename to storage/sealer/ffiwrapper/sealer_cgo.go index cfabaadfbc1..e2c2d03afc9 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/storage/sealer/ffiwrapper/sealer_cgo.go @@ -28,11 +28,11 @@ import ( "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/fr32" - "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - nr "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" "github.com/filecoin-project/lotus/lib/nullreader" + nr "github.com/filecoin-project/lotus/storage/pipeline/lib/nullreader" + "github.com/filecoin-project/lotus/storage/sealer/fr32" + "github.com/filecoin-project/lotus/storage/sealer/partialfile" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var _ Storage = &Sealer{} diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/storage/sealer/ffiwrapper/sealer_test.go similarity index 98% rename from extern/sector-storage/ffiwrapper/sealer_test.go rename to storage/sealer/ffiwrapper/sealer_test.go index e878d92d107..f539209f9d9 100644 --- a/extern/sector-storage/ffiwrapper/sealer_test.go +++ b/storage/sealer/ffiwrapper/sealer_test.go @@ -23,15 +23,15 @@ import ( ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/filecoin-ffi/cgo" commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" - paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-state-types/abi" prooftypes "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/chain/actors/policy" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper/basicfs" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" + "github.com/filecoin-project/lotus/storage/pipeline/lib/nullreader" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper/basicfs" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func init() { diff --git a/extern/sector-storage/ffiwrapper/types.go b/storage/sealer/ffiwrapper/types.go similarity index 93% rename from extern/sector-storage/ffiwrapper/types.go rename to storage/sealer/ffiwrapper/types.go index bba8104543f..fcf71ab8283 100644 --- a/extern/sector-storage/ffiwrapper/types.go +++ b/storage/sealer/ffiwrapper/types.go @@ -10,8 +10,8 @@ import ( "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper/basicfs" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper/basicfs" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type Validator interface { diff --git a/extern/sector-storage/ffiwrapper/unseal_ranges.go b/storage/sealer/ffiwrapper/unseal_ranges.go similarity index 85% rename from extern/sector-storage/ffiwrapper/unseal_ranges.go rename to storage/sealer/ffiwrapper/unseal_ranges.go index 5e0dd759cc4..22b41659c2c 100644 --- a/extern/sector-storage/ffiwrapper/unseal_ranges.go +++ b/storage/sealer/ffiwrapper/unseal_ranges.go @@ -6,8 +6,8 @@ import ( rlepluslazy "github.com/filecoin-project/go-bitfield/rle" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/partialfile" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // merge gaps between ranges which are close to each other diff --git a/extern/sector-storage/ffiwrapper/verifier_cgo.go b/storage/sealer/ffiwrapper/verifier_cgo.go similarity index 98% rename from extern/sector-storage/ffiwrapper/verifier_cgo.go rename to storage/sealer/ffiwrapper/verifier_cgo.go index 3f689e62e4f..2c21ebce68b 100644 --- a/extern/sector-storage/ffiwrapper/verifier_cgo.go +++ b/storage/sealer/ffiwrapper/verifier_cgo.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/specs-actors/v7/actors/runtime/proof" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.ExtendedSectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, error) { diff --git a/extern/sector-storage/fr32/fr32.go b/storage/sealer/fr32/fr32.go similarity index 100% rename from extern/sector-storage/fr32/fr32.go rename to storage/sealer/fr32/fr32.go diff --git a/extern/sector-storage/fr32/fr32_ffi_cmp_test.go b/storage/sealer/fr32/fr32_ffi_cmp_test.go similarity index 95% rename from extern/sector-storage/fr32/fr32_ffi_cmp_test.go rename to storage/sealer/fr32/fr32_ffi_cmp_test.go index ff657ee293a..7dece47232e 100644 --- a/extern/sector-storage/fr32/fr32_ffi_cmp_test.go +++ b/storage/sealer/fr32/fr32_ffi_cmp_test.go @@ -13,7 +13,7 @@ import ( commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/fr32" + "github.com/filecoin-project/lotus/storage/sealer/fr32" ) func TestWriteTwoPcs(t *testing.T) { diff --git a/extern/sector-storage/fr32/fr32_test.go b/storage/sealer/fr32/fr32_test.go similarity index 98% rename from extern/sector-storage/fr32/fr32_test.go rename to storage/sealer/fr32/fr32_test.go index 4793de64599..f9150e550cd 100644 --- a/extern/sector-storage/fr32/fr32_test.go +++ b/storage/sealer/fr32/fr32_test.go @@ -14,7 +14,7 @@ import ( commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/fr32" + "github.com/filecoin-project/lotus/storage/sealer/fr32" ) func padFFI(buf []byte) []byte { diff --git a/extern/sector-storage/fr32/readers.go b/storage/sealer/fr32/readers.go similarity index 100% rename from extern/sector-storage/fr32/readers.go rename to storage/sealer/fr32/readers.go diff --git a/extern/sector-storage/fr32/readers_test.go b/storage/sealer/fr32/readers_test.go similarity index 91% rename from extern/sector-storage/fr32/readers_test.go rename to storage/sealer/fr32/readers_test.go index 2411955529a..21a5cd9cd83 100644 --- a/extern/sector-storage/fr32/readers_test.go +++ b/storage/sealer/fr32/readers_test.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/fr32" + "github.com/filecoin-project/lotus/storage/sealer/fr32" ) func TestUnpadReader(t *testing.T) { diff --git a/extern/sector-storage/fr32/utils.go b/storage/sealer/fr32/utils.go similarity index 100% rename from extern/sector-storage/fr32/utils.go rename to storage/sealer/fr32/utils.go diff --git a/extern/sector-storage/fsutil/dealloc_linux.go b/storage/sealer/fsutil/dealloc_linux.go similarity index 100% rename from extern/sector-storage/fsutil/dealloc_linux.go rename to storage/sealer/fsutil/dealloc_linux.go diff --git a/extern/sector-storage/fsutil/dealloc_other.go b/storage/sealer/fsutil/dealloc_other.go similarity index 100% rename from extern/sector-storage/fsutil/dealloc_other.go rename to storage/sealer/fsutil/dealloc_other.go diff --git a/extern/sector-storage/fsutil/filesize_unix.go b/storage/sealer/fsutil/filesize_unix.go similarity index 100% rename from extern/sector-storage/fsutil/filesize_unix.go rename to storage/sealer/fsutil/filesize_unix.go diff --git a/extern/sector-storage/fsutil/statfs.go b/storage/sealer/fsutil/statfs.go similarity index 100% rename from extern/sector-storage/fsutil/statfs.go rename to storage/sealer/fsutil/statfs.go diff --git a/extern/sector-storage/fsutil/statfs_unix.go b/storage/sealer/fsutil/statfs_unix.go similarity index 100% rename from extern/sector-storage/fsutil/statfs_unix.go rename to storage/sealer/fsutil/statfs_unix.go diff --git a/extern/sector-storage/fsutil/statfs_windows.go b/storage/sealer/fsutil/statfs_windows.go similarity index 100% rename from extern/sector-storage/fsutil/statfs_windows.go rename to storage/sealer/fsutil/statfs_windows.go diff --git a/extern/sector-storage/manager.go b/storage/sealer/manager.go similarity index 97% rename from extern/sector-storage/manager.go rename to storage/sealer/manager.go index b898d0e8950..cbfcfec79db 100644 --- a/extern/sector-storage/manager.go +++ b/storage/sealer/manager.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -9,7 +9,7 @@ import ( "github.com/google/uuid" "github.com/hashicorp/go-multierror" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "go.uber.org/multierr" @@ -19,11 +19,11 @@ import ( "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("advmgr") @@ -55,11 +55,11 @@ type SectorManager interface { var ClosedWorkerID = uuid.UUID{} type Manager struct { - ls stores.LocalStorage - storage stores.Store - localStore *stores.Local - remoteHnd *stores.FetchHandler - index stores.SectorIndex + ls paths.LocalStorage + storage paths.Store + localStore *paths.Local + remoteHnd *paths.FetchHandler + index paths.SectorIndex sched *Scheduler windowPoStSched *poStScheduler @@ -133,7 +133,7 @@ type StorageAuth http.Header type WorkerStateStore *statestore.StateStore type ManagerStateStore *statestore.StateStore -func New(ctx context.Context, lstor *stores.Local, stor stores.Store, ls stores.LocalStorage, si stores.SectorIndex, sc Config, wss WorkerStateStore, mss ManagerStateStore) (*Manager, error) { +func New(ctx context.Context, lstor *paths.Local, stor paths.Store, ls paths.LocalStorage, si paths.SectorIndex, sc Config, wss WorkerStateStore, mss ManagerStateStore) (*Manager, error) { prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor, index: si}) if err != nil { return nil, xerrors.Errorf("creating prover instance: %w", err) @@ -148,7 +148,7 @@ func New(ctx context.Context, lstor *stores.Local, stor stores.Store, ls stores. ls: ls, storage: stor, localStore: lstor, - remoteHnd: &stores.FetchHandler{Local: lstor, PfHandler: &stores.DefaultPartialFileHandler{}}, + remoteHnd: &paths.FetchHandler{Local: lstor, PfHandler: &paths.DefaultPartialFileHandler{}}, index: si, sched: sh, @@ -222,8 +222,8 @@ func (m *Manager) AddLocalStorage(ctx context.Context, path string) error { return xerrors.Errorf("opening local path: %w", err) } - if err := m.ls.SetStorage(func(sc *stores.StorageConfig) { - sc.StoragePaths = append(sc.StoragePaths, stores.LocalPath{Path: path}) + if err := m.ls.SetStorage(func(sc *paths.StorageConfig) { + sc.StoragePaths = append(sc.StoragePaths, paths.LocalPath{Path: path}) }); err != nil { return xerrors.Errorf("get storage config: %w", err) } diff --git a/extern/sector-storage/manager_calltracker.go b/storage/sealer/manager_calltracker.go similarity index 98% rename from extern/sector-storage/manager_calltracker.go rename to storage/sealer/manager_calltracker.go index 7c8703e898c..1c6a7cf1b51 100644 --- a/extern/sector-storage/manager_calltracker.go +++ b/storage/sealer/manager_calltracker.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -11,8 +11,8 @@ import ( "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type WorkID struct { diff --git a/extern/sector-storage/manager_post.go b/storage/sealer/manager_post.go similarity index 98% rename from extern/sector-storage/manager_post.go rename to storage/sealer/manager_post.go index 70000c20f25..e88807a29f0 100644 --- a/extern/sector-storage/manager_post.go +++ b/storage/sealer/manager_post.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/specs-actors/v6/actors/builtin" "github.com/filecoin-project/specs-actors/v7/actors/runtime/proof" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func (m *Manager) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.ExtendedSectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, error) { diff --git a/extern/sector-storage/manager_test.go b/storage/sealer/manager_test.go similarity index 95% rename from extern/sector-storage/manager_test.go rename to storage/sealer/manager_test.go index 61c9cbf1d58..f5695fc118f 100644 --- a/extern/sector-storage/manager_test.go +++ b/storage/sealer/manager_test.go @@ -1,5 +1,5 @@ //stm: #unit -package sectorstorage +package sealer import ( "bytes" @@ -29,29 +29,29 @@ import ( proof7 "github.com/filecoin-project/specs-actors/v7/actors/runtime/proof" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func init() { logging.SetAllLoggers(logging.LevelDebug) } -type testStorage stores.StorageConfig +type testStorage paths.StorageConfig func (t testStorage) DiskUsage(path string) (int64, error) { return 1, nil // close enough } func newTestStorage(t *testing.T) *testStorage { - tp, err := ioutil.TempDir(os.TempDir(), "sector-storage-test-") + tp, err := ioutil.TempDir(os.TempDir(), "sealer-test-") require.NoError(t, err) { - b, err := json.MarshalIndent(&stores.LocalStorageMeta{ + b, err := json.MarshalIndent(&paths.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), Weight: 1, CanSeal: true, @@ -64,7 +64,7 @@ func newTestStorage(t *testing.T) *testStorage { } return &testStorage{ - StoragePaths: []stores.LocalPath{ + StoragePaths: []paths.LocalPath{ {Path: tp}, }, } @@ -83,12 +83,12 @@ func (t testStorage) cleanup() { } } -func (t testStorage) GetStorage() (stores.StorageConfig, error) { - return stores.StorageConfig(t), nil +func (t testStorage) GetStorage() (paths.StorageConfig, error) { + return paths.StorageConfig(t), nil } -func (t *testStorage) SetStorage(f func(*stores.StorageConfig)) error { - f((*stores.StorageConfig)(t)) +func (t *testStorage) SetStorage(f func(*paths.StorageConfig)) error { + f((*paths.StorageConfig)(t)) return nil } @@ -96,20 +96,20 @@ func (t *testStorage) Stat(path string) (fsutil.FsStat, error) { return fsutil.Statfs(path) } -var _ stores.LocalStorage = &testStorage{} +var _ paths.LocalStorage = &testStorage{} -func newTestMgr(ctx context.Context, t *testing.T, ds datastore.Datastore) (*Manager, *stores.Local, *stores.Remote, *stores.Index, func()) { +func newTestMgr(ctx context.Context, t *testing.T, ds datastore.Datastore) (*Manager, *paths.Local, *paths.Remote, *paths.Index, func()) { st := newTestStorage(t) - si := stores.NewIndex() + si := paths.NewIndex() - lstor, err := stores.NewLocal(ctx, st, si, nil) + lstor, err := paths.NewLocal(ctx, st, si, nil) require.NoError(t, err) prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor, index: si}) require.NoError(t, err) - stor := stores.NewRemote(lstor, si, nil, 6000, &stores.DefaultPartialFileHandler{}) + stor := paths.NewRemote(lstor, si, nil, 6000, &paths.DefaultPartialFileHandler{}) sh, err := newScheduler("") require.NoError(t, err) @@ -118,7 +118,7 @@ func newTestMgr(ctx context.Context, t *testing.T, ds datastore.Datastore) (*Man ls: st, storage: stor, localStore: lstor, - remoteHnd: &stores.FetchHandler{Local: lstor}, + remoteHnd: &paths.FetchHandler{Local: lstor}, index: si, sched: sh, @@ -692,7 +692,7 @@ func TestRestartWorker(t *testing.T) { func TestReenableWorker(t *testing.T) { logging.SetAllLoggers(logging.LevelDebug) - stores.HeartbeatInterval = 5 * time.Millisecond + paths.HeartbeatInterval = 5 * time.Millisecond ctx, done := context.WithCancel(context.Background()) defer done() diff --git a/extern/sector-storage/mock/mock.go b/storage/sealer/mock/mock.go similarity index 98% rename from extern/sector-storage/mock/mock.go rename to storage/sealer/mock/mock.go index 0f9bbc9af97..ae35f969551 100644 --- a/extern/sector-storage/mock/mock.go +++ b/storage/sealer/mock/mock.go @@ -15,14 +15,14 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/dagstore/mount" - ffiwrapper2 "github.com/filecoin-project/go-commp-utils/ffiwrapper" + commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-state-types/abi" prooftypes "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("sbmock") @@ -90,7 +90,7 @@ func (mgr *SectorMgr) AddPiece(ctx context.Context, sectorID storage.SectorRef, var b bytes.Buffer tr := io.TeeReader(r, &b) - c, err := ffiwrapper2.GeneratePieceCIDFromFile(sectorID.ProofType, tr, size) + c, err := commpffi.GeneratePieceCIDFromFile(sectorID.ProofType, tr, size) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("failed to generate piece cid: %w", err) } diff --git a/extern/sector-storage/mock/mock_test.go b/storage/sealer/mock/mock_test.go similarity index 100% rename from extern/sector-storage/mock/mock_test.go rename to storage/sealer/mock/mock_test.go diff --git a/storage/mockstorage/preseal.go b/storage/sealer/mock/util.go similarity index 82% rename from storage/mockstorage/preseal.go rename to storage/sealer/mock/util.go index 07e8646f5ab..5e6e3ecd30d 100644 --- a/storage/mockstorage/preseal.go +++ b/storage/sealer/mock/util.go @@ -1,4 +1,4 @@ -package mockstorage +package mock import ( "fmt" @@ -10,16 +10,23 @@ import ( commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - markettypes "github.com/filecoin-project/go-state-types/builtin/v8/market" + "github.com/filecoin-project/go-state-types/builtin/v8/market" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/extern/sector-storage/mock" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/genesis" ) +func CommDR(in []byte) (out [32]byte) { + for i, b := range in { + out[i] = ^b + } + + return out +} + func PreSeal(spt abi.RegisteredSealProof, maddr address.Address, sectors int) (*genesis.Miner, *types.KeyInfo, error) { - k, err := wallet.GenerateKey(types.KTBLS) + k, err := key.GenerateKey(types.KTBLS) if err != nil { return nil, nil, err } @@ -40,7 +47,7 @@ func PreSeal(spt abi.RegisteredSealProof, maddr address.Address, sectors int) (* } for i := range genm.Sectors { - label, err := markettypes.NewLabelFromString(fmt.Sprintf("%d", i)) + label, err := market.NewLabelFromString(fmt.Sprintf("%d", i)) if err != nil { return nil, nil, xerrors.Errorf("failed to create label: %w", err) } @@ -50,10 +57,10 @@ func PreSeal(spt abi.RegisteredSealProof, maddr address.Address, sectors int) (* preseal.ProofType = spt preseal.CommD = zerocomm.ZeroPieceCommitment(abi.PaddedPieceSize(ssize).Unpadded()) d, _ := commcid.CIDToPieceCommitmentV1(preseal.CommD) - r := mock.CommDR(d) + r := CommDR(d) preseal.CommR, _ = commcid.ReplicaCommitmentV1ToCID(r[:]) preseal.SectorID = abi.SectorNumber(i + 1) - preseal.Deal = markettypes.DealProposal{ + preseal.Deal = market.DealProposal{ PieceCID: preseal.CommD, PieceSize: abi.PaddedPieceSize(ssize), Client: k.Address, diff --git a/extern/sector-storage/partialfile/partialfile.go b/storage/sealer/partialfile/partialfile.go similarity index 98% rename from extern/sector-storage/partialfile/partialfile.go rename to storage/sealer/partialfile/partialfile.go index 49f84fe815f..6e8c2d84331 100644 --- a/extern/sector-storage/partialfile/partialfile.go +++ b/storage/sealer/partialfile/partialfile.go @@ -13,8 +13,8 @@ import ( rlepluslazy "github.com/filecoin-project/go-bitfield/rle" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("partialfile") diff --git a/extern/sector-storage/piece_provider.go b/storage/sealer/piece_provider.go similarity index 95% rename from extern/sector-storage/piece_provider.go rename to storage/sealer/piece_provider.go index 32f0c902892..e3e1ca23a4b 100644 --- a/extern/sector-storage/piece_provider.go +++ b/storage/sealer/piece_provider.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "bufio" @@ -12,9 +12,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/fr32" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/fr32" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type Unsealer interface { @@ -35,12 +35,12 @@ type PieceProvider interface { var _ PieceProvider = &pieceProvider{} type pieceProvider struct { - storage *stores.Remote - index stores.SectorIndex + storage *paths.Remote + index paths.SectorIndex uns Unsealer } -func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unsealer) PieceProvider { +func NewPieceProvider(storage *paths.Remote, index paths.SectorIndex, uns Unsealer) PieceProvider { return &pieceProvider{ storage: storage, index: index, diff --git a/extern/sector-storage/piece_provider_test.go b/storage/sealer/piece_provider_test.go similarity index 92% rename from extern/sector-storage/piece_provider_test.go rename to storage/sealer/piece_provider_test.go index 7a67cf9ad82..6d9525b0ac6 100644 --- a/extern/sector-storage/piece_provider_test.go +++ b/storage/sealer/piece_provider_test.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "bytes" @@ -22,9 +22,9 @@ import ( "github.com/filecoin-project/go-statestore" specstorage "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // TestPieceProviderReadPiece verifies that the ReadPiece method works correctly @@ -180,13 +180,13 @@ func TestReadPieceRemoteWorkers(t *testing.T) { type pieceProviderTestHarness struct { ctx context.Context - index *stores.Index + index *paths.Index pp PieceProvider sector specstorage.SectorRef mgr *Manager ticket abi.SealRandomness commD cid.Cid - localStores []*stores.Local + localStores []*paths.Local servers []*http.Server @@ -207,11 +207,11 @@ func newPieceProviderTestHarness(t *testing.T, mgrConfig Config, sectorProofType require.NoError(t, err) // create index, storage, local store & remote store. - index := stores.NewIndex() + index := paths.NewIndex() storage := newTestStorage(t) - localStore, err := stores.NewLocal(ctx, storage, index, []string{"http://" + nl.Addr().String() + "/remote"}) + localStore, err := paths.NewLocal(ctx, storage, index, []string{"http://" + nl.Addr().String() + "/remote"}) require.NoError(t, err) - remoteStore := stores.NewRemote(localStore, index, nil, 6000, &stores.DefaultPartialFileHandler{}) + remoteStore := paths.NewRemote(localStore, index, nil, 6000, &paths.DefaultPartialFileHandler{}) // data stores for state tracking. dstore := ds_sync.MutexWrap(datastore.NewMapDatastore()) @@ -261,12 +261,12 @@ func (p *pieceProviderTestHarness) addRemoteWorker(t *testing.T, tasks []sealtas nl, err := net.Listen("tcp", address) require.NoError(t, err) - localStore, err := stores.NewLocal(p.ctx, newTestStorage(t), p.index, []string{"http://" + nl.Addr().String() + "/remote"}) + localStore, err := paths.NewLocal(p.ctx, newTestStorage(t), p.index, []string{"http://" + nl.Addr().String() + "/remote"}) require.NoError(t, err) - fh := &stores.FetchHandler{ + fh := &paths.FetchHandler{ Local: localStore, - PfHandler: &stores.DefaultPartialFileHandler{}, + PfHandler: &paths.DefaultPartialFileHandler{}, } mux := mux.NewRouter() @@ -280,8 +280,8 @@ func (p *pieceProviderTestHarness) addRemoteWorker(t *testing.T, tasks []sealtas _ = svc.Serve(nl) }() - remote := stores.NewRemote(localStore, p.index, nil, 1000, - &stores.DefaultPartialFileHandler{}) + remote := paths.NewRemote(localStore, p.index, nil, 1000, + &paths.DefaultPartialFileHandler{}) dstore := ds_sync.MutexWrap(datastore.NewMapDatastore()) csts := statestore.New(namespace.Wrap(dstore, datastore.NewKey("/stmgr/calls"))) diff --git a/extern/sector-storage/piece_reader.go b/storage/sealer/piece_reader.go similarity index 99% rename from extern/sector-storage/piece_reader.go rename to storage/sealer/piece_reader.go index 56e19142c64..6ef616e4121 100644 --- a/extern/sector-storage/piece_reader.go +++ b/storage/sealer/piece_reader.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "bufio" diff --git a/extern/sector-storage/request_queue.go b/storage/sealer/request_queue.go similarity index 97% rename from extern/sector-storage/request_queue.go rename to storage/sealer/request_queue.go index 87a05792360..1107fb90ee6 100644 --- a/extern/sector-storage/request_queue.go +++ b/storage/sealer/request_queue.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import "sort" diff --git a/extern/sector-storage/request_queue_test.go b/storage/sealer/request_queue_test.go similarity index 92% rename from extern/sector-storage/request_queue_test.go rename to storage/sealer/request_queue_test.go index c95548c43e7..5a48586a02f 100644 --- a/extern/sector-storage/request_queue_test.go +++ b/storage/sealer/request_queue_test.go @@ -1,10 +1,10 @@ -package sectorstorage +package sealer import ( "fmt" "testing" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" ) func TestRequestQueue(t *testing.T) { diff --git a/extern/sector-storage/roprov.go b/storage/sealer/roprov.go similarity index 82% rename from extern/sector-storage/roprov.go rename to storage/sealer/roprov.go index ebc7610d78d..e57f6a2c1b7 100644 --- a/extern/sector-storage/roprov.go +++ b/storage/sealer/roprov.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -7,13 +7,13 @@ import ( "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type readonlyProvider struct { - index stores.SectorIndex - stor *stores.Local + index paths.SectorIndex + stor *paths.Local } func (l *readonlyProvider) AcquireSector(ctx context.Context, id storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, sealing storiface.PathType) (storiface.SectorPaths, func(), error) { diff --git a/extern/sector-storage/sched.go b/storage/sealer/sched.go similarity index 98% rename from extern/sector-storage/sched.go rename to storage/sealer/sched.go index 32f91b4e73f..192e415c8d5 100644 --- a/extern/sector-storage/sched.go +++ b/storage/sealer/sched.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -11,8 +11,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type schedPrioCtxKey int diff --git a/extern/sector-storage/sched_assigner_common.go b/storage/sealer/sched_assigner_common.go similarity index 99% rename from extern/sector-storage/sched_assigner_common.go rename to storage/sealer/sched_assigner_common.go index 41cf866b2d3..09ff82a8921 100644 --- a/extern/sector-storage/sched_assigner_common.go +++ b/storage/sealer/sched_assigner_common.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" diff --git a/extern/sector-storage/sched_assigner_spread.go b/storage/sealer/sched_assigner_spread.go similarity index 95% rename from extern/sector-storage/sched_assigner_spread.go rename to storage/sealer/sched_assigner_spread.go index c62cd42b220..f00d24d82b0 100644 --- a/extern/sector-storage/sched_assigner_spread.go +++ b/storage/sealer/sched_assigner_spread.go @@ -1,9 +1,9 @@ -package sectorstorage +package sealer import ( "math" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func NewSpreadAssigner() Assigner { diff --git a/extern/sector-storage/sched_assigner_utilization.go b/storage/sealer/sched_assigner_utilization.go similarity index 96% rename from extern/sector-storage/sched_assigner_utilization.go rename to storage/sealer/sched_assigner_utilization.go index 6cea72a1c11..2d051d0007a 100644 --- a/extern/sector-storage/sched_assigner_utilization.go +++ b/storage/sealer/sched_assigner_utilization.go @@ -1,9 +1,9 @@ -package sectorstorage +package sealer import ( "math" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func NewLowestUtilizationAssigner() Assigner { diff --git a/extern/sector-storage/sched_post.go b/storage/sealer/sched_post.go similarity index 92% rename from extern/sector-storage/sched_post.go rename to storage/sealer/sched_post.go index 9c3838e0865..64986726f09 100644 --- a/extern/sector-storage/sched_post.go +++ b/storage/sealer/sched_post.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -6,13 +6,13 @@ import ( "sync" "time" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" - sealtasks "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type poStScheduler struct { @@ -155,7 +155,7 @@ func (ps *poStScheduler) enable(wid storiface.WorkerID) { } func (ps *poStScheduler) watch(wid storiface.WorkerID, worker *WorkerHandle) { - heartbeatTimer := time.NewTicker(stores.HeartbeatInterval) + heartbeatTimer := time.NewTicker(paths.HeartbeatInterval) defer heartbeatTimer.Stop() ctx, cancel := context.WithCancel(context.TODO()) @@ -169,7 +169,7 @@ func (ps *poStScheduler) watch(wid storiface.WorkerID, worker *WorkerHandle) { }() for { - sctx, scancel := context.WithTimeout(ctx, stores.HeartbeatInterval/2) + sctx, scancel := context.WithTimeout(ctx, paths.HeartbeatInterval/2) curSes, err := worker.workerRpc.Session(sctx) scancel() if err != nil { diff --git a/extern/sector-storage/sched_resources.go b/storage/sealer/sched_resources.go similarity index 97% rename from extern/sector-storage/sched_resources.go rename to storage/sealer/sched_resources.go index 0f1f259b7c9..88725f6bacc 100644 --- a/extern/sector-storage/sched_resources.go +++ b/storage/sealer/sched_resources.go @@ -1,12 +1,12 @@ -package sectorstorage +package sealer import ( "context" "sync" "time" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type ActiveResources struct { diff --git a/extern/sector-storage/sched_test.go b/storage/sealer/sched_test.go similarity index 94% rename from extern/sector-storage/sched_test.go rename to storage/sealer/sched_test.go index dc87bc74409..8ebaa317aaf 100644 --- a/extern/sector-storage/sched_test.go +++ b/storage/sealer/sched_test.go @@ -1,5 +1,5 @@ //stm: #unit -package sectorstorage +package sealer import ( "context" @@ -20,10 +20,10 @@ import ( prooftypes "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func init() { @@ -183,7 +183,7 @@ func (s *schedTestWorker) Close() error { var _ Worker = &schedTestWorker{} -func addTestWorker(t *testing.T, sched *Scheduler, index *stores.Index, name string, taskTypes map[sealtasks.TaskType]struct{}, resources storiface.WorkerResources, ignoreResources bool) { +func addTestWorker(t *testing.T, sched *Scheduler, index *paths.Index, name string, taskTypes map[sealtasks.TaskType]struct{}, resources storiface.WorkerResources, ignoreResources bool) { w := &schedTestWorker{ name: name, taskTypes: taskTypes, @@ -227,7 +227,7 @@ func TestSchedStartStop(t *testing.T) { require.NoError(t, err) go sched.runSched() - addTestWorker(t, sched, stores.NewIndex(), "fred", nil, decentWorkerResources, false) + addTestWorker(t, sched, paths.NewIndex(), "fred", nil, decentWorkerResources, false) require.NoError(t, sched.Close(context.TODO())) } @@ -260,13 +260,13 @@ func TestSched(t *testing.T) { wg sync.WaitGroup } - type task func(*testing.T, *Scheduler, *stores.Index, *runMeta) + type task func(*testing.T, *Scheduler, *paths.Index, *runMeta) sched := func(taskName, expectWorker string, sid abi.SectorNumber, taskType sealtasks.TaskType) task { _, _, l, _ := runtime.Caller(1) _, _, l2, _ := runtime.Caller(2) - return func(t *testing.T, sched *Scheduler, index *stores.Index, rm *runMeta) { + return func(t *testing.T, sched *Scheduler, index *paths.Index, rm *runMeta) { done := make(chan struct{}) rm.done[taskName] = done @@ -315,7 +315,7 @@ func TestSched(t *testing.T) { taskStarted := func(name string) task { _, _, l, _ := runtime.Caller(1) _, _, l2, _ := runtime.Caller(2) - return func(t *testing.T, sched *Scheduler, index *stores.Index, rm *runMeta) { + return func(t *testing.T, sched *Scheduler, index *paths.Index, rm *runMeta) { select { case rm.done[name] <- struct{}{}: case <-ctx.Done(): @@ -327,7 +327,7 @@ func TestSched(t *testing.T) { taskDone := func(name string) task { _, _, l, _ := runtime.Caller(1) _, _, l2, _ := runtime.Caller(2) - return func(t *testing.T, sched *Scheduler, index *stores.Index, rm *runMeta) { + return func(t *testing.T, sched *Scheduler, index *paths.Index, rm *runMeta) { select { case rm.done[name] <- struct{}{}: case <-ctx.Done(): @@ -340,7 +340,7 @@ func TestSched(t *testing.T) { taskNotScheduled := func(name string) task { _, _, l, _ := runtime.Caller(1) _, _, l2, _ := runtime.Caller(2) - return func(t *testing.T, sched *Scheduler, index *stores.Index, rm *runMeta) { + return func(t *testing.T, sched *Scheduler, index *paths.Index, rm *runMeta) { select { case rm.done[name] <- struct{}{}: t.Fatal("not expected", l, l2) @@ -351,7 +351,7 @@ func TestSched(t *testing.T) { testFunc := func(workers []workerSpec, tasks []task) func(t *testing.T) { return func(t *testing.T) { - index := stores.NewIndex() + index := paths.NewIndex() sched, err := newScheduler("") require.NoError(t, err) @@ -380,7 +380,7 @@ func TestSched(t *testing.T) { } multTask := func(tasks ...task) task { - return func(t *testing.T, s *Scheduler, index *stores.Index, meta *runMeta) { + return func(t *testing.T, s *Scheduler, index *paths.Index, meta *runMeta) { for _, tsk := range tasks { tsk(t, s, index, meta) } @@ -494,7 +494,7 @@ func TestSched(t *testing.T) { } diag := func() task { - return func(t *testing.T, s *Scheduler, index *stores.Index, meta *runMeta) { + return func(t *testing.T, s *Scheduler, index *paths.Index, meta *runMeta) { time.Sleep(20 * time.Millisecond) for _, request := range s.diag().Requests { log.Infof("!!! sDIAG: sid(%d) task(%s)", request.Sector.Number, request.TaskType) diff --git a/extern/sector-storage/sched_worker.go b/storage/sealer/sched_worker.go similarity index 97% rename from extern/sector-storage/sched_worker.go rename to storage/sealer/sched_worker.go index c9b019ad065..0cd87651113 100644 --- a/extern/sector-storage/sched_worker.go +++ b/storage/sealer/sched_worker.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -6,9 +6,9 @@ import ( "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type schedWorker struct { @@ -66,7 +66,7 @@ func (sh *Scheduler) runWorker(ctx context.Context, wid storiface.WorkerID, work wid: wid, - heartbeatTimer: time.NewTicker(stores.HeartbeatInterval), + heartbeatTimer: time.NewTicker(paths.HeartbeatInterval), scheduledWindows: make(chan *SchedWindow, SchedWindows), taskDone: make(chan struct{}, 1), @@ -200,7 +200,7 @@ func (sw *schedWorker) disable(ctx context.Context) error { func (sw *schedWorker) checkSession(ctx context.Context) bool { for { - sctx, scancel := context.WithTimeout(ctx, stores.HeartbeatInterval/2) + sctx, scancel := context.WithTimeout(ctx, paths.HeartbeatInterval/2) curSes, err := sw.worker.workerRpc.Session(sctx) scancel() if err != nil { diff --git a/extern/sector-storage/sealtasks/task.go b/storage/sealer/sealtasks/task.go similarity index 100% rename from extern/sector-storage/sealtasks/task.go rename to storage/sealer/sealtasks/task.go diff --git a/extern/sector-storage/selector_alloc.go b/storage/sealer/selector_alloc.go similarity index 79% rename from extern/sector-storage/selector_alloc.go rename to storage/sealer/selector_alloc.go index ffdf35b5ee7..6418dcb853a 100644 --- a/extern/sector-storage/selector_alloc.go +++ b/storage/sealer/selector_alloc.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -7,18 +7,18 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type allocSelector struct { - index stores.SectorIndex + index paths.SectorIndex alloc storiface.SectorFileType ptype storiface.PathType } -func newAllocSelector(index stores.SectorIndex, alloc storiface.SectorFileType, ptype storiface.PathType) *allocSelector { +func newAllocSelector(index paths.SectorIndex, alloc storiface.SectorFileType, ptype storiface.PathType) *allocSelector { return &allocSelector{ index: index, alloc: alloc, diff --git a/extern/sector-storage/selector_existing.go b/storage/sealer/selector_existing.go similarity index 79% rename from extern/sector-storage/selector_existing.go rename to storage/sealer/selector_existing.go index 78a3b1988d0..02f5eb55abd 100644 --- a/extern/sector-storage/selector_existing.go +++ b/storage/sealer/selector_existing.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -7,19 +7,19 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type existingSelector struct { - index stores.SectorIndex + index paths.SectorIndex sector abi.SectorID alloc storiface.SectorFileType allowFetch bool } -func newExistingSelector(index stores.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, allowFetch bool) *existingSelector { +func newExistingSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, allowFetch bool) *existingSelector { return &existingSelector{ index: index, sector: sector, diff --git a/extern/sector-storage/selector_move.go b/storage/sealer/selector_move.go similarity index 84% rename from extern/sector-storage/selector_move.go rename to storage/sealer/selector_move.go index 7c63691b143..63fd2deb53b 100644 --- a/extern/sector-storage/selector_move.go +++ b/storage/sealer/selector_move.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -7,20 +7,20 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type moveSelector struct { - index stores.SectorIndex + index paths.SectorIndex sector abi.SectorID alloc storiface.SectorFileType destPtype storiface.PathType allowRemote bool } -func newMoveSelector(index stores.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, destPtype storiface.PathType, allowRemote bool) *moveSelector { +func newMoveSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, destPtype storiface.PathType, allowRemote bool) *moveSelector { return &moveSelector{ index: index, sector: sector, diff --git a/extern/sector-storage/selector_task.go b/storage/sealer/selector_task.go similarity index 88% rename from extern/sector-storage/selector_task.go rename to storage/sealer/selector_task.go index d5ea9618bd2..cc12b514e56 100644 --- a/extern/sector-storage/selector_task.go +++ b/storage/sealer/selector_task.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -7,8 +7,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type taskSelector struct { diff --git a/extern/sector-storage/stats.go b/storage/sealer/stats.go similarity index 94% rename from extern/sector-storage/stats.go rename to storage/sealer/stats.go index 56866793bbd..f34109d78c8 100644 --- a/extern/sector-storage/stats.go +++ b/storage/sealer/stats.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -6,8 +6,8 @@ import ( "github.com/google/uuid" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func (m *Manager) WorkerStats(ctx context.Context) map[uuid.UUID]storiface.WorkerStats { diff --git a/extern/sector-storage/storiface/cbor_gen.go b/storage/sealer/storiface/cbor_gen.go similarity index 100% rename from extern/sector-storage/storiface/cbor_gen.go rename to storage/sealer/storiface/cbor_gen.go diff --git a/extern/sector-storage/storiface/ffi.go b/storage/sealer/storiface/ffi.go similarity index 100% rename from extern/sector-storage/storiface/ffi.go rename to storage/sealer/storiface/ffi.go diff --git a/extern/sector-storage/storiface/filetype.go b/storage/sealer/storiface/filetype.go similarity index 100% rename from extern/sector-storage/storiface/filetype.go rename to storage/sealer/storiface/filetype.go diff --git a/extern/sector-storage/storiface/index.go b/storage/sealer/storiface/index.go similarity index 94% rename from extern/sector-storage/storiface/index.go rename to storage/sealer/storiface/index.go index d198c91f6b2..fdac7057fc8 100644 --- a/extern/sector-storage/storiface/index.go +++ b/storage/sealer/storiface/index.go @@ -5,7 +5,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" ) // ID identifies sector storage by UUID. One sector storage should map to one diff --git a/extern/sector-storage/storiface/resources.go b/storage/sealer/storiface/resources.go similarity index 99% rename from extern/sector-storage/storiface/resources.go rename to storage/sealer/storiface/resources.go index cd55924d57b..f3a658e3437 100644 --- a/extern/sector-storage/storiface/resources.go +++ b/storage/sealer/storiface/resources.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" ) type Resources struct { diff --git a/extern/sector-storage/storiface/resources_test.go b/storage/sealer/storiface/resources_test.go similarity index 96% rename from extern/sector-storage/storiface/resources_test.go rename to storage/sealer/storiface/resources_test.go index fd035911615..1040ef96ef1 100644 --- a/extern/sector-storage/storiface/resources_test.go +++ b/storage/sealer/storiface/resources_test.go @@ -8,7 +8,7 @@ import ( stabi "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" ) func TestListResourceVars(t *testing.T) { diff --git a/extern/sector-storage/storiface/storage.go b/storage/sealer/storiface/storage.go similarity index 100% rename from extern/sector-storage/storiface/storage.go rename to storage/sealer/storiface/storage.go diff --git a/extern/sector-storage/storiface/worker.go b/storage/sealer/storiface/worker.go similarity index 99% rename from extern/sector-storage/storiface/worker.go rename to storage/sealer/storiface/worker.go index d5a243fc757..e03c546eca0 100644 --- a/extern/sector-storage/storiface/worker.go +++ b/storage/sealer/storiface/worker.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" ) type WorkerID uuid.UUID // worker session UUID diff --git a/extern/sector-storage/tarutil/systar.go b/storage/sealer/tarutil/systar.go similarity index 100% rename from extern/sector-storage/tarutil/systar.go rename to storage/sealer/tarutil/systar.go diff --git a/extern/sector-storage/teststorage_test.go b/storage/sealer/teststorage_test.go similarity index 96% rename from extern/sector-storage/teststorage_test.go rename to storage/sealer/teststorage_test.go index b687e37a26e..e1e59c0b046 100644 --- a/extern/sector-storage/teststorage_test.go +++ b/storage/sealer/teststorage_test.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -10,8 +10,8 @@ import ( "github.com/filecoin-project/specs-actors/v7/actors/runtime/proof" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type apres struct { diff --git a/extern/sector-storage/testworker_test.go b/storage/sealer/testworker_test.go similarity index 90% rename from extern/sector-storage/testworker_test.go rename to storage/sealer/testworker_test.go index 608a6d5c5e4..35169b84500 100644 --- a/extern/sector-storage/testworker_test.go +++ b/storage/sealer/testworker_test.go @@ -1,24 +1,24 @@ -package sectorstorage +package sealer import ( "context" "sync" "github.com/google/uuid" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/mock" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/mock" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type testWorker struct { acceptTasks map[sealtasks.TaskType]struct{} - lstor *stores.Local + lstor *paths.Local ret storiface.WorkerReturn mockSeal *mock.SectorMgr @@ -32,7 +32,7 @@ type testWorker struct { Worker } -func newTestWorker(wcfg WorkerConfig, lstor *stores.Local, ret storiface.WorkerReturn) *testWorker { +func newTestWorker(wcfg WorkerConfig, lstor *paths.Local, ret storiface.WorkerReturn) *testWorker { acceptTasks := map[sealtasks.TaskType]struct{}{} for _, taskType := range wcfg.TaskTypes { acceptTasks[taskType] = struct{}{} diff --git a/extern/sector-storage/worker_calltracker.go b/storage/sealer/worker_calltracker.go similarity index 95% rename from extern/sector-storage/worker_calltracker.go rename to storage/sealer/worker_calltracker.go index 049c604c1e2..eb5bab145ce 100644 --- a/extern/sector-storage/worker_calltracker.go +++ b/storage/sealer/worker_calltracker.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "fmt" @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type workerCallTracker struct { diff --git a/extern/sector-storage/worker_local.go b/storage/sealer/worker_local.go similarity index 97% rename from extern/sector-storage/worker_local.go rename to storage/sealer/worker_local.go index 13e00a41400..b2691b6ddaf 100644 --- a/extern/sector-storage/worker_local.go +++ b/storage/sealer/worker_local.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -23,10 +23,10 @@ import ( "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var pathTypes = []storiface.SectorFileType{storiface.FTUnsealed, storiface.FTSealed, storiface.FTCache, storiface.FTUpdate, storiface.FTUpdateCache} @@ -49,9 +49,9 @@ type ExecutorFunc func() (ffiwrapper.Storage, error) type EnvFunc func(string) (string, bool) type LocalWorker struct { - storage stores.Store - localStore *stores.Local - sindex stores.SectorIndex + storage paths.Store + localStore *paths.Local + sindex paths.SectorIndex ret storiface.WorkerReturn executor ExecutorFunc noSwap bool @@ -73,7 +73,7 @@ type LocalWorker struct { closing chan struct{} } -func newLocalWorker(executor ExecutorFunc, wcfg WorkerConfig, envLookup EnvFunc, store stores.Store, local *stores.Local, sindex stores.SectorIndex, ret storiface.WorkerReturn, cst *statestore.StateStore) *LocalWorker { +func newLocalWorker(executor ExecutorFunc, wcfg WorkerConfig, envLookup EnvFunc, store paths.Store, local *paths.Local, sindex paths.SectorIndex, ret storiface.WorkerReturn, cst *statestore.StateStore) *LocalWorker { acceptTasks := map[sealtasks.TaskType]struct{}{} for _, taskType := range wcfg.TaskTypes { acceptTasks[taskType] = struct{}{} @@ -135,7 +135,7 @@ func newLocalWorker(executor ExecutorFunc, wcfg WorkerConfig, envLookup EnvFunc, return w } -func NewLocalWorker(wcfg WorkerConfig, store stores.Store, local *stores.Local, sindex stores.SectorIndex, ret storiface.WorkerReturn, cst *statestore.StateStore) *LocalWorker { +func NewLocalWorker(wcfg WorkerConfig, store paths.Store, local *paths.Local, sindex paths.SectorIndex, ret storiface.WorkerReturn, cst *statestore.StateStore) *LocalWorker { return newLocalWorker(nil, wcfg, os.LookupEnv, store, local, sindex, ret, cst) } diff --git a/extern/sector-storage/worker_local_test.go b/storage/sealer/worker_local_test.go similarity index 88% rename from extern/sector-storage/worker_local_test.go rename to storage/sealer/worker_local_test.go index 3ca4544e5c9..8624f2d918b 100644 --- a/extern/sector-storage/worker_local_test.go +++ b/storage/sealer/worker_local_test.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -11,12 +11,12 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type hangStore struct { - stores.Store + paths.Store challengeReads chan struct{} unhang chan struct{} diff --git a/extern/sector-storage/worker_tracked.go b/storage/sealer/worker_tracked.go similarity index 96% rename from extern/sector-storage/worker_tracked.go rename to storage/sealer/worker_tracked.go index e3ce0a46a45..e051b14549e 100644 --- a/extern/sector-storage/worker_tracked.go +++ b/storage/sealer/worker_tracked.go @@ -1,4 +1,4 @@ -package sectorstorage +package sealer import ( "context" @@ -13,9 +13,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type trackedWork struct { @@ -175,7 +175,9 @@ func (t *trackedWorker) SealPreCommit2(ctx context.Context, sector storage.Secto } func (t *trackedWorker) SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) { - return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTCommit1, func() (storiface.CallID, error) { return t.Worker.SealCommit1(ctx, sector, ticket, seed, pieces, cids) }) + return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTCommit1, func() (storiface.CallID, error) { + return t.Worker.SealCommit1(ctx, sector, ticket, seed, pieces, cids) + }) } func (t *trackedWorker) SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error) { diff --git a/storage/wdpost_changehandler.go b/storage/wdpost/wdpost_changehandler.go similarity index 99% rename from storage/wdpost_changehandler.go rename to storage/wdpost/wdpost_changehandler.go index e4f0e3c5c98..5778aaf9dbc 100644 --- a/storage/wdpost_changehandler.go +++ b/storage/wdpost/wdpost_changehandler.go @@ -1,4 +1,4 @@ -package storage +package wdpost import ( "context" diff --git a/storage/wdpost_changehandler_test.go b/storage/wdpost/wdpost_changehandler_test.go similarity index 99% rename from storage/wdpost_changehandler_test.go rename to storage/wdpost/wdpost_changehandler_test.go index 97a297b166b..fc44091ee0b 100644 --- a/storage/wdpost_changehandler_test.go +++ b/storage/wdpost/wdpost_changehandler_test.go @@ -1,5 +1,5 @@ //stm: #unit -package storage +package wdpost import ( "context" diff --git a/storage/wdpost_journal.go b/storage/wdpost/wdpost_journal.go similarity index 99% rename from storage/wdpost_journal.go rename to storage/wdpost/wdpost_journal.go index aefad937632..b4c790eb0c2 100644 --- a/storage/wdpost_journal.go +++ b/storage/wdpost/wdpost_journal.go @@ -1,4 +1,4 @@ -package storage +package wdpost import ( "github.com/ipfs/go-cid" diff --git a/storage/wdpost_nextdl_test.go b/storage/wdpost/wdpost_nextdl_test.go similarity index 98% rename from storage/wdpost_nextdl_test.go rename to storage/wdpost/wdpost_nextdl_test.go index 41e07b64fbe..2902cb4fc3c 100644 --- a/storage/wdpost_nextdl_test.go +++ b/storage/wdpost/wdpost_nextdl_test.go @@ -1,5 +1,5 @@ //stm: #unit -package storage +package wdpost import ( "testing" diff --git a/storage/wdpost_run.go b/storage/wdpost/wdpost_run.go similarity index 99% rename from storage/wdpost_run.go rename to storage/wdpost/wdpost_run.go index fe1a3912601..5f31541518b 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost/wdpost_run.go @@ -1,4 +1,4 @@ -package storage +package wdpost import ( "bytes" diff --git a/storage/wdpost_run_test.go b/storage/wdpost/wdpost_run_test.go similarity index 78% rename from storage/wdpost_run_test.go rename to storage/wdpost/wdpost_run_test.go index a1a1ab261f2..db3e535ddc4 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost/wdpost_run_test.go @@ -1,5 +1,5 @@ //stm: #unit -package storage +package wdpost import ( "bytes" @@ -27,17 +27,17 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" lbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/journal" + "github.com/filecoin-project/lotus/storage/ctladdr" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type mockStorageMinerAPI struct { partitions []api.Partition pushedMessages chan *types.Message - fullNodeFilteredAPI + NodeAPI } func newMockStorageMinerAPI() *mockStorageMinerAPI { @@ -236,7 +236,7 @@ func TestWDPostDoPost(t *testing.T) { proofType: proofType, actor: postAct, journal: journal.NilJournal(), - addrSel: &AddressSelector{}, + addrSel: &ctladdr.AddressSelector{}, } di := &dline.Info{ @@ -292,26 +292,6 @@ func mockTipSet(t *testing.T) *types.TipSet { // All the mock methods below here are unused // -func (m *mockStorageMinerAPI) StateCall(ctx context.Context, message *types.Message, key types.TipSetKey) (*api.InvocResult, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok types.TipSetKey) ([]api.Deadline, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) StateSectorPreCommitInfo(ctx context.Context, address address.Address, number abi.SectorNumber, key types.TipSetKey) (minertypes.SectorPreCommitOnChainInfo, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) StateSectorGetInfo(ctx context.Context, address address.Address, number abi.SectorNumber, key types.TipSetKey) (*minertypes.SectorOnChainInfo, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) { - panic("implement me") -} - func (m *mockStorageMinerAPI) StateMinerProvingDeadline(ctx context.Context, address address.Address, key types.TipSetKey) (*dline.Info, error) { return &dline.Info{ CurrentEpoch: 0, @@ -329,18 +309,6 @@ func (m *mockStorageMinerAPI) StateMinerProvingDeadline(ctx context.Context, add }, nil } -func (m *mockStorageMinerAPI) StateMinerPreCommitDepositForPower(ctx context.Context, address address.Address, info minertypes.SectorPreCommitInfo, key types.TipSetKey) (types.BigInt, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) StateMinerInitialPledgeCollateral(ctx context.Context, address address.Address, info minertypes.SectorPreCommitInfo, key types.TipSetKey) (types.BigInt, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) { - panic("implement me") -} - func (m *mockStorageMinerAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { code, err := lbuiltin.GetMinerActorCodeID(actors.Version7) if err != nil { @@ -351,22 +319,6 @@ func (m *mockStorageMinerAPI) StateGetActor(ctx context.Context, actor address.A }, nil } -func (m *mockStorageMinerAPI) StateGetReceipt(ctx context.Context, cid cid.Cid, key types.TipSetKey) (*types.MessageReceipt, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) StateMarketStorageDeal(ctx context.Context, id abi.DealID, key types.TipSetKey) (*api.MarketDeal, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) StateMinerFaults(ctx context.Context, address address.Address, key types.TipSetKey) (bitfield.BitField, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) StateMinerRecoveries(ctx context.Context, address address.Address, key types.TipSetKey) (bitfield.BitField, error) { - panic("implement me") -} - func (m *mockStorageMinerAPI) StateAccountKey(ctx context.Context, address address.Address, key types.TipSetKey) (address.Address, error) { return address, nil } @@ -383,30 +335,6 @@ func (m *mockStorageMinerAPI) ChainHead(ctx context.Context) (*types.TipSet, err return nil, nil } -func (m *mockStorageMinerAPI) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) ChainGetBlockMessages(ctx context.Context, cid cid.Cid) (*api.BlockMessages, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) ChainReadObj(ctx context.Context, cid cid.Cid) ([]byte, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) ChainHasObj(ctx context.Context, cid cid.Cid) (bool, error) { - panic("implement me") -} - -func (m *mockStorageMinerAPI) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { - panic("implement me") -} - func (m *mockStorageMinerAPI) WalletSign(ctx context.Context, address address.Address, bytes []byte) (*crypto.Signature, error) { return nil, nil } @@ -419,4 +347,4 @@ func (m *mockStorageMinerAPI) WalletHas(ctx context.Context, address address.Add return true, nil } -var _ fullNodeFilteredAPI = &mockStorageMinerAPI{} +var _ NodeAPI = &mockStorageMinerAPI{} diff --git a/storage/wdpost_sched.go b/storage/wdpost/wdpost_sched.go similarity index 65% rename from storage/wdpost_sched.go rename to storage/wdpost/wdpost_sched.go index cd10c59efe1..99f14c691eb 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost/wdpost_sched.go @@ -1,27 +1,61 @@ -package storage +package wdpost import ( "context" "time" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v8/miner" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/storage/ctladdr" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) +var log = logging.Logger("wdpost") + +type NodeAPI interface { + ChainHead(context.Context) (*types.TipSet, error) + ChainNotify(context.Context) (<-chan []*api.HeadChange, error) + + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) + StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) + StateMinerSectors(context.Context, address.Address, *bitfield.BitField, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) + StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) + StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) + StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) + StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) + StateMinerPartitions(context.Context, address.Address, uint64, types.TipSetKey) ([]api.Partition, error) + StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) + StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) + + MpoolPushMessage(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error) + + GasEstimateMessageGas(context.Context, *types.Message, *api.MessageSendSpec, types.TipSetKey) (*types.Message, error) + GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) + GasEstimateGasPremium(_ context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) + + WalletBalance(context.Context, address.Address) (types.BigInt, error) + WalletHas(context.Context, address.Address) (bool, error) +} + // WindowPoStScheduler is the coordinator for WindowPoSt submissions, fault // declaration, and recovery declarations. It watches the chain for reverts and // applies, and schedules/run those processes as partition deadlines arrive. @@ -29,12 +63,12 @@ import ( // WindowPoStScheduler watches the chain though the changeHandler, which in turn // turn calls the scheduler when the time arrives to do work. type WindowPoStScheduler struct { - api fullNodeFilteredAPI + api NodeAPI feeCfg config.MinerFeeConfig - addrSel *AddressSelector + addrSel *ctladdr.AddressSelector prover storage.Prover verifier ffiwrapper.Verifier - faultTracker sectorstorage.FaultTracker + faultTracker sealer.FaultTracker proofType abi.RegisteredPoStProof partitionSectors uint64 ch *changeHandler @@ -49,12 +83,12 @@ type WindowPoStScheduler struct { } // NewWindowedPoStScheduler creates a new WindowPoStScheduler scheduler. -func NewWindowedPoStScheduler(api fullNodeFilteredAPI, +func NewWindowedPoStScheduler(api NodeAPI, cfg config.MinerFeeConfig, - as *AddressSelector, + as *ctladdr.AddressSelector, sp storage.Prover, verif ffiwrapper.Verifier, - ft sectorstorage.FaultTracker, + ft sealer.FaultTracker, j journal.Journal, actor address.Address) (*WindowPoStScheduler, error) { mi, err := api.StateMinerInfo(context.TODO(), actor, types.EmptyTSK) @@ -88,7 +122,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { // callbacks is a union of the fullNodeFilteredAPI and ourselves. callbacks := struct { - fullNodeFilteredAPI + NodeAPI *WindowPoStScheduler }{s.api, s} diff --git a/testplans/lotus-soup/rfwp/chain_state.go b/testplans/lotus-soup/rfwp/chain_state.go index 929870e26b1..96ad9174397 100644 --- a/testplans/lotus-soup/rfwp/chain_state.go +++ b/testplans/lotus-soup/rfwp/chain_state.go @@ -26,7 +26,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + sealing "github.com/filecoin-project/lotus/storage/pipeline" "github.com/filecoin-project/lotus/testplans/lotus-soup/testkit" tsync "github.com/filecoin-project/lotus/tools/stats/sync" ) diff --git a/testplans/lotus-soup/testkit/node.go b/testplans/lotus-soup/testkit/node.go index cd9e0bb787c..4d8d04aeb53 100644 --- a/testplans/lotus-soup/testkit/node.go +++ b/testplans/lotus-soup/testkit/node.go @@ -18,7 +18,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/beacon" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node" @@ -36,11 +36,11 @@ type LotusNode struct { FullApi api.FullNode MinerApi api.StorageMiner StopFn node.StopFunc - Wallet *wallet.Key + Wallet *key.Key MineOne func(context.Context, miner.MineReq) error } -func (n *LotusNode) setWallet(ctx context.Context, walletKey *wallet.Key) error { +func (n *LotusNode) setWallet(ctx context.Context, walletKey *key.Key) error { _, err := n.FullApi.WalletImport(ctx, &walletKey.KeyInfo) if err != nil { return err diff --git a/testplans/lotus-soup/testkit/role_client.go b/testplans/lotus-soup/testkit/role_client.go index 595284493f2..afde274202a 100644 --- a/testplans/lotus-soup/testkit/role_client.go +++ b/testplans/lotus-soup/testkit/role_client.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/repo" ) @@ -44,7 +44,7 @@ func PrepareClient(t *TestEnvironment) (*LotusClient, error) { } // first create a wallet - walletKey, err := wallet.GenerateKey(types.KTBLS) + walletKey, err := key.GenerateKey(types.KTBLS) if err != nil { return nil, err } diff --git a/testplans/lotus-soup/testkit/role_miner.go b/testplans/lotus-soup/testkit/role_miner.go index a30af367121..59cb9dc712d 100644 --- a/testplans/lotus-soup/testkit/role_miner.go +++ b/testplans/lotus-soup/testkit/role_miner.go @@ -32,10 +32,8 @@ import ( "github.com/filecoin-project/lotus/chain/actors" genesis_chain "github.com/filecoin-project/lotus/chain/gen/genesis" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/key" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node" @@ -43,6 +41,8 @@ import ( "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const ( @@ -78,7 +78,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { } // first create a wallet - walletKey, err := wallet.GenerateKey(types.KTBLS) + walletKey, err := key.GenerateKey(types.KTBLS) if err != nil { return nil, err } @@ -198,9 +198,9 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { } } - var localPaths []stores.LocalPath + var localPaths []paths.LocalPath - b, err := json.MarshalIndent(&stores.LocalStorageMeta{ + b, err := json.MarshalIndent(&paths.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), Weight: 10, CanSeal: true, @@ -214,11 +214,11 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { return nil, fmt.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) } - localPaths = append(localPaths, stores.LocalPath{ + localPaths = append(localPaths, paths.LocalPath{ Path: lr.Path(), }) - if err := lr.SetStorage(func(sc *stores.StorageConfig) { + if err := lr.SetStorage(func(sc *paths.StorageConfig) { sc.StoragePaths = append(sc.StoragePaths, localPaths...) }); err != nil { return nil, err diff --git a/tools/stats/influx/influx.go b/tools/stats/influx/influx.go index 39f25edbeed..6c589fd575d 100644 --- a/tools/stats/influx/influx.go +++ b/tools/stats/influx/influx.go @@ -7,7 +7,7 @@ import ( "time" _ "github.com/influxdata/influxdb1-client" - models "github.com/influxdata/influxdb1-client/models" + "github.com/influxdata/influxdb1-client/models" client "github.com/influxdata/influxdb1-client/v2" "github.com/filecoin-project/lotus/build"