Support standalone miner-market process

cmd/lotus-storage-miner/init_restore.go

@@ -31,6 +31,50 @@ import (
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
+var restoreCmd = &cli.Command{
+	Name:  "restore",
+	Usage: "Initialize a lotus miner repo from a backup",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "nosync",
+			Usage: "don't check full-node sync status",
+		},
+		&cli.StringFlag{
+			Name:  "config",
+			Usage: "config file (config.toml)",
+		},
+		&cli.StringFlag{
+			Name:  "storage-config",
+			Usage: "storage paths config (storage.json)",
+		},
+	},
+	ArgsUsage: "[backupFile]",
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+		log.Info("Initializing lotus miner using a backup")
+
+		if err := restore(ctx, cctx, nil, func(api lapi.FullNode, maddr address.Address, peerid peer.ID, mi miner.MinerInfo) error {
+			log.Info("Checking proof parameters")
+
+			if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(mi.SectorSize)); err != nil {
+				return xerrors.Errorf("fetching proof parameters: %w", err)
+			}
+
+			log.Info("Configuring miner actor")
+
+			if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil {
+				return err
+			}
+
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}
+
 func restore(ctx context.Context, cctx *cli.Context, manageConfig func(*config.StorageMiner) error, after func(api lapi.FullNode, addr address.Address, peerid peer.ID, mi miner.MinerInfo) error) error {
 	if cctx.Args().Len() != 1 {
 		return xerrors.Errorf("expected 1 argument")
@@ -246,47 +290,3 @@ func restore(ctx context.Context, cctx *cli.Context, manageConfig func(*config.S
 
 	return after(api, maddr, peerid, mi)
 }
-
-var restoreCmd = &cli.Command{
-	Name:  "restore",
-	Usage: "Initialize a lotus miner repo from a backup",
-	Flags: []cli.Flag{
-		&cli.BoolFlag{
-			Name:  "nosync",
-			Usage: "don't check full-node sync status",
-		},
-		&cli.StringFlag{
-			Name:  "config",
-			Usage: "config file (config.toml)",
-		},
-		&cli.StringFlag{
-			Name:  "storage-config",
-			Usage: "storage paths config (storage.json)",
-		},
-	},
-	ArgsUsage: "[backupFile]",
-	Action: func(cctx *cli.Context) error {
-		ctx := lcli.ReqContext(cctx)
-		log.Info("Initializing lotus miner using a backup")
-
-		if err := restore(ctx, cctx, nil, func(api lapi.FullNode, maddr address.Address, peerid peer.ID, mi miner.MinerInfo) error {
-			log.Info("Checking proof parameters")
-
-			if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(mi.SectorSize)); err != nil {
-				return xerrors.Errorf("fetching proof parameters: %w", err)
-			}
-
-			log.Info("Configuring miner actor")
-
-			if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil {
-				return err
-			}
-
-			return nil
-		}); err != nil {
-			return err
-		}
-
-		return nil
-	},
-}

cmd/lotus-storage-miner/init_service.go

@@ -17,33 +17,9 @@ import (
 	"golang.org/x/xerrors"
 )
 
-func checkApiInfo(ctx context.Context, ai string) (string, error) {
-	ai = strings.TrimPrefix(strings.TrimSpace(ai), "MINER_API_INFO=")
-	info := cliutil.ParseApiInfo(ai)
-	addr, err := info.DialArgs("v0")
-	if err != nil {
-		return "", xerrors.Errorf("could not get DialArgs: %w", err)
-	}
-
-	log.Infof("Checking api version of %s", addr)
-
-	api, closer, err := client.NewStorageMinerRPCV0(ctx, addr, info.AuthHeader())
-	if err != nil {
-		return "", err
-	}
-	defer closer()
-
-	v, err := api.Version(ctx)
-	if err != nil {
-		return "", xerrors.Errorf("checking version: %w", err)
-	}
-
-	if !v.APIVersion.EqMajorMinor(lapi.MinerAPIVersion0) {
-		return "", xerrors.Errorf("remote service API version didn't match (expected %s, remote %s)", lapi.MinerAPIVersion0, v.APIVersion)
-	}
-
-	return ai, nil
-}
+const (
+	MarketsService = "markets"
+)
 
 var serviceCmd = &cli.Command{
 	Name:  "service",
@@ -63,12 +39,10 @@ var serviceCmd = &cli.Command{
 			Name:  "nosync",
 			Usage: "don't check full-node sync status",
 		},
-
-		&cli.BoolFlag{
-			Name:  "enable-market",
-			Usage: "enable market module",
+		&cli.StringSliceFlag{
+			Name:  "name",
+			Usage: "services to be enabled",
 		},
-
 		&cli.StringFlag{
 			Name:  "api-sealer",
 			Usage: "sealer API info (lotus-miner auth api-info --perm=admin)",
@@ -83,10 +57,17 @@ var serviceCmd = &cli.Command{
 		ctx := lcli.ReqContext(cctx)
 		log.Info("Initializing lotus miner service")
 
-		if !cctx.Bool("enable-market") {
+		es := EnabledServices(cctx.StringSlice("name"))
+
+		if len(es) == 0 {
 			return xerrors.Errorf("at least one module must be enabled")
 		}
 
+		// we should remove this as soon as we have more service types and not just `markets`
+		if !es.Contains(MarketsService) {
+			return xerrors.Errorf("markets module must be enabled")
+		}
+
 		if !cctx.IsSet("api-sealer") {
 			return xerrors.Errorf("--api-sealer is required without the sealer module enabled")
 		}
@@ -95,7 +76,7 @@ var serviceCmd = &cli.Command{
 		}
 
 		if err := restore(ctx, cctx, func(cfg *config.StorageMiner) error {
-			cfg.Subsystems.EnableStorageMarket = cctx.Bool("enable-market")
+			cfg.Subsystems.EnableMarkets = es.Contains(MarketsService)
 			cfg.Subsystems.EnableMining = false
 			cfg.Subsystems.EnableSealing = false
 			cfg.Subsystems.EnableSectorStorage = false
@@ -118,7 +99,7 @@ var serviceCmd = &cli.Command{
 
 			return nil
 		}, func(api lapi.FullNode, maddr address.Address, peerid peer.ID, mi miner.MinerInfo) error {
-			if cctx.Bool("enable-market") {
+			if es.Contains(MarketsService) {
 				log.Info("Configuring miner actor")
 
 				if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil {
@@ -134,3 +115,42 @@ var serviceCmd = &cli.Command{
 		return nil
 	},
 }
+
+type EnabledServices []string
+
+func (es EnabledServices) Contains(name string) bool {
+	for _, s := range es {
+		if s == name {
+			return true
+		}
+	}
+	return false
+}
+
+func checkApiInfo(ctx context.Context, ai string) (string, error) {
+	ai = strings.TrimPrefix(strings.TrimSpace(ai), "MINER_API_INFO=")
+	info := cliutil.ParseApiInfo(ai)
+	addr, err := info.DialArgs("v0")
+	if err != nil {
+		return "", xerrors.Errorf("could not get DialArgs: %w", err)
+	}
+
+	log.Infof("Checking api version of %s", addr)
+
+	api, closer, err := client.NewStorageMinerRPCV0(ctx, addr, info.AuthHeader())
+	if err != nil {
+		return "", err
+	}
+	defer closer()
+
+	v, err := api.Version(ctx)
+	if err != nil {
+		return "", xerrors.Errorf("checking version: %w", err)
+	}
+
+	if !v.APIVersion.EqMajorMinor(lapi.MinerAPIVersion0) {
+		return "", xerrors.Errorf("remote service API version didn't match (expected %s, remote %s)", lapi.MinerAPIVersion0, v.APIVersion)
+	}
+
+	return ai, nil
+}

cmd/lotus-storage-miner/run.go

@@ -132,7 +132,7 @@ var runCmd = &cli.Command{
 			return xerrors.Errorf("invalid config for repo, got: %T", c)
 		}
 
-		bootstrapLibP2P := cfg.Subsystems.EnableStorageMarket
+		bootstrapLibP2P := cfg.Subsystems.EnableMarkets
 
 		err = lr.Close()
 		if err != nil {

documentation/en/cli-lotus-miner.md

@@ -106,7 +106,7 @@ OPTIONS:
    --config value            config file (config.toml)
    --storage-config value    storage paths config (storage.json)
    --nosync                  don't check full-node sync status (default: false)
-   --enable-market           enable market module (default: false)
+   --name value              services to be enabled
    --api-sealer value        sealer API info (lotus-miner auth api-info --perm=admin)
    --api-sector-index value  sector Index API info (lotus-miner auth api-info --perm=admin)
    --help, -h                show help (default: false)

extern/sector-storage/manager.go

@@ -723,4 +723,3 @@ func (m *Manager) Close(ctx context.Context) error {
 
 var _ Unsealer = &Manager{}
 var _ SectorManager = &Manager{}
-var _ Unsealer = &Manager{}

itests/deals_publish_test.go

@@ -55,6 +55,11 @@ func TestPublishDealsBatching(t *testing.T) {
 	client, miner, ens := kit.EnsembleMinimal(t, kit.Account(publisherKey, types.FromFil(10)), kit.MockProofs(), kit.ConstructorOpts(opts))
 	ens.InterconnectAll().BeginMining(10 * time.Millisecond)
 
+	_, err = client.WalletImport(ctx, &publisherKey.KeyInfo)
+	require.NoError(t, err)
+
+	miner.SetControlAddresses(publisherKey.Address)
+
 	dh := kit.NewDealHarness(t, client, miner, miner)
 
 	// Starts a deal and waits until it's published

itests/kit/ensemble.go

@@ -424,7 +424,7 @@ func (n *Ensemble) Start() *Ensemble {
 			n.t.Fatalf("invalid config from repo, got: %T", c)
 		}
 		cfg.Common.API.RemoteListenAddress = m.RemoteListener.Addr().String()
-		cfg.Subsystems.EnableStorageMarket = m.options.subsystems.Has(SStorageMarket)
+		cfg.Subsystems.EnableMarkets = m.options.subsystems.Has(SMarkets)
 		cfg.Subsystems.EnableMining = m.options.subsystems.Has(SMining)
 		cfg.Subsystems.EnableSealing = m.options.subsystems.Has(SSealing)
 		cfg.Subsystems.EnableSectorStorage = m.options.subsystems.Has(SSectorStorage)

itests/kit/ensemble_presets.go

@@ -38,7 +38,7 @@ func EnsembleWithMinerAndMarketNodes(t *testing.T, opts ...interface{}) (*TestFu
 	ens := NewEnsemble(t, eopts...).FullNode(&fullnode, nopts...).Miner(&main, &fullnode, mainNodeOpts...).Start()
 	ens.BeginMining(blockTime)
 
-	marketNodeOpts := []NodeOpt{OwnerAddr(fullnode.DefaultKey), MainMiner(&main), WithSubsystems(SStorageMarket)}
+	marketNodeOpts := []NodeOpt{OwnerAddr(fullnode.DefaultKey), MainMiner(&main), WithSubsystems(SMarkets)}
 	marketNodeOpts = append(marketNodeOpts, nopts...)
 
 	ens.Miner(&market, &fullnode, marketNodeOpts...).Start().Connect(market, fullnode)

itests/kit/node_miner.go

@@ -31,7 +31,7 @@ import (
 type MinerSubsystem int
 
 const (
-	SStorageMarket MinerSubsystem = 1 << iota
+	SMarkets MinerSubsystem = 1 << iota
 	SMining
 	SSealing
 	SSectorStorage

itests/kit/node_opts.go

@@ -49,7 +49,7 @@ type NodeOpt func(opts *nodeOpts) error
 
 func WithAllSubsystems() NodeOpt {
 	return func(opts *nodeOpts) error {
-		opts.subsystems = opts.subsystems.Add(SStorageMarket)
+		opts.subsystems = opts.subsystems.Add(SMarkets)
 		opts.subsystems = opts.subsystems.Add(SMining)
 		opts.subsystems = opts.subsystems.Add(SSealing)
 		opts.subsystems = opts.subsystems.Add(SSectorStorage)

node/builder_miner.go

@@ -67,7 +67,7 @@ func ConfigStorageMiner(c interface{}) Option {
 		return Error(xerrors.New("retrieval pricing policy must be either default or external"))
 	}
 
-	enableLibp2pNode := cfg.Subsystems.EnableStorageMarket // we enable libp2p nodes if the storage market subsystem is enabled, otherwise we don't
+	enableLibp2pNode := cfg.Subsystems.EnableMarkets // we enable libp2p nodes if the storage market subsystem is enabled, otherwise we don't
 
 	return Options(
 		ConfigCommon(&cfg.Common, enableLibp2pNode),
@@ -127,7 +127,7 @@ func ConfigStorageMiner(c interface{}) Option {
 			Override(new(stores.SectorIndex), From(new(modules.MinerSealingService))),
 		),
 
-		If(cfg.Subsystems.EnableStorageMarket,
+		If(cfg.Subsystems.EnableMarkets,
 			// Markets
 			Override(new(dtypes.StagingMultiDstore), modules.StagingMultiDatastore),
 			Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore),
@@ -209,7 +209,7 @@ func StorageMiner(out *api.StorageMiner, subsystemsCfg config.MinerSubsystemConf
 
 		func(s *Settings) error {
 			s.nodeType = repo.StorageMiner
-			s.enableLibp2pNode = subsystemsCfg.EnableStorageMarket
+			s.enableLibp2pNode = subsystemsCfg.EnableMarkets
 			return nil
 		},
 

node/config/def.go

@@ -62,7 +62,7 @@ type MinerSubsystemConfig struct {
 	EnableMining        bool
 	EnableSealing       bool
 	EnableSectorStorage bool
-	EnableStorageMarket bool
+	EnableMarkets       bool
 
 	SealerApiInfo      string // if EnableSealing == false
 	SectorIndexApiInfo string // if EnableSectorStorage == false
@@ -400,7 +400,7 @@ func DefaultStorageMiner() *StorageMiner {
 			EnableMining:        true,
 			EnableSealing:       true,
 			EnableSectorStorage: true,
-			EnableStorageMarket: true,
+			EnableMarkets:       true,
 		},
 
 		Fees: MinerFeeConfig{

node/modules/storageminer.go

@@ -677,7 +677,6 @@ func RetrievalProvider(
 	pieceStore dtypes.ProviderPieceStore,
 	mds dtypes.StagingMultiDstore,
 	dt dtypes.ProviderDataTransfer,
-	pieceProvider sectorstorage.PieceProvider,
 	pricingFnc dtypes.RetrievalPricingFunc,
 	userFilter dtypes.RetrievalDealFilter,
 ) (retrievalmarket.RetrievalProvider, error) {
@@ -727,11 +726,11 @@ func StorageAuth(ctx helpers.MetricsCtx, ca v0api.Common) (sectorstorage.Storage
 	return sectorstorage.StorageAuth(headers), nil
 }
 
-func StorageAuthWithURL(url string) func(ctx helpers.MetricsCtx, ca v0api.Common) (sectorstorage.StorageAuth, error) {
+func StorageAuthWithURL(apiInfo string) func(ctx helpers.MetricsCtx, ca v0api.Common) (sectorstorage.StorageAuth, error) {
 	return func(ctx helpers.MetricsCtx, ca v0api.Common) (sectorstorage.StorageAuth, error) {
-		s := strings.Split(url, ":")
+		s := strings.Split(apiInfo, ":")
 		if len(s) != 2 {
-			return nil, errors.New("unexpected format of URL")
+			return nil, errors.New("unexpected format of `apiInfo`")
 		}
 		headers := http.Header{}
 		headers.Add("Authorization", "Bearer "+s[0])