diff --git a/CHANGELOG.md b/CHANGELOG.md index 5de6d8f9df4..1d382436648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,11 @@ - Add Magik's bootstrap node. ([filecoin-project/lotus#12792](https://github.com/filecoin-project/lotus/pull/12792)) - Lotus now reports the network name as a tag in most metrics. Some untagged metrics will be completed in a follow-up at a later date. ([filecoin-project/lotus#12733](https://github.com/filecoin-project/lotus/pull/12733)) - Refactored Ethereum API implementation into smaller, more manageable modules in a new `github.com/filecoin-project/lotus/node/impl/eth` package. ([filecoin-project/lotus#12796](https://github.com/filecoin-project/lotus/pull/12796)) +- Generate the cli docs directly from the code instead compiling and executing binaries' `help` output. ([filecoin-project/lotus#12717](https://github.com/filecoin-project/lotus/pull/12717)) +- Add `lotus-shed msg --gas-stats` to show summarised gas stats for a given message. ([filecoin-project/lotus#12817](https://github.com/filecoin-project/lotus/pull/12817)) # UNRELEASED v.1.32.0 - See https://github.com/filecoin-project/lotus/blob/release/v1.32.0/CHANGELOG.md # Node and Miner v1.31.0 / 2024-12-02 diff --git a/LOTUS_RELEASE_FLOW.md b/LOTUS_RELEASE_FLOW.md index d4e77faf732..5a307a3a57b 100644 --- a/LOTUS_RELEASE_FLOW.md +++ b/LOTUS_RELEASE_FLOW.md @@ -96,7 +96,7 @@ The specific steps executed for Lotus software releases are captured in the [Rel ## Release Candidates (RCs) - For regular (i.e., no critical security patch) releases with no accompanying network upgrade, the RC period is typically around 1 week. -- For releases accompanying network upgrades, the release candiadte period is a lot longer to allow for more extensive testing, usually around 5 to 6 weeks. +- For releases accompanying network upgrades, the release candidate period is a lot longer to allow for more extensive testing, usually around 5 to 6 weeks. - Releases rushing out a critical security patch will likely have an RC period on the order of hours or days, or may even forgo the RC phase. To compensate for the release speed, these releases will include the minimum delta necessary, meaning they'll be a patch on top of an existing release rather than taking the latest changes in the `master` branch. ## Security Fix Policy diff --git a/Makefile b/Makefile index 6387a1da66a..4ea9dcf1fec 100644 --- a/Makefile +++ b/Makefile @@ -351,8 +351,6 @@ snap: lotus lotus-miner lotus-worker ## Build snap package # separate from gen because it needs binaries docsgen-cli: lotus lotus-miner lotus-worker ## Generate CLI documentation $(GOCC) run ./scripts/docsgen-cli - ./lotus config default > documentation/en/default-lotus-config.toml - ./lotus-miner config default > documentation/en/default-lotus-miner-config.toml .PHONY: docsgen-cli print-%: ## Print variable value diff --git a/api/api_storage.go b/api/api_storage.go index 35221058291..45c0120fb84 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -186,7 +186,7 @@ type StorageMiner interface { // // If allowFetch is set, list of paths to which the sector can be fetched will also be returned. // - Paths which have sector files locally (don't require fetching) will be listed first. - // - Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes. + // - Paths which have sector files locally will not be filtered based on AllowTypes/DenyTypes. // - Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple // file types are specified, each type will be considered individually, and a union of all paths // which can accommodate each file type will be returned. diff --git a/build/openrpc/miner.json b/build/openrpc/miner.json index 5467df0dce3..573a86d6a4d 100644 --- a/build/openrpc/miner.json +++ b/build/openrpc/miner.json @@ -6731,7 +6731,7 @@ { "name": "Filecoin.StorageFindSector", "description": "```go\nfunc (s *StorageMinerStruct) StorageFindSector(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]storiface.SectorStorageInfo, error) {\n\tif s.Internal.StorageFindSector == nil {\n\t\treturn *new([]storiface.SectorStorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageFindSector(p0, p1, p2, p3, p4)\n}\n```", - "summary": "StorageFindSector returns list of paths where the specified sector files exist.\n\nIf allowFetch is set, list of paths to which the sector can be fetched will also be returned.\n- Paths which have sector files locally (don't require fetching) will be listed first.\n- Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes.\n- Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple\n file types are specified, each type will be considered individually, and a union of all paths\n which can accommodate each file type will be returned.\n", + "summary": "StorageFindSector returns list of paths where the specified sector files exist.\n\nIf allowFetch is set, list of paths to which the sector can be fetched will also be returned.\n- Paths which have sector files locally (don't require fetching) will be listed first.\n- Paths which have sector files locally will not be filtered based on AllowTypes/DenyTypes.\n- Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple\n file types are specified, each type will be considered individually, and a union of all paths\n which can accommodate each file type will be returned.\n", "paramStructure": "by-position", "params": [ { diff --git a/chain/index/reconcile.go b/chain/index/reconcile.go index 72a1e6ecaa6..1e2940068c0 100644 --- a/chain/index/reconcile.go +++ b/chain/index/reconcile.go @@ -236,10 +236,10 @@ func (si *SqliteIndexer) backfillIndex(ctx context.Context, tx *sql.Tx, head *ty log.Infof("reached stop height %d; backfilled %d tipsets", stopAfter, totalApplied) return nil } - + height := currTs.Height() currTs, err = si.cs.GetTipSetFromKey(ctx, currTs.Parents()) if err != nil { - return xerrors.Errorf("failed to walk chain at height %d: %w", currTs.Height(), err) + return xerrors.Errorf("failed to walk chain beyond height %d: %w", height, err) } } diff --git a/chain/lf3/signer.go b/chain/lf3/signer.go index 3f4d517a471..8f016a832a3 100644 --- a/chain/lf3/signer.go +++ b/chain/lf3/signer.go @@ -18,7 +18,7 @@ type signer struct { } // Sign signs a message with the private key corresponding to a public key. -// The the key must be known by the wallet and be of BLS type. +// The key must be known by the wallet and be of BLS type. func (s *signer) Sign(ctx context.Context, sender gpbft.PubKey, msg []byte) ([]byte, error) { addr, err := address.NewBLSAddress(sender) if err != nil { diff --git a/chain/types/tipset_key.go b/chain/types/tipset_key.go index 703ff2a4c43..81a9795717d 100644 --- a/chain/types/tipset_key.go +++ b/chain/types/tipset_key.go @@ -37,7 +37,7 @@ func init() { type TipSetKey struct { // The internal representation is a concatenation of the bytes of the CIDs, which are // self-describing, wrapped as a string. - // These gymnastics make the a TipSetKey usable as a map key. + // These gymnastics make a TipSetKey usable as a map key. // The empty key has value "". value string } diff --git a/cli/backup.go b/cli/backup.go index e0495678cfd..c5122f57c67 100644 --- a/cli/backup.go +++ b/cli/backup.go @@ -112,7 +112,7 @@ func BackupCmd(repoFlag string, rt repo.RepoType, getApi BackupApiFn) *cli.Comma Description: `The backup command writes a copy of node metadata under the specified path Online backups: -For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set +For security reasons, the daemon must have LOTUS_BACKUP_BASE_PATH env var set to a path where backup files are supposed to be saved, and the path specified in this command must be within this base path`, Flags: []cli.Flag{ diff --git a/cli/cmd.go b/cli/cmd.go index 9ae8c14b75e..2c7f69f176d 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -57,10 +57,10 @@ var GetStorageMinerAPI = cliutil.GetStorageMinerAPI var GetWorkerAPI = cliutil.GetWorkerAPI var CommonCommands = []*cli.Command{ - AuthCmd, - LogCmd, - WaitApiCmd, - FetchParamCmd, + WithCategory("developer", AuthCmd), + WithCategory("developer", LogCmd), + WithCategory("developer", WaitApiCmd), + WithCategory("developer", FetchParamCmd), PprofCmd, VersionCmd, } diff --git a/cmd/lotus/backup.go b/cli/lotus/backup.go similarity index 99% rename from cmd/lotus/backup.go rename to cli/lotus/backup.go index 0fbdf29622c..ed664a370aa 100644 --- a/cmd/lotus/backup.go +++ b/cli/lotus/backup.go @@ -1,4 +1,4 @@ -package main +package lotus import ( "os" diff --git a/cmd/lotus/config.go b/cli/lotus/config.go similarity index 99% rename from cmd/lotus/config.go rename to cli/lotus/config.go index 4b323bfb227..9528cc05363 100644 --- a/cmd/lotus/config.go +++ b/cli/lotus/config.go @@ -1,4 +1,4 @@ -package main +package lotus import ( "fmt" diff --git a/cmd/lotus/daemon.go b/cli/lotus/daemon.go similarity index 99% rename from cmd/lotus/daemon.go rename to cli/lotus/daemon.go index 97d2294b79b..f4e2c4b7fa2 100644 --- a/cmd/lotus/daemon.go +++ b/cli/lotus/daemon.go @@ -1,7 +1,7 @@ //go:build !nodaemon // +build !nodaemon -package main +package lotus import ( "bufio" diff --git a/cmd/lotus/daemon_nodaemon.go b/cli/lotus/daemon_nodaemon.go similarity index 96% rename from cmd/lotus/daemon_nodaemon.go rename to cli/lotus/daemon_nodaemon.go index 7cf12dac41a..be3f8721e58 100644 --- a/cmd/lotus/daemon_nodaemon.go +++ b/cli/lotus/daemon_nodaemon.go @@ -1,7 +1,7 @@ //go:build nodaemon // +build nodaemon -package main +package lotus import ( "errors" diff --git a/cmd/lotus/debug_advance.go b/cli/lotus/debug_advance.go similarity index 99% rename from cmd/lotus/debug_advance.go rename to cli/lotus/debug_advance.go index 422f14e5a2e..915af49d668 100644 --- a/cmd/lotus/debug_advance.go +++ b/cli/lotus/debug_advance.go @@ -1,7 +1,7 @@ //go:build debug // +build debug -package main +package lotus import ( "encoding/binary" diff --git a/cli/lotus/lotus.go b/cli/lotus/lotus.go new file mode 100644 index 00000000000..1bdaf6fd217 --- /dev/null +++ b/cli/lotus/lotus.go @@ -0,0 +1,123 @@ +package lotus + +import ( + "context" + "os" + + "github.com/fatih/color" + logging "github.com/ipfs/go-log/v2" + "github.com/mattn/go-isatty" + "github.com/urfave/cli/v2" + "go.opencensus.io/trace" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/cli/clicommands" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/lib/tracing" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("lotus") + +var AdvanceBlockCmd *cli.Command + +func App() *cli.App { + api.RunningNodeType = api.NodeFull + + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + DaemonCmd, + backupCmd, + configCmd, + } + if AdvanceBlockCmd != nil { + local = append(local, AdvanceBlockCmd) + } + + jaeger := tracing.SetupJaegerTracing("lotus") + defer func() { + if jaeger != nil { + _ = jaeger.ForceFlush(context.Background()) + } + }() + + for _, cmd := range local { + cmd := cmd + originBefore := cmd.Before + cmd.Before = func(cctx *cli.Context) error { + if jaeger != nil { + _ = jaeger.Shutdown(cctx.Context) + } + jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) + + if cctx.IsSet("color") { + color.NoColor = !cctx.Bool("color") + } + + if originBefore != nil { + return originBefore(cctx) + } + return nil + } + } + ctx, span := trace.StartSpan(context.Background(), "/cli") + defer span.End() + + interactiveDef := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) + + app := &cli.App{ + Name: "lotus", + Usage: "Filecoin decentralized storage network client", + Version: string(build.NodeUserVersion()), + EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "panic-reports", + EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, + Hidden: true, + Value: "~/.lotus", // should follow --repo default + }, + &cli.BoolFlag{ + // examined in the Before above + Name: "color", + Usage: "use color in display output", + DefaultText: "depends on output being a TTY", + }, + &cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Hidden: true, + Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME + }, + &cli.BoolFlag{ + Name: "interactive", + Usage: "setting to false will disable interactive functionality of commands", + Value: interactiveDef, + }, + &cli.BoolFlag{ + Name: "force-send", + Usage: "if true, will ignore pre-send checks", + }, + cliutil.FlagVeryVerbose, + }, + After: func(c *cli.Context) error { + if r := recover(); r != nil { + // Generate report in LOTUS_PATH and re-raise panic + build.GenerateNodePanicReport(c.String("panic-reports"), c.String("repo"), c.App.Name) + panic(r) + } + return nil + }, + + Commands: append(local, clicommands.Commands...), + } + + app.Setup() + app.Metadata["traceContext"] = ctx + app.Metadata["repoType"] = repo.FullNode + + return app +} diff --git a/cmd/lotus-miner/actor.go b/cli/miner/actor.go similarity index 99% rename from cmd/lotus-miner/actor.go rename to cli/miner/actor.go index 1330d883924..89eae061304 100644 --- a/cmd/lotus-miner/actor.go +++ b/cli/miner/actor.go @@ -1,4 +1,4 @@ -package main +package miner import ( "fmt" diff --git a/cmd/lotus-miner/actor_test.go b/cli/miner/actor_test.go similarity index 99% rename from cmd/lotus-miner/actor_test.go rename to cli/miner/actor_test.go index 39d6a5ba25e..07e14b3c843 100644 --- a/cmd/lotus-miner/actor_test.go +++ b/cli/miner/actor_test.go @@ -1,4 +1,4 @@ -package main +package miner import ( "bytes" diff --git a/cmd/lotus-miner/allinfo_test.go b/cli/miner/allinfo_test.go similarity index 98% rename from cmd/lotus-miner/allinfo_test.go rename to cli/miner/allinfo_test.go index 001ffb3af3d..5141fc582a1 100644 --- a/cmd/lotus-miner/allinfo_test.go +++ b/cli/miner/allinfo_test.go @@ -1,4 +1,4 @@ -package main +package miner import ( "flag" diff --git a/cmd/lotus-miner/backup.go b/cli/miner/backup.go similarity index 96% rename from cmd/lotus-miner/backup.go rename to cli/miner/backup.go index cf8c9f9125c..b7c8cc9de50 100644 --- a/cmd/lotus-miner/backup.go +++ b/cli/miner/backup.go @@ -1,4 +1,4 @@ -package main +package miner import ( "github.com/urfave/cli/v2" diff --git a/cmd/lotus-miner/config.go b/cli/miner/config.go similarity index 99% rename from cmd/lotus-miner/config.go rename to cli/miner/config.go index b7af1b2e51e..16e3061ac0c 100644 --- a/cmd/lotus-miner/config.go +++ b/cli/miner/config.go @@ -1,4 +1,4 @@ -package main +package miner import ( "fmt" diff --git a/cmd/lotus-miner/info.go b/cli/miner/info.go similarity index 99% rename from cmd/lotus-miner/info.go rename to cli/miner/info.go index cdc0cba00bf..ad6f3b21351 100644 --- a/cmd/lotus-miner/info.go +++ b/cli/miner/info.go @@ -1,4 +1,4 @@ -package main +package miner import ( "context" diff --git a/cmd/lotus-miner/info_all.go b/cli/miner/info_all.go similarity index 98% rename from cmd/lotus-miner/info_all.go rename to cli/miner/info_all.go index 253d2befc67..c05cda49cbb 100644 --- a/cmd/lotus-miner/info_all.go +++ b/cli/miner/info_all.go @@ -1,4 +1,4 @@ -package main +package miner import ( "flag" @@ -194,7 +194,7 @@ var infoAllCmd = &cli.Command{ if !_test { fmt.Println("\n#: Goroutines") - if err := lcli.PprofGoroutines.Action(cctx); err != nil { + if err := lcli.PprofGoroutinesCmd.Action(cctx); err != nil { fmt.Println("ERROR: ", err) } } diff --git a/cmd/lotus-miner/init.go b/cli/miner/init.go similarity index 99% rename from cmd/lotus-miner/init.go rename to cli/miner/init.go index 6d6fe7362a5..4f84b9c1c84 100644 --- a/cmd/lotus-miner/init.go +++ b/cli/miner/init.go @@ -1,4 +1,4 @@ -package main +package miner import ( "bytes" diff --git a/cmd/lotus-miner/init_restore.go b/cli/miner/init_restore.go similarity index 99% rename from cmd/lotus-miner/init_restore.go rename to cli/miner/init_restore.go index 05721f6cf56..e8e8970a1e8 100644 --- a/cmd/lotus-miner/init_restore.go +++ b/cli/miner/init_restore.go @@ -1,4 +1,4 @@ -package main +package miner import ( "context" diff --git a/cli/miner/miner.go b/cli/miner/miner.go new file mode 100644 index 00000000000..3d432ec99b3 --- /dev/null +++ b/cli/miner/miner.go @@ -0,0 +1,168 @@ +package miner + +import ( + "context" + "fmt" + + "github.com/fatih/color" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/lib/tracing" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("lotus-miner") + +const ( + FlagMinerRepo = "miner-repo" +) + +// TODO remove after deprecation period +const FlagMinerRepoDeprecation = "storagerepo" + +func App() *cli.App { + api.RunningNodeType = api.NodeMiner + + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + initCmd, + runCmd, + stopCmd, + configCmd, + backupCmd, + lcli.WithCategory("chain", actorCmd), + lcli.WithCategory("chain", infoCmd), + lcli.WithCategory("storage", sectorsCmd), + lcli.WithCategory("storage", provingCmd), + lcli.WithCategory("storage", storageCmd), + lcli.WithCategory("storage", sealingCmd), + } + + jaeger := tracing.SetupJaegerTracing("lotus") + defer func() { + if jaeger != nil { + _ = jaeger.ForceFlush(context.Background()) + } + }() + + for _, cmd := range local { + cmd := cmd + originBefore := cmd.Before + cmd.Before = func(cctx *cli.Context) error { + if jaeger != nil { + _ = jaeger.Shutdown(cctx.Context) + } + jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) + + if cctx.IsSet("color") { + color.NoColor = !cctx.Bool("color") + } + + if originBefore != nil { + return originBefore(cctx) + } + + return nil + } + } + + app := &cli.App{ + Name: "lotus-miner", + Usage: "Filecoin decentralized storage network miner", + Version: string(build.MinerUserVersion()), + EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "actor", + Value: "", + Usage: "specify other actor to query / manipulate", + Aliases: []string{"a"}, + }, + &cli.BoolFlag{ + // examined in the Before above + Name: "color", + Usage: "use color in display output", + DefaultText: "depends on output being a TTY", + }, + &cli.StringFlag{ + Name: "panic-reports", + EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, + Hidden: true, + Value: "~/.lotusminer", // should follow --repo default + }, + &cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Hidden: true, + Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME + }, + &cli.StringFlag{ + Name: FlagMinerRepo, + Aliases: []string{FlagMinerRepoDeprecation}, + EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, + Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME + Usage: fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation), + }, + cliutil.FlagVeryVerbose, + }, + Commands: append(local, lcli.CommonCommands...), + After: func(c *cli.Context) error { + if r := recover(); r != nil { + // Generate report in LOTUS_PATH and re-raise panic + build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagMinerRepo), c.App.Name) + panic(r) + } + return nil + }, + } + app.Setup() + app.Metadata["repoType"] = repo.StorageMiner + return app +} + +func getActorAddress(ctx context.Context, cctx *cli.Context) (maddr address.Address, err error) { + if cctx.IsSet("actor") { + maddr, err = address.NewFromString(cctx.String("actor")) + if err != nil { + return maddr, err + } + return + } + + minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return address.Undef, err + } + defer closer() + + maddr, err = minerApi.ActorAddress(ctx) + if err != nil { + return maddr, xerrors.Errorf("getting actor address: %w", err) + } + + return maddr, nil +} + +func LMActorOrEnvGetter(cctx *cli.Context) (address.Address, error) { + return getActorAddress(cctx.Context, cctx) +} + +func LMActorGetter(cctx *cli.Context) (address.Address, error) { + minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return address.Undef, err + } + defer closer() + + return minerApi.ActorAddress(cctx.Context) +} diff --git a/cmd/lotus-miner/proving.go b/cli/miner/proving.go similarity index 99% rename from cmd/lotus-miner/proving.go rename to cli/miner/proving.go index f5e4d59c0e2..02cefd1b6c2 100644 --- a/cmd/lotus-miner/proving.go +++ b/cli/miner/proving.go @@ -1,4 +1,4 @@ -package main +package miner import ( "encoding/json" diff --git a/cmd/lotus-miner/run.go b/cli/miner/run.go similarity index 99% rename from cmd/lotus-miner/run.go rename to cli/miner/run.go index be670287e1a..e84e098799b 100644 --- a/cmd/lotus-miner/run.go +++ b/cli/miner/run.go @@ -1,4 +1,4 @@ -package main +package miner import ( "fmt" diff --git a/cmd/lotus-miner/sealing.go b/cli/miner/sealing.go similarity index 99% rename from cmd/lotus-miner/sealing.go rename to cli/miner/sealing.go index 6d5f4ed7cbb..06b669a9964 100644 --- a/cmd/lotus-miner/sealing.go +++ b/cli/miner/sealing.go @@ -1,4 +1,4 @@ -package main +package miner import ( "encoding/hex" diff --git a/cmd/lotus-miner/sectors.go b/cli/miner/sectors.go similarity index 99% rename from cmd/lotus-miner/sectors.go rename to cli/miner/sectors.go index fa23e5f268f..c9eb8e777e5 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cli/miner/sectors.go @@ -1,4 +1,4 @@ -package main +package miner import ( "bufio" diff --git a/cmd/lotus-miner/stop.go b/cli/miner/stop.go similarity index 96% rename from cmd/lotus-miner/stop.go rename to cli/miner/stop.go index 186a156169e..f5bfeaa4889 100644 --- a/cmd/lotus-miner/stop.go +++ b/cli/miner/stop.go @@ -1,4 +1,4 @@ -package main +package miner import ( _ "net/http/pprof" diff --git a/cmd/lotus-miner/storage.go b/cli/miner/storage.go similarity index 99% rename from cmd/lotus-miner/storage.go rename to cli/miner/storage.go index 609f2ead674..2a72813ef1a 100644 --- a/cmd/lotus-miner/storage.go +++ b/cli/miner/storage.go @@ -1,4 +1,4 @@ -package main +package miner import ( "context" diff --git a/cli/pprof.go b/cli/pprof.go index ae4016e1108..28b40dad170 100644 --- a/cli/pprof.go +++ b/cli/pprof.go @@ -15,11 +15,11 @@ var PprofCmd = &cli.Command{ Name: "pprof", Hidden: true, Subcommands: []*cli.Command{ - PprofGoroutines, + PprofGoroutinesCmd, }, } -var PprofGoroutines = &cli.Command{ +var PprofGoroutinesCmd = &cli.Command{ Name: "goroutines", Usage: "Get goroutine stacks", Action: func(cctx *cli.Context) error { diff --git a/cmd/lotus-worker/info.go b/cli/worker/info.go similarity index 99% rename from cmd/lotus-worker/info.go rename to cli/worker/info.go index 338345c2857..a8197141928 100644 --- a/cmd/lotus-worker/info.go +++ b/cli/worker/info.go @@ -1,4 +1,4 @@ -package main +package worker import ( "fmt" diff --git a/cmd/lotus-worker/resources.go b/cli/worker/resources.go similarity index 98% rename from cmd/lotus-worker/resources.go rename to cli/worker/resources.go index 45fd01e702b..6d79f9523be 100644 --- a/cmd/lotus-worker/resources.go +++ b/cli/worker/resources.go @@ -1,4 +1,4 @@ -package main +package worker import ( "fmt" diff --git a/cmd/lotus-worker/cli.go b/cli/worker/set.go similarity index 98% rename from cmd/lotus-worker/cli.go rename to cli/worker/set.go index 393ece18c86..bdbe15ef71c 100644 --- a/cmd/lotus-worker/cli.go +++ b/cli/worker/set.go @@ -1,4 +1,4 @@ -package main +package worker import ( "fmt" diff --git a/cmd/lotus-worker/storage.go b/cli/worker/storage.go similarity index 99% rename from cmd/lotus-worker/storage.go rename to cli/worker/storage.go index 8da35c94418..42e7b5ae3c1 100644 --- a/cmd/lotus-worker/storage.go +++ b/cli/worker/storage.go @@ -1,4 +1,4 @@ -package main +package worker import ( "encoding/json" diff --git a/cmd/lotus-worker/tasks.go b/cli/worker/tasks.go similarity index 99% rename from cmd/lotus-worker/tasks.go rename to cli/worker/tasks.go index 73ca5eb0a01..a67a53f0221 100644 --- a/cmd/lotus-worker/tasks.go +++ b/cli/worker/tasks.go @@ -1,4 +1,4 @@ -package main +package worker import ( "context" diff --git a/cli/worker/worker.go b/cli/worker/worker.go new file mode 100644 index 00000000000..64580821284 --- /dev/null +++ b/cli/worker/worker.go @@ -0,0 +1,874 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/google/uuid" + "github.com/ipfs/go-datastore/namespace" + logging "github.com/ipfs/go-log/v2" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + "github.com/urfave/cli/v2" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-jsonrpc/auth" + "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-statestore" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/build/buildconstants" + lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" + "github.com/filecoin-project/lotus/lib/ulimit" + "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" +) + +const FlagWorkerRepo = "worker-repo" + +// TODO remove after deprecation period +const FlagWorkerRepoDeprecation = "workerrepo" + +var log = logging.Logger("worker") + +func App() *cli.App { + local := []*cli.Command{ + runCmd, + stopCmd, + infoCmd, + storageCmd, + setCmd, + waitQuietCmd, + resourcesCmd, + tasksCmd, + } + + app := &cli.App{ + Name: "lotus-worker", + Usage: "Remote miner worker", + Version: string(build.MinerUserVersion()), + EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: FlagWorkerRepo, + Aliases: []string{FlagWorkerRepoDeprecation}, + EnvVars: []string{"LOTUS_WORKER_PATH", "WORKER_PATH"}, + Value: "~/.lotusworker", // TODO: Consider XDG_DATA_HOME + Usage: fmt.Sprintf("Specify worker repo path. flag %s and env WORKER_PATH are DEPRECATION, will REMOVE SOON", FlagWorkerRepoDeprecation), + }, + &cli.StringFlag{ + Name: "panic-reports", + EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, + Hidden: true, + Value: "~/.lotusworker", // should follow --repo default + }, + &cli.StringFlag{ + Name: "miner-repo", + Aliases: []string{"storagerepo"}, + EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, + Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME + Usage: fmt.Sprintf("Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are DEPRECATION, will REMOVE SOON"), + }, + &cli.BoolFlag{ + Name: "enable-gpu-proving", + Usage: "enable use of GPU for mining operations", + Value: true, + EnvVars: []string{"LOTUS_WORKER_ENABLE_GPU_PROVING"}, + }, + }, + + After: func(c *cli.Context) error { + if r := recover(); r != nil { + // Generate report in LOTUS_PANIC_REPORT_PATH and re-raise panic + build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagWorkerRepo), c.App.Name) + panic(r) + } + return nil + }, + Commands: local, + } + + app.Setup() + app.Metadata["repoType"] = repo.Worker + return app +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "Start lotus worker", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Usage: "host address and port the worker api will listen on", + Value: "0.0.0.0:3456", + EnvVars: []string{"LOTUS_WORKER_LISTEN"}, + }, + &cli.StringFlag{ + Name: "address", + Hidden: true, + }, + &cli.BoolFlag{ + Name: "no-local-storage", + Usage: "don't use storageminer repo for sector storage", + EnvVars: []string{"LOTUS_WORKER_NO_LOCAL_STORAGE"}, + }, + &cli.BoolFlag{ + Name: "no-swap", + Usage: "don't use swap", + Value: false, + EnvVars: []string{"LOTUS_WORKER_NO_SWAP"}, + }, + &cli.StringFlag{ + Name: "name", + Usage: "custom worker name", + EnvVars: []string{"LOTUS_WORKER_NAME"}, + DefaultText: "hostname", + }, + &cli.BoolFlag{ + Name: "addpiece", + Usage: "enable addpiece", + Value: true, + EnvVars: []string{"LOTUS_WORKER_ADDPIECE"}, + }, + &cli.BoolFlag{ + Name: "precommit1", + Usage: "enable precommit1", + Value: true, + EnvVars: []string{"LOTUS_WORKER_PRECOMMIT1"}, + }, + &cli.BoolFlag{ + Name: "unseal", + Usage: "enable unsealing", + Value: true, + EnvVars: []string{"LOTUS_WORKER_UNSEAL"}, + }, + &cli.BoolFlag{ + Name: "precommit2", + Usage: "enable precommit2", + Value: true, + EnvVars: []string{"LOTUS_WORKER_PRECOMMIT2"}, + }, + &cli.BoolFlag{ + Name: "commit", + Usage: "enable commit", + Value: true, + EnvVars: []string{"LOTUS_WORKER_COMMIT"}, + }, + &cli.BoolFlag{ + Name: "replica-update", + Usage: "enable replica update", + Value: true, + EnvVars: []string{"LOTUS_WORKER_REPLICA_UPDATE"}, + }, + &cli.BoolFlag{ + Name: "prove-replica-update2", + Usage: "enable prove replica update 2", + Value: true, + EnvVars: []string{"LOTUS_WORKER_PROVE_REPLICA_UPDATE2"}, + }, + &cli.BoolFlag{ + Name: "regen-sector-key", + Usage: "enable regen sector key", + Value: true, + EnvVars: []string{"LOTUS_WORKER_REGEN_SECTOR_KEY"}, + }, + &cli.BoolFlag{ + Name: "sector-download", + Usage: "enable external sector data download", + Value: false, + EnvVars: []string{"LOTUS_WORKER_SECTOR_DOWNLOAD"}, + }, + &cli.BoolFlag{ + Name: "windowpost", + Usage: "enable window post", + Value: false, + EnvVars: []string{"LOTUS_WORKER_WINDOWPOST"}, + }, + &cli.BoolFlag{ + Name: "winningpost", + Usage: "enable winning post", + Value: false, + EnvVars: []string{"LOTUS_WORKER_WINNINGPOST"}, + }, + &cli.BoolFlag{ + Name: "no-default", + Usage: "disable all default compute tasks, use the worker for storage/fetching only", + Value: false, + EnvVars: []string{"LOTUS_WORKER_NO_DEFAULT"}, + }, + &cli.IntFlag{ + Name: "parallel-fetch-limit", + Usage: "maximum fetch operations to run in parallel", + Value: 5, + EnvVars: []string{"LOTUS_WORKER_PARALLEL_FETCH_LIMIT"}, + }, + &cli.IntFlag{ + Name: "post-parallel-reads", + Usage: "maximum number of parallel challenge reads (0 = no limit)", + Value: 32, + EnvVars: []string{"LOTUS_WORKER_POST_PARALLEL_READS"}, + }, + &cli.DurationFlag{ + Name: "post-read-timeout", + Usage: "time limit for reading PoSt challenges (0 = no limit)", + Value: 0, + EnvVars: []string{"LOTUS_WORKER_POST_READ_TIMEOUT"}, + }, + &cli.StringFlag{ + Name: "timeout", + Usage: "used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function", + Value: "30m", + EnvVars: []string{"LOTUS_WORKER_TIMEOUT"}, + }, + &cli.StringFlag{ + Name: "http-server-timeout", + Value: "30s", + }, + &cli.BoolFlag{ + Name: "data-cid", + Usage: "Run the data-cid task. true|false", + Value: true, + DefaultText: "inherits --addpiece", + }, + &cli.StringFlag{ + Name: "external-pc2", + Usage: "command for computing PC2 externally", + }, + }, + Description: `Run lotus-worker. + +--external-pc2 can be used to compute the PreCommit2 inputs externally. +The flag behaves similarly to the related lotus-worker flag, using it in +lotus-bench may be useful for testing if the external PreCommit2 command is +invoked correctly. + +The command will be called with a number of environment variables set: +* EXTSEAL_PC2_SECTOR_NUM: the sector number +* EXTSEAL_PC2_SECTOR_MINER: the miner id +* EXTSEAL_PC2_PROOF_TYPE: the proof type +* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes +* EXTSEAL_PC2_CACHE: the path to the cache directory +* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) +* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) + +The command is expected to: +* Create cache sc-02-data-tree-r* files +* Create cache sc-02-data-tree-c* files +* Create cache p_aux / t_aux files +* Transform the sealed file in place + +Example invocation of lotus-bench as external executor: +'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT' +`, + Before: func(cctx *cli.Context) error { + if cctx.IsSet("address") { + log.Warnf("The '--address' flag is deprecated, it has been replaced by '--listen'") + if err := cctx.Set("listen", cctx.String("address")); err != nil { + return err + } + } + + return nil + }, + Action: func(cctx *cli.Context) error { + log.Info("Starting lotus worker") + + if !cctx.Bool("enable-gpu-proving") { + if err := os.Setenv("BELLMAN_NO_GPU", "true"); err != nil { + return xerrors.Errorf("could not set no-gpu env: %+v", err) + } + } + + // ensure tmpdir exists + td := os.TempDir() + if err := os.MkdirAll(td, 0755); err != nil { + return xerrors.Errorf("ensuring temp dir %s exists: %w", td, err) + } + + // Check file descriptor limit + limit, _, err := ulimit.GetLimit() + switch { + case err == ulimit.ErrUnsupported: + log.Errorw("checking file descriptor limit failed", "error", err) + case err != nil: + return xerrors.Errorf("checking fd limit: %w", err) + default: + if limit < buildconstants.MinerFDLimit { + return xerrors.Errorf("soft file descriptor limit (ulimit -n) too low, want %d, current %d", buildconstants.MinerFDLimit, limit) + } + } + + // Check DC-environment variable + sectorSizes := []string{"2KiB", "8MiB", "512MiB", "32GiB", "64GiB"} + resourcesType := reflect.TypeOf(storiface.Resources{}) + + for _, sectorSize := range sectorSizes { + for i := 0; i < resourcesType.NumField(); i++ { + field := resourcesType.Field(i) + envName := field.Tag.Get("envname") + if envName != "" { + // Check if DC_[SectorSize]_[ResourceRestriction] is set + envVar, ok := os.LookupEnv("DC_" + sectorSize + "_" + envName) + if ok { + // If it is set, convert it to DC_[ResourceRestriction] + err := os.Setenv("DC_"+envName, envVar) + if err != nil { + log.Fatalf("Error setting environment variable: %v", err) + } + log.Warnf("Converted DC_%s_%s to DC_%s, because DC is a sector-size independent job", sectorSize, envName, envName) + } + } + } + } + + // Connect to storage-miner + ctx := lcli.ReqContext(cctx) + + // Create a new context with cancel function + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Listen for interrupt signals + go func() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + <-c + cancel() + }() + + var nodeApi api.StorageMiner + var closer func() + for { + nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) + if err == nil { + _, err = nodeApi.Version(ctx) + if err == nil { + break + } + } + fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err) + select { + case <-ctx.Done(): + return xerrors.New("Interrupted by user") + case <-time.After(time.Second): + } + } + defer closer() + // Register all metric views + if err := view.Register( + metrics.DefaultViews..., + ); err != nil { + log.Fatalf("Cannot register the view: %v", err) + } + + v, err := nodeApi.Version(ctx) + if err != nil { + return err + } + if v.APIVersion != api.MinerAPIVersion0 { + return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.APIVersion{APIVersion: api.MinerAPIVersion0}) + } + log.Infof("Remote version %s", v) + + // Check params + + act, err := nodeApi.ActorAddress(ctx) + if err != nil { + return err + } + ssize, err := nodeApi.ActorSectorSize(ctx, act) + if err != nil { + return err + } + + var taskTypes []sealtasks.TaskType + var workerType string + var needParams bool + + if cctx.Bool("windowpost") { + needParams = true + workerType = sealtasks.WorkerWindowPoSt + taskTypes = append(taskTypes, sealtasks.TTGenerateWindowPoSt) + } + if cctx.Bool("winningpost") { + needParams = true + workerType = sealtasks.WorkerWinningPoSt + taskTypes = append(taskTypes, sealtasks.TTGenerateWinningPoSt) + } + + if workerType == "" { + taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeUnsealed, sealtasks.TTFinalizeReplicaUpdate) + + if !cctx.Bool("no-default") { + workerType = sealtasks.WorkerSealing + } + } + + ttDataCidDefault := false + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") { + taskTypes = append(taskTypes, sealtasks.TTAddPiece) + ttDataCidDefault = true + } + if workerType == sealtasks.WorkerSealing { + if cctx.IsSet("data-cid") { + if cctx.Bool("data-cid") { + taskTypes = append(taskTypes, sealtasks.TTDataCid) + } + } else if ttDataCidDefault { + taskTypes = append(taskTypes, sealtasks.TTDataCid) + } + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("sector-download")) && cctx.Bool("sector-download") { + taskTypes = append(taskTypes, sealtasks.TTDownloadSector) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit1")) && cctx.Bool("precommit1") { + taskTypes = append(taskTypes, sealtasks.TTPreCommit1) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("unseal")) && cctx.Bool("unseal") { + taskTypes = append(taskTypes, sealtasks.TTUnseal) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit2")) && cctx.Bool("precommit2") { + taskTypes = append(taskTypes, sealtasks.TTPreCommit2) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("commit")) && cctx.Bool("commit") { + needParams = true + taskTypes = append(taskTypes, sealtasks.TTCommit2) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("replica-update")) && cctx.Bool("replica-update") { + taskTypes = append(taskTypes, sealtasks.TTReplicaUpdate) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("prove-replica-update2")) && cctx.Bool("prove-replica-update2") { + needParams = true + taskTypes = append(taskTypes, sealtasks.TTProveReplicaUpdate2) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("regen-sector-key")) && cctx.Bool("regen-sector-key") { + taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey) + } + + if cctx.Bool("no-default") && workerType == "" { + workerType = sealtasks.WorkerSealing + } + + if len(taskTypes) == 0 { + return xerrors.Errorf("no task types specified") + } + for _, taskType := range taskTypes { + if taskType.WorkerType() != workerType { + return xerrors.Errorf("expected all task types to be for %s worker, but task %s is for %s worker", workerType, taskType, taskType.WorkerType()) + } + } + + if needParams { + if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(ssize)); err != nil { + return xerrors.Errorf("get params: %w", err) + } + } + + // Open repo + + repoPath := cctx.String(FlagWorkerRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + if err := r.Init(repo.Worker); err != nil { + return err + } + + lr, err := r.Lock(repo.Worker) + if err != nil { + return err + } + + var localPaths []storiface.LocalPath + + if !cctx.Bool("no-local-storage") { + b, err := json.MarshalIndent(&storiface.LocalStorageMeta{ + ID: storiface.ID(uuid.New().String()), + Weight: 10, + CanSeal: true, + CanStore: false, + }, "", " ") + if err != nil { + return xerrors.Errorf("marshaling storage config: %w", err) + } + + if err := os.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { + return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) + } + + localPaths = append(localPaths, storiface.LocalPath{ + Path: lr.Path(), + }) + } + + if err := lr.SetStorage(func(sc *storiface.StorageConfig) { + sc.StoragePaths = append(sc.StoragePaths, localPaths...) + }); err != nil { + return xerrors.Errorf("set storage config: %w", err) + } + + { + // init datastore for r.Exists + _, err := lr.Datastore(context.Background(), "/metadata") + if err != nil { + return err + } + } + if err := lr.Close(); err != nil { + return xerrors.Errorf("close repo: %w", err) + } + } + + lr, err := r.Lock(repo.Worker) + if err != nil { + return err + } + defer func() { + if err := lr.Close(); err != nil { + log.Error("closing repo", err) + } + }() + ds, err := lr.Datastore(context.Background(), "/metadata") + if err != nil { + return err + } + + log.Info("Opening local storage; connecting to master") + const unspecifiedAddress = "0.0.0.0" + + address := cctx.String("listen") + host, port, err := net.SplitHostPort(address) + if err != nil { + return err + } + + if ip := net.ParseIP(host); ip != nil { + if ip.String() == unspecifiedAddress { + timeout, err := time.ParseDuration(cctx.String("timeout")) + if err != nil { + return err + } + rip, err := extractRoutableIP(timeout) + if err != nil { + return err + } + host = rip + } + } + + var newAddress string + + // Check if the IP address is IPv6 + ip := net.ParseIP(host) + if ip.To4() == nil && ip.To16() != nil { + newAddress = "[" + host + "]:" + port + } else { + newAddress = host + ":" + port + } + + localStore, err := paths.NewLocal(ctx, lr, nodeApi, []string{"http://" + newAddress + "/remote"}) + if err != nil { + return err + } + + // Setup remote sector store + sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + if err != nil { + return xerrors.Errorf("could not get api info: %w", err) + } + + remote := paths.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"), + &paths.DefaultPartialFileHandler{}) + + fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}} + remoteHandler := func(w http.ResponseWriter, r *http.Request) { + if !auth.HasPerm(r.Context(), nil, api.PermAdmin) { + w.WriteHeader(401) + _ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"}) + return + } + + fh.ServeHTTP(w, r) + } + + // Parse ffi executor flags + + var ffiOpts []ffiwrapper.FFIWrapperOpt + + if cctx.IsSet("external-pc2") { + extSeal := ffiwrapper.ExternalSealer{ + PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")), + } + + ffiOpts = append(ffiOpts, ffiwrapper.WithExternalSealCalls(extSeal)) + } + + // Create / expose the worker + + wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix)) + + workerApi := &sealworker.Worker{ + LocalWorker: sealer.NewLocalWorkerWithExecutor( + sealer.FFIExec(ffiOpts...), + sealer.WorkerConfig{ + TaskTypes: taskTypes, + NoSwap: cctx.Bool("no-swap"), + MaxParallelChallengeReads: cctx.Int("post-parallel-reads"), + ChallengeReadTimeout: cctx.Duration("post-read-timeout"), + Name: cctx.String("name"), + }, os.LookupEnv, remote, localStore, nodeApi, nodeApi, wsts), + LocalStore: localStore, + Storage: lr, + } + + log.Info("Setting up control endpoint at " + newAddress) + + timeout, err := time.ParseDuration(cctx.String("http-server-timeout")) + if err != nil { + return xerrors.Errorf("invalid time string %s: %x", cctx.String("http-server-timeout"), err) + } + + srv := &http.Server{ + Handler: sealworker.WorkerHandler(nodeApi.AuthVerify, remoteHandler, workerApi, true), + ReadHeaderTimeout: timeout, + BaseContext: func(listener net.Listener) context.Context { + ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker")) + return ctx + }, + } + + go func() { + <-ctx.Done() + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + + nl, err := net.Listen("tcp", newAddress) + if err != nil { + return err + } + + { + a, err := net.ResolveTCPAddr("tcp", newAddress) + if err != nil { + return xerrors.Errorf("parsing address: %w", err) + } + + ma, err := manet.FromNetAddr(a) + if err != nil { + return xerrors.Errorf("creating api multiaddress: %w", err) + } + + if err := lr.SetAPIEndpoint(ma); err != nil { + return xerrors.Errorf("setting api endpoint: %w", err) + } + + ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + if err != nil { + return xerrors.Errorf("could not get miner API info: %w", err) + } + + // TODO: ideally this would be a token with some permissions dropped + if err := lr.SetAPIToken(ainfo.Token); err != nil { + return xerrors.Errorf("setting api token: %w", err) + } + } + + minerSession, err := nodeApi.Session(ctx) + if err != nil { + return xerrors.Errorf("getting miner session: %w", err) + } + + waitQuietCh := func() chan struct{} { + out := make(chan struct{}) + go func() { + workerApi.LocalWorker.WaitQuiet() + close(out) + }() + return out + } + + go func() { + heartbeats := time.NewTicker(paths.HeartbeatInterval) + defer heartbeats.Stop() + + var redeclareStorage bool + var readyCh chan struct{} + for { + // If we're reconnecting, redeclare storage first + if redeclareStorage { + log.Info("Redeclaring local storage") + + if err := localStore.Redeclare(ctx, nil, false); err != nil { + log.Errorf("Redeclaring local storage failed: %+v", err) + + select { + case <-ctx.Done(): + return // graceful shutdown + case <-heartbeats.C: + } + continue + } + } + + // TODO: we could get rid of this, but that requires tracking resources for restarted tasks correctly + if readyCh == nil { + log.Info("Making sure no local tasks are running") + readyCh = waitQuietCh() + } + + for { + curSession, err := nodeApi.Session(ctx) + if err != nil { + log.Errorf("heartbeat: checking remote session failed: %+v", err) + } else { + if curSession != minerSession { + minerSession = curSession + break + } + } + + select { + case <-readyCh: + if err := nodeApi.WorkerConnect(ctx, "http://"+newAddress+"/rpc/v0"); err != nil { + log.Errorf("Registering worker failed: %+v", err) + cancel() + return + } + + log.Info("Worker registered successfully, waiting for tasks") + + readyCh = nil + case <-heartbeats.C: + case <-ctx.Done(): + return // graceful shutdown + } + } + + log.Errorf("LOTUS-MINER CONNECTION LOST") + + redeclareStorage = true + } + }() + + go func() { + <-workerApi.Done() + // Wait 20s to allow the miner to unregister the worker on next heartbeat + time.Sleep(20 * time.Second) + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + + return srv.Serve(nl) + }, +} + +var stopCmd = &cli.Command{ + Name: "stop", + Usage: "Stop a running lotus worker", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetWorkerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := lcli.ReqContext(cctx) + + // Detach any storage associated with this worker + err = api.StorageDetachAll(ctx) + if err != nil { + return err + } + + err = api.Shutdown(ctx) + if err != nil { + return err + } + + return nil + }, +} + +func extractRoutableIP(timeout time.Duration) (string, error) { + minerMultiAddrKey := "MINER_API_INFO" + deprecatedMinerMultiAddrKey := "STORAGE_API_INFO" + env, ok := os.LookupEnv(minerMultiAddrKey) + if !ok { + _, ok = os.LookupEnv(deprecatedMinerMultiAddrKey) + if ok { + log.Warnf("Using a deprecated env(%s) value, please use env(%s) instead.", deprecatedMinerMultiAddrKey, minerMultiAddrKey) + } + return "", xerrors.New("MINER_API_INFO environment variable required to extract IP") + } + + // Splitting the env to separate the JWT from the multiaddress + splitEnv := strings.SplitN(env, ":", 2) + if len(splitEnv) < 2 { + return "", xerrors.Errorf("invalid MINER_API_INFO format") + } + // Only take the multiaddress part + maddrStr := splitEnv[1] + + maddr, err := multiaddr.NewMultiaddr(maddrStr) + if err != nil { + return "", err + } + + minerIP, _ := maddr.ValueForProtocol(multiaddr.P_IP6) + if minerIP == "" { + minerIP, _ = maddr.ValueForProtocol(multiaddr.P_IP4) + } + minerPort, _ := maddr.ValueForProtocol(multiaddr.P_TCP) + + // Format the address appropriately + addressToDial := net.JoinHostPort(minerIP, minerPort) + + conn, err := net.DialTimeout("tcp", addressToDial, timeout) + if err != nil { + return "", err + } + + defer func() { + if cerr := conn.Close(); cerr != nil { + log.Errorf("Error closing connection: %v", cerr) + } + }() + + localAddr := conn.LocalAddr().(*net.TCPAddr) + return localAddr.IP.String(), nil +} diff --git a/cmd/lotus-miner/main.go b/cmd/lotus-miner/main.go index 76b8c0deb05..55ab1e6a132 100644 --- a/cmd/lotus-miner/main.go +++ b/cmd/lotus-miner/main.go @@ -1,168 +1,11 @@ package main import ( - "context" - "fmt" - - "github.com/fatih/color" - logging "github.com/ipfs/go-log/v2" - "github.com/urfave/cli/v2" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" - cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/lib/lotuslog" - "github.com/filecoin-project/lotus/lib/tracing" - "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/cli/miner" ) -var log = logging.Logger("main") - -const ( - FlagMinerRepo = "miner-repo" -) - -// TODO remove after deprecation period -const FlagMinerRepoDeprecation = "storagerepo" - func main() { - api.RunningNodeType = api.NodeMiner - - lotuslog.SetupLogLevels() - - local := []*cli.Command{ - initCmd, - runCmd, - stopCmd, - configCmd, - backupCmd, - lcli.WithCategory("chain", actorCmd), - lcli.WithCategory("chain", infoCmd), - lcli.WithCategory("storage", sectorsCmd), - lcli.WithCategory("storage", provingCmd), - lcli.WithCategory("storage", storageCmd), - lcli.WithCategory("storage", sealingCmd), - } - - jaeger := tracing.SetupJaegerTracing("lotus") - defer func() { - if jaeger != nil { - _ = jaeger.ForceFlush(context.Background()) - } - }() - - for _, cmd := range local { - cmd := cmd - originBefore := cmd.Before - cmd.Before = func(cctx *cli.Context) error { - if jaeger != nil { - _ = jaeger.Shutdown(cctx.Context) - } - jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) - - if cctx.IsSet("color") { - color.NoColor = !cctx.Bool("color") - } - - if originBefore != nil { - return originBefore(cctx) - } - - return nil - } - } - - app := &cli.App{ - Name: "lotus-miner", - Usage: "Filecoin decentralized storage network miner", - Version: string(build.MinerUserVersion()), - EnableBashCompletion: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "actor", - Value: "", - Usage: "specify other actor to query / manipulate", - Aliases: []string{"a"}, - }, - &cli.BoolFlag{ - // examined in the Before above - Name: "color", - Usage: "use color in display output", - DefaultText: "depends on output being a TTY", - }, - &cli.StringFlag{ - Name: "panic-reports", - EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, - Hidden: true, - Value: "~/.lotusminer", // should follow --repo default - }, - &cli.StringFlag{ - Name: "repo", - EnvVars: []string{"LOTUS_PATH"}, - Hidden: true, - Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME - }, - &cli.StringFlag{ - Name: FlagMinerRepo, - Aliases: []string{FlagMinerRepoDeprecation}, - EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, - Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME - Usage: fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation), - }, - cliutil.FlagVeryVerbose, - }, - Commands: append(local, lcli.CommonCommands...), - After: func(c *cli.Context) error { - if r := recover(); r != nil { - // Generate report in LOTUS_PATH and re-raise panic - build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagMinerRepo), c.App.Name) - panic(r) - } - return nil - }, - } - app.Setup() - app.Metadata["repoType"] = repo.StorageMiner + app := miner.App() lcli.RunApp(app) } - -func getActorAddress(ctx context.Context, cctx *cli.Context) (maddr address.Address, err error) { - if cctx.IsSet("actor") { - maddr, err = address.NewFromString(cctx.String("actor")) - if err != nil { - return maddr, err - } - return - } - - minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return address.Undef, err - } - defer closer() - - maddr, err = minerApi.ActorAddress(ctx) - if err != nil { - return maddr, xerrors.Errorf("getting actor address: %w", err) - } - - return maddr, nil -} - -func LMActorOrEnvGetter(cctx *cli.Context) (address.Address, error) { - return getActorAddress(cctx.Context, cctx) -} - -func LMActorGetter(cctx *cli.Context) (address.Address, error) { - minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return address.Undef, err - } - defer closer() - - return minerApi.ActorAddress(cctx.Context) -} diff --git a/cmd/lotus-shed/msg.go b/cmd/lotus-shed/msg.go index 35f8eed35c1..4de7789a85c 100644 --- a/cmd/lotus-shed/msg.go +++ b/cmd/lotus-shed/msg.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" + "sort" "github.com/fatih/color" "github.com/ipfs/go-cid" @@ -19,6 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/consensus" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/tablewriter" ) var msgCmd = &cli.Command{ @@ -27,10 +30,19 @@ var msgCmd = &cli.Command{ Usage: "Translate message between various formats", ArgsUsage: "Message in any form", Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "show-message", + Usage: "Print the message details", + Value: true, + }, &cli.BoolFlag{ Name: "exec-trace", Usage: "Print the execution trace", }, + &cli.BoolFlag{ + Name: "gas-stats", + Usage: "Print a summary of gas charges", + }, }, Action: func(cctx *cli.Context) error { if cctx.NArg() != 1 { @@ -82,16 +94,61 @@ var msgCmd = &cli.Command{ fmt.Printf("Return: %x\n", res.MsgRct.Return) fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed) } + + if cctx.Bool("gas-stats") { + var printTrace func(descPfx string, trace types.ExecutionTrace) error + printTrace = func(descPfx string, trace types.ExecutionTrace) error { + typ := "Message" + if descPfx != "" { + typ = "Subcall" + } + _, _ = fmt.Fprintln(cctx.App.Writer, color.New(color.Bold).Sprint(fmt.Sprintf("%s (%s%s) gas charges:", typ, descPfx, trace.Msg.To))) + if err := statsTable(cctx.App.Writer, trace, false); err != nil { + return err + } + for _, subtrace := range trace.Subcalls { + _, _ = fmt.Fprintln(cctx.App.Writer) + if err := printTrace(descPfx+trace.Msg.To.String()+"➜", subtrace); err != nil { + return err + } + } + return nil + } + if err := printTrace("", res.ExecutionTrace); err != nil { + return err + } + if len(res.ExecutionTrace.Subcalls) > 0 { + _, _ = fmt.Fprintln(cctx.App.Writer) + _, _ = fmt.Fprintln(cctx.App.Writer, color.New(color.Bold).Sprint("Total gas charges:")) + if err := statsTable(cctx.App.Writer, res.ExecutionTrace, true); err != nil { + return err + } + perCallTrace := gasTracesPerCall(res.ExecutionTrace) + _, _ = fmt.Fprintln(cctx.App.Writer) + _, _ = fmt.Fprintln(cctx.App.Writer, color.New(color.Bold).Sprint("Gas charges per call:")) + if err := statsTable(cctx.App.Writer, perCallTrace, false); err != nil { + return err + } + } + } } - switch msg := msg.(type) { - case *types.SignedMessage: - return printSignedMessage(cctx, msg) - case *types.Message: - return printMessage(cctx, msg) - default: - return xerrors.Errorf("this error message can't be printed") + if cctx.Bool("show-message") { + switch msg := msg.(type) { + case *types.SignedMessage: + if err := printSignedMessage(cctx, msg); err != nil { + return err + } + case *types.Message: + if err := printMessage(cctx, msg); err != nil { + return err + } + default: + return xerrors.Errorf("this error message can't be printed") + } } + + return nil }, } @@ -335,3 +392,105 @@ func messageFromCID(cctx *cli.Context, c cid.Cid) (types.ChainMsg, error) { return messageFromBytes(cctx, msgb) } + +type gasTally struct { + storageGas int64 + computeGas int64 + count int +} + +func accumGasTallies(charges map[string]*gasTally, totals *gasTally, trace types.ExecutionTrace, recurse bool) { + for _, charge := range trace.GasCharges { + name := charge.Name + if _, ok := charges[name]; !ok { + charges[name] = &gasTally{} + } + charges[name].computeGas += charge.ComputeGas + charges[name].storageGas += charge.StorageGas + charges[name].count++ + totals.computeGas += charge.ComputeGas + totals.storageGas += charge.StorageGas + totals.count++ + } + if recurse { + for _, subtrace := range trace.Subcalls { + accumGasTallies(charges, totals, subtrace, recurse) + } + } +} + +func statsTable(out io.Writer, trace types.ExecutionTrace, recurse bool) error { + tw := tablewriter.New( + tablewriter.Col("Type"), + tablewriter.Col("Count", tablewriter.RightAlign()), + tablewriter.Col("Storage Gas", tablewriter.RightAlign()), + tablewriter.Col("S%", tablewriter.RightAlign()), + tablewriter.Col("Compute Gas", tablewriter.RightAlign()), + tablewriter.Col("C%", tablewriter.RightAlign()), + tablewriter.Col("Total Gas", tablewriter.RightAlign()), + tablewriter.Col("T%", tablewriter.RightAlign()), + ) + + totals := &gasTally{} + charges := make(map[string]*gasTally) + accumGasTallies(charges, totals, trace, recurse) + + // Sort by name + names := make([]string, 0, len(charges)) + for name := range charges { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + charge := charges[name] + tw.Write(map[string]interface{}{ + "Type": name, + "Count": charge.count, + "Storage Gas": charge.storageGas, + "S%": fmt.Sprintf("%.2f", float64(charge.storageGas)/float64(totals.storageGas)*100), + "Compute Gas": charge.computeGas, + "C%": fmt.Sprintf("%.2f", float64(charge.computeGas)/float64(totals.computeGas)*100), + "Total Gas": charge.storageGas + charge.computeGas, + "T%": fmt.Sprintf("%.2f", float64(charge.storageGas+charge.computeGas)/float64(totals.storageGas+totals.computeGas)*100), + }) + } + tw.Write(map[string]interface{}{ + "Type": "Total", + "Count": totals.count, + "Storage Gas": totals.storageGas, + "S%": "100.00", + "Compute Gas": totals.computeGas, + "C%": "100.00", + "Total Gas": totals.storageGas + totals.computeGas, + "T%": "100.00", + }) + return tw.Flush(out, tablewriter.WithBorders()) +} + +// Takes an execution trace and returns a new trace that groups all the gas charges by the message +// they were charged in, with the gas charges named per message; the output is partial and only +// suitable for calling statsTable() with. +func gasTracesPerCall(inTrace types.ExecutionTrace) types.ExecutionTrace { + outTrace := types.ExecutionTrace{ + GasCharges: []*types.GasTrace{}, + } + count := 1 + var accum func(name string, trace types.ExecutionTrace) + accum = func(name string, trace types.ExecutionTrace) { + totals := &gasTally{} + charges := make(map[string]*gasTally) + accumGasTallies(charges, totals, trace, false) + outTrace.GasCharges = append(outTrace.GasCharges, &types.GasTrace{ + Name: fmt.Sprintf("#%d %s", count, name), + ComputeGas: totals.computeGas, + StorageGas: totals.storageGas, + }) + count++ + for _, subtrace := range trace.Subcalls { + accum(name+"➜"+subtrace.Msg.To.String(), subtrace) + } + } + accum(inTrace.Msg.To.String(), inTrace) + return outTrace +} diff --git a/cmd/lotus-worker/main.go b/cmd/lotus-worker/main.go index 6d70b8240e8..7b120499a7d 100644 --- a/cmd/lotus-worker/main.go +++ b/cmd/lotus-worker/main.go @@ -1,882 +1,25 @@ package main import ( - "context" - "encoding/json" - "fmt" - "net" - "net/http" "os" - "os/signal" - "path/filepath" - "reflect" - "strings" - "time" - "github.com/google/uuid" - "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log/v2" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" - "github.com/urfave/cli/v2" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-jsonrpc/auth" - "github.com/filecoin-project/go-paramfetch" - "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/build/buildconstants" - lcli "github.com/filecoin-project/lotus/cli" - cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" + "github.com/filecoin-project/lotus/cli/worker" "github.com/filecoin-project/lotus/lib/lotuslog" - "github.com/filecoin-project/lotus/lib/ulimit" - "github.com/filecoin-project/lotus/metrics" - "github.com/filecoin-project/lotus/node/modules" - "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/paths" - "github.com/filecoin-project/lotus/storage/sealer" - "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" - "github.com/filecoin-project/lotus/storage/sealer/sealtasks" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("main") -const FlagWorkerRepo = "worker-repo" - -// TODO remove after deprecation period -const FlagWorkerRepoDeprecation = "workerrepo" - func main() { api.RunningNodeType = api.NodeWorker lotuslog.SetupLogLevels() - local := []*cli.Command{ - runCmd, - stopCmd, - infoCmd, - storageCmd, - setCmd, - waitQuietCmd, - resourcesCmd, - tasksCmd, - } - - app := &cli.App{ - Name: "lotus-worker", - Usage: "Remote miner worker", - Version: string(build.MinerUserVersion()), - EnableBashCompletion: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: FlagWorkerRepo, - Aliases: []string{FlagWorkerRepoDeprecation}, - EnvVars: []string{"LOTUS_WORKER_PATH", "WORKER_PATH"}, - Value: "~/.lotusworker", // TODO: Consider XDG_DATA_HOME - Usage: fmt.Sprintf("Specify worker repo path. flag %s and env WORKER_PATH are DEPRECATION, will REMOVE SOON", FlagWorkerRepoDeprecation), - }, - &cli.StringFlag{ - Name: "panic-reports", - EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, - Hidden: true, - Value: "~/.lotusworker", // should follow --repo default - }, - &cli.StringFlag{ - Name: "miner-repo", - Aliases: []string{"storagerepo"}, - EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, - Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME - Usage: fmt.Sprintf("Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are DEPRECATION, will REMOVE SOON"), - }, - &cli.BoolFlag{ - Name: "enable-gpu-proving", - Usage: "enable use of GPU for mining operations", - Value: true, - EnvVars: []string{"LOTUS_WORKER_ENABLE_GPU_PROVING"}, - }, - }, - - After: func(c *cli.Context) error { - if r := recover(); r != nil { - // Generate report in LOTUS_PANIC_REPORT_PATH and re-raise panic - build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagWorkerRepo), c.App.Name) - panic(r) - } - return nil - }, - Commands: local, - } - app.Setup() - app.Metadata["repoType"] = repo.Worker - + app := worker.App() if err := app.Run(os.Args); err != nil { log.Warnf("%+v", err) return } } - -var stopCmd = &cli.Command{ - Name: "stop", - Usage: "Stop a running lotus worker", - Flags: []cli.Flag{}, - Action: func(cctx *cli.Context) error { - api, closer, err := lcli.GetWorkerAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := lcli.ReqContext(cctx) - - // Detach any storage associated with this worker - err = api.StorageDetachAll(ctx) - if err != nil { - return err - } - - err = api.Shutdown(ctx) - if err != nil { - return err - } - - return nil - }, -} - -var runCmd = &cli.Command{ - Name: "run", - Usage: "Start lotus worker", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "listen", - Usage: "host address and port the worker api will listen on", - Value: "0.0.0.0:3456", - EnvVars: []string{"LOTUS_WORKER_LISTEN"}, - }, - &cli.StringFlag{ - Name: "address", - Hidden: true, - }, - &cli.BoolFlag{ - Name: "no-local-storage", - Usage: "don't use storageminer repo for sector storage", - EnvVars: []string{"LOTUS_WORKER_NO_LOCAL_STORAGE"}, - }, - &cli.BoolFlag{ - Name: "no-swap", - Usage: "don't use swap", - Value: false, - EnvVars: []string{"LOTUS_WORKER_NO_SWAP"}, - }, - &cli.StringFlag{ - Name: "name", - Usage: "custom worker name", - EnvVars: []string{"LOTUS_WORKER_NAME"}, - DefaultText: "hostname", - }, - &cli.BoolFlag{ - Name: "addpiece", - Usage: "enable addpiece", - Value: true, - EnvVars: []string{"LOTUS_WORKER_ADDPIECE"}, - }, - &cli.BoolFlag{ - Name: "precommit1", - Usage: "enable precommit1", - Value: true, - EnvVars: []string{"LOTUS_WORKER_PRECOMMIT1"}, - }, - &cli.BoolFlag{ - Name: "unseal", - Usage: "enable unsealing", - Value: true, - EnvVars: []string{"LOTUS_WORKER_UNSEAL"}, - }, - &cli.BoolFlag{ - Name: "precommit2", - Usage: "enable precommit2", - Value: true, - EnvVars: []string{"LOTUS_WORKER_PRECOMMIT2"}, - }, - &cli.BoolFlag{ - Name: "commit", - Usage: "enable commit", - Value: true, - EnvVars: []string{"LOTUS_WORKER_COMMIT"}, - }, - &cli.BoolFlag{ - Name: "replica-update", - Usage: "enable replica update", - Value: true, - EnvVars: []string{"LOTUS_WORKER_REPLICA_UPDATE"}, - }, - &cli.BoolFlag{ - Name: "prove-replica-update2", - Usage: "enable prove replica update 2", - Value: true, - EnvVars: []string{"LOTUS_WORKER_PROVE_REPLICA_UPDATE2"}, - }, - &cli.BoolFlag{ - Name: "regen-sector-key", - Usage: "enable regen sector key", - Value: true, - EnvVars: []string{"LOTUS_WORKER_REGEN_SECTOR_KEY"}, - }, - &cli.BoolFlag{ - Name: "sector-download", - Usage: "enable external sector data download", - Value: false, - EnvVars: []string{"LOTUS_WORKER_SECTOR_DOWNLOAD"}, - }, - &cli.BoolFlag{ - Name: "windowpost", - Usage: "enable window post", - Value: false, - EnvVars: []string{"LOTUS_WORKER_WINDOWPOST"}, - }, - &cli.BoolFlag{ - Name: "winningpost", - Usage: "enable winning post", - Value: false, - EnvVars: []string{"LOTUS_WORKER_WINNINGPOST"}, - }, - &cli.BoolFlag{ - Name: "no-default", - Usage: "disable all default compute tasks, use the worker for storage/fetching only", - Value: false, - EnvVars: []string{"LOTUS_WORKER_NO_DEFAULT"}, - }, - &cli.IntFlag{ - Name: "parallel-fetch-limit", - Usage: "maximum fetch operations to run in parallel", - Value: 5, - EnvVars: []string{"LOTUS_WORKER_PARALLEL_FETCH_LIMIT"}, - }, - &cli.IntFlag{ - Name: "post-parallel-reads", - Usage: "maximum number of parallel challenge reads (0 = no limit)", - Value: 32, - EnvVars: []string{"LOTUS_WORKER_POST_PARALLEL_READS"}, - }, - &cli.DurationFlag{ - Name: "post-read-timeout", - Usage: "time limit for reading PoSt challenges (0 = no limit)", - Value: 0, - EnvVars: []string{"LOTUS_WORKER_POST_READ_TIMEOUT"}, - }, - &cli.StringFlag{ - Name: "timeout", - Usage: "used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function", - Value: "30m", - EnvVars: []string{"LOTUS_WORKER_TIMEOUT"}, - }, - &cli.StringFlag{ - Name: "http-server-timeout", - Value: "30s", - }, - &cli.BoolFlag{ - Name: "data-cid", - Usage: "Run the data-cid task. true|false", - Value: true, - DefaultText: "inherits --addpiece", - }, - &cli.StringFlag{ - Name: "external-pc2", - Usage: "command for computing PC2 externally", - }, - }, - Description: `Run lotus-worker. - ---external-pc2 can be used to compute the PreCommit2 inputs externally. -The flag behaves similarly to the related lotus-worker flag, using it in -lotus-bench may be useful for testing if the external PreCommit2 command is -invoked correctly. - -The command will be called with a number of environment variables set: -* EXTSEAL_PC2_SECTOR_NUM: the sector number -* EXTSEAL_PC2_SECTOR_MINER: the miner id -* EXTSEAL_PC2_PROOF_TYPE: the proof type -* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes -* EXTSEAL_PC2_CACHE: the path to the cache directory -* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) -* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) - -The command is expected to: -* Create cache sc-02-data-tree-r* files -* Create cache sc-02-data-tree-c* files -* Create cache p_aux / t_aux files -* Transform the sealed file in place - -Example invocation of lotus-bench as external executor: -'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT' -`, - Before: func(cctx *cli.Context) error { - if cctx.IsSet("address") { - log.Warnf("The '--address' flag is deprecated, it has been replaced by '--listen'") - if err := cctx.Set("listen", cctx.String("address")); err != nil { - return err - } - } - - return nil - }, - Action: func(cctx *cli.Context) error { - log.Info("Starting lotus worker") - - if !cctx.Bool("enable-gpu-proving") { - if err := os.Setenv("BELLMAN_NO_GPU", "true"); err != nil { - return xerrors.Errorf("could not set no-gpu env: %+v", err) - } - } - - // ensure tmpdir exists - td := os.TempDir() - if err := os.MkdirAll(td, 0755); err != nil { - return xerrors.Errorf("ensuring temp dir %s exists: %w", td, err) - } - - // Check file descriptor limit - limit, _, err := ulimit.GetLimit() - switch { - case err == ulimit.ErrUnsupported: - log.Errorw("checking file descriptor limit failed", "error", err) - case err != nil: - return xerrors.Errorf("checking fd limit: %w", err) - default: - if limit < buildconstants.MinerFDLimit { - return xerrors.Errorf("soft file descriptor limit (ulimit -n) too low, want %d, current %d", buildconstants.MinerFDLimit, limit) - } - } - - // Check DC-environment variable - sectorSizes := []string{"2KiB", "8MiB", "512MiB", "32GiB", "64GiB"} - resourcesType := reflect.TypeOf(storiface.Resources{}) - - for _, sectorSize := range sectorSizes { - for i := 0; i < resourcesType.NumField(); i++ { - field := resourcesType.Field(i) - envName := field.Tag.Get("envname") - if envName != "" { - // Check if DC_[SectorSize]_[ResourceRestriction] is set - envVar, ok := os.LookupEnv("DC_" + sectorSize + "_" + envName) - if ok { - // If it is set, convert it to DC_[ResourceRestriction] - err := os.Setenv("DC_"+envName, envVar) - if err != nil { - log.Fatalf("Error setting environment variable: %v", err) - } - log.Warnf("Converted DC_%s_%s to DC_%s, because DC is a sector-size independent job", sectorSize, envName, envName) - } - } - } - } - - // Connect to storage-miner - ctx := lcli.ReqContext(cctx) - - // Create a new context with cancel function - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - // Listen for interrupt signals - go func() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - <-c - cancel() - }() - - var nodeApi api.StorageMiner - var closer func() - for { - nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) - if err == nil { - _, err = nodeApi.Version(ctx) - if err == nil { - break - } - } - fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err) - select { - case <-ctx.Done(): - return xerrors.New("Interrupted by user") - case <-time.After(time.Second): - } - } - defer closer() - // Register all metric views - if err := view.Register( - metrics.DefaultViews..., - ); err != nil { - log.Fatalf("Cannot register the view: %v", err) - } - - v, err := nodeApi.Version(ctx) - if err != nil { - return err - } - if v.APIVersion != api.MinerAPIVersion0 { - return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.APIVersion{APIVersion: api.MinerAPIVersion0}) - } - log.Infof("Remote version %s", v) - - // Check params - - act, err := nodeApi.ActorAddress(ctx) - if err != nil { - return err - } - ssize, err := nodeApi.ActorSectorSize(ctx, act) - if err != nil { - return err - } - - var taskTypes []sealtasks.TaskType - var workerType string - var needParams bool - - if cctx.Bool("windowpost") { - needParams = true - workerType = sealtasks.WorkerWindowPoSt - taskTypes = append(taskTypes, sealtasks.TTGenerateWindowPoSt) - } - if cctx.Bool("winningpost") { - needParams = true - workerType = sealtasks.WorkerWinningPoSt - taskTypes = append(taskTypes, sealtasks.TTGenerateWinningPoSt) - } - - if workerType == "" { - taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeUnsealed, sealtasks.TTFinalizeReplicaUpdate) - - if !cctx.Bool("no-default") { - workerType = sealtasks.WorkerSealing - } - } - - ttDataCidDefault := false - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") { - taskTypes = append(taskTypes, sealtasks.TTAddPiece) - ttDataCidDefault = true - } - if workerType == sealtasks.WorkerSealing { - if cctx.IsSet("data-cid") { - if cctx.Bool("data-cid") { - taskTypes = append(taskTypes, sealtasks.TTDataCid) - } - } else if ttDataCidDefault { - taskTypes = append(taskTypes, sealtasks.TTDataCid) - } - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("sector-download")) && cctx.Bool("sector-download") { - taskTypes = append(taskTypes, sealtasks.TTDownloadSector) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit1")) && cctx.Bool("precommit1") { - taskTypes = append(taskTypes, sealtasks.TTPreCommit1) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("unseal")) && cctx.Bool("unseal") { - taskTypes = append(taskTypes, sealtasks.TTUnseal) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit2")) && cctx.Bool("precommit2") { - taskTypes = append(taskTypes, sealtasks.TTPreCommit2) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("commit")) && cctx.Bool("commit") { - needParams = true - taskTypes = append(taskTypes, sealtasks.TTCommit2) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("replica-update")) && cctx.Bool("replica-update") { - taskTypes = append(taskTypes, sealtasks.TTReplicaUpdate) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("prove-replica-update2")) && cctx.Bool("prove-replica-update2") { - needParams = true - taskTypes = append(taskTypes, sealtasks.TTProveReplicaUpdate2) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("regen-sector-key")) && cctx.Bool("regen-sector-key") { - taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey) - } - - if cctx.Bool("no-default") && workerType == "" { - workerType = sealtasks.WorkerSealing - } - - if len(taskTypes) == 0 { - return xerrors.Errorf("no task types specified") - } - for _, taskType := range taskTypes { - if taskType.WorkerType() != workerType { - return xerrors.Errorf("expected all task types to be for %s worker, but task %s is for %s worker", workerType, taskType, taskType.WorkerType()) - } - } - - if needParams { - if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(ssize)); err != nil { - return xerrors.Errorf("get params: %w", err) - } - } - - // Open repo - - repoPath := cctx.String(FlagWorkerRepo) - r, err := repo.NewFS(repoPath) - if err != nil { - return err - } - - ok, err := r.Exists() - if err != nil { - return err - } - if !ok { - if err := r.Init(repo.Worker); err != nil { - return err - } - - lr, err := r.Lock(repo.Worker) - if err != nil { - return err - } - - var localPaths []storiface.LocalPath - - if !cctx.Bool("no-local-storage") { - b, err := json.MarshalIndent(&storiface.LocalStorageMeta{ - ID: storiface.ID(uuid.New().String()), - Weight: 10, - CanSeal: true, - CanStore: false, - }, "", " ") - if err != nil { - return xerrors.Errorf("marshaling storage config: %w", err) - } - - if err := os.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { - return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) - } - - localPaths = append(localPaths, storiface.LocalPath{ - Path: lr.Path(), - }) - } - - if err := lr.SetStorage(func(sc *storiface.StorageConfig) { - sc.StoragePaths = append(sc.StoragePaths, localPaths...) - }); err != nil { - return xerrors.Errorf("set storage config: %w", err) - } - - { - // init datastore for r.Exists - _, err := lr.Datastore(context.Background(), "/metadata") - if err != nil { - return err - } - } - if err := lr.Close(); err != nil { - return xerrors.Errorf("close repo: %w", err) - } - } - - lr, err := r.Lock(repo.Worker) - if err != nil { - return err - } - defer func() { - if err := lr.Close(); err != nil { - log.Error("closing repo", err) - } - }() - ds, err := lr.Datastore(context.Background(), "/metadata") - if err != nil { - return err - } - - log.Info("Opening local storage; connecting to master") - const unspecifiedAddress = "0.0.0.0" - - address := cctx.String("listen") - host, port, err := net.SplitHostPort(address) - if err != nil { - return err - } - - if ip := net.ParseIP(host); ip != nil { - if ip.String() == unspecifiedAddress { - timeout, err := time.ParseDuration(cctx.String("timeout")) - if err != nil { - return err - } - rip, err := extractRoutableIP(timeout) - if err != nil { - return err - } - host = rip - } - } - - var newAddress string - - // Check if the IP address is IPv6 - ip := net.ParseIP(host) - if ip.To4() == nil && ip.To16() != nil { - newAddress = "[" + host + "]:" + port - } else { - newAddress = host + ":" + port - } - - localStore, err := paths.NewLocal(ctx, lr, nodeApi, []string{"http://" + newAddress + "/remote"}) - if err != nil { - return err - } - - // Setup remote sector store - sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) - if err != nil { - return xerrors.Errorf("could not get api info: %w", err) - } - - remote := paths.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"), - &paths.DefaultPartialFileHandler{}) - - fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}} - remoteHandler := func(w http.ResponseWriter, r *http.Request) { - if !auth.HasPerm(r.Context(), nil, api.PermAdmin) { - w.WriteHeader(401) - _ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"}) - return - } - - fh.ServeHTTP(w, r) - } - - // Parse ffi executor flags - - var ffiOpts []ffiwrapper.FFIWrapperOpt - - if cctx.IsSet("external-pc2") { - extSeal := ffiwrapper.ExternalSealer{ - PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")), - } - - ffiOpts = append(ffiOpts, ffiwrapper.WithExternalSealCalls(extSeal)) - } - - // Create / expose the worker - - wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix)) - - workerApi := &sealworker.Worker{ - LocalWorker: sealer.NewLocalWorkerWithExecutor( - sealer.FFIExec(ffiOpts...), - sealer.WorkerConfig{ - TaskTypes: taskTypes, - NoSwap: cctx.Bool("no-swap"), - MaxParallelChallengeReads: cctx.Int("post-parallel-reads"), - ChallengeReadTimeout: cctx.Duration("post-read-timeout"), - Name: cctx.String("name"), - }, os.LookupEnv, remote, localStore, nodeApi, nodeApi, wsts), - LocalStore: localStore, - Storage: lr, - } - - log.Info("Setting up control endpoint at " + newAddress) - - timeout, err := time.ParseDuration(cctx.String("http-server-timeout")) - if err != nil { - return xerrors.Errorf("invalid time string %s: %x", cctx.String("http-server-timeout"), err) - } - - srv := &http.Server{ - Handler: sealworker.WorkerHandler(nodeApi.AuthVerify, remoteHandler, workerApi, true), - ReadHeaderTimeout: timeout, - BaseContext: func(listener net.Listener) context.Context { - ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker")) - return ctx - }, - } - - go func() { - <-ctx.Done() - log.Warn("Shutting down...") - if err := srv.Shutdown(context.TODO()); err != nil { - log.Errorf("shutting down RPC server failed: %s", err) - } - log.Warn("Graceful shutdown successful") - }() - - nl, err := net.Listen("tcp", newAddress) - if err != nil { - return err - } - - { - a, err := net.ResolveTCPAddr("tcp", newAddress) - if err != nil { - return xerrors.Errorf("parsing address: %w", err) - } - - ma, err := manet.FromNetAddr(a) - if err != nil { - return xerrors.Errorf("creating api multiaddress: %w", err) - } - - if err := lr.SetAPIEndpoint(ma); err != nil { - return xerrors.Errorf("setting api endpoint: %w", err) - } - - ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) - if err != nil { - return xerrors.Errorf("could not get miner API info: %w", err) - } - - // TODO: ideally this would be a token with some permissions dropped - if err := lr.SetAPIToken(ainfo.Token); err != nil { - return xerrors.Errorf("setting api token: %w", err) - } - } - - minerSession, err := nodeApi.Session(ctx) - if err != nil { - return xerrors.Errorf("getting miner session: %w", err) - } - - waitQuietCh := func() chan struct{} { - out := make(chan struct{}) - go func() { - workerApi.LocalWorker.WaitQuiet() - close(out) - }() - return out - } - - go func() { - heartbeats := time.NewTicker(paths.HeartbeatInterval) - defer heartbeats.Stop() - - var redeclareStorage bool - var readyCh chan struct{} - for { - // If we're reconnecting, redeclare storage first - if redeclareStorage { - log.Info("Redeclaring local storage") - - if err := localStore.Redeclare(ctx, nil, false); err != nil { - log.Errorf("Redeclaring local storage failed: %+v", err) - - select { - case <-ctx.Done(): - return // graceful shutdown - case <-heartbeats.C: - } - continue - } - } - - // TODO: we could get rid of this, but that requires tracking resources for restarted tasks correctly - if readyCh == nil { - log.Info("Making sure no local tasks are running") - readyCh = waitQuietCh() - } - - for { - curSession, err := nodeApi.Session(ctx) - if err != nil { - log.Errorf("heartbeat: checking remote session failed: %+v", err) - } else { - if curSession != minerSession { - minerSession = curSession - break - } - } - - select { - case <-readyCh: - if err := nodeApi.WorkerConnect(ctx, "http://"+newAddress+"/rpc/v0"); err != nil { - log.Errorf("Registering worker failed: %+v", err) - cancel() - return - } - - log.Info("Worker registered successfully, waiting for tasks") - - readyCh = nil - case <-heartbeats.C: - case <-ctx.Done(): - return // graceful shutdown - } - } - - log.Errorf("LOTUS-MINER CONNECTION LOST") - - redeclareStorage = true - } - }() - - go func() { - <-workerApi.Done() - // Wait 20s to allow the miner to unregister the worker on next heartbeat - time.Sleep(20 * time.Second) - log.Warn("Shutting down...") - if err := srv.Shutdown(context.TODO()); err != nil { - log.Errorf("shutting down RPC server failed: %s", err) - } - log.Warn("Graceful shutdown successful") - }() - - return srv.Serve(nl) - }, -} - -func extractRoutableIP(timeout time.Duration) (string, error) { - minerMultiAddrKey := "MINER_API_INFO" - deprecatedMinerMultiAddrKey := "STORAGE_API_INFO" - env, ok := os.LookupEnv(minerMultiAddrKey) - if !ok { - _, ok = os.LookupEnv(deprecatedMinerMultiAddrKey) - if ok { - log.Warnf("Using a deprecated env(%s) value, please use env(%s) instead.", deprecatedMinerMultiAddrKey, minerMultiAddrKey) - } - return "", xerrors.New("MINER_API_INFO environment variable required to extract IP") - } - - // Splitting the env to separate the JWT from the multiaddress - splitEnv := strings.SplitN(env, ":", 2) - if len(splitEnv) < 2 { - return "", xerrors.Errorf("invalid MINER_API_INFO format") - } - // Only take the multiaddress part - maddrStr := splitEnv[1] - - maddr, err := multiaddr.NewMultiaddr(maddrStr) - if err != nil { - return "", err - } - - minerIP, _ := maddr.ValueForProtocol(multiaddr.P_IP6) - if minerIP == "" { - minerIP, _ = maddr.ValueForProtocol(multiaddr.P_IP4) - } - minerPort, _ := maddr.ValueForProtocol(multiaddr.P_TCP) - - // Format the address appropriately - addressToDial := net.JoinHostPort(minerIP, minerPort) - - conn, err := net.DialTimeout("tcp", addressToDial, timeout) - if err != nil { - return "", err - } - - defer func() { - if cerr := conn.Close(); cerr != nil { - log.Errorf("Error closing connection: %v", cerr) - } - }() - - localAddr := conn.LocalAddr().(*net.TCPAddr) - return localAddr.IP.String(), nil -} diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index 6b29f6cf83a..4778689d20e 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -1,124 +1,11 @@ package main import ( - "context" - "os" - - "github.com/fatih/color" - logging "github.com/ipfs/go-log/v2" - "github.com/mattn/go-isatty" - "github.com/urfave/cli/v2" - "go.opencensus.io/trace" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/cli/clicommands" - cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/lib/lotuslog" - "github.com/filecoin-project/lotus/lib/tracing" - "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/cli/lotus" ) -var log = logging.Logger("main") - -var AdvanceBlockCmd *cli.Command - func main() { - api.RunningNodeType = api.NodeFull - - lotuslog.SetupLogLevels() - - local := []*cli.Command{ - DaemonCmd, - backupCmd, - configCmd, - } - if AdvanceBlockCmd != nil { - local = append(local, AdvanceBlockCmd) - } - - jaeger := tracing.SetupJaegerTracing("lotus") - defer func() { - if jaeger != nil { - _ = jaeger.ForceFlush(context.Background()) - } - }() - - for _, cmd := range local { - cmd := cmd - originBefore := cmd.Before - cmd.Before = func(cctx *cli.Context) error { - if jaeger != nil { - _ = jaeger.Shutdown(cctx.Context) - } - jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) - - if cctx.IsSet("color") { - color.NoColor = !cctx.Bool("color") - } - - if originBefore != nil { - return originBefore(cctx) - } - return nil - } - } - ctx, span := trace.StartSpan(context.Background(), "/cli") - defer span.End() - - interactiveDef := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) - - app := &cli.App{ - Name: "lotus", - Usage: "Filecoin decentralized storage network client", - Version: string(build.NodeUserVersion()), - EnableBashCompletion: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "panic-reports", - EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, - Hidden: true, - Value: "~/.lotus", // should follow --repo default - }, - &cli.BoolFlag{ - // examined in the Before above - Name: "color", - Usage: "use color in display output", - DefaultText: "depends on output being a TTY", - }, - &cli.StringFlag{ - Name: "repo", - EnvVars: []string{"LOTUS_PATH"}, - Hidden: true, - Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME - }, - &cli.BoolFlag{ - Name: "interactive", - Usage: "setting to false will disable interactive functionality of commands", - Value: interactiveDef, - }, - &cli.BoolFlag{ - Name: "force-send", - Usage: "if true, will ignore pre-send checks", - }, - cliutil.FlagVeryVerbose, - }, - After: func(c *cli.Context) error { - if r := recover(); r != nil { - // Generate report in LOTUS_PATH and re-raise panic - build.GenerateNodePanicReport(c.String("panic-reports"), c.String("repo"), c.App.Name) - panic(r) - } - return nil - }, - - Commands: append(local, clicommands.Commands...), - } - - app.Setup() - app.Metadata["traceContext"] = ctx - app.Metadata["repoType"] = repo.FullNode - + app := lotus.App() lcli.RunApp(app) } diff --git a/documentation/changelog/CHANGELOG_1.2x.md b/documentation/changelog/CHANGELOG_1.2x.md index a98c77585a6..efe1f4392bc 100644 --- a/documentation/changelog/CHANGELOG_1.2x.md +++ b/documentation/changelog/CHANGELOG_1.2x.md @@ -38,7 +38,7 @@ This is the stable release for Lotus node v1.29.2. Key updates in this release i - **New API Support:** Added support for `EthGetBlockReceipts` RPC method to retrieve transaction receipts for a specified block. This method allows users to obtain Ethereum format receipts of all transactions included in a given tipset as specified by its Ethereum block equivalent. ([filecoin-project/lotus#12478](https://github.com/filecoin-project/lotus/pull/12478)) - **Dependency Update:** Upgraded go-libp2p to version v0.35.5 ([filecoin-project/lotus#12511](https://github.com/filecoin-project/lotus/pull/12511)), and go-multiaddr-dns to v0.4.0 ([filecoin-project/lotus#12540](https://github.com/filecoin-project/lotus/pull/12540)). -- **Bug Fix:** Legacy/historical Drand lookups via `StateGetBeaconEntry` now work again for all historical epochs. `StateGetBeaconEntry` now uses the on-chain beacon entries and follows the same rules for historical Drand round matching as `StateGetRandomnessFromBeacon` and the `get_beacon_randomness` FVM syscall. Be aware that there will be some some variance in matching Filecoin epochs to Drand rounds where null Filecoin rounds are involved prior to network version 14. ([filecoin-project/lotus#12428](https://github.com/filecoin-project/lotus/pull/12428)). +- **Bug Fix:** Legacy/historical Drand lookups via `StateGetBeaconEntry` now work again for all historical epochs. `StateGetBeaconEntry` now uses the on-chain beacon entries and follows the same rules for historical Drand round matching as `StateGetRandomnessFromBeacon` and the `get_beacon_randomness` FVM syscall. Be aware that there will be some variance in matching Filecoin epochs to Drand rounds where null Filecoin rounds are involved prior to network version 14. ([filecoin-project/lotus#12428](https://github.com/filecoin-project/lotus/pull/12428)). ## ☢️ Upgrade Warnings ☢️ @@ -78,7 +78,7 @@ This is the stable release for Lotus node v1.29.2. Key updates in this release i - **New API Support:** Added support for `EthGetBlockReceipts` RPC method to retrieve transaction receipts for a specified block. This method allows users to obtain Ethereum format receipts of all transactions included in a given tipset as specified by its Ethereum block equivalent. ([filecoin-project/lotus#12478](https://github.com/filecoin-project/lotus/pull/12478)) - **Dependency Update:** Upgraded go-libp2p to version v0.35.5 ([filecoin-project/lotus#12511](https://github.com/filecoin-project/lotus/pull/12511)), and go-multiaddr-dns to v0.4.0 ([filecoin-project/lotus#12540](https://github.com/filecoin-project/lotus/pull/12540)). -- **Bug Fix:** Legacy/historical Drand lookups via `StateGetBeaconEntry` now work again for all historical epochs. `StateGetBeaconEntry` now uses the on-chain beacon entries and follows the same rules for historical Drand round matching as `StateGetRandomnessFromBeacon` and the `get_beacon_randomness` FVM syscall. Be aware that there will be some some variance in matching Filecoin epochs to Drand rounds where null Filecoin rounds are involved prior to network version 14. ([filecoin-project/lotus#12428](https://github.com/filecoin-project/lotus/pull/12428)). +- **Bug Fix:** Legacy/historical Drand lookups via `StateGetBeaconEntry` now work again for all historical epochs. `StateGetBeaconEntry` now uses the on-chain beacon entries and follows the same rules for historical Drand round matching as `StateGetRandomnessFromBeacon` and the `get_beacon_randomness` FVM syscall. Be aware that there will be some variance in matching Filecoin epochs to Drand rounds where null Filecoin rounds are involved prior to network version 14. ([filecoin-project/lotus#12428](https://github.com/filecoin-project/lotus/pull/12428)). ## ☢️ Upgrade Warnings ☢️ - This release requires a minimum Go version of v1.22.7 or higher ([filecoin-project/lotus#12459](https://github.com/filecoin-project/lotus/pull/12459)) @@ -899,7 +899,7 @@ All node operators, including storage providers, should be aware that ONE pre-mi We recommend node operators (who haven't enabled splitstore discard mode) that do not care about historical chain states, to prune the chain blockstore by syncing from a snapshot 1-2 days before the upgrade. -You can test out the migration by running running the [`benchmarking a network migration` tutorial.](https://lotus.filecoin.io/kb/test-migration/) +You can test out the migration by running the [`benchmarking a network migration` tutorial.](https://lotus.filecoin.io/kb/test-migration/) For certain node operators, such as full archival nodes or systems that need to keep large amounts of state (RPC providers), completing the pre-migration in time before the network upgrade might not be achievable. For those node operators, it is recommended to skip the pre-migration and run the non-cached migration (i.e., just running the migration at the exact upgrade epoch), and schedule for some downtime during the upgrade epoch. Operators of such nodes can read the [`How to disable premigration in network upgrade` tutorial.](https://lotus.filecoin.io/kb/disable-premigration/) @@ -1761,7 +1761,7 @@ Please read carefully through the **upgrade warnings** section if you are upgrad - This feature release requires a **minimum Go version of v1.19.7 or higher to successfully build Lotus**. Additionally, Go version v1.20 and higher is now also supported. - **Storage Providers:** The proofs libraries now have CUDA enabled by default, which requires you to install [CUDA](https://lotus.filecoin.io/tutorials/lotus-miner/cuda/) if you haven't already done so. If you prefer to use OpenCL on your GPUs instead, you can use the `FFI_USE_OPENCL=1` flag when building from source. On the other hand, if you want to disable GPUs altogether, you can use the `FFI_NO_GPU=1` environment variable when building from source. - **Storage Providers:** The `lotus-miner sectors extend` command has been refactored to the functionality of `lotus-miner sectors renew`. -- **Exchanges/Node operators/RPC-providers::** Execution traces (returned from `lotus state exec-trace`, `lotus state replay`, etc.), has changed to account for changes introduced by the by the FVM. **Please make sure to read the `Execution trace format change` section carefully, as these are interface breaking changes** +- **Exchanges/Node operators/RPC-providers::** Execution traces (returned from `lotus state exec-trace`, `lotus state replay`, etc.), has changed to account for changes introduced by the FVM. **Please make sure to read the `Execution trace format change` section carefully, as these are interface breaking changes** - **Syncing issues:** If you have been struggling with syncing issues in normal operations you can try to adjust the amount of threads used for more concurrent FMV execution through via the `LOTUS_FVM_CONCURRENCY` enviroment variable. It is set to 4 threads by default. Recommended formula for concurrency == YOUR_RAM/4 , but max during a network upgrade is 24. If you are a Storage Provider and are pushing many messages within a short period of time, exporting `LOTUS_SKIP_APPLY_TS_MESSAGE_CALL_WITH_GAS=1` will also help with keeping in sync. - **Catching up from a Snapshot:** Users have noticed that catching up sync from a snapshot is taking a lot longer these day. This is largely related to the built-in market actor consuming a lot of computational demand for block validation. A FIP for a short-term mitigation for this is currently in Last Call and will be included network version 19 upgrade if accepted. You [can read the FIP here.](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0060.md) @@ -2012,9 +2012,9 @@ The `lotus-miner sector list` is now running in parallel - which should speed up - backport: fix: miner: correctly count sector extensions (10555) ([filecoin-project/lotus#10555](https://github.com/filecoin-project/lotus/pull/10555)) - Fixes the issue with sector extensions. -- fix: proving: Initialize slice with with same length as partition (#10574) ([filecoin-project/lotus#10574])(https://github.com/filecoin-project/lotus/pull/10574) +- fix: proving: Initialize slice with with same length as partition (#10574) ([filecoin-project/lotus#10574](https://github.com/filecoin-project/lotus/pull/10574)) - Fixes an issue where `lotus-miner proving compute window-post` paniced when trying to make skipped sectors human readable. -- feat: stmgr: speed up calculation of genesis circ supply (#10553) ([filecoin-project/lotus#10553])(https://github.com/filecoin-project/lotus/pull/10553) +- feat: stmgr: speed up calculation of genesis circ supply (#10553) ([filecoin-project/lotus#10553](https://github.com/filecoin-project/lotus/pull/10553)) - perf: eth: gas estimate set applyTsMessages false (#10546) ([filecoin-project/lotus#10456](https://github.com/filecoin-project/lotus/pull/10546)) - feat: config: Force existing users to opt into new defaults (#10488) ([filecoin-project/lotus#10488](https://github.com/filecoin-project/lotus/pull/10488)) - Force existing users to opt into the new SplitStore defaults. diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index 9edb3ff7da4..4dfc8356244 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -2243,7 +2243,7 @@ StorageFindSector returns list of paths where the specified sector files exist. If allowFetch is set, list of paths to which the sector can be fetched will also be returned. - Paths which have sector files locally (don't require fetching) will be listed first. -- Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes. +- Paths which have sector files locally will not be filtered based on AllowTypes/DenyTypes. - Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple file types are specified, each type will be considered individually, and a union of all paths which can accommodate each file type will be returned. diff --git a/documentation/en/chain-indexer-overview-for-operators.md b/documentation/en/chain-indexer-overview-for-operators.md index cafa24872dc..1c6e3a238a5 100644 --- a/documentation/en/chain-indexer-overview-for-operators.md +++ b/documentation/en/chain-indexer-overview-for-operators.md @@ -212,7 +212,7 @@ There is no automated migration from [pre-ChainIndexer indices](#previous-indexi ### Backfill Timing -Backfilling the new `ChainIndexer` was [benchmarked to take approximately ~12 hours per month of epochs on a sample archival node doing no other work](https://github.com/filecoin-project/lotus/issues/12453#issuecomment-2405306468). Your results will vary depending on hardware, network, and competing processes. This means if one is upgrading a FEVM archival node, they should plan on the node being out of production service for ~10 days. Additional nodes to update don't need to go throuh the same time-intensive process though. They can get a `${LOTUS_PATH}/chainindex/chainindex.db` copied from a trusted node per the [upgrade steps](#upgrade). +Backfilling the new `ChainIndexer` was [benchmarked to take approximately ~12 hours per month of epochs on a sample archival node doing no other work](https://github.com/filecoin-project/lotus/issues/12453#issuecomment-2405306468). Your results will vary depending on hardware, network, and competing processes. This means if one is upgrading a FEVM archival node, they should plan on the node being out of production service for ~10 days. Additional nodes to update don't need to go through the same time-intensive process though. They can get a `${LOTUS_PATH}/chainindex/chainindex.db` copied from a trusted node per the [upgrade steps](#upgrade). ### Backfill Disk Space Requirements @@ -318,7 +318,7 @@ In case you need to downgrade to the [previous indexing system](#previous-indexi The decision to not invest here ultimately comes down to the development-time cost vs. benefit ratio. -For achival nodes, we don't have the confidence that the [previous indexing system](#previous-indexing-system) has the correct data to bootstrap from. In 2024, Lotus maintainers have fixed multiple bugs in the [previous indexing system](#previous-indexing-system), but they still see reports of missing data, mismatched event index counts, etc. Investing here in a migration isn't guaranteed to yield a correct index. As a result, one would still need to perform the [backfill steps](#backfill) to validate and correct the data anyway. While this should be faster having partially correct data than no data, it would still require an archival node to take an outage on the order of days which isn't good enough. +For archival nodes, we don't have the confidence that the [previous indexing system](#previous-indexing-system) has the correct data to bootstrap from. In 2024, Lotus maintainers have fixed multiple bugs in the [previous indexing system](#previous-indexing-system), but they still see reports of missing data, mismatched event index counts, etc. Investing here in a migration isn't guaranteed to yield a correct index. As a result, one would still need to perform the [backfill steps](#backfill) to validate and correct the data anyway. While this should be faster having partially correct data than no data, it would still require an archival node to take an outage on the order of days which isn't good enough. The schemas of [the old fragmented Indices](#previous-indexing-system) don't naturally map to the schema of the [ChainIndexer](#chainindexer-indexing-system). There would be additional data wrangling work to ultimately get this right. diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index c483ac7b1b0..ff427ea09e9 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -1,4 +1,5 @@ # lotus-miner + ``` NAME: lotus-miner - Filecoin decentralized storage network miner @@ -10,20 +11,21 @@ VERSION: 1.32.1-dev COMMANDS: - init Initialize a lotus miner repo - run Start a lotus miner process - stop Stop a running lotus miner - config Manage node config - backup Create node metadata backup - auth Manage RPC permissions - log Manage logging - wait-api Wait for lotus api to come online - fetch-params Fetch proving parameters - version Print version - help, h Shows a list of commands or help for one command + init Initialize a lotus miner repo + run Start a lotus miner process + stop Stop a running lotus miner + config Manage node config + backup Create node metadata backup + version Print version + help, h Shows a list of commands or help for one command CHAIN: actor manipulate the miner actor info Print miner info + DEVELOPER: + auth Manage RPC permissions + log Manage logging + wait-api Wait for lotus api to come online + fetch-params Fetch proving parameters STORAGE: sectors interact with sector store proving View proving information @@ -40,6 +42,7 @@ GLOBAL OPTIONS: ``` ## lotus-miner init + ``` NAME: lotus-miner init - Initialize a lotus miner repo @@ -69,6 +72,7 @@ OPTIONS: ``` ### lotus-miner init restore + ``` NAME: lotus-miner init restore - Initialize a lotus miner repo from a backup @@ -84,6 +88,7 @@ OPTIONS: ``` ## lotus-miner run + ``` NAME: lotus-miner run - Start a lotus miner process @@ -100,6 +105,7 @@ OPTIONS: ``` ## lotus-miner stop + ``` NAME: lotus-miner stop - Stop a running lotus miner @@ -112,6 +118,7 @@ OPTIONS: ``` ## lotus-miner config + ``` NAME: lotus-miner config - Manage node config @@ -129,6 +136,7 @@ OPTIONS: ``` ### lotus-miner config default + ``` NAME: lotus-miner config default - Print default node config @@ -142,6 +150,7 @@ OPTIONS: ``` ### lotus-miner config updated + ``` NAME: lotus-miner config updated - Print updated node config @@ -155,6 +164,7 @@ OPTIONS: ``` ## lotus-miner backup + ``` NAME: lotus-miner backup - Create node metadata backup @@ -166,7 +176,7 @@ DESCRIPTION: The backup command writes a copy of node metadata under the specified path Online backups: - For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set + For security reasons, the daemon must have LOTUS_BACKUP_BASE_PATH env var set to a path where backup files are supposed to be saved, and the path specified in this command must be within this base path @@ -175,151 +185,8 @@ OPTIONS: --help, -h show help ``` -## lotus-miner auth -``` -NAME: - lotus-miner auth - Manage RPC permissions - -USAGE: - lotus-miner auth command [command options] [arguments...] - -COMMANDS: - create-token Create token - api-info Get token with API info required to connect to this node - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help -``` - -### lotus-miner auth create-token -``` -NAME: - lotus-miner auth create-token - Create token - -USAGE: - lotus-miner auth create-token [command options] [arguments...] - -OPTIONS: - --perm value permission to assign to the token, one of: read, write, sign, admin - --help, -h show help -``` - -### lotus-miner auth api-info -``` -NAME: - lotus-miner auth api-info - Get token with API info required to connect to this node - -USAGE: - lotus-miner auth api-info [command options] [arguments...] - -OPTIONS: - --perm value permission to assign to the token, one of: read, write, sign, admin - --help, -h show help -``` - -## lotus-miner log -``` -NAME: - lotus-miner log - Manage logging - -USAGE: - lotus-miner log command [command options] [arguments...] - -COMMANDS: - list List log systems - set-level Set log level - alerts Get alert states - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help -``` - -### lotus-miner log list -``` -NAME: - lotus-miner log list - List log systems - -USAGE: - lotus-miner log list [command options] [arguments...] - -OPTIONS: - --help, -h show help -``` - -### lotus-miner log set-level -``` -NAME: - lotus-miner log set-level - Set log level - -USAGE: - lotus-miner log set-level [command options] [level] - -DESCRIPTION: - Set the log level for logging systems: - - The system flag can be specified multiple times. - - eg) log set-level --system chain --system chainxchg debug - - Available Levels: - debug - info - warn - error - - Environment Variables: - GOLOG_LOG_LEVEL - Default log level for all log systems - GOLOG_LOG_FMT - Change output log format (json, nocolor) - GOLOG_FILE - Write logs to file - GOLOG_OUTPUT - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr - - -OPTIONS: - --system value [ --system value ] limit to log system - --help, -h show help -``` - -### lotus-miner log alerts -``` -NAME: - lotus-miner log alerts - Get alert states - -USAGE: - lotus-miner log alerts [command options] [arguments...] - -OPTIONS: - --all get all (active and inactive) alerts (default: false) - --help, -h show help -``` - -## lotus-miner wait-api -``` -NAME: - lotus-miner wait-api - Wait for lotus api to come online - -USAGE: - lotus-miner wait-api [command options] [arguments...] - -OPTIONS: - --timeout value duration to wait till fail (default: 30s) - --help, -h show help -``` - -## lotus-miner fetch-params -``` -NAME: - lotus-miner fetch-params - Fetch proving parameters - -USAGE: - lotus-miner fetch-params [command options] [sectorSize] - -OPTIONS: - --help, -h show help -``` - ## lotus-miner version + ``` NAME: lotus-miner version - Print version @@ -332,6 +199,7 @@ OPTIONS: ``` ## lotus-miner actor + ``` NAME: lotus-miner actor - manipulate the miner actor @@ -359,6 +227,7 @@ OPTIONS: ``` ### lotus-miner actor set-addresses + ``` NAME: lotus-miner actor set-addresses - set addresses that your miner can be publicly dialed on @@ -374,6 +243,7 @@ OPTIONS: ``` ### lotus-miner actor settle-deal + ``` NAME: lotus-miner actor settle-deal - Settle deals manually, if dealIds are not provided all deals will be settled @@ -387,6 +257,7 @@ OPTIONS: ``` ### lotus-miner actor withdraw + ``` NAME: lotus-miner actor withdraw - withdraw available balance to beneficiary @@ -401,6 +272,7 @@ OPTIONS: ``` ### lotus-miner actor repay-debt + ``` NAME: lotus-miner actor repay-debt - pay down a miner's debt @@ -414,6 +286,7 @@ OPTIONS: ``` ### lotus-miner actor set-peer-id + ``` NAME: lotus-miner actor set-peer-id - set the peer id of your miner @@ -427,6 +300,7 @@ OPTIONS: ``` ### lotus-miner actor set-owner + ``` NAME: lotus-miner actor set-owner - Set owner address (this command should be invoked twice, first with the old owner as the senderAddress, and then with the new owner) @@ -440,6 +314,7 @@ OPTIONS: ``` ### lotus-miner actor control + ``` NAME: lotus-miner actor control - Manage control addresses @@ -457,6 +332,7 @@ OPTIONS: ``` #### lotus-miner actor control list + ``` NAME: lotus-miner actor control list - Get currently set control addresses @@ -470,6 +346,7 @@ OPTIONS: ``` #### lotus-miner actor control set + ``` NAME: lotus-miner actor control set - Set control address(-es) @@ -483,6 +360,7 @@ OPTIONS: ``` ### lotus-miner actor propose-change-worker + ``` NAME: lotus-miner actor propose-change-worker - Propose a worker address change @@ -496,6 +374,7 @@ OPTIONS: ``` ### lotus-miner actor confirm-change-worker + ``` NAME: lotus-miner actor confirm-change-worker - Confirm a worker address change @@ -509,6 +388,7 @@ OPTIONS: ``` ### lotus-miner actor compact-allocated + ``` NAME: lotus-miner actor compact-allocated - compact allocated sectors bitfield @@ -524,6 +404,7 @@ OPTIONS: ``` ### lotus-miner actor propose-change-beneficiary + ``` NAME: lotus-miner actor propose-change-beneficiary - Propose a beneficiary address change @@ -539,6 +420,7 @@ OPTIONS: ``` ### lotus-miner actor confirm-change-beneficiary + ``` NAME: lotus-miner actor confirm-change-beneficiary - Confirm a beneficiary address change @@ -554,6 +436,7 @@ OPTIONS: ``` ## lotus-miner info + ``` NAME: lotus-miner info - Print miner info @@ -572,6 +455,7 @@ OPTIONS: ``` ### lotus-miner info all + ``` NAME: lotus-miner info all - dump all related miner info @@ -584,6 +468,7 @@ OPTIONS: ``` ## lotus-miner sectors + ``` NAME: lotus-miner sectors - interact with sector store @@ -620,6 +505,7 @@ OPTIONS: ``` ### lotus-miner sectors status + ``` NAME: lotus-miner sectors status - Get the seal status of a sector by its number @@ -636,6 +522,7 @@ OPTIONS: ``` ### lotus-miner sectors list + ``` NAME: lotus-miner sectors list - List sectors @@ -660,6 +547,7 @@ OPTIONS: ``` #### lotus-miner sectors list upgrade-bounds + ``` NAME: lotus-miner sectors list upgrade-bounds - Output upgrade bounds for available sectors @@ -675,6 +563,7 @@ OPTIONS: ``` ### lotus-miner sectors refs + ``` NAME: lotus-miner sectors refs - List References to sectors @@ -687,6 +576,7 @@ OPTIONS: ``` ### lotus-miner sectors update-state + ``` NAME: lotus-miner sectors update-state - ADVANCED: manually update the state of a sector, this may aid in error recovery @@ -700,6 +590,7 @@ OPTIONS: ``` ### lotus-miner sectors pledge + ``` NAME: lotus-miner sectors pledge - store random data in a sector @@ -712,6 +603,7 @@ OPTIONS: ``` ### lotus-miner sectors numbers + ``` NAME: lotus-miner sectors numbers - manage sector number assignments @@ -731,6 +623,7 @@ OPTIONS: ``` #### lotus-miner sectors numbers info + ``` NAME: lotus-miner sectors numbers info - view sector assigner state @@ -743,6 +636,7 @@ OPTIONS: ``` #### lotus-miner sectors numbers reservations + ``` NAME: lotus-miner sectors numbers reservations - list sector number reservations @@ -755,6 +649,7 @@ OPTIONS: ``` #### lotus-miner sectors numbers reserve + ``` NAME: lotus-miner sectors numbers reserve - create sector number reservations @@ -768,6 +663,7 @@ OPTIONS: ``` #### lotus-miner sectors numbers free + ``` NAME: lotus-miner sectors numbers free - remove sector number reservations @@ -780,6 +676,7 @@ OPTIONS: ``` ### lotus-miner sectors precommits + ``` NAME: lotus-miner sectors precommits - Print on-chain precommit info @@ -792,6 +689,7 @@ OPTIONS: ``` ### lotus-miner sectors check-expire + ``` NAME: lotus-miner sectors check-expire - Inspect expiring sectors @@ -805,6 +703,7 @@ OPTIONS: ``` ### lotus-miner sectors expired + ``` NAME: lotus-miner sectors expired - Get or cleanup expired sectors @@ -820,6 +719,7 @@ OPTIONS: ``` ### lotus-miner sectors extend + ``` NAME: lotus-miner sectors extend - Extend expiring sectors while not exceeding each sector's max life @@ -844,6 +744,7 @@ OPTIONS: ``` ### lotus-miner sectors terminate + ``` NAME: lotus-miner sectors terminate - Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector) @@ -862,6 +763,7 @@ OPTIONS: ``` #### lotus-miner sectors terminate flush + ``` NAME: lotus-miner sectors terminate flush - Send a terminate message if there are sectors queued for termination @@ -874,6 +776,7 @@ OPTIONS: ``` #### lotus-miner sectors terminate pending + ``` NAME: lotus-miner sectors terminate pending - List sector numbers of sectors pending termination @@ -886,6 +789,7 @@ OPTIONS: ``` ### lotus-miner sectors remove + ``` NAME: lotus-miner sectors remove - Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector (use 'terminate' for lower penalty)) @@ -899,6 +803,7 @@ OPTIONS: ``` ### lotus-miner sectors snap-up + ``` NAME: lotus-miner sectors snap-up - Mark a committed capacity sector to be filled with deals @@ -911,6 +816,7 @@ OPTIONS: ``` ### lotus-miner sectors abort-upgrade + ``` NAME: lotus-miner sectors abort-upgrade - Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before @@ -924,6 +830,7 @@ OPTIONS: ``` ### lotus-miner sectors seal + ``` NAME: lotus-miner sectors seal - Manually start sealing a sector (filling any unused space with junk) @@ -936,6 +843,7 @@ OPTIONS: ``` ### lotus-miner sectors set-seal-delay + ``` NAME: lotus-miner sectors set-seal-delay - Set the time (in minutes) that a new sector waits for deals before sealing starts @@ -949,6 +857,7 @@ OPTIONS: ``` ### lotus-miner sectors get-cc-collateral + ``` NAME: lotus-miner sectors get-cc-collateral - Get the collateral required to pledge a committed capacity sector @@ -962,6 +871,7 @@ OPTIONS: ``` ### lotus-miner sectors batching + ``` NAME: lotus-miner sectors batching - manage batch sector operations @@ -979,6 +889,7 @@ OPTIONS: ``` #### lotus-miner sectors batching commit + ``` NAME: lotus-miner sectors batching commit - list sectors waiting in commit batch queue @@ -992,6 +903,7 @@ OPTIONS: ``` #### lotus-miner sectors batching precommit + ``` NAME: lotus-miner sectors batching precommit - list sectors waiting in precommit batch queue @@ -1005,6 +917,7 @@ OPTIONS: ``` ### lotus-miner sectors match-pending-pieces + ``` NAME: lotus-miner sectors match-pending-pieces - force a refreshed match of pending pieces to open sectors without manually waiting for more deals @@ -1017,6 +930,7 @@ OPTIONS: ``` ### lotus-miner sectors compact-partitions + ``` NAME: lotus-miner sectors compact-partitions - removes dead sectors from partitions and reduces the number of partitions used if possible @@ -1032,6 +946,7 @@ OPTIONS: ``` ### lotus-miner sectors unseal + ``` NAME: lotus-miner sectors unseal - unseal a sector @@ -1044,6 +959,7 @@ OPTIONS: ``` ## lotus-miner proving + ``` NAME: lotus-miner proving - View proving information @@ -1067,6 +983,7 @@ OPTIONS: ``` ### lotus-miner proving info + ``` NAME: lotus-miner proving info - View current state information @@ -1079,6 +996,7 @@ OPTIONS: ``` ### lotus-miner proving deadlines + ``` NAME: lotus-miner proving deadlines - View the current proving period deadlines information @@ -1092,6 +1010,7 @@ OPTIONS: ``` ### lotus-miner proving deadline + ``` NAME: lotus-miner proving deadline - View the current proving period deadline information by its index @@ -1106,6 +1025,7 @@ OPTIONS: ``` ### lotus-miner proving faults + ``` NAME: lotus-miner proving faults - View the currently known proving faulty sectors information @@ -1118,6 +1038,7 @@ OPTIONS: ``` ### lotus-miner proving check + ``` NAME: lotus-miner proving check - Check sectors provable @@ -1134,6 +1055,7 @@ OPTIONS: ``` ### lotus-miner proving workers + ``` NAME: lotus-miner proving workers - list workers @@ -1146,6 +1068,7 @@ OPTIONS: ``` ### lotus-miner proving compute + ``` NAME: lotus-miner proving compute - Compute simulated proving tasks @@ -1162,6 +1085,7 @@ OPTIONS: ``` #### lotus-miner proving compute windowed-post + ``` NAME: lotus-miner proving compute windowed-post - Compute WindowPoSt for a specific deadline @@ -1178,6 +1102,7 @@ OPTIONS: ``` ### lotus-miner proving recover-faults + ``` NAME: lotus-miner proving recover-faults - Manually recovers faulty sectors on chain @@ -1191,6 +1116,7 @@ OPTIONS: ``` ## lotus-miner storage + ``` NAME: lotus-miner storage - manage sector storage @@ -1219,6 +1145,7 @@ OPTIONS: ``` ### lotus-miner storage attach + ``` NAME: lotus-miner storage attach - attach local storage path @@ -1259,6 +1186,7 @@ OPTIONS: ``` ### lotus-miner storage detach + ``` NAME: lotus-miner storage detach - detach local storage path @@ -1272,6 +1200,7 @@ OPTIONS: ``` ### lotus-miner storage redeclare + ``` NAME: lotus-miner storage redeclare - redeclare sectors in a local storage path @@ -1287,6 +1216,7 @@ OPTIONS: ``` ### lotus-miner storage list + ``` NAME: lotus-miner storage list - list local storage paths @@ -1303,6 +1233,7 @@ OPTIONS: ``` #### lotus-miner storage list sectors + ``` NAME: lotus-miner storage list sectors - get list of all sector files @@ -1315,6 +1246,7 @@ OPTIONS: ``` ### lotus-miner storage find + ``` NAME: lotus-miner storage find - find sector in the storage system @@ -1327,6 +1259,7 @@ OPTIONS: ``` ### lotus-miner storage cleanup + ``` NAME: lotus-miner storage cleanup - trigger cleanup actions @@ -1340,6 +1273,7 @@ OPTIONS: ``` ### lotus-miner storage locks + ``` NAME: lotus-miner storage locks - show active sector locks @@ -1352,6 +1286,7 @@ OPTIONS: ``` ## lotus-miner sealing + ``` NAME: lotus-miner sealing - interact with sealing pipeline @@ -1372,6 +1307,7 @@ OPTIONS: ``` ### lotus-miner sealing jobs + ``` NAME: lotus-miner sealing jobs - list running jobs @@ -1385,6 +1321,7 @@ OPTIONS: ``` ### lotus-miner sealing workers + ``` NAME: lotus-miner sealing workers - list workers @@ -1397,6 +1334,7 @@ OPTIONS: ``` ### lotus-miner sealing sched-diag + ``` NAME: lotus-miner sealing sched-diag - Dump internal scheduler state @@ -1410,6 +1348,7 @@ OPTIONS: ``` ### lotus-miner sealing abort + ``` NAME: lotus-miner sealing abort - Abort a running job @@ -1423,6 +1362,7 @@ OPTIONS: ``` ### lotus-miner sealing data-cid + ``` NAME: lotus-miner sealing data-cid - Compute data CID using workers @@ -1434,3 +1374,162 @@ OPTIONS: --file-size value real file size (default: 0) --help, -h show help ``` + +## lotus-miner auth + +``` +NAME: + lotus-miner auth - Manage RPC permissions + +USAGE: + lotus-miner auth command [command options] [arguments...] + +COMMANDS: + create-token Create token + api-info Get token with API info required to connect to this node + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help +``` + +### lotus-miner auth create-token + +``` +NAME: + lotus-miner auth create-token - Create token + +USAGE: + lotus-miner auth create-token [command options] [arguments...] + +OPTIONS: + --perm value permission to assign to the token, one of: read, write, sign, admin + --help, -h show help +``` + +### lotus-miner auth api-info + +``` +NAME: + lotus-miner auth api-info - Get token with API info required to connect to this node + +USAGE: + lotus-miner auth api-info [command options] [arguments...] + +OPTIONS: + --perm value permission to assign to the token, one of: read, write, sign, admin + --help, -h show help +``` + +## lotus-miner log + +``` +NAME: + lotus-miner log - Manage logging + +USAGE: + lotus-miner log command [command options] [arguments...] + +COMMANDS: + list List log systems + set-level Set log level + alerts Get alert states + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help +``` + +### lotus-miner log list + +``` +NAME: + lotus-miner log list - List log systems + +USAGE: + lotus-miner log list [command options] [arguments...] + +OPTIONS: + --help, -h show help +``` + +### lotus-miner log set-level + +``` +NAME: + lotus-miner log set-level - Set log level + +USAGE: + lotus-miner log set-level [command options] [level] + +DESCRIPTION: + Set the log level for logging systems: + + The system flag can be specified multiple times. + + eg) log set-level --system chain --system chainxchg debug + + Available Levels: + debug + info + warn + error + + Environment Variables: + GOLOG_LOG_LEVEL - Default log level for all log systems + GOLOG_LOG_FMT - Change output log format (json, nocolor) + GOLOG_FILE - Write logs to file + GOLOG_OUTPUT - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr + + +OPTIONS: + --system value [ --system value ] limit to log system + --help, -h show help +``` + +### lotus-miner log alerts + +``` +NAME: + lotus-miner log alerts - Get alert states + +USAGE: + lotus-miner log alerts [command options] [arguments...] + +OPTIONS: + --all get all (active and inactive) alerts (default: false) + --help, -h show help +``` + +## lotus-miner wait-api + +``` +NAME: + lotus-miner wait-api - Wait for lotus api to come online + +USAGE: + lotus-miner wait-api [command options] [arguments...] + +CATEGORY: + DEVELOPER + +OPTIONS: + --timeout value duration to wait till fail (default: 30s) + --help, -h show help +``` + +## lotus-miner fetch-params + +``` +NAME: + lotus-miner fetch-params - Fetch proving parameters + +USAGE: + lotus-miner fetch-params [command options] [sectorSize] + +CATEGORY: + DEVELOPER + +OPTIONS: + --help, -h show help +``` diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index 81b639f728b..7e5cf8c4781 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -1,4 +1,5 @@ # lotus-worker + ``` NAME: lotus-worker - Remote miner worker @@ -27,6 +28,7 @@ GLOBAL OPTIONS: ``` ## lotus-worker run + ``` NAME: lotus-worker run - Start lotus worker @@ -89,6 +91,7 @@ OPTIONS: ``` ## lotus-worker stop + ``` NAME: lotus-worker stop - Stop a running lotus worker @@ -101,6 +104,7 @@ OPTIONS: ``` ## lotus-worker info + ``` NAME: lotus-worker info - Print worker info @@ -113,6 +117,7 @@ OPTIONS: ``` ## lotus-worker storage + ``` NAME: lotus-worker storage - manage sector storage @@ -131,6 +136,7 @@ OPTIONS: ``` ### lotus-worker storage attach + ``` NAME: lotus-worker storage attach - attach local storage path @@ -150,6 +156,7 @@ OPTIONS: ``` ### lotus-worker storage detach + ``` NAME: lotus-worker storage detach - detach local storage path @@ -163,6 +170,7 @@ OPTIONS: ``` ### lotus-worker storage redeclare + ``` NAME: lotus-worker storage redeclare - redeclare sectors in a local storage path @@ -178,6 +186,7 @@ OPTIONS: ``` ## lotus-worker resources + ``` NAME: lotus-worker resources - Manage resource table overrides @@ -192,6 +201,7 @@ OPTIONS: ``` ## lotus-worker tasks + ``` NAME: lotus-worker tasks - Manage task processing @@ -209,6 +219,7 @@ OPTIONS: ``` ### lotus-worker tasks enable + ``` NAME: lotus-worker tasks enable - Enable a task type @@ -222,6 +233,7 @@ OPTIONS: ``` ### lotus-worker tasks disable + ``` NAME: lotus-worker tasks disable - Disable a task type diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 1237c7e44ce..748f7f62621 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -1,4 +1,5 @@ # lotus + ``` NAME: lotus - Filecoin decentralized storage network client @@ -49,6 +50,7 @@ GLOBAL OPTIONS: ``` ## lotus daemon + ``` NAME: lotus daemon - Start a lotus daemon process @@ -80,6 +82,7 @@ OPTIONS: ``` ### lotus daemon stop + ``` NAME: lotus daemon stop - Stop a running lotus daemon @@ -92,6 +95,7 @@ OPTIONS: ``` ## lotus backup + ``` NAME: lotus backup - Create node metadata backup @@ -103,7 +107,7 @@ DESCRIPTION: The backup command writes a copy of node metadata under the specified path Online backups: - For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set + For security reasons, the daemon must have LOTUS_BACKUP_BASE_PATH env var set to a path where backup files are supposed to be saved, and the path specified in this command must be within this base path @@ -113,6 +117,7 @@ OPTIONS: ``` ## lotus config + ``` NAME: lotus config - Manage node config @@ -130,6 +135,7 @@ OPTIONS: ``` ### lotus config default + ``` NAME: lotus config default - Print default node config @@ -143,6 +149,7 @@ OPTIONS: ``` ### lotus config updated + ``` NAME: lotus config updated - Print updated node config @@ -156,6 +163,7 @@ OPTIONS: ``` ## lotus version + ``` NAME: lotus version - Print version @@ -168,6 +176,7 @@ OPTIONS: ``` ## lotus send + ``` NAME: lotus send - Send funds between accounts @@ -193,6 +202,7 @@ OPTIONS: ``` ## lotus wallet + ``` NAME: lotus wallet - Manage wallet @@ -219,6 +229,7 @@ OPTIONS: ``` ### lotus wallet new + ``` NAME: lotus wallet new - Generate a new key of the given type @@ -231,6 +242,7 @@ OPTIONS: ``` ### lotus wallet list + ``` NAME: lotus wallet list - List wallet address @@ -246,6 +258,7 @@ OPTIONS: ``` ### lotus wallet balance + ``` NAME: lotus wallet balance - Get account balance @@ -258,6 +271,7 @@ OPTIONS: ``` ### lotus wallet export + ``` NAME: lotus wallet export - export keys @@ -270,6 +284,7 @@ OPTIONS: ``` ### lotus wallet import + ``` NAME: lotus wallet import - import keys @@ -284,6 +299,7 @@ OPTIONS: ``` ### lotus wallet default + ``` NAME: lotus wallet default - Get default wallet address @@ -296,6 +312,7 @@ OPTIONS: ``` ### lotus wallet set-default + ``` NAME: lotus wallet set-default - Set default wallet address @@ -308,6 +325,7 @@ OPTIONS: ``` ### lotus wallet sign + ``` NAME: lotus wallet sign - sign a message @@ -320,6 +338,7 @@ OPTIONS: ``` ### lotus wallet verify + ``` NAME: lotus wallet verify - verify the signature of a message @@ -332,6 +351,7 @@ OPTIONS: ``` ### lotus wallet delete + ``` NAME: lotus wallet delete - Soft delete an address from the wallet - hard deletion needed for permanent removal @@ -344,6 +364,7 @@ OPTIONS: ``` ### lotus wallet market + ``` NAME: lotus wallet market - Interact with market balances @@ -361,6 +382,7 @@ OPTIONS: ``` #### lotus wallet market withdraw + ``` NAME: lotus wallet market withdraw - Withdraw funds from the Storage Market Actor @@ -376,6 +398,7 @@ OPTIONS: ``` #### lotus wallet market add + ``` NAME: lotus wallet market add - Add funds to the Storage Market Actor @@ -390,6 +413,7 @@ OPTIONS: ``` ## lotus info + ``` NAME: lotus info - Print node info @@ -405,6 +429,7 @@ OPTIONS: ``` ## lotus msig + ``` NAME: lotus msig - Interact with a multisig wallet @@ -438,6 +463,7 @@ OPTIONS: ``` ### lotus msig create + ``` NAME: lotus msig create - Create a new multisig wallet @@ -454,6 +480,7 @@ OPTIONS: ``` ### lotus msig inspect + ``` NAME: lotus msig inspect - Inspect a multisig wallet @@ -468,6 +495,7 @@ OPTIONS: ``` ### lotus msig propose + ``` NAME: lotus msig propose - Propose a multisig transaction @@ -481,6 +509,7 @@ OPTIONS: ``` ### lotus msig propose-remove + ``` NAME: lotus msig propose-remove - Propose to remove a signer @@ -495,6 +524,7 @@ OPTIONS: ``` ### lotus msig approve + ``` NAME: lotus msig approve - Approve a multisig message @@ -508,6 +538,7 @@ OPTIONS: ``` ### lotus msig cancel + ``` NAME: lotus msig cancel - Cancel a multisig message @@ -521,6 +552,7 @@ OPTIONS: ``` ### lotus msig add-propose + ``` NAME: lotus msig add-propose - Propose to add a signer @@ -535,6 +567,7 @@ OPTIONS: ``` ### lotus msig add-approve + ``` NAME: lotus msig add-approve - Approve a message to add a signer @@ -548,6 +581,7 @@ OPTIONS: ``` ### lotus msig add-cancel + ``` NAME: lotus msig add-cancel - Cancel a message to add a signer @@ -561,6 +595,7 @@ OPTIONS: ``` ### lotus msig swap-propose + ``` NAME: lotus msig swap-propose - Propose to swap signers @@ -574,6 +609,7 @@ OPTIONS: ``` ### lotus msig swap-approve + ``` NAME: lotus msig swap-approve - Approve a message to swap signers @@ -587,6 +623,7 @@ OPTIONS: ``` ### lotus msig swap-cancel + ``` NAME: lotus msig swap-cancel - Cancel a message to swap signers @@ -600,6 +637,7 @@ OPTIONS: ``` ### lotus msig lock-propose + ``` NAME: lotus msig lock-propose - Propose to lock up some balance @@ -613,6 +651,7 @@ OPTIONS: ``` ### lotus msig lock-approve + ``` NAME: lotus msig lock-approve - Approve a message to lock up some balance @@ -626,6 +665,7 @@ OPTIONS: ``` ### lotus msig lock-cancel + ``` NAME: lotus msig lock-cancel - Cancel a message to lock up some balance @@ -639,6 +679,7 @@ OPTIONS: ``` ### lotus msig vested + ``` NAME: lotus msig vested - Gets the amount vested in an msig between two epochs @@ -653,6 +694,7 @@ OPTIONS: ``` ### lotus msig propose-threshold + ``` NAME: lotus msig propose-threshold - Propose setting a different signing threshold on the account @@ -666,6 +708,7 @@ OPTIONS: ``` ## lotus filplus + ``` NAME: lotus filplus - Interact with the verified registry actor used by Filplus @@ -692,6 +735,7 @@ OPTIONS: ``` ### lotus filplus grant-datacap + ``` NAME: lotus filplus grant-datacap - give allowance to the specified verified client address @@ -705,6 +749,7 @@ OPTIONS: ``` ### lotus filplus list-notaries + ``` NAME: lotus filplus list-notaries - list all notaries @@ -717,6 +762,7 @@ OPTIONS: ``` ### lotus filplus list-clients + ``` NAME: lotus filplus list-clients - list all verified clients @@ -729,6 +775,7 @@ OPTIONS: ``` ### lotus filplus check-client-datacap + ``` NAME: lotus filplus check-client-datacap - check verified client remaining bytes @@ -741,6 +788,7 @@ OPTIONS: ``` ### lotus filplus check-notary-datacap + ``` NAME: lotus filplus check-notary-datacap - check a notary's remaining bytes @@ -753,6 +801,7 @@ OPTIONS: ``` ### lotus filplus sign-remove-data-cap-proposal + ``` NAME: lotus filplus sign-remove-data-cap-proposal - allows a notary to sign a Remove Data Cap Proposal @@ -766,6 +815,7 @@ OPTIONS: ``` ### lotus filplus list-allocations + ``` NAME: lotus filplus list-allocations - List allocations available in verified registry actor or made by a client if specified @@ -780,6 +830,7 @@ OPTIONS: ``` ### lotus filplus list-claims + ``` NAME: lotus filplus list-claims - List claims available in verified registry actor or made by provider if specified @@ -794,6 +845,7 @@ OPTIONS: ``` ### lotus filplus remove-expired-allocations + ``` NAME: lotus filplus remove-expired-allocations - remove expired allocations (if no allocations are specified all eligible allocations are removed) @@ -807,6 +859,7 @@ OPTIONS: ``` ### lotus filplus remove-expired-claims + ``` NAME: lotus filplus remove-expired-claims - remove expired claims (if no claims are specified all eligible claims are removed) @@ -820,6 +873,7 @@ OPTIONS: ``` ### lotus filplus extend-claim + ``` NAME: lotus filplus extend-claim - extends claim expiration (TermMax) @@ -842,6 +896,7 @@ OPTIONS: ``` ## lotus paych + ``` NAME: lotus paych - Manage payment channels @@ -864,6 +919,7 @@ OPTIONS: ``` ### lotus paych add-funds + ``` NAME: lotus paych add-funds - Add funds to the payment channel between fromAddress and toAddress. Creates the payment channel if it doesn't already exist. @@ -877,6 +933,7 @@ OPTIONS: ``` ### lotus paych list + ``` NAME: lotus paych list - List all locally registered payment channels @@ -889,6 +946,7 @@ OPTIONS: ``` ### lotus paych voucher + ``` NAME: lotus paych voucher - Interact with payment channel vouchers @@ -910,6 +968,7 @@ OPTIONS: ``` #### lotus paych voucher create + ``` NAME: lotus paych voucher create - Create a signed payment channel voucher @@ -923,6 +982,7 @@ OPTIONS: ``` #### lotus paych voucher check + ``` NAME: lotus paych voucher check - Check validity of payment channel voucher @@ -935,6 +995,7 @@ OPTIONS: ``` #### lotus paych voucher add + ``` NAME: lotus paych voucher add - Add payment channel voucher to local datastore @@ -947,6 +1008,7 @@ OPTIONS: ``` #### lotus paych voucher list + ``` NAME: lotus paych voucher list - List stored vouchers for a given payment channel @@ -960,6 +1022,7 @@ OPTIONS: ``` #### lotus paych voucher best-spendable + ``` NAME: lotus paych voucher best-spendable - Print vouchers with highest value that is currently spendable for each lane @@ -973,6 +1036,7 @@ OPTIONS: ``` #### lotus paych voucher submit + ``` NAME: lotus paych voucher submit - Submit voucher to chain to update payment channel state @@ -985,6 +1049,7 @@ OPTIONS: ``` ### lotus paych settle + ``` NAME: lotus paych settle - Settle a payment channel @@ -997,6 +1062,7 @@ OPTIONS: ``` ### lotus paych status + ``` NAME: lotus paych status - Show the status of an outbound payment channel @@ -1009,6 +1075,7 @@ OPTIONS: ``` ### lotus paych status-by-from-to + ``` NAME: lotus paych status-by-from-to - Show the status of an active outbound payment channel by from/to addresses @@ -1021,6 +1088,7 @@ OPTIONS: ``` ### lotus paych collect + ``` NAME: lotus paych collect - Collect funds for a payment channel @@ -1033,6 +1101,7 @@ OPTIONS: ``` ## lotus auth + ``` NAME: lotus auth - Manage RPC permissions @@ -1050,6 +1119,7 @@ OPTIONS: ``` ### lotus auth create-token + ``` NAME: lotus auth create-token - Create token @@ -1063,6 +1133,7 @@ OPTIONS: ``` ### lotus auth api-info + ``` NAME: lotus auth api-info - Get token with API info required to connect to this node @@ -1076,6 +1147,7 @@ OPTIONS: ``` ## lotus mpool + ``` NAME: lotus mpool - Manage message pool @@ -1099,6 +1171,7 @@ OPTIONS: ``` ### lotus mpool pending + ``` NAME: lotus mpool pending - Get pending messages @@ -1115,6 +1188,7 @@ OPTIONS: ``` ### lotus mpool sub + ``` NAME: lotus mpool sub - Subscribe to mpool changes @@ -1127,6 +1201,7 @@ OPTIONS: ``` ### lotus mpool stat + ``` NAME: lotus mpool stat - print mempool stats @@ -1141,6 +1216,7 @@ OPTIONS: ``` ### lotus mpool replace + ``` NAME: lotus mpool replace - replace a message in the mempool @@ -1158,6 +1234,7 @@ OPTIONS: ``` ### lotus mpool find + ``` NAME: lotus mpool find - find a message in the mempool @@ -1173,6 +1250,7 @@ OPTIONS: ``` ### lotus mpool config + ``` NAME: lotus mpool config - get or set current mpool configuration @@ -1185,6 +1263,7 @@ OPTIONS: ``` ### lotus mpool gas-perf + ``` NAME: lotus mpool gas-perf - Check gas performance of messages in mempool @@ -1198,6 +1277,7 @@ OPTIONS: ``` ### lotus mpool manage + ``` NAME: lotus mpool manage @@ -1210,6 +1290,7 @@ OPTIONS: ``` ## lotus state + ``` NAME: lotus state - Interact with and query filecoin chain state @@ -1250,6 +1331,7 @@ OPTIONS: ``` ### lotus state power + ``` NAME: lotus state power - Query network or miner power @@ -1262,6 +1344,7 @@ OPTIONS: ``` ### lotus state sectors + ``` NAME: lotus state sectors - Query the sector set of a miner @@ -1274,6 +1357,7 @@ OPTIONS: ``` ### lotus state active-sectors + ``` NAME: lotus state active-sectors - Query the active sector set of a miner @@ -1286,6 +1370,7 @@ OPTIONS: ``` ### lotus state list-actors + ``` NAME: lotus state list-actors - list all actors in the network @@ -1298,6 +1383,7 @@ OPTIONS: ``` ### lotus state list-miners + ``` NAME: lotus state list-miners - list all miners in the network @@ -1311,6 +1397,7 @@ OPTIONS: ``` ### lotus state circulating-supply + ``` NAME: lotus state circulating-supply - Get the exact current circulating supply of Filecoin @@ -1324,6 +1411,7 @@ OPTIONS: ``` ### lotus state sector + ``` NAME: lotus state sector - Get miner sector info @@ -1336,6 +1424,7 @@ OPTIONS: ``` ### lotus state get-actor + ``` NAME: lotus state get-actor - Print actor information @@ -1348,6 +1437,7 @@ OPTIONS: ``` ### lotus state lookup + ``` NAME: lotus state lookup - Find corresponding ID address @@ -1361,6 +1451,7 @@ OPTIONS: ``` ### lotus state replay + ``` NAME: lotus state replay - Replay a particular message @@ -1375,6 +1466,7 @@ OPTIONS: ``` ### lotus state sector-size + ``` NAME: lotus state sector-size - Look up miners sector size @@ -1387,6 +1479,7 @@ OPTIONS: ``` ### lotus state read-state + ``` NAME: lotus state read-state - View a json representation of an actors state @@ -1399,6 +1492,7 @@ OPTIONS: ``` ### lotus state list-messages + ``` NAME: lotus state list-messages - list messages on chain matching given criteria @@ -1415,6 +1509,7 @@ OPTIONS: ``` ### lotus state compute-state + ``` NAME: lotus state compute-state - Perform state computations @@ -1434,6 +1529,7 @@ OPTIONS: ``` ### lotus state call + ``` NAME: lotus state call - Invoke a method on an actor locally @@ -1450,6 +1546,7 @@ OPTIONS: ``` ### lotus state get-deal + ``` NAME: lotus state get-deal - View on-chain deal info @@ -1462,6 +1559,7 @@ OPTIONS: ``` ### lotus state wait-msg + ``` NAME: lotus state wait-msg - Wait for a message to appear on chain @@ -1475,6 +1573,7 @@ OPTIONS: ``` ### lotus state search-msg + ``` NAME: lotus state search-msg - Search to see whether a message has appeared on chain @@ -1487,6 +1586,7 @@ OPTIONS: ``` ### lotus state miner-info + ``` NAME: lotus state miner-info - Retrieve miner information @@ -1499,6 +1599,7 @@ OPTIONS: ``` ### lotus state market + ``` NAME: lotus state market - Inspect the storage market actor @@ -1516,6 +1617,7 @@ OPTIONS: ``` #### lotus state market balance + ``` NAME: lotus state market balance - Get the market balance (locked and escrowed) for a given account @@ -1528,6 +1630,7 @@ OPTIONS: ``` #### lotus state market proposal-pending + ``` NAME: lotus state market proposal-pending - check if a given proposal CID is pending in the market actor @@ -1540,6 +1643,7 @@ OPTIONS: ``` ### lotus state exec-trace + ``` NAME: lotus state exec-trace - Get the execution trace of a given message @@ -1552,6 +1656,7 @@ OPTIONS: ``` ### lotus state network-version + ``` NAME: lotus state network-version - Returns the network version @@ -1564,6 +1669,7 @@ OPTIONS: ``` ### lotus state miner-proving-deadline + ``` NAME: lotus state miner-proving-deadline - Retrieve information about a given miner's proving deadline @@ -1576,6 +1682,7 @@ OPTIONS: ``` ### lotus state actor-cids + ``` NAME: lotus state actor-cids - Returns the built-in actor bundle manifest ID & system actor cids @@ -1589,6 +1696,7 @@ OPTIONS: ``` ## lotus chain + ``` NAME: lotus chain - Interact with filecoin blockchain @@ -1623,6 +1731,7 @@ OPTIONS: ``` ### lotus chain head + ``` NAME: lotus chain head - Print chain head @@ -1636,6 +1745,7 @@ OPTIONS: ``` ### lotus chain get-block + ``` NAME: lotus chain get-block - Get a block and print its details @@ -1649,6 +1759,7 @@ OPTIONS: ``` ### lotus chain read-obj + ``` NAME: lotus chain read-obj - Read the raw bytes of an object @@ -1661,6 +1772,7 @@ OPTIONS: ``` ### lotus chain delete-obj + ``` NAME: lotus chain delete-obj - Delete an object from the chain blockstore @@ -1677,6 +1789,7 @@ OPTIONS: ``` ### lotus chain stat-obj + ``` NAME: lotus chain stat-obj - Collect size and ipld link counts for objs @@ -1697,6 +1810,7 @@ OPTIONS: ``` ### lotus chain getmessage + ``` NAME: lotus chain getmessage - Get and print a message by its cid @@ -1709,6 +1823,7 @@ OPTIONS: ``` ### lotus chain sethead + ``` NAME: lotus chain sethead - manually set the local nodes head tipset (Caution: normally only used for recovery) @@ -1723,6 +1838,7 @@ OPTIONS: ``` ### lotus chain list + ``` NAME: lotus chain list - View a segment of the chain @@ -1740,6 +1856,7 @@ OPTIONS: ``` ### lotus chain get + ``` NAME: lotus chain get - Get chain DAG node by path @@ -1786,6 +1903,7 @@ OPTIONS: ``` ### lotus chain bisect + ``` NAME: lotus chain bisect - bisect chain for an event @@ -1813,6 +1931,7 @@ OPTIONS: ``` ### lotus chain export + ``` NAME: lotus chain export - export chain to a car file @@ -1828,6 +1947,7 @@ OPTIONS: ``` ### lotus chain export-range + ``` NAME: lotus chain export-range - export chain to a car file @@ -1847,6 +1967,7 @@ OPTIONS: ``` ### lotus chain slash-consensus + ``` NAME: lotus chain slash-consensus - Report consensus fault @@ -1861,6 +1982,7 @@ OPTIONS: ``` ### lotus chain gas-price + ``` NAME: lotus chain gas-price - Estimate gas prices @@ -1873,6 +1995,7 @@ OPTIONS: ``` ### lotus chain inspect-usage + ``` NAME: lotus chain inspect-usage - Inspect block space usage of a given tipset @@ -1888,6 +2011,7 @@ OPTIONS: ``` ### lotus chain decode + ``` NAME: lotus chain decode - decode various types @@ -1904,6 +2028,7 @@ OPTIONS: ``` #### lotus chain decode params + ``` NAME: lotus chain decode params - Decode message params @@ -1918,6 +2043,7 @@ OPTIONS: ``` ### lotus chain encode + ``` NAME: lotus chain encode - encode various types @@ -1934,6 +2060,7 @@ OPTIONS: ``` #### lotus chain encode params + ``` NAME: lotus chain encode params - Encodes the given JSON params @@ -1949,6 +2076,7 @@ OPTIONS: ``` ### lotus chain disputer + ``` NAME: lotus chain disputer - interact with the window post disputer @@ -1968,6 +2096,7 @@ OPTIONS: ``` #### lotus chain disputer start + ``` NAME: lotus chain disputer start - Start the window post disputer @@ -1981,6 +2110,7 @@ OPTIONS: ``` #### lotus chain disputer dispute + ``` NAME: lotus chain disputer dispute - Send a specific DisputeWindowedPoSt message @@ -1993,6 +2123,7 @@ OPTIONS: ``` ### lotus chain prune + ``` NAME: lotus chain prune - splitstore gc @@ -2011,6 +2142,7 @@ OPTIONS: ``` #### lotus chain prune compact-cold + ``` NAME: lotus chain prune compact-cold - force splitstore compaction on cold store state and run gc @@ -2026,6 +2158,7 @@ OPTIONS: ``` #### lotus chain prune hot + ``` NAME: lotus chain prune hot - run online (badger vlog) garbage collection on hotstore @@ -2040,6 +2173,7 @@ OPTIONS: ``` #### lotus chain prune hot-moving + ``` NAME: lotus chain prune hot-moving - run moving gc on hotstore @@ -2052,6 +2186,7 @@ OPTIONS: ``` ## lotus log + ``` NAME: lotus log - Manage logging @@ -2070,6 +2205,7 @@ OPTIONS: ``` ### lotus log list + ``` NAME: lotus log list - List log systems @@ -2082,6 +2218,7 @@ OPTIONS: ``` ### lotus log set-level + ``` NAME: lotus log set-level - Set log level @@ -2115,6 +2252,7 @@ OPTIONS: ``` ### lotus log alerts + ``` NAME: lotus log alerts - Get alert states @@ -2128,6 +2266,7 @@ OPTIONS: ``` ## lotus wait-api + ``` NAME: lotus wait-api - Wait for lotus api to come online @@ -2144,6 +2283,7 @@ OPTIONS: ``` ## lotus fetch-params + ``` NAME: lotus fetch-params - Fetch proving parameters @@ -2159,6 +2299,7 @@ OPTIONS: ``` ## lotus evm + ``` NAME: lotus evm - Commands related to the Filecoin EVM runtime @@ -2180,6 +2321,7 @@ OPTIONS: ``` ### lotus evm deploy + ``` NAME: lotus evm deploy - Deploy an EVM smart contract and return its address @@ -2194,6 +2336,7 @@ OPTIONS: ``` ### lotus evm invoke + ``` NAME: lotus evm invoke - Invoke an EVM smart contract using the specified CALLDATA @@ -2208,6 +2351,7 @@ OPTIONS: ``` ### lotus evm stat + ``` NAME: lotus evm stat - Print eth/filecoin addrs and code cid @@ -2220,6 +2364,7 @@ OPTIONS: ``` ### lotus evm call + ``` NAME: lotus evm call - Simulate an eth contract call @@ -2232,6 +2377,7 @@ OPTIONS: ``` ### lotus evm contract-address + ``` NAME: lotus evm contract-address - Generate contract address from smart contract code @@ -2244,6 +2390,7 @@ OPTIONS: ``` ### lotus evm bytecode + ``` NAME: lotus evm bytecode - Write the bytecode of a smart contract to a file @@ -2257,6 +2404,7 @@ OPTIONS: ``` ## lotus index + ``` NAME: lotus index - Commands related to managing the chainindex @@ -2273,6 +2421,7 @@ OPTIONS: ``` ### lotus index validate-backfill + ``` NAME: lotus index validate-backfill - Validates and optionally backfills the chainindex for a range of epochs @@ -2325,6 +2474,7 @@ OPTIONS: ``` ## lotus net + ``` NAME: lotus net - Manage P2P Network @@ -2356,6 +2506,7 @@ OPTIONS: ``` ### lotus net peers + ``` NAME: lotus net peers - Print peers @@ -2370,6 +2521,7 @@ OPTIONS: ``` ### lotus net ping + ``` NAME: lotus net ping - Ping peers @@ -2384,6 +2536,7 @@ OPTIONS: ``` ### lotus net connect + ``` NAME: lotus net connect - Connect to a peer @@ -2396,6 +2549,7 @@ OPTIONS: ``` ### lotus net disconnect + ``` NAME: lotus net disconnect - Disconnect from a peer @@ -2408,6 +2562,7 @@ OPTIONS: ``` ### lotus net listen + ``` NAME: lotus net listen - List listen addresses @@ -2420,6 +2575,7 @@ OPTIONS: ``` ### lotus net id + ``` NAME: lotus net id - Get node identity @@ -2432,6 +2588,7 @@ OPTIONS: ``` ### lotus net find-peer + ``` NAME: lotus net find-peer - Find the addresses of a given peerID @@ -2444,6 +2601,7 @@ OPTIONS: ``` ### lotus net scores + ``` NAME: lotus net scores - Print peers' pubsub scores @@ -2457,6 +2615,7 @@ OPTIONS: ``` ### lotus net reachability + ``` NAME: lotus net reachability - Print information about reachability from the internet @@ -2469,6 +2628,7 @@ OPTIONS: ``` ### lotus net bandwidth + ``` NAME: lotus net bandwidth - Print bandwidth usage information @@ -2483,6 +2643,7 @@ OPTIONS: ``` ### lotus net block + ``` NAME: lotus net block - Manage network connection gating rules @@ -2501,6 +2662,7 @@ OPTIONS: ``` #### lotus net block add + ``` NAME: lotus net block add - Add connection gating rules @@ -2519,6 +2681,7 @@ OPTIONS: ``` ##### lotus net block add peer + ``` NAME: lotus net block add peer - Block a peer @@ -2531,6 +2694,7 @@ OPTIONS: ``` ##### lotus net block add ip + ``` NAME: lotus net block add ip - Block an IP address @@ -2543,6 +2707,7 @@ OPTIONS: ``` ##### lotus net block add subnet + ``` NAME: lotus net block add subnet - Block an IP subnet @@ -2555,6 +2720,7 @@ OPTIONS: ``` #### lotus net block remove + ``` NAME: lotus net block remove - Remove connection gating rules @@ -2573,6 +2739,7 @@ OPTIONS: ``` ##### lotus net block remove peer + ``` NAME: lotus net block remove peer - Unblock a peer @@ -2585,6 +2752,7 @@ OPTIONS: ``` ##### lotus net block remove ip + ``` NAME: lotus net block remove ip - Unblock an IP address @@ -2597,6 +2765,7 @@ OPTIONS: ``` ##### lotus net block remove subnet + ``` NAME: lotus net block remove subnet - Unblock an IP subnet @@ -2609,6 +2778,7 @@ OPTIONS: ``` #### lotus net block list + ``` NAME: lotus net block list - list connection gating rules @@ -2621,6 +2791,7 @@ OPTIONS: ``` ### lotus net stat + ``` NAME: lotus net stat - Report resource usage for a scope @@ -2646,6 +2817,7 @@ OPTIONS: ``` ### lotus net limit + ``` NAME: lotus net limit - Get or set resource limits for a scope @@ -2672,6 +2844,7 @@ OPTIONS: ``` ### lotus net protect + ``` NAME: lotus net protect - Add one or more peer IDs to the list of protected peer connections @@ -2684,6 +2857,7 @@ OPTIONS: ``` ### lotus net unprotect + ``` NAME: lotus net unprotect - Remove one or more peer IDs from the list of protected peer connections. @@ -2696,6 +2870,7 @@ OPTIONS: ``` ### lotus net list-protected + ``` NAME: lotus net list-protected - List the peer IDs with protected connection. @@ -2708,6 +2883,7 @@ OPTIONS: ``` ## lotus sync + ``` NAME: lotus sync - Inspect or interact with the chain syncer @@ -2729,6 +2905,7 @@ OPTIONS: ``` ### lotus sync status + ``` NAME: lotus sync status - check sync status @@ -2741,6 +2918,7 @@ OPTIONS: ``` ### lotus sync wait + ``` NAME: lotus sync wait - Wait for sync to be complete @@ -2754,6 +2932,7 @@ OPTIONS: ``` ### lotus sync mark-bad + ``` NAME: lotus sync mark-bad - Mark the given block as bad, will prevent syncing to a chain that contains it @@ -2766,6 +2945,7 @@ OPTIONS: ``` ### lotus sync unmark-bad + ``` NAME: lotus sync unmark-bad - Unmark the given block as bad, makes it possible to sync to a chain containing it @@ -2779,6 +2959,7 @@ OPTIONS: ``` ### lotus sync check-bad + ``` NAME: lotus sync check-bad - check if the given block was marked bad, and for what reason @@ -2791,6 +2972,7 @@ OPTIONS: ``` ### lotus sync checkpoint + ``` NAME: lotus sync checkpoint - mark a certain tipset as checkpointed; the node will never fork away from this tipset @@ -2804,6 +2986,7 @@ OPTIONS: ``` ## lotus f3 + ``` NAME: lotus f3 - Manages Filecoin Fast Finality (F3) interactions @@ -2824,6 +3007,7 @@ OPTIONS: ``` ### lotus f3 list-miners + ``` NAME: lotus f3 list-miners - Lists the miners that currently participate in F3 via this node. @@ -2836,6 +3020,7 @@ OPTIONS: ``` ### lotus f3 powertable + ``` NAME: lotus f3 powertable @@ -2853,6 +3038,7 @@ OPTIONS: ``` #### lotus f3 powertable get + ``` NAME: lotus f3 powertable get - Get F3 power table at a specific instance ID or latest instance if none is specified. @@ -2866,6 +3052,7 @@ OPTIONS: ``` #### lotus f3 powertable get-proportion + ``` NAME: lotus f3 powertable get-proportion - Gets the total proportion of power for a list of actors at a given instance. @@ -2880,6 +3067,7 @@ OPTIONS: ``` ### lotus f3 certs + ``` NAME: lotus f3 certs - Manages interactions with F3 finality certificates. @@ -2932,6 +3120,7 @@ OPTIONS: ``` #### lotus f3 certs get + ``` NAME: lotus f3 certs get - Gets an F3 finality certificate to a given instance ID, or the latest certificate if no instance is specified. @@ -2945,6 +3134,7 @@ OPTIONS: ``` #### lotus f3 certs list + ``` NAME: lotus f3 certs list - Lists a range of F3 finality certificates. @@ -2995,6 +3185,7 @@ OPTIONS: ``` ### lotus f3 manifest + ``` NAME: lotus f3 manifest - Gets the current manifest used by F3. @@ -3008,6 +3199,7 @@ OPTIONS: ``` ### lotus f3 status + ``` NAME: lotus f3 status - Checks the F3 status. @@ -3020,6 +3212,7 @@ OPTIONS: ``` ## lotus status + ``` NAME: lotus status - Check node status diff --git a/documentation/en/data-onboarding-visibility.md b/documentation/en/data-onboarding-visibility.md index 3225753dd1d..346867a58fc 100644 --- a/documentation/en/data-onboarding-visibility.md +++ b/documentation/en/data-onboarding-visibility.md @@ -32,7 +32,7 @@ There are two possible additions to this flow: 💡 **The builtin market actor should not be used as single a source of truth regarding data onboarding activities.** The builtin market actor is only a source of truth for data onboarding mediated by the builtin market actor. -💡 **The builtin market actor should not be used as a source of truth regarding verified claims and metrics related to FIL+ usage (size, clients, profiders).** The `VerifiedClaim` property of `DealState` has been removed from the builtin market actor. Instead, the verified registry should be used as the only source of truth regarding both allocations and claims. +💡 **The builtin market actor should not be used as a source of truth regarding verified claims and metrics related to FIL+ usage (size, clients, providers).** The `VerifiedClaim` property of `DealState` has been removed from the builtin market actor. Instead, the verified registry should be used as the only source of truth regarding both allocations and claims. 💡 **Sector data commitments and their constituent pieces are only stored on chain in the verified registry claims in the case of verified data (pieces) onboarded in any mechanism (DDO and/or builtin market actor).** Piece information for data onboarded that is not verified ("sparkling data") and not mediated through the builtin market actor will only appear in messages and actor events. Messages and actor events may be used as a source of truth for data sector commitments. diff --git a/documentation/en/default-lotus-config.toml b/documentation/en/default-lotus-config.toml index 41bc082da38..1a915593edb 100644 --- a/documentation/en/default-lotus-config.toml +++ b/documentation/en/default-lotus-config.toml @@ -377,4 +377,3 @@ # env var: LOTUS_FAULTREPORTER_CONSENSUSFAULTREPORTERADDRESS #ConsensusFaultReporterAddress = "" - diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index aaf58fec1cb..e03f9579af9 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -650,4 +650,3 @@ # env var: LOTUS_HARMONYDB_PORT #Port = "5433" - diff --git a/documentation/misc/Building_a_network_skeleton.md b/documentation/misc/Building_a_network_skeleton.md index 1fae76b923b..cb53b4698a7 100644 --- a/documentation/misc/Building_a_network_skeleton.md +++ b/documentation/misc/Building_a_network_skeleton.md @@ -101,7 +101,7 @@ There is a network skeleton in Lotus, which bubbles up all the other dependencie You can take a look at [this Ref-FVM PR as a reference](https://github.com/filecoin-project/ref-fvm/pull/2029), which added the skeleton for network version 24. You can also check out the [releasing primary FVM crates checklist here](https://github.com/filecoin-project/ref-fvm/blob/master/CONTRIBUTING.md#primary-fvm-crates) -2. In a seperate PR bump the Ref-FVM version: +2. In a separate PR bump the Ref-FVM version: - Bump the version in the root Cargo.toml file. - Bump the fvm, fvm_shared and fvm_sdk versions in the `workspace` section in `ref-fvm/cargo.toml` @@ -179,7 +179,7 @@ You can take a look at [this PR as a reference](https://github.com/filecoin-proj 👉 You can take a look at this [Filecoin-FFI PR as a reference](https://github.com/filecoin-project/filecoin-ffi/pull/481), which was for network version 24. -Note: one only needs to update `filecion-ffi`'s dependency on `go-state-types` when a network upgrade is introducing new types in `go-state-types` (see [below](#new-types-in-go-state-types)). Otherwise, `filecion-ffi`'s dependency on `go-state-types` is just updated when doing fiinal releases before the network upgrade. +Note: one only needs to update `filecion-ffi`'s dependency on `go-state-types` when a network upgrade is introducing new types in `go-state-types` (see [below](#new-types-in-go-state-types)). Otherwise, `filecion-ffi`'s dependency on `go-state-types` is just updated when doing final releases before the network upgrade. ## Lotus Checklist diff --git a/itests/sector_miner_collateral_test.go b/itests/sector_miner_collateral_test.go index 98bd98f8dcf..5bb2e79df8e 100644 --- a/itests/sector_miner_collateral_test.go +++ b/itests/sector_miner_collateral_test.go @@ -326,7 +326,7 @@ func TestPledgeCalculations(t *testing.T) { return markettypes.ClientDealProposal{Proposal: dp, ClientSignature: *sig} } - // make a deal of a given size that may or may not be be verified (and require an allocation) + // make a deal of a given size that may or may not be verified (and require an allocation) makeDealOfSize := func(paddedSize int, verified bool) (abi.PieceInfo, abi.DealID) { marketPieceSize := abi.PaddedPieceSize(paddedSize) pieceData := make([]byte, marketPieceSize) diff --git a/lib/tablewriter/tablewriter.go b/lib/tablewriter/tablewriter.go index 75e089938bd..b826ac5ce5c 100644 --- a/lib/tablewriter/tablewriter.go +++ b/lib/tablewriter/tablewriter.go @@ -13,6 +13,31 @@ type Column struct { Name string SeparateLine bool Lines int + RightAlign bool +} + +type tableCfg struct { + borders bool +} + +type TableOption func(*tableCfg) + +func WithBorders() TableOption { + return func(c *tableCfg) { + c.borders = true + } +} + +type columnCfg struct { + rightAlign bool +} + +type ColumnOption func(*columnCfg) + +func RightAlign() ColumnOption { + return func(c *columnCfg) { + c.rightAlign = true + } } type TableWriter struct { @@ -20,10 +45,15 @@ type TableWriter struct { rows []map[int]string } -func Col(name string) Column { +func Col(name string, opts ...ColumnOption) Column { + cfg := &columnCfg{} + for _, o := range opts { + o(cfg) + } return Column{ Name: name, SeparateLine: false, + RightAlign: cfg.rightAlign, } } @@ -69,7 +99,12 @@ cloop: w.rows = append(w.rows, byColID) } -func (w *TableWriter) Flush(out io.Writer) error { +func (w *TableWriter) Flush(out io.Writer, opts ...TableOption) error { + cfg := &tableCfg{} + for _, o := range opts { + o(cfg) + } + colLengths := make([]int, len(w.cols)) header := map[int]string{} @@ -99,21 +134,62 @@ func (w *TableWriter) Flush(out io.Writer) error { } } - for _, row := range w.rows { + if cfg.borders { + // top line + if _, err := fmt.Fprint(out, "┌"); err != nil { + return err + } + for ci, col := range w.cols { + if col.Lines == 0 { + continue + } + if _, err := fmt.Fprint(out, strings.Repeat("─", colLengths[ci]+2)); err != nil { + return err + } + if ci != len(w.cols)-1 { + if _, err := fmt.Fprint(out, "┬"); err != nil { + return err + } + } + } + if _, err := fmt.Fprintln(out, "┐"); err != nil { + return err + } + } + + for lineNumber, row := range w.rows { cols := make([]string, len(w.cols)) + if cfg.borders { + if _, err := fmt.Fprint(out, "│ "); err != nil { + return err + } + } + for ci, col := range w.cols { if col.Lines == 0 { continue } - e, _ := row[ci] + e := row[ci] pad := colLengths[ci] - cliStringLength(e) + 2 + if cfg.borders { + pad-- + } if !col.SeparateLine && col.Lines > 0 { - e = e + strings.Repeat(" ", pad) + if col.RightAlign { + e = strings.Repeat(" ", pad-1) + e + " " + } else { + e = e + strings.Repeat(" ", pad) + } if _, err := fmt.Fprint(out, e); err != nil { return err } + if cfg.borders { + if _, err := fmt.Fprint(out, "│ "); err != nil { + return err + } + } } cols[ci] = e @@ -132,6 +208,53 @@ func (w *TableWriter) Flush(out io.Writer) error { return err } } + + if lineNumber == 0 && cfg.borders { + // print bottom of header + if _, err := fmt.Fprint(out, "├"); err != nil { + return err + } + for ci, col := range w.cols { + if col.Lines == 0 { + continue + } + + if _, err := fmt.Fprint(out, strings.Repeat("─", colLengths[ci]+2)); err != nil { + return err + } + if ci != len(w.cols)-1 { + if _, err := fmt.Fprint(out, "┼"); err != nil { + return err + } + } + } + if _, err := fmt.Fprintln(out, "┤"); err != nil { + return err + } + } + } + + if cfg.borders { + // bottom line + if _, err := fmt.Fprint(out, "└"); err != nil { + return err + } + for ci, col := range w.cols { + if col.Lines == 0 { + continue + } + if _, err := fmt.Fprint(out, strings.Repeat("─", colLengths[ci]+2)); err != nil { + return err + } + if ci != len(w.cols)-1 { + if _, err := fmt.Fprint(out, "┴"); err != nil { + return err + } + } + } + if _, err := fmt.Fprintln(out, "┘"); err != nil { + return err + } } return nil diff --git a/scripts/docsgen-cli/doc_generator.go b/scripts/docsgen-cli/doc_generator.go new file mode 100644 index 00000000000..3638ac92cda --- /dev/null +++ b/scripts/docsgen-cli/doc_generator.go @@ -0,0 +1,156 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/urfave/cli/v2" +) + +// DocGenerator handles CLI documentation generation +type DocGenerator struct { + app *cli.App + outputDir string + writer io.Writer +} + +// NewDocGenerator creates a new documentation generator +func NewDocGenerator(outputDir string, app *cli.App) *DocGenerator { + return &DocGenerator{ + outputDir: outputDir, + app: app, + } +} + +// Generate generates documentation for the CLI app +func (g *DocGenerator) Generate(name string) error { + file, err := g.createMarkdownFile(name) + if err != nil { + return fmt.Errorf("failed to create markdown file: %w", err) + } + defer func() { + if err := file.Close(); err != nil { + fmt.Printf("failed to close markdown file: %v\n", err) + } + }() + + return g.generateContent(file, name) +} + +// createMarkdownFile creates a new markdown file for output. +func (g *DocGenerator) createMarkdownFile(name string) (*os.File, error) { + filePath := filepath.Join(g.outputDir, fmt.Sprintf("cli-%s.md", name)) + return os.Create(filePath) +} + +func (g *DocGenerator) generateContent(file *os.File, name string) error { + bufferedWriter := bufio.NewWriter(file) + g.writer = bufferedWriter + g.app.Writer = bufferedWriter + + if err := g.generateDocs(name); err != nil { + return fmt.Errorf("failed to generate documentation: %w", err) + } + + return bufferedWriter.Flush() +} + +// generateDocs orchestrates the documentation generation process +func (g *DocGenerator) generateDocs(name string) error { + if err := g.writeAppHeader(); err != nil { + return fmt.Errorf("failed to write app header: %w", err) + } + + return g.writeCommandDocs(g.app.Commands, name, 0) +} + +// writeAppHeader writes the application header documentation +func (g *DocGenerator) writeAppHeader() error { + if _, err := g.writer.Write([]byte(fmt.Sprintf("# %s\n\n```\n", g.app.Name))); err != nil { + return err + } + + if err := g.app.Run(getHelpArgs("", "")); err != nil { + return fmt.Errorf("failed to write command docs: %w", err) + } + + if _, err := g.writer.Write([]byte("```\n")); err != nil { + return err + } + + return nil +} + +func (g *DocGenerator) writeCommandDocs(commands cli.Commands, rootName string, depth int) error { + uncategorizedCmds, categorizedCmds := separateCommands(commands) + + // Write uncategorized commands first + if err := g.writeCommands(uncategorizedCmds, rootName, depth); err != nil { + return fmt.Errorf("failed to write uncategorized commands: %w", err) + } + + // Write categorized commands next + if err := g.writeCommands(categorizedCmds, rootName, depth); err != nil { + return fmt.Errorf("failed to write categorized commands: %w", err) + } + + return nil +} + +// separateCommands separates commands into uncategorized and categorized +func separateCommands(commands []*cli.Command) (uncategorized cli.Commands, categorized cli.Commands) { + for _, cmd := range commands { + if cmd.Category == "" { + uncategorized = append(uncategorized, cmd) + } else { + categorized = append(categorized, cmd) + } + } + + return uncategorized, categorized +} + +// writeCommands writes documentation for all commands recursively +func (g *DocGenerator) writeCommands(commands cli.Commands, rootName string, depth int) error { + for _, cmd := range commands { + if cmd.Name == "help" || cmd.Hidden { + continue + } + + cmdName := fmt.Sprintf("%s %s", rootName, cmd.Name) + + if _, err := g.writer.Write([]byte(fmt.Sprintf("\n%s %s\n\n```\n", strings.Repeat("#", depth+2), cmdName))); err != nil { + return err + } + + if err := g.app.Run(getHelpArgs(rootName, cmd.Name)); err != nil { + return fmt.Errorf("failed to write command docs: %w", err) + } + + if _, err := g.writer.Write([]byte("```\n")); err != nil { + return err + } + + if len(cmd.Subcommands) > 0 { + if err := g.writeCommands(cmd.Subcommands, rootName+" "+cmd.Name, depth+1); err != nil { + return err + } + } + } + return nil +} + +func getHelpArgs(rootName string, cmdName string) []string { + if rootName == "" && cmdName == "" { + return []string{"-h"} + } + + args := strings.Split(rootName, " ") + args = append(args, cmdName) + args = append(args, "-h") + return args +} diff --git a/scripts/docsgen-cli/main.go b/scripts/docsgen-cli/main.go index 95d3c5170d3..e6e562e7790 100644 --- a/scripts/docsgen-cli/main.go +++ b/scripts/docsgen-cli/main.go @@ -3,11 +3,14 @@ package main import ( "fmt" "os" - "os/exec" "path/filepath" - "strings" - "golang.org/x/sync/errgroup" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/cli/lotus" + "github.com/filecoin-project/lotus/cli/miner" + "github.com/filecoin-project/lotus/cli/worker" + "github.com/filecoin-project/lotus/node/config" ) const ( @@ -44,90 +47,93 @@ func main() { os.Exit(1) } + cliApps := loadCLIApps() + fmt.Println("Generating CLI documentation...") - var eg errgroup.Group - for _, cmd := range []string{"lotus", "lotus-miner", "lotus-worker"} { - eg.Go(func() error { - err := generateMarkdownForCLI(cmd) - if err != nil { - fmt.Printf(" ❌ %s: %v\n", cmd, err) - } else { - fmt.Printf(" ✅ %s\n", cmd) - } - return err - }) - } - if err := eg.Wait(); err != nil { - fmt.Printf("Failed to generate CLI documentation: %v\n", err) + failed := generateCLIDocumentation(cliApps) + + fmt.Println("Generating default config files...") + failed = generateDefaultConfigs() || failed + + if failed { + fmt.Println("Documentation generation failed.") os.Exit(1) } fmt.Println("Documentation generation complete.") } -func generateMarkdownForCLI(cli string) error { - md := filepath.Join(outputDir, fmt.Sprintf("cli-%s.md", cli)) - out, err := os.Create(md) - if err != nil { - return err +func loadCLIApps() map[string]*cli.App { + // Some help output is generated based on whether the output is a terminal or not. To make stable + // output text, we set Stdout to not be a terminal while we load the CLI apps and reset it + // before generating the documentation. + _, w, _ := os.Pipe() + stdout := os.Stdout + os.Stdout = w + + cliApps := map[string]*cli.App{ + "lotus": lotus.App(), + "lotus-worker": worker.App(), + "lotus-miner": miner.App(), } - defer func() { _ = out.Close() }() - return writeCommandDocs(out, cli, 0) + + _ = w.Close() + os.Stdout = stdout + + return cliApps } -func writeCommandDocs(file *os.File, command string, depth int) error { - // For sanity, fail fast if depth exceeds some arbitrarily large number. In which - // case, chances are there is a bug in this script. - if depth > depthRecursionLimit { - return fmt.Errorf("recursion exceeded limit of %d", depthRecursionLimit) - } +func generateCLIDocumentation(cliApps map[string]*cli.App) bool { + var failed bool + for name, app := range cliApps { + resetCommandHelpName(app) - // Get usage from the command. - usage, err := exec.Command("sh", "-c", "./"+command+" -h").Output() - if err != nil { - return fmt.Errorf("failed to run '%s': %v", command, err) + generator := NewDocGenerator(outputDir, app) + if err := generator.Generate(name); err != nil { + fmt.Printf(" ❌ %s: %v\n", name, err) + failed = true + continue + } + fmt.Printf(" ✅ %s\n", name) } + return failed +} - // Skip the first new line since the docs do not start with a newline at the very - // top. - if depth != 0 { - if _, err := file.WriteString("\n"); err != nil { - return err +// resetCommandHelpName resets the HelpName of all commands to include the parent command names. +// This is needed for the case where Commands are shared between apps. +func resetCommandHelpName(app *cli.App) { + var fix func(cmds []*cli.Command, helpName string) + fix = func(cmds []*cli.Command, helpName string) { + for _, cmd := range cmds { + cmd.HelpName = fmt.Sprintf("%s %s", helpName, cmd.Name) + fix(cmd.Subcommands, cmd.HelpName) } } + fix(app.Commands, app.HelpName) +} - // Write out command header and usage. - header := fmt.Sprintf("%s# %s\n", strings.Repeat("#", depth), command) - if _, err := file.WriteString(header); err != nil { - return err - } else if _, err := file.WriteString("```\n"); err != nil { - return err - } else if _, err := file.Write(usage); err != nil { - return err - } else if _, err := file.WriteString("```\n"); err != nil { - return err +func generateDefaultConfigs() bool { + var failed bool + if err := generateDefaultConfig(config.DefaultFullNode(), "default-lotus-config.toml"); err != nil { + fmt.Printf(" ❌ %s: %v\n", "lotus", err) + failed = true + } else { + fmt.Printf(" ✅ %s\n", "lotus") } - // Recurse sub-commands. - commands := false - lines := strings.Split(string(usage), "\n") - for _, line := range lines { - switch line = strings.TrimSpace(line); { - case line == "": - commands = false - case line == "COMMANDS:": - commands = true - case strings.HasPrefix(line, "help, h"): - // Skip usage command. - case commands: - // Find the sub command and trim any potential comma in case of alias. - subCommand := strings.TrimSuffix(strings.Fields(line)[0], ",") - // Skip sections in usage that have no command. - if !strings.Contains(subCommand, ":") { - if err := writeCommandDocs(file, command+" "+subCommand, depth+1); err != nil { - return err - } - } - } + if err := generateDefaultConfig(config.DefaultStorageMiner(), "default-lotus-miner-config.toml"); err != nil { + fmt.Printf(" ❌ %s: %v\n", "lotus-miner", err) + failed = true + } else { + fmt.Printf(" ✅ %s\n", "lotus-miner") + } + return failed +} + +func generateDefaultConfig(c interface{}, file string) error { + cb, err := config.ConfigUpdate(c, nil, config.Commented(true), config.DefaultKeepUncommented()) + if err != nil { + return err } - return nil + output := filepath.Join(outputDir, file) + return os.WriteFile(output, cb, 0644) } diff --git a/storage/sealer/README.md b/storage/sealer/README.md index b90a9ebd0b7..186e536913b 100644 --- a/storage/sealer/README.md +++ b/storage/sealer/README.md @@ -5,7 +5,7 @@ > a concrete implementation of the [specs-storage](https://github.com/filecoin-project/specs-storage) interface -The sector-storage project provides a implementation-nonspecific reference implementation of the [specs-storage](https://github.com/filecoin-project/specs-storage) interface. +The sector-storage project provides an implementation-nonspecific reference implementation of the [specs-storage](https://github.com/filecoin-project/specs-storage) interface. ## Disclaimer diff --git a/tools/kibana/README.md b/tools/kibana/README.md index c556ae10cf5..728a9894c39 100644 --- a/tools/kibana/README.md +++ b/tools/kibana/README.md @@ -6,7 +6,7 @@ throughout Filecoin network. ### Importing index template Index template needs to be imported into Elasticsearch for score weights and to -prevent Elasticsearch from infering wrong field type. +prevent Elasticsearch from inferring wrong field type. The [template](./index-template.json) is loaded via [Kibana Index Management](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-mgmt.html) and pasted into newly created Index Template.