From 2656b441b01e6850e0b77ecb0e80cfa8e9f9b256 Mon Sep 17 00:00:00 2001 From: aarshkshah1992 Date: Mon, 21 Oct 2024 11:57:25 +0530 Subject: [PATCH 1/3] fix backfilling UX --- chain/index/api.go | 22 ++++++---------------- cmd/lotus-shed/chain_index.go | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/chain/index/api.go b/chain/index/api.go index 4ec7f34cfa9..ed243db941d 100644 --- a/chain/index/api.go +++ b/chain/index/api.go @@ -6,6 +6,7 @@ import ( "errors" "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" @@ -285,12 +286,6 @@ func (si *SqliteIndexer) verifyIndexedData(ctx context.Context, ts *types.TipSet } func (si *SqliteIndexer) backfillMissingTipset(ctx context.Context, ts *types.TipSet) (*types.IndexValidation, error) { - // backfill the tipset in the Index - parentTs, err := si.cs.GetTipSetFromKey(ctx, ts.Parents()) - if err != nil { - return nil, xerrors.Errorf("failed to get parent tipset at height %d: %w", ts.Height(), err) - } - executionTs, err := si.getNextTipset(ctx, ts) if err != nil { return nil, xerrors.Errorf("failed to get next tipset at height %d: %w", ts.Height(), err) @@ -298,20 +293,15 @@ func (si *SqliteIndexer) backfillMissingTipset(ctx context.Context, ts *types.Ti backfillFunc := func() error { return withTx(ctx, si.db, func(tx *sql.Tx) error { - if err := si.indexTipsetWithParentEvents(ctx, tx, ts, executionTs); err != nil { - return xerrors.Errorf("error indexing (ts, executionTs): %w", err) - } - - if err := si.indexTipsetWithParentEvents(ctx, tx, parentTs, ts); err != nil { - return xerrors.Errorf("error indexing (parentTs, ts): %w", err) - } - - return nil + return si.indexTipsetWithParentEvents(ctx, tx, ts, executionTs) }) } if err := backfillFunc(); err != nil { - return nil, xerrors.Errorf("failed to backfill tipset: %w", err) + if ipld.IsNotFound(err) { + return nil, xerrors.Errorf("failed to backfill tipset at epoch %d: chain store does not contain data", ts.Height()) + } + return nil, xerrors.Errorf("failed to backfill tipset at epoch %d; err: %w", ts.Height(), err) } indexedData, err := si.getIndexedTipSetData(ctx, ts) diff --git a/cmd/lotus-shed/chain_index.go b/cmd/lotus-shed/chain_index.go index 411b5fd27a2..7d721b56719 100644 --- a/cmd/lotus-shed/chain_index.go +++ b/cmd/lotus-shed/chain_index.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "strings" "time" "github.com/urfave/cli/v2" @@ -135,8 +136,10 @@ number of failed RPC calls. Otherwise, it will exit with a zero status. _, _ = fmt.Fprintf(cctx.App.Writer, "%s starting chainindex validation; from epoch: %d; to epoch: %d; backfill: %t; log-good: %t\n", currentTimeString(), fromEpoch, toEpoch, backfill, logGood) } - totalEpochs := fromEpoch - toEpoch + 1 + haltNoData := false + haltHeight := -1 + for epoch := fromEpoch; epoch >= toEpoch; epoch-- { if ctx.Err() != nil { return ctx.Err() @@ -153,6 +156,12 @@ number of failed RPC calls. Otherwise, it will exit with a zero status. indexValidateResp, err := api.ChainValidateIndex(ctx, abi.ChainEpoch(epoch), backfill) if err != nil { + if strings.Contains(err.Error(), "chain store does not contain data") { + haltHeight = epoch + haltNoData = true + break + } + _, _ = fmt.Fprintf(cctx.App.Writer, "%s ✗ Epoch %d; failure: %s\n", currentTimeString(), epoch, err) failedRPCs++ continue @@ -190,7 +199,9 @@ number of failed RPC calls. Otherwise, it will exit with a zero status. _, _ = fmt.Fprintf(cctx.App.Writer, "Total successful Null round validations: %d\n", successfulNullRounds) } - if failedRPCs > 0 { + if haltNoData { + return fmt.Errorf("chain index validation and backfilled halted at height %d as chain state does contain data for that height", haltHeight) + } else if failedRPCs > 0 { return fmt.Errorf("chain index validation failed with %d RPC errors", failedRPCs) } From 4cb958b4a5b127cdfea952b59bd414d1e3dfd295 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Mon, 21 Oct 2024 11:12:51 +0400 Subject: [PATCH 2/3] Update chain/index/api.go Co-authored-by: Rod Vagg --- chain/index/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/index/api.go b/chain/index/api.go index ed243db941d..9588383ea1f 100644 --- a/chain/index/api.go +++ b/chain/index/api.go @@ -299,7 +299,7 @@ func (si *SqliteIndexer) backfillMissingTipset(ctx context.Context, ts *types.Ti if err := backfillFunc(); err != nil { if ipld.IsNotFound(err) { - return nil, xerrors.Errorf("failed to backfill tipset at epoch %d: chain store does not contain data", ts.Height()) + return nil, xerrors.Errorf("failed to backfill tipset at epoch %d: chain store does not contain data: %w", ts.Height(), err) } return nil, xerrors.Errorf("failed to backfill tipset at epoch %d; err: %w", ts.Height(), err) } From dcce2f94f508e3e0d1ee1d06dc66be728ab78dd7 Mon Sep 17 00:00:00 2001 From: aarshkshah1992 Date: Mon, 21 Oct 2024 12:47:14 +0530 Subject: [PATCH 3/3] address review --- cmd/lotus-shed/chain_index.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/lotus-shed/chain_index.go b/cmd/lotus-shed/chain_index.go index 7d721b56719..53cc06d2b61 100644 --- a/cmd/lotus-shed/chain_index.go +++ b/cmd/lotus-shed/chain_index.go @@ -137,7 +137,6 @@ number of failed RPC calls. Otherwise, it will exit with a zero status. fromEpoch, toEpoch, backfill, logGood) } totalEpochs := fromEpoch - toEpoch + 1 - haltNoData := false haltHeight := -1 for epoch := fromEpoch; epoch >= toEpoch; epoch-- { @@ -158,7 +157,6 @@ number of failed RPC calls. Otherwise, it will exit with a zero status. if err != nil { if strings.Contains(err.Error(), "chain store does not contain data") { haltHeight = epoch - haltNoData = true break } @@ -199,7 +197,7 @@ number of failed RPC calls. Otherwise, it will exit with a zero status. _, _ = fmt.Fprintf(cctx.App.Writer, "Total successful Null round validations: %d\n", successfulNullRounds) } - if haltNoData { + if haltHeight >= 0 { return fmt.Errorf("chain index validation and backfilled halted at height %d as chain state does contain data for that height", haltHeight) } else if failedRPCs > 0 { return fmt.Errorf("chain index validation failed with %d RPC errors", failedRPCs)