fix(chainindexer): backfilling should halt when chain state data is missing and not backfill parents

chain/index/api.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
@@ -285,33 +286,22 @@ func (si *SqliteIndexer) verifyIndexedData(ctx context.Context, ts *types.TipSet
 }
 
 func (si *SqliteIndexer) backfillMissingTipset(ctx context.Context, ts *types.TipSet) (*types.IndexValidation, error) {
-	// backfill the tipset in the Index
-	parentTs, err := si.cs.GetTipSetFromKey(ctx, ts.Parents())
-	if err != nil {
-		return nil, xerrors.Errorf("failed to get parent tipset at height %d: %w", ts.Height(), err)
-	}
-
 	executionTs, err := si.getNextTipset(ctx, ts)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to get next tipset at height %d: %w", ts.Height(), err)
 	}
 
 	backfillFunc := func() error {
 		return withTx(ctx, si.db, func(tx *sql.Tx) error {
-			if err := si.indexTipsetWithParentEvents(ctx, tx, ts, executionTs); err != nil {
-				return xerrors.Errorf("error indexing (ts, executionTs): %w", err)
-			}
-
-			if err := si.indexTipsetWithParentEvents(ctx, tx, parentTs, ts); err != nil {
-				return xerrors.Errorf("error indexing (parentTs, ts): %w", err)
-			}
-
-			return nil
+			return si.indexTipsetWithParentEvents(ctx, tx, ts, executionTs)
 		})
 	}
 
 	if err := backfillFunc(); err != nil {
-		return nil, xerrors.Errorf("failed to backfill tipset: %w", err)
+		if ipld.IsNotFound(err) {
+			return nil, xerrors.Errorf("failed to backfill tipset at epoch %d: chain store does not contain data: %w", ts.Height(), err)
+		}
+		return nil, xerrors.Errorf("failed to backfill tipset at epoch %d; err: %w", ts.Height(), err)
 	}
 
 	indexedData, err := si.getIndexedTipSetData(ctx, ts)

cmd/lotus-shed/chain_index.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/urfave/cli/v2"
@@ -135,8 +136,9 @@ number of failed RPC calls. Otherwise, it will exit with a zero status.
 			_, _ = fmt.Fprintf(cctx.App.Writer, "%s starting chainindex validation; from epoch: %d; to epoch: %d; backfill: %t; log-good: %t\n", currentTimeString(),
 				fromEpoch, toEpoch, backfill, logGood)
 		}
-
 		totalEpochs := fromEpoch - toEpoch + 1
+		haltHeight := -1
+
 		for epoch := fromEpoch; epoch >= toEpoch; epoch-- {
 			if ctx.Err() != nil {
 				return ctx.Err()
@@ -153,6 +155,11 @@ number of failed RPC calls. Otherwise, it will exit with a zero status.
 
 			indexValidateResp, err := api.ChainValidateIndex(ctx, abi.ChainEpoch(epoch), backfill)
 			if err != nil {
+				if strings.Contains(err.Error(), "chain store does not contain data") {
+					haltHeight = epoch
+					break
+				}
+
 				_, _ = fmt.Fprintf(cctx.App.Writer, "%s ✗ Epoch %d; failure: %s\n", currentTimeString(), epoch, err)
 				failedRPCs++
 				continue
@@ -190,7 +197,9 @@ number of failed RPC calls. Otherwise, it will exit with a zero status.
 			_, _ = fmt.Fprintf(cctx.App.Writer, "Total successful Null round validations: %d\n", successfulNullRounds)
 		}
 
-		if failedRPCs > 0 {
+		if haltHeight >= 0 {
+			return fmt.Errorf("chain index validation and backfilled halted at height %d as chain state does contain data for that height", haltHeight)
+		} else if failedRPCs > 0 {
 			return fmt.Errorf("chain index validation failed with %d RPC errors", failedRPCs)
 		}