sealing: Address review

api/api_storage.go

@@ -563,11 +563,11 @@ type RemoteSectorMeta struct {
 	// Sector urls - lotus will use those for fetching files into local storage
 
 	// Required in all states
-	DataUnsealed *storiface.SectorData
+	DataUnsealed *storiface.SectorLocation
 
 	// Required in PreCommitting and later
-	DataSealed *storiface.SectorData
-	DataCache  *storiface.SectorData
+	DataSealed *storiface.SectorLocation
+	DataCache  *storiface.SectorLocation
 
 	////////
 	// SEALING SERVICE HOOKS

api/api_worker.go

@@ -49,7 +49,7 @@ type Worker interface {
 	MoveStorage(ctx context.Context, sector storiface.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                   //perm:admin
 	UnsealPiece(context.Context, storiface.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                             //perm:admin
 	Fetch(context.Context, storiface.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                               //perm:admin
-	DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorData) (storiface.CallID, error)                                     //perm:admin
+	DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorLocation) (storiface.CallID, error)                                 //perm:admin
 
 	GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error)                           //perm:admin
 	GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (storiface.WindowPoStResult, error) //perm:admin

api/docgen/docgen.go

@@ -345,7 +345,7 @@ func init() {
 		"Authorization": []string{"Bearer ey.."},
 	})
 
-	addExample(map[storiface.SectorFileType]storiface.SectorData{
+	addExample(map[storiface.SectorFileType]storiface.SectorLocation{
 		storiface.FTSealed: {
 			Local:   false,
 			URL:     "https://example.com/sealingservice/sectors/s-f0123-12345",

gen/main.go

@@ -104,7 +104,7 @@ func main() {
 	err = gen.WriteMapEncodersToFile("./storage/sealer/storiface/cbor_gen.go", "storiface",
 		storiface.CallID{},
 		storiface.SecDataHttpHeader{},
-		storiface.SectorData{},
+		storiface.SectorLocation{},
 	)
 	if err != nil {
 		fmt.Println(err)

itests/sector_import_full_test.go

@@ -113,7 +113,7 @@ func TestSectorImport(t *testing.T) {
 			})
 			require.NoError(t, err)
 
-			// CRETE THE UNSEALED FILE
+			// CREATE THE UNSEALED FILE
 
 			// create a reader for all-zero (CC) data
 			dataReader := bytes.NewReader(bytes.Repeat([]byte{0}, int(pieceSize.Unpadded())))
@@ -194,15 +194,15 @@ func TestSectorImport(t *testing.T) {
 				CommD: &scids.Unsealed,
 				CommR: &scids.Sealed,
 
-				DataUnsealed: &storiface.SectorData{
+				DataUnsealed: &storiface.SectorLocation{
 					Local: false,
 					URL:   unsealedURL,
 				},
-				DataSealed: &storiface.SectorData{
+				DataSealed: &storiface.SectorLocation{
 					Local: false,
 					URL:   sealedURL,
 				},
-				DataCache: &storiface.SectorData{
+				DataCache: &storiface.SectorLocation{
 					Local: false,
 					URL:   cacheURL,
 				},

itests/sector_import_simple_test.go

@@ -87,7 +87,7 @@ func TestSectorImportAfterPC2(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	// CRETE THE UNSEALED FILE
+	// CREATE THE UNSEALED FILE
 
 	// create a reader for all-zero (CC) data
 	dataReader := bytes.NewReader(bytes.Repeat([]byte{0}, int(pieceSize.Unpadded())))
@@ -169,15 +169,15 @@ func TestSectorImportAfterPC2(t *testing.T) {
 		CommD: &scids.Unsealed,
 		CommR: &scids.Sealed,
 
-		DataUnsealed: &storiface.SectorData{
+		DataUnsealed: &storiface.SectorLocation{
 			Local: false,
 			URL:   unsealedURL,
 		},
-		DataSealed: &storiface.SectorData{
+		DataSealed: &storiface.SectorLocation{
 			Local: false,
 			URL:   sealedURL,
 		},
-		DataCache: &storiface.SectorData{
+		DataCache: &storiface.SectorLocation{
 			Local: false,
 			URL:   cacheURL,
 		},

storage/paths/fetch.go

@@ -72,6 +72,7 @@ func fetch(ctx context.Context, url, outname string, header http.Header) (rerr e
 }
 
 // FetchWithTemp fetches data into a temp 'fetching' directory, then moves the file to destination
+// The set of URLs must refer to the same object, if one fails, another one will be tried.
 func FetchWithTemp(ctx context.Context, urls []string, dest string, header http.Header) (string, error) {
 	var merr error
 	for _, url := range urls {

storage/pipeline/receive.go

@@ -91,19 +91,28 @@ func (m *Sealing) checkSectorMeta(ctx context.Context, meta api.RemoteSectorMeta
 
 	switch SectorState(meta.State) {
 	case Proving, Available:
-		// todo possibly check
-		info.CommitMessage = meta.CommitMessage
+		if meta.CommitMessage != nil {
+			if err := checkMessagePrefix(*meta.CommitMessage); err != nil {
+				return SectorInfo{}, xerrors.Errorf("commit message prefix: %w", err)
+			}
+
+			info.CommitMessage = meta.CommitMessage
+		}
 
 		fallthrough
 	case SubmitCommit:
 		if meta.PreCommitDeposit == nil {
 			return SectorInfo{}, xerrors.Errorf("sector PreCommitDeposit was null")
 		}
 
-		info.PreCommitInfo = meta.PreCommitInfo
 		info.PreCommitDeposit = *meta.PreCommitDeposit
-		info.PreCommitMessage = meta.PreCommitMessage
 		info.PreCommitTipSet = meta.PreCommitTipSet
+		if info.PreCommitMessage != nil {
+			if err := checkMessagePrefix(*meta.PreCommitMessage); err != nil {
+				return SectorInfo{}, xerrors.Errorf("commit message prefix: %w", err)
+			}
+			info.PreCommitMessage = meta.PreCommitMessage
+		}
 
 		// check provided seed
 		if len(meta.SeedValue) != abi.RandomnessLength {
@@ -256,9 +265,9 @@ func (m *Sealing) checkSectorMeta(ctx context.Context, meta api.RemoteSectorMeta
 }
 
 func (m *Sealing) handleReceiveSector(ctx statemachine.Context, sector SectorInfo) error {
-	toFetch := map[storiface.SectorFileType]storiface.SectorData{}
+	toFetch := map[storiface.SectorFileType]storiface.SectorLocation{}
 
-	for fileType, data := range map[storiface.SectorFileType]*storiface.SectorData{
+	for fileType, data := range map[storiface.SectorFileType]*storiface.SectorLocation{
 		storiface.FTUnsealed: sector.RemoteDataUnsealed,
 		storiface.FTSealed:   sector.RemoteDataSealed,
 		storiface.FTCache:    sector.RemoteDataCache,
@@ -285,3 +294,11 @@ func (m *Sealing) handleReceiveSector(ctx statemachine.Context, sector SectorInf
 
 	return ctx.Send(SectorReceived{})
 }
+
+func checkMessagePrefix(c cid.Cid) error {
+	p := c.Prefix()
+	if p.Version != 1 || p.MhLength != 32 || p.MhType != multihash.BLAKE2B_MIN+31 || p.Codec != cid.DagCBOR {
+		return xerrors.New("invalid message prefix")
+	}
+	return nil
+}

storage/pipeline/types.go

@@ -94,9 +94,9 @@ type SectorInfo struct {
 	TerminatedAt     abi.ChainEpoch
 
 	// Remote import
-	RemoteDataUnsealed        *storiface.SectorData
-	RemoteDataSealed          *storiface.SectorData
-	RemoteDataCache           *storiface.SectorData
+	RemoteDataUnsealed        *storiface.SectorLocation
+	RemoteDataSealed          *storiface.SectorLocation
+	RemoteDataCache           *storiface.SectorLocation
 	RemoteCommit1Endpoint     string
 	RemoteCommit2Endpoint     string
 	RemoteSealingDoneEndpoint string

storage/sealer/ffiwrapper/sealer_cgo.go

@@ -1169,7 +1169,7 @@ func (sb *Sealer) Remove(ctx context.Context, sector storiface.SectorRef) error
 	return xerrors.Errorf("not supported at this layer") // happens in localworker
 }
 
-func (sb *Sealer) DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorData) error {
+func (sb *Sealer) DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorLocation) error {
 	var todo storiface.SectorFileType
 	for fileType := range src {
 		todo |= fileType

storage/sealer/manager.go

@@ -1089,7 +1089,7 @@ func (m *Manager) ProveReplicaUpdate2(ctx context.Context, sector storiface.Sect
 	return out, waitErr
 }
 
-func (m *Manager) DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorData) error {
+func (m *Manager) DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorLocation) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -1098,7 +1098,7 @@ func (m *Manager) DownloadSectorData(ctx context.Context, sector storiface.Secto
 	// get a sorted list of sectors files to make a consistent work key from
 	ents := make([]struct {
 		T storiface.SectorFileType
-		S storiface.SectorData
+		S storiface.SectorLocation
 	}, 0, len(src))
 	for fileType, data := range src {
 		if len(fileType.AllSet()) != 1 {
@@ -1109,7 +1109,7 @@ func (m *Manager) DownloadSectorData(ctx context.Context, sector storiface.Secto
 
 		ents = append(ents, struct {
 			T storiface.SectorFileType
-			S storiface.SectorData
+			S storiface.SectorLocation
 		}{T: fileType, S: data})
 	}
 	sort.Slice(ents, func(i, j int) bool {

storage/sealer/mock/mock.go

@@ -517,7 +517,7 @@ func (mgr *SectorMgr) ReleaseSectorKey(ctx context.Context, sector storiface.Sec
 	return nil
 }
 
-func (mgr *SectorMgr) DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorData) error {
+func (mgr *SectorMgr) DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorLocation) error {
 	return xerrors.Errorf("not supported")
 }
 

storage/sealer/sched_test.go

@@ -67,7 +67,7 @@ type schedTestWorker struct {
 	ignoreResources bool
 }
 
-func (s *schedTestWorker) DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorData) (storiface.CallID, error) {
+func (s *schedTestWorker) DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorLocation) (storiface.CallID, error) {
 	panic("implement me")
 }