itest flakiness: concurrent map access in mock sealer

[magik6k]

fatal error: concurrent map read and map write

goroutine 74803 [running]:
runtime.throw({0x3e43a4f, 0x0})
	/usr/local/go/src/runtime/panic.go:1198 +0x71 fp=0xc00690b278 sp=0xc00690b248 pc=0x69ccf1
runtime.mapaccess1_faststr(0x38e1d80, 0xc00f2def90, {0xc18478fe60, 0x27})
	/usr/local/go/src/runtime/map_faststr.go:21 +0x3a5 fp=0xc00690b2e0 sp=0xc00690b278 pc=0x6784a5
github.com/filecoin-project/lotus/storage/sealer/mock.(*SectorMgr).ReadPiece(0xc00f2df020, {0x40, 0xc00690b5b8}, {{0x3e8, 0x9}, 0x5}, 0x1fc, 0x1fc, {0x0, 0x0, ...}, ...)
	/home/circleci/project/storage/sealer/mock/mock.go:447 +0x265 fp=0xc00690b398 sp=0xc00690b2e0 pc=0x20c7965
github.com/filecoin-project/lotus/markets/sectoraccessor.(*sectorAccessor).UnsealSectorAt(0xc002f6fc00, {0x43eb490, 0xc00013e008}, 0xc00690b8c0, 0x18dafe5, 0x6cf58e)
	/home/circleci/project/markets/sectoraccessor/sectoraccessor.go:72 +0x40b fp=0xc00690b888 sp=0xc00690b398 pc=0x20b4bcb
github.com/filecoin-project/lotus/markets/dagstore.(*minerAPI).FetchUnsealedPiece.func3({0x43eb490, 0xc00013e008})
	/home/circleci/project/markets/dagstore/miner_api.go:172 +0x6d fp=0xc00690b8d0 sp=0xc00690b888 pc=0x2022bcd
github.com/filecoin-project/dagstore/throttle.(*throttler).Do(0xc01f0ce710, {0x43eb490, 0xc00013e008}, 0xc184c87b40)
	/home/circleci/go/pkg/mod/github.com/filecoin-project/dagstore@v0.5.2/throttle/throttler.go:38 +0x118 fp=0xc00690b970 sp=0xc00690b8d0 pc=0x18daef8
github.com/filecoin-project/lotus/markets/dagstore.(*minerAPI).FetchUnsealedPiece(0xc00bc6fb80, {0x43eb490, 0xc00013e008}, {{0xc154e8e030, 0x1}})
	/home/circleci/project/markets/dagstore/miner_api.go:170 +0x62a fp=0xc00690bad0 sp=0xc00690b970 pc=0x20229aa
github.com/filecoin-project/lotus/markets/dagstore.(*LotusMount).Fetch(0xc000c01be0, {0x43eb490, 0xc00013e008})
	/home/circleci/project/markets/dagstore/mount.go:59 +0x36 fp=0xc00690bb08 sp=0xc00690bad0 pc=0x20233f6
github.com/filecoin-project/dagstore/mount.(*Upgrader).Fetch(0xc15e2391e0, {0x43eb490, 0xc00013e008})
	/home/circleci/go/pkg/mod/github.com/filecoin-project/dagstore@v0.5.2/mount/upgrader.go:93 +0xa30 fp=0xc00690bdb8 sp=0xc00690bb08 pc=0x18dcfb0
github.com/filecoin-project/dagstore.(*DAGStore).initializeShard(0xc00bc02dc0, {0x43eb490, 0xc00013e008}, 0xc184d56630, {0x43f91b8, 0xc15e2391e0})
	/home/circleci/go/pkg/mod/github.com/filecoin-project/dagstore@v0.5.2/dagstore_async.go:108 +0x76 fp=0xc00690bfa0 sp=0xc00690bdb8 pc=0x1a00476
github.com/filecoin-project/dagstore.(*DAGStore).control·dwrap·8()
	/home/circleci/go/pkg/mod/github.com/filecoin-project/dagstore@v0.5.2/dagstore_control.go:101 +0x3a fp=0xc00690bfe0 sp=0xc00690bfa0 pc=0x1a03c5a
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc00690bfe8 sp=0xc00690bfe0 pc=0x6d1761
created by github.com/filecoin-project/dagstore.(*DAGStore).control
	/home/circleci/go/pkg/mod/github.com/filecoin-project/dagstore@v0.5.2/dagstore_control.go:101 +0x705

Made itest-batch_deal flake - https://app.circleci.com/pipelines/github/filecoin-project/lotus/22741/workflows/2b047c42-02ce-4b20-acd2-03dcfddd148a/jobs/560618/steps

[RobQuistNL]

I just had this happen on production, too..

{"level":"info","ts":"2023-01-26T17:59:13.602Z","logger":"stores","caller":"paths/index.go:181","msg":"New sector storage: pc1-018-storage0"}
{"level":"info","ts":"2023-01-26T17:59:13.602Z","logger":"stores","caller":"paths/index.go:181","msg":"New sector storage: pc1-017-storage3"}
fatal error: concurrent map read and map write
{"level":"info","ts":"2023-01-26T17:59:13.602Z","logger":"stores","caller":"paths/index.go:181","msg":"New sector storage: pc1-029-storage2"}

goroutine 15171 [running]:
runtime.throw({0x427e118?, 0xa0?})
        /usr/local/go/src/runtime/panic.go:992 +0x71 fp=0xc009d126f0 sp=0xc009d126c0 pc=0x702411
runtime.mapaccess1_faststr(0x3cc9660?, 0xc000796600?, {0xc034361518, 0x15})
        /usr/local/go/src/runtime/map_faststr.go:22 +0x3a5 fp=0xc009d12758 sp=0xc009d126f0 pc=0x6dca65
github.com/filecoin-project/lotus/storage/paths.(*Index).StorageAttach(0xc0017ea2c0, {0xc010d723f0?, 0x66ec3e0?}, {{0xc034361518, 0x15}, {0xc018472a00, 0x1, 0x4}, 0xa, 0x0, ...}, ...)