some tasks are sent to multiple seal workers repeatedly for execution

[arajasek]

Reported by a community member, logs attached

2020-05-16T20:57:24.512 INFO storage_proofs_porep::stacked::vanilla::proof > building base tree_c 3/8
2020-05-16T21:05:07.486 INFO storage_proofs_porep::stacked::vanilla::proof > building base tree_c 4/8
2020-05-16T21:12:07.398 INFO storage_proofs_porep::stacked::vanilla::proof > building base tree_c 5/8
2020-05-16T21:19:09.821 INFO storage_proofs_porep::stacked::vanilla::proof > building base tree_c 6/8
2020-05-16T21:26:10.191 INFO storage_proofs_porep::stacked::vanilla::proof > building base tree_c 7/8
2020-05-16T21:33:11.695 INFO storage_proofs_porep::stacked::vanilla::proof > building base tree_c 8/8
2020-05-16T21:33:52.063 INFO storage_proofs_porep::stacked::vanilla::proof > tree_c done
2020-05-16T21:33:52.064 INFO storage_proofs_porep::stacked::vanilla::proof > building tree_r_last
2020-05-16T21:33:55.434 INFO storage_proofs_porep::stacked::vanilla::proof > building base tree_r_last 1/8
2020-05-16T21:34:44.173 INFO storage_proofs_porep::stacked::vanilla::proof > building base tree_r_last 2/8
2020-05-16T22:29:06.817 INFO filcrypto::proofs::api > seal_pre_commit_phase2: finish
2020-05-16T22:29:06.819+0800	WARN	rpc	jsonrpc/handler.go:225	error in RPC call to 'Filecoin.SealPreCommit2': presealing sector 37 ():
    github.com/filecoin-project/sector-storage/ffiwrapper.(*Sealer).SealPreCommit2
        /root/go/pkg/mod/github.com/filecoin-project/sector-storage@v0.0.0-20200515123304-20817dc51db5/ffiwrapper/sealer_cgo.go:261
  - Stale file handle (os error 116)
    
       0: <merkletree::store::level_cache::LevelCacheStore<E,R> as merkletree::store::Store<E>>::build
       1: <merkletree::merkle::MerkleTree<E,A,S,BaseTreeArity,SubTreeArity,TopTreeArity> as merkletree::merkle::FromIndexedParallelIterator<E,BaseTreeArity>>::from_par_iter_with_config
       2: storage_proofs_core::measurements::measure_op
       3: storage_proofs_porep::stacked::vanilla::proof::StackedDrg<Tree,G>::transform_and_replicate_layers_inner
       4: storage_proofs_porep::stacked::vanilla::proof::StackedDrg<Tree,G>::replicate_phase2
       5: filecoin_proofs::api::seal::seal_pre_commit_phase2
       6: filecoin_proofs_api::seal::seal_pre_commit_phase2_inner
       7: filecoin_proofs_api::seal::seal_pre_commit_phase2
       8: <std::panic::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
       9: ffi_toolkit::catch_panic_response
      10: fil_seal_pre_commit_phase2
      11: _cgo_66afc6c6cd88_Cfunc_fil_seal_pre_commit_phase2
                 at /tmp/go-build/cgo-gcc-prolog:987
      12: runtime.asmcgocall
                 at /usr/lib/go-1.14/src/runtime/asm_amd64.s:655
    github.com/filecoin-project/filecoin-ffi.SealPreCommitPhase2
    	/usr/local/lotus/extern/filecoin-ffi/proofs.go:304
    github.com/filecoin-project/sector-storage/ffiwrapper.(*Sealer).SealPreCommit2
    	/root/go/pkg/mod/github.com/filecoin-project/sector-storage@v0.0.0-20200515123304-20817dc51db5/ffiwrapper/sealer_cgo.go:259
    github.com/filecoin-project/sector-storage.(*LocalWorker).SealPreCommit2
    	/root/go/pkg/mod/github.com/filecoin-project/sector-storage@v0.0.0-20200515123304-20817dc51db5/localworker.go:164
    reflect.Value.call
    	/usr/lib/go-1.14/src/reflect/value.go:460
    reflect.Value.Call
    	/usr/lib/go-1.14/src/reflect/value.go:321
    github.com/filecoin-project/lotus/api/apistruct.permissionedAny.func1
    	/usr/local/lotus/api/apistruct/permissioned.go:94
    reflect.callReflect
    	/usr/lib/go-1.14/src/reflect/value.go:549
    reflect.makeFuncStub
    	/usr/lib/go-1.14/src/reflect/asm_amd64.s:20
    github.com/filecoin-project/lotus/api/apistruct.(*WorkerStruct).SealPreCommit2
    	/usr/local/lotus/api/apistruct/struct.go:843
    reflect.Value.call
    	/usr/lib/go-1.14/src/reflect/value.go:460
    reflect.Value.Call
    	/usr/lib/go-1.14/src/reflect/value.go:321
    github.com/filecoin-project/lotus/lib/jsonrpc.doCall
    	/usr/local/lotus/lib/jsonrpc/handler.go:129
    github.com/filecoin-project/lotus/lib/jsonrpc.handlers.handle
    	/usr/local/lotus/lib/jsonrpc/handler.go:205
    runtime.goexit
    	/usr/lib/go-1.14/src/runtime/asm_amd64.s:1373
2020-05-16T22:29:06.820+0800	ERROR	rpc	jsonrpc/websocket.go:120	handle me:write tcp 192.168.0.24:2345->192.168.0.14:34770: write: broken pipe

root@idc:~# lotus-storage-miner sectors status --log 37
SectorID:       37
Status: PreCommit1
CommD:          <nil>
CommR:          <nil>
Ticket:         dced52ef3c4fe8c88dabf1a90846e090cded1c49c3c1e1c2e2b090fc1b8ba64b
TicketH:                4203
Seed:   
SeedH:          0
Proof:  
Deals:          [0]
Retries:                0
--------
Event Log:
0.      2020-05-16 11:04:30 +0800 CST:  [event;sealing.SectorStart]     {"User":{"ID":37,"SectorType":1,"Pieces":[{"Piece":{"Size":34359738368,"PieceCID":{"/":"bafk4chzaa57f7xrvyufjga5fkae6gsmkj27n7444ik3rbnzq3dwhvr5puy7a"}},"DealInfo":null}]}}
1.      2020-05-16 11:04:30 +0800 CST:  [event;sealing.SectorPacked]    {"User":{"FillerPieces":null}}
2.      2020-05-16 14:34:25 +0800 CST:  [event;sealing.SectorSealPreCommitFailed]       {"User":{}}
        seal pre commit(1) failed: handler: websocket connection closed
3.      2020-05-16 14:35:25 +0800 CST:  [event;sealing.SectorRetrySeal] {"User":{}}
4.      2020-05-16 14:48:43 +0800 CST:  [event;sealing.SectorSealPreCommitFailed]       {"User":{}}
        seal pre commit(1) failed: handler: websocket connection closed
5.      2020-05-16 14:49:43 +0800 CST:  [event;sealing.SectorRetrySeal] {"User":{}}
6.      2020-05-16 19:42:27 +0800 CST:  [event;sealing.SectorPreCommit1]        {"User":{"PreCommit1Out":"eyJyZWdpc3RlcmVkX3Byb29mIjoiU3RhY2tlZERyZzMyR2lCVjEiLCJsYWJlbHMiOnsiU3RhY2tlZERyZzMyR2lCVjEiOnsibGFiZWxzIjpbeyJwYXRoIjoiL2RhdGEvd29ya2VyMjBfMS9jYWNoZS9zLXQwMTIzMTE1LTM3IiwiaWQiOiJsYXllci0xIiwic2l6ZSI6MTA3Mzc0MTgyNCwibGV2ZWxzIjo3fSx7InBhdGgiOiIvZGF0YS93b3JrZXIyMF8xL2NhY2hlL3MtdDAxMjMxMTUtMzciLCJpZCI6ImxheWVyLTIiLCJzaXplIjoxMDczNzQxODI0LCJsZXZlbHMiOjd9LHsicGF0aCI6Ii9kYXRhL3dvcmtlcjIwXzEvY2FjaGUvcy10MDEyMzExNS0zNyIsImlkIjoibGF5ZXItMyIsInNpemUiOjEwNzM3NDE4MjQsImxldmVscyI6N30seyJwYXRoIjoiL2RhdGEvd29ya2VyMjBfMS9jYWNoZS9zLXQwMTIzMTE1LTM3IiwiaWQiOiJsYXllci00Iiwic2l6ZSI6MTA3Mzc0MTgyNCwibGV2ZWxzIjo3fSx7InBhdGgiOiIvZGF0YS93b3JrZXIyMF8xL2NhY2hlL3MtdDAxMjMxMTUtMzciLCJpZCI6ImxheWVyLTUiLCJzaXplIjoxMDczNzQxODI0LCJsZXZlbHMiOjd9LHsicGF0aCI6Ii9kYXRhL3dvcmtlcjIwXzEvY2FjaGUvcy10MDEyMzExNS0zNyIsImlkIjoibGF5ZXItNiIsInNpemUiOjEwNzM3NDE4MjQsImxldmVscyI6N30seyJwYXRoIjoiL2RhdGEvd29ya2VyMjBfMS9jYWNoZS9zLXQwMTIzMTE1LTM3IiwiaWQiOiJsYXllci03Iiwic2l6ZSI6MTA3Mzc0MTgyNCwibGV2ZWxzIjo3fSx7InBhdGgiOiIvZGF0YS93b3JrZXIyMF8xL2NhY2hlL3MtdDAxMjMxMTUtMzciLCJpZCI6ImxheWVyLTgiLCJzaXplIjoxMDczNzQxODI0LCJsZXZlbHMiOjd9LHsicGF0aCI6Ii9kYXRhL3dvcmtlcjIwXzEvY2FjaGUvcy10MDEyMzExNS0zNyIsImlkIjoibGF5ZXItOSIsInNpemUiOjEwNzM3NDE4MjQsImxldmVscyI6N30seyJwYXRoIjoiL2RhdGEvd29ya2VyMjBfMS9jYWNoZS9zLXQwMTIzMTE1LTM3IiwiaWQiOiJsYXllci0xMCIsInNpemUiOjEwNzM3NDE4MjQsImxldmVscyI6N30seyJwYXRoIjoiL2RhdGEvd29ya2VyMjBfMS9jYWNoZS9zLXQwMTIzMTE1LTM3IiwiaWQiOiJsYXllci0xMSIsInNpemUiOjEwNzM3NDE4MjQsImxldmVscyI6N31dLCJfaCI6bnVsbH19LCJjb25maWciOnsicGF0aCI6Ii9kYXRhL3dvcmtlcjIwXzEvY2FjaGUvcy10MDEyMzExNS0zNyIsImlkIjoidHJlZS1kIiwic2l6ZSI6MjE0NzQ4MzY0NywibGV2ZWxzIjo3fSwiY29tbV9kIjpbNywxMjYsOTUsMjIyLDUzLDE5NywxMCwxNDcsMywxNjUsODAsOSwyMjcsNzMsMTM4LDc4LDE5MCwyMjMsMjQzLDE1Niw2NiwxODMsMTYsMTgzLDQ4LDIxNiwyMzYsMTIyLDE5OSwxNzUsMTY2LDYyXX0=","TicketValue":"3O1S7zxP6MiNq/GpCEbgkM3tHEnDweHC4rCQ/BuLpks=","TicketEpoch":4203}}
7.      2020-05-16 21:41:42 +0800 CST:  [event;sealing.SectorSealPreCommitFailed]       {"User":{}}
        seal pre commit(2) failed: handler: websocket connection closed
8.      2020-05-16 21:42:42 +0800 CST:  [event;sealing.SectorRetrySeal] {"User":{}}
root@idc:~# lotus-storage-miner sectors list |grep 37
37: PreCommit1    sSet: NO  pSet: NO  tktH: 4203 seedH: 0    deals: [0]