ci: deadlock while running unit tests (concurrent drain operation)

[rolinh]

A deadlock situation occurred when the CI ran the unit tests as of commit 625c1b2.
The part of the tests that check errors during concurrent drain operations is likely the culprit.

Stack trace (click to expand)

  go test -v -race -cover
  shell: /usr/bin/bash -e {0}
  env:
    GOROOT: /opt/hostedtoolcache/go/1.15.7/x64
=== RUN   TestWorkerPool
Error:     workerpool_test.go:87: draining failed: drain operation in progress
Error:     workerpool_test.go:90: missing tasks results: got '0', want '4'
coverage: 55.8% of statements
panic: test timed out after 10m0s

goroutine 27 [running]:
testing.(*M).startAlarm.func1()
	/opt/hostedtoolcache/go/1.15.7/x64/src/testing/testing.go:1618 +0x11f
created by time.goFunc
	/opt/hostedtoolcache/go/1.15.7/x64/src/time/sleep.go:167 +0x52

goroutine 1 [chan receive]:
testing.(*T).Run(0xc000082780, 0x60e990, 0xe, 0x616ff0, 0x0)
	/opt/hostedtoolcache/go/1.15.7/x64/src/testing/testing.go:1169 +0x5f4
testing.runTests.func1(0xc000082780)
	/opt/hostedtoolcache/go/1.15.7/x64/src/testing/testing.go:1439 +0xa7
testing.tRunner(0xc000082780, 0xc0000b1c78)
	/opt/hostedtoolcache/go/1.15.7/x64/src/testing/testing.go:1123 +0x203
testing.runTests(0xc0000be020, 0x72edd0, 0x1, 0x1, 0xbffe806d52edc167, 0x8bb2d252a1, 0x736e80, 0x449645)
	/opt/hostedtoolcache/go/1.15.7/x64/src/testing/testing.go:1437 +0x613
testing.(*M).Run(0xc0000d6000, 0x0)
	/opt/hostedtoolcache/go/1.15.7/x64/src/testing/testing.go:1345 +0x3b4
main.main()
	_testmain.go:93 +0x357

goroutine 21 [semacquire]:
sync.runtime_Semacquire(0xc000084630)
	/opt/hostedtoolcache/go/1.15.7/x64/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc000084628)
	/opt/hostedtoolcache/go/1.15.7/x64/src/sync/waitgroup.go:130 +0xe5
github.com/cilium/workerpool.(*WorkerPool).Drain(0xc000084600, 0x0, 0xc0000a6090, 0xc00008a420, 0xc000084600, 0xc000082900)
	/home/runner/work/workerpool/workerpool/workerpool.go:121 +0x145
github.com/cilium/workerpool.TestWorkerPool(0xc000082900)
	/home/runner/work/workerpool/workerpool/workerpool_test.go:104 +0x7b0
testing.tRunner(0xc000082900, 0x616ff0)
	/opt/hostedtoolcache/go/1.15.7/x64/src/testing/testing.go:1123 +0x203
created by testing.(*T).Run
	/opt/hostedtoolcache/go/1.15.7/x64/src/testing/testing.go:1168 +0x5bc

goroutine 22 [chan send]:
github.com/cilium/workerpool.(*WorkerPool).run(0xc000084600)
	/home/runner/work/workerpool/workerpool/workerpool.go:66 +0xb7
created by github.com/cilium/workerpool.New
	/home/runner/work/workerpool/workerpool/workerpool.go:56 +0x172

goroutine 23 [chan send]:
github.com/cilium/workerpool.TestWorkerPool.func1(0x0, 0x0)
	/home/runner/work/workerpool/workerpool/workerpool_test.go:47 +0xbd
github.com/cilium/workerpool.(*WorkerPool).run.func1(0xc000084600, 0xc00009e330)
	/home/runner/work/workerpool/workerpool/workerpool.go:69 +0xb0
created by github.com/cilium/workerpool.(*WorkerPool).run
	/home/runner/work/workerpool/workerpool/workerpool.go:67 +0xe6

goroutine 24 [chan send]:
github.com/cilium/workerpool.TestWorkerPool.func1(0x0, 0x0)
	/home/runner/work/workerpool/workerpool/workerpool_test.go:47 +0xbd
github.com/cilium/workerpool.(*WorkerPool).run.func1(0xc000084600, 0xc00009e360)
	/home/runner/work/workerpool/workerpool/workerpool.go:69 +0xb0
created by github.com/cilium/workerpool.(*WorkerPool).run
	/home/runner/work/workerpool/workerpool/workerpool.go:67 +0xe6

goroutine 25 [chan send]:
github.com/cilium/workerpool.(*WorkerPool).Submit(0xc000084600, 0xc0000a60b0, 0x8, 0xc0000be0c0, 0x1, 0xc0000a60b0)
	/home/runner/work/workerpool/workerpool/workerpool.go:95 +0x1e5
github.com/cilium/workerpool.TestWorkerPool.func2(0x4, 0xc00008a420, 0xc000084600, 0xc0000a6090, 0xc00008a360, 0xc00008a480)
	/home/runner/work/workerpool/workerpool/workerpool_test.go:66 +0x17d
created by github.com/cilium/workerpool.TestWorkerPool
	/home/runner/work/workerpool/workerpool/workerpool_test.go:63 +0x5f7
exit status 2
FAIL	github.com/cilium/workerpool	600.015s
Error: Process completed with exit code 1.