Fix flaky test due to map iteration randomness in Golang and add stress test options

Makefile

@@ -163,6 +163,15 @@ deps-all: ## Check for all dependency updates
 integTests:
 	$Q go test -v ./integ
 
+integTestsWithStress:
+	$Q go test -v ./integ -repeat=10 interval=100
+
+integTestsCadence:
+	$Q go test -v ./integ -repeat=10 -temporal=false interval=100
+
+integTestsTemporal:
+	$Q go test -v ./integ -repeat=10 -cadence=false interval=100
+
 help:
 	@# print help first, so it's visible
 	@printf "\033[36m%-20s\033[0m %s\n" 'help' 'Prints a help message showing any specially-commented targets'

go.sum

@@ -288,7 +288,6 @@ github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDf
 github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
 github.com/cactus/go-statsd-client/statsd v0.0.0-20191106001114-12b4e2b38748/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -596,7 +595,6 @@ github.com/uber/ringpop-go v0.8.5/go.mod h1:zVI6eGO6L7pG14GkntHsSOfmUAWQ7B4lvmzl
 github.com/uber/tchannel-go v1.16.0/go.mod h1:Rrgz1eL8kMjW/nEzZos0t+Heq0O4LhnUJVA32OvWKHo=
 github.com/uber/tchannel-go v1.22.2 h1:NKA5FVESYh6Ij6V+tujK+IFZnBKDyUHdsBY264UYhgk=
 github.com/uber/tchannel-go v1.22.2/go.mod h1:Rrgz1eL8kMjW/nEzZos0t+Heq0O4LhnUJVA32OvWKHo=
-github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo=
 github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
 github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=

integ/any_command_close_test.go

@@ -12,11 +12,23 @@ import (
 )
 
 func TestAnyCommandCloseWorkflowTemporal(t *testing.T) {
-	doTestAnyCommandCloseWorkflow(t, service.BackendTypeTemporal)
+	if !*temporalIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestAnyCommandCloseWorkflow(t, service.BackendTypeTemporal)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func TestAnyCommandCloseWorkflowCadence(t *testing.T) {
-	doTestAnyCommandCloseWorkflow(t, service.BackendTypeCadence)
+	if !*cadenceIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestAnyCommandCloseWorkflow(t, service.BackendTypeCadence)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func doTestAnyCommandCloseWorkflow(t *testing.T, backendType service.BackendType) {

integ/any_timer_signal_test.go

@@ -12,12 +12,24 @@ import (
 )
 
 func TestAnyTimerSignalWorkflowTemporal(t *testing.T) {
-	doTestAnyTimerSignalWorkflow(t, service.BackendTypeTemporal)
+	if !*temporalIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestAnyTimerSignalWorkflow(t, service.BackendTypeTemporal)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 // TODO this bug in Cadence SDK may cause the test to fail https://github.com/uber-go/cadence-client/issues/1198
 func TestAnyTimerSignalWorkflowCadence(t *testing.T) {
-	doTestAnyTimerSignalWorkflow(t, service.BackendTypeCadence)
+	if !*cadenceIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestAnyTimerSignalWorkflow(t, service.BackendTypeCadence)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func doTestAnyTimerSignalWorkflow(t *testing.T, backendType service.BackendType) {

integ/basic_test.go

@@ -16,11 +16,33 @@ import (
 )
 
 func TestBasicWorkflowTemporal(t *testing.T) {
-	doTestBasicWorkflow(t, service.BackendTypeTemporal)
+	if !*temporalIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestBasicWorkflow(t, service.BackendTypeTemporal)
+		// NOTE: basic wf is too fast so we have to make sure to have enough interval
+		du := time.Millisecond * time.Duration(*repeatInterval)
+		if *repeatIntegTest > 1 && du < time.Second {
+			du = time.Second
+		}
+		time.Sleep(du)
+	}
 }
 
 func TestBasicWorkflowCadence(t *testing.T) {
-	doTestBasicWorkflow(t, service.BackendTypeCadence)
+	if !*cadenceIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestBasicWorkflow(t, service.BackendTypeCadence)
+		// NOTE: basic wf is too fast so we have to make sure to have enough interval
+		du := time.Millisecond * time.Duration(*repeatInterval)
+		if *repeatIntegTest > 1 && du < time.Second {
+			du = time.Second
+		}
+		time.Sleep(du)
+	}
 }
 
 func doTestBasicWorkflow(t *testing.T, backendType service.BackendType) {

integ/flag.go

@@ -0,0 +1,13 @@
+package integ
+
+import (
+	"flag"
+)
+
+var repeatIntegTest = flag.Int("repeat", 1, "the number of repeat")
+
+var repeatInterval = flag.Int("interval", 0, "the interval between test in milliseconds")
+
+var cadenceIntegTest = flag.Bool("cadence", true, "run integ test against cadence")
+
+var temporalIntegTest = flag.Bool("temporal", true, "run integ test against temporal")

integ/interstate_test.go

@@ -14,11 +14,23 @@ import (
 )
 
 func TestInterStateWorkflowTemporal(t *testing.T) {
-	doTestInterStateWorkflow(t, service.BackendTypeTemporal)
+	if !*temporalIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestInterStateWorkflow(t, service.BackendTypeTemporal)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func TestInterStateWorkflowCadence(t *testing.T) {
-	doTestInterStateWorkflow(t, service.BackendTypeCadence)
+	if !*cadenceIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestInterStateWorkflow(t, service.BackendTypeCadence)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func doTestInterStateWorkflow(t *testing.T, backendType service.BackendType) {

integ/parallel_test.go

@@ -14,11 +14,23 @@ import (
 )
 
 func TestParallelWorkflowTemporal(t *testing.T) {
-	doTestParallelWorkflow(t, service.BackendTypeTemporal)
+	if !*temporalIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestParallelWorkflow(t, service.BackendTypeTemporal)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func TestParallelWorkflowCadence(t *testing.T) {
-	doTestParallelWorkflow(t, service.BackendTypeCadence)
+	if !*cadenceIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestParallelWorkflow(t, service.BackendTypeCadence)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func doTestParallelWorkflow(t *testing.T, backendType service.BackendType) {

integ/persistence_test.go

@@ -16,11 +16,23 @@ import (
 )
 
 func TestPersistenceWorkflowTemporal(t *testing.T) {
-	doTestPersistenceWorkflow(t, service.BackendTypeTemporal)
+	if !*temporalIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestPersistenceWorkflow(t, service.BackendTypeTemporal)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func TestPersistenceWorkflowCadence(t *testing.T) {
-	doTestPersistenceWorkflow(t, service.BackendTypeCadence)
+	if !*cadenceIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestPersistenceWorkflow(t, service.BackendTypeCadence)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func doTestPersistenceWorkflow(t *testing.T, backendType service.BackendType) {
@@ -147,7 +159,7 @@ func doTestPersistenceWorkflow(t *testing.T, backendType service.BackendType) {
 			"S2_start_kwSaFounds":   1,
 
 			"S1_decide_localAttFound": true,
-			"S1_decide_queryAttFound": true,
+			"S1_decide_queryAttFound": 2,
 			"S2_decide_queryAttFound": true,
 			"S2_start_queryAttFound":  true,
 		}, data)
@@ -164,7 +176,7 @@ func doTestPersistenceWorkflow(t *testing.T, backendType service.BackendType) {
 			"S2_start_kwSaFounds":   0,
 
 			"S1_decide_localAttFound": true,
-			"S1_decide_queryAttFound": true,
+			"S1_decide_queryAttFound": 2,
 			"S2_decide_queryAttFound": true,
 			"S2_start_queryAttFound":  true,
 		}, data)
@@ -186,8 +198,8 @@ func doTestPersistenceWorkflow(t *testing.T, backendType service.BackendType) {
 			Value: &persistence.TestDataObjectVal1,
 		},
 	}
-	assertions.Equal(expected1, queryResult1.GetObjects())
-	assertions.Equal(expected2, queryResult2.GetObjects())
+	assertions.ElementsMatch(expected1, queryResult1.GetObjects())
+	assertions.ElementsMatch(expected2, queryResult2.GetObjects())
 
 	expectedSearchAttributeInt := iwfidl.SearchAttribute{
 		Key:          iwfidl.PtrString(persistence.TestSearchAttributeIntKey),

integ/signal_test.go

@@ -15,11 +15,23 @@ import (
 )
 
 func TestSignalWorkflowTemporal(t *testing.T) {
-	doTestSignalWorkflow(t, service.BackendTypeTemporal)
+	if !*temporalIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestSignalWorkflow(t, service.BackendTypeTemporal)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func TestSignalWorkflowCadence(t *testing.T) {
-	doTestSignalWorkflow(t, service.BackendTypeCadence)
+	if !*cadenceIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestSignalWorkflow(t, service.BackendTypeCadence)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func doTestSignalWorkflow(t *testing.T, backendType service.BackendType) {

integ/timer_test.go

@@ -14,11 +14,23 @@ import (
 )
 
 func TestTimerWorkflowTemporal(t *testing.T) {
-	doTestTimerWorkflow(t, service.BackendTypeTemporal)
+	if !*temporalIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestTimerWorkflow(t, service.BackendTypeTemporal)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func TestTimerWorkflowCadence(t *testing.T) {
-	doTestTimerWorkflow(t, service.BackendTypeCadence)
+	if !*cadenceIntegTest {
+		t.Skip()
+	}
+	for i := 0; i < *repeatIntegTest; i++ {
+		doTestTimerWorkflow(t, service.BackendTypeCadence)
+		time.Sleep(time.Millisecond * time.Duration(*repeatInterval))
+	}
 }
 
 func doTestTimerWorkflow(t *testing.T, backendType service.BackendType) {

integ/workflow/persistence/routers.go

@@ -214,17 +214,23 @@ func (h *handler) ApiV1WorkflowStateDecide(c *gin.Context) {
 			h.invokeData["S1_decide_kwSaFounds"] = kwSaFounds
 			h.invokeData["S1_decide_intSaFounds"] = intSaFounds
 
-			queryAttFound := false
-			queryAtt := req.GetDataObjects()[0]
-			value := queryAtt.GetValue()
-			if queryAtt.GetKey() == TestDataObjectKey && value.GetData() == TestDataObjectVal1.GetData() && value.GetEncoding() == TestDataObjectVal1.GetEncoding() {
-				queryAttFound = true
+			queryAttFound := 0
+			queryAtts := req.GetDataObjects()
+
+			for _, queryAtt := range queryAtts {
+				value := queryAtt.GetValue()
+				if queryAtt.GetKey() == TestDataObjectKey && value.GetData() == TestDataObjectVal1.GetData() && value.GetEncoding() == TestDataObjectVal1.GetEncoding() {
+					queryAttFound++
+				}
+				if queryAtt.GetKey() == TestDataObjectKey2 && value.GetData() == TestDataObjectVal1.GetData() && value.GetEncoding() == TestDataObjectVal1.GetEncoding() {
+					queryAttFound++
+				}
 			}
 			h.invokeData["S1_decide_queryAttFound"] = queryAttFound
 
 			localAttFound := false
 			localAtt := req.GetStateLocals()[0]
-			value = localAtt.GetValue()
+			value := localAtt.GetValue()
 			if localAtt.GetKey() == TestStateLocalKey && value.GetData() == testStateLocalVal.GetData() && value.GetEncoding() == testStateLocalVal.GetEncoding() {
 				localAttFound = true
 			}