IWF-118: Refactor package structure (#433)

.github/workflows/ci-cadence-temporal-integ-test.yml → .github/workflows/ci-all-tests.yml

@@ -1,4 +1,4 @@
-name: Temporal/Cadence 2-in-1 Integration Test
+name: All tests
 on:
   pull_request:
   push:
@@ -7,18 +7,18 @@ on:
 
 jobs:
   tests:
-    name: "Integration testing"
+    name: "All tests"
     runs-on: ubuntu-latest
     permissions:
       # Give the default GITHUB_TOKEN write permission to commit and push the
       # added or changed files to the repository.
       contents: write
     steps:
       - uses: actions/checkout@v3
-      - name: "Set up Cadence/Temporal environment"
+      - name: "Set up Cadence/Temporal environment for integration tests"
         run: docker compose -f docker-compose/ci-cadence-temporal-dependencies.yml up -d
-      - name: "Test against cadence/temporal"
-        run: make ci-cadence-temporal-integ-test
+      - name: "Run all tests against cadence/temporal"
+        run: make ci-all-tests
       - name: Dump docker logs
         if: always()
         uses: jwalton/gh-docker-logs@v2

Makefile

@@ -180,10 +180,10 @@ ci-cadence-integ-test:
 	$Q go test -v ./integ -search=false -temporal=false -dependencyWaitSeconds=180
 
 ci-temporal-integ-test:
-	$Q go test -v -cover ./integ -coverprofile coverage.out -coverpkg ./service/... -search=false -cadence=false -dependencyWaitSeconds=60
+	$Q go test -v ./integ  -cover -coverprofile coverage.out -coverpkg ./service/... -search=false -cadence=false -dependencyWaitSeconds=60
 
-ci-cadence-temporal-integ-test:
-	$Q go test -v -cover ./integ -coverprofile coverage.out -coverpkg ./service/... ./cmd/... ./integ/... -dependencyWaitSeconds=60
+ci-all-tests:
+	$Q go test -v ./... -cover -coverprofile coverage.out -coverpkg ./service/...
 
 integTestsNoSearch:
 	$Q go test -v ./integ -search=false

cmd/server/iwf/iwf.go

@@ -22,6 +22,7 @@ package iwf
 
 import (
 	"fmt"
+	"github.com/indeedeng/iwf/config"
 	rawLog "log"
 	"strings"
 	"sync"
@@ -32,7 +33,6 @@ import (
 	cadenceapi "github.com/indeedeng/iwf/service/api/cadence"
 	temporalapi "github.com/indeedeng/iwf/service/api/temporal"
 	uclient "github.com/indeedeng/iwf/service/client"
-	"github.com/indeedeng/iwf/service/common/config"
 	"github.com/indeedeng/iwf/service/common/log"
 	"github.com/indeedeng/iwf/service/common/log/loggerimpl"
 	"github.com/indeedeng/iwf/service/common/log/tag"
@@ -161,7 +161,10 @@ func start(c *cli.Context) {
 	wg.Wait()
 }
 
-func launchTemporalService(svcName string, config config.Config, unifiedClient uclient.UnifiedClient, temporalClient client.Client, logger log.Logger) {
+func launchTemporalService(
+	svcName string, config config.Config, unifiedClient uclient.UnifiedClient, temporalClient client.Client,
+	logger log.Logger,
+) {
 	switch svcName {
 	case serviceAPI:
 		svc := api.NewService(config, unifiedClient, logger.WithTags(tag.Service(svcName)))
@@ -181,7 +184,8 @@ func launchCadenceService(
 	service workflowserviceclient.Interface,
 	domain string,
 	closeFunc func(),
-	logger log.Logger) {
+	logger log.Logger,
+) {
 	switch svcName {
 	case serviceAPI:
 		svc := api.NewService(config, unifiedClient, logger.WithTags(tag.Service(svcName)))

integ/README.md

@@ -0,0 +1,7 @@
+### Integration tests
+
+This directory contains integration tests for the iWF service.
+
+* [How to run](../CONTRIBUTING.md#how-to-run-server-or-integration-test)
+* The integration tests are written without iWF SDKs. The workflows are implemented in REST API routes. e.g. [this basic workflow](./workflow/basic/routers.go)
+

integ/config.go

@@ -1,7 +1,7 @@
 package integ
 
 import (
-	"github.com/indeedeng/iwf/service/common/config"
+	"github.com/indeedeng/iwf/config"
 )
 
 const testWorkflowServerPort = "9714"

integ/persistence_loading_policy_test.go

@@ -57,7 +57,9 @@ func TestPersistenceLoadingPolicy_NONE(t *testing.T) {
 	}
 }
 
-func doTestPersistenceLoadingPolicy(t *testing.T, backendType service.BackendType, loadingType iwfidl.PersistenceLoadingType, rpcUseMemo bool) {
+func doTestPersistenceLoadingPolicy(
+	t *testing.T, backendType service.BackendType, loadingType iwfidl.PersistenceLoadingType, rpcUseMemo bool,
+) {
 	assertions := assert.New(t)
 
 	wfHandler := persistence_loading_policy.NewHandler()

integ/signal_test.go

@@ -3,10 +3,10 @@ package integ
 import (
 	"context"
 	"fmt"
+	config2 "github.com/indeedeng/iwf/config"
 	"github.com/indeedeng/iwf/gen/iwfidl"
 	"github.com/indeedeng/iwf/integ/workflow/signal"
 	"github.com/indeedeng/iwf/service"
-	config2 "github.com/indeedeng/iwf/service/common/config"
 	"github.com/indeedeng/iwf/service/common/ptr"
 	"github.com/stretchr/testify/assert"
 	"net/http"

replayTests/README.md

@@ -0,0 +1,20 @@
+### Replay Tests
+
+### Why
+Replay tests are special and unique in Cadence/Temporal programming model.
+It's ensuring the [determinism](https://docs.temporal.io/workflows#deterministic-constraints) is not being broken by the changes in the workflow logic.
+
+See more about replay tests in the [Cadence documentation](https://cadenceworkflow.io/docs/go-client/workflow-replay-shadowing/#workflow-replayer)
+and Temporal documentation [here](https://docs.temporal.io/develop/go/testing-suite#replay).
+
+To simplify the work, we only use Temporal replay tests in iWF.
+
+### Global versioning design pattern
+In iWF, we are using the [global versioning design pattern](https://medium.com/@qlong/how-to-overcome-some-maintenance-challenges-of-temporal-cadence-workflow-versioning-f893815dd18d) 
+to ensure the determinism of the workflow.
+The pattern makes it simple to manage the workflow versioning and replay tests.
+
+* For every new [global version](../service/interpreter/versions/versions.go), we add at least a new [history file](./history) in the replay_test.
+* For each version, we may need to have multiple history files to cover different scenarios(code paths).
+* To get the JSON history file, start and run a workflow that will use the code path that you want to protect the determinism. Then download the JSON from WebUI.
+* Usually, the workflow is from an integration test 

integ/replay_test.go → replayTests/replay_test.go

@@ -1,4 +1,4 @@
-package integ
+package replayTests
 
 import (
 	"github.com/indeedeng/iwf/service/interpreter/temporal"

service/api/handler.go

@@ -2,11 +2,11 @@ package api
 
 import (
 	"encoding/json"
+	"github.com/indeedeng/iwf/config"
 	"net/http"
 
 	"github.com/indeedeng/iwf/service"
 	uclient "github.com/indeedeng/iwf/service/client"
-	"github.com/indeedeng/iwf/service/common/config"
 	"github.com/indeedeng/iwf/service/common/errors"
 	"github.com/indeedeng/iwf/service/common/log"
 	"github.com/indeedeng/iwf/service/common/log/tag"

service/api/routers.go

@@ -2,8 +2,8 @@ package api
 
 import (
 	"github.com/gin-gonic/gin"
+	"github.com/indeedeng/iwf/config"
 	uclient "github.com/indeedeng/iwf/service/client"
-	"github.com/indeedeng/iwf/service/common/config"
 	"github.com/indeedeng/iwf/service/common/log"
 )
 

service/api/service.go

@@ -5,6 +5,7 @@ import (
 	"crypto/md5"
 	"encoding/json"
 	"fmt"
+	"github.com/indeedeng/iwf/config"
 	"github.com/indeedeng/iwf/service/interpreter/versions"
 	"math"
 	"net/http"
@@ -18,7 +19,6 @@ import (
 	"github.com/indeedeng/iwf/service/common/utils"
 	"github.com/indeedeng/iwf/service/interpreter"
 
-	"github.com/indeedeng/iwf/service/common/config"
 	"github.com/indeedeng/iwf/service/common/errors"
 	"github.com/indeedeng/iwf/service/common/log"
 	"github.com/indeedeng/iwf/service/common/log/tag"

service/interpreter/activityImpl.go

@@ -3,10 +3,10 @@ package interpreter
 import (
 	"context"
 	"fmt"
+	"github.com/indeedeng/iwf/config"
 	"github.com/indeedeng/iwf/gen/iwfidl"
 	"github.com/indeedeng/iwf/service"
 	"github.com/indeedeng/iwf/service/common/compatibility"
-	"github.com/indeedeng/iwf/service/common/config"
 	"github.com/indeedeng/iwf/service/common/rpc"
 	"github.com/indeedeng/iwf/service/common/urlautofix"
 	"github.com/indeedeng/iwf/service/interpreter/env"

service/interpreter/cadence/worker.go

@@ -1,10 +1,10 @@
 package cadence
 
 import (
+	"github.com/indeedeng/iwf/config"
 	"log"
 
 	uclient "github.com/indeedeng/iwf/service/client"
-	"github.com/indeedeng/iwf/service/common/config"
 	"github.com/indeedeng/iwf/service/interpreter"
 	"github.com/indeedeng/iwf/service/interpreter/env"
 	"go.uber.org/cadence/.gen/go/cadence/workflowserviceclient"
@@ -19,7 +19,10 @@ type InterpreterWorker struct {
 	tasklist  string
 }
 
-func NewInterpreterWorker(config config.Config, service workflowserviceclient.Interface, domain, tasklist string, closeFunc func(), unifiedClient uclient.UnifiedClient) *InterpreterWorker {
+func NewInterpreterWorker(
+	config config.Config, service workflowserviceclient.Interface, domain, tasklist string, closeFunc func(),
+	unifiedClient uclient.UnifiedClient,
+) *InterpreterWorker {
 	env.SetSharedEnv(config, false, nil, unifiedClient, tasklist)
 	return &InterpreterWorker{
 		service:   service,

service/interpreter/env/env.go

@@ -1,8 +1,8 @@
 package env
 
 import (
+	"github.com/indeedeng/iwf/config"
 	uclient "github.com/indeedeng/iwf/service/client"
-	"github.com/indeedeng/iwf/service/common/config"
 	"go.temporal.io/sdk/converter"
 )
 

service/interpreter/temporal/worker.go

@@ -1,10 +1,10 @@
 package temporal
 
 import (
+	"github.com/indeedeng/iwf/config"
 	"log"
 
 	uclient "github.com/indeedeng/iwf/service/client"
-	"github.com/indeedeng/iwf/service/common/config"
 	"github.com/indeedeng/iwf/service/interpreter"
 	"github.com/indeedeng/iwf/service/interpreter/env"
 	"go.temporal.io/sdk/client"
@@ -18,7 +18,10 @@ type InterpreterWorker struct {
 	taskQueue      string
 }
 
-func NewInterpreterWorker(config config.Config, temporalClient client.Client, taskQueue string, memoEncryption bool, memoEncryptionConverter converter.DataConverter, unifiedClient uclient.UnifiedClient) *InterpreterWorker {
+func NewInterpreterWorker(
+	config config.Config, temporalClient client.Client, taskQueue string, memoEncryption bool,
+	memoEncryptionConverter converter.DataConverter, unifiedClient uclient.UnifiedClient,
+) *InterpreterWorker {
 	env.SetSharedEnv(config, memoEncryption, memoEncryptionConverter, unifiedClient, taskQueue)
 
 	return &InterpreterWorker{