From 9c440a74f538ee39e17d81882480976c12314e3b Mon Sep 17 00:00:00 2001
From: Yasser Elsayed <yasser.ebrahim@gmail.com>
Date: Wed, 12 Dec 2018 14:40:03 -0800
Subject: [PATCH 1/3] skip integration tests when unit test flag is set to true

---
 backend/test/cli_test.go            | 31 ++++++++++++++---------------
 backend/test/experiment_api_test.go |  5 +++++
 backend/test/job_api_test.go        |  5 +++++
 backend/test/pipeline_api_test.go   |  9 +++++++--
 backend/test/run_api_test.go        |  5 +++++
 5 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/backend/test/cli_test.go b/backend/test/cli_test.go
index 56f3aab4443..4c542f1a27d 100644
--- a/backend/test/cli_test.go
+++ b/backend/test/cli_test.go
@@ -20,18 +20,14 @@ const (
 	// If false, run as an integration test calling the service.
 	// This is useful to test locally before running an e2e test (which takes a while).
 	// IMPORTANT: This should always be set to FALSE in the checked-in code.
-	runAsUnitTest = false
+	runAsUnitTest = true
 )
 
 func GetRealRootCommand() (*cmd.RootCommand, cmd.ClientFactoryInterface) {
-	if runAsUnitTest {
-		return cmd.GetFakeRootCommand()
-	} else {
-		clientFactory := cmd.NewClientFactoryWithByteBuffer()
-		rootCmd := cmd.NewRootCmd(clientFactory)
-		rootCmd = cmd.CreateSubCommands(rootCmd, defaultPageSize)
-		return rootCmd, clientFactory
-	}
+	clientFactory := cmd.NewClientFactoryWithByteBuffer()
+	rootCmd := cmd.NewRootCmd(clientFactory)
+	rootCmd = cmd.CreateSubCommands(rootCmd, defaultPageSize)
+	return rootCmd, clientFactory
 }
 
 type CLIIntegrationTest struct {
@@ -41,15 +37,18 @@ type CLIIntegrationTest struct {
 
 // Check the cluster namespace has Kubeflow pipelines installed and ready.
 func (c *CLIIntegrationTest) SetupTest() {
+	if runAsUnitTest {
+		c.T().SkipNow()
+		return
+	}
+
 	c.namespace = *namespace
 
-	if !runAsUnitTest {
-		// Wait for the system to be ready.
-		err := waitForReady(c.namespace, *initializeTimeout)
-		if err != nil {
-			glog.Exitf("Cluster namespace '%s' is still not ready after timeout. Error: %s", c.namespace,
-				err.Error())
-		}
+	// Wait for the system to be ready.
+	err := waitForReady(c.namespace, *initializeTimeout)
+	if err != nil {
+		glog.Exitf("Cluster namespace '%s' is still not ready after timeout. Error: %s", c.namespace,
+			err.Error())
 	}
 }
 
diff --git a/backend/test/experiment_api_test.go b/backend/test/experiment_api_test.go
index d813d50d326..23603b9b520 100644
--- a/backend/test/experiment_api_test.go
+++ b/backend/test/experiment_api_test.go
@@ -22,6 +22,11 @@ type ExperimentApiTest struct {
 
 // Check the namespace have ML job installed and ready
 func (s *ExperimentApiTest) SetupTest() {
+	if runAsUnitTest {
+		s.T().SkipNow()
+		return
+	}
+
 	err := waitForReady(*namespace, *initializeTimeout)
 	if err != nil {
 		glog.Exitf("Failed to initialize test. Error: %s", err.Error())
diff --git a/backend/test/job_api_test.go b/backend/test/job_api_test.go
index a63b307e204..7ca00b44a7e 100644
--- a/backend/test/job_api_test.go
+++ b/backend/test/job_api_test.go
@@ -34,6 +34,11 @@ type JobApiTestSuite struct {
 
 // Check the namespace have ML pipeline installed and ready
 func (s *JobApiTestSuite) SetupTest() {
+	if runAsUnitTest {
+		s.T().SkipNow()
+		return
+	}
+
 	err := waitForReady(*namespace, *initializeTimeout)
 	if err != nil {
 		glog.Exitf("Failed to initialize test. Error: %s", err.Error())
diff --git a/backend/test/pipeline_api_test.go b/backend/test/pipeline_api_test.go
index 211ce6e5786..737d2e25b95 100644
--- a/backend/test/pipeline_api_test.go
+++ b/backend/test/pipeline_api_test.go
@@ -32,6 +32,11 @@ type PipelineApiTest struct {
 
 // Check the namespace have ML job installed and ready
 func (s *PipelineApiTest) SetupTest() {
+	if runAsUnitTest {
+		s.T().SkipNow()
+		return
+	}
+
 	err := waitForReady(*namespace, *initializeTimeout)
 	if err != nil {
 		glog.Exitf("Failed to initialize test. Error: %s", err.Error())
@@ -65,7 +70,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 	/* ---------- Import pipeline YAML by URL ---------- */
 	time.Sleep(1 * time.Second)
 	sequentialPipeline, err := s.pipelineClient.Create(&params.CreatePipelineParams{
-		Body:  &pipeline_model.APIPipeline{URL:&pipeline_model.APIURL{
+		Body: &pipeline_model.APIPipeline{URL: &pipeline_model.APIURL{
 			PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/sequential.yaml"}}})
 	assert.Nil(t, err)
 	assert.Equal(t, "sequential.yaml", sequentialPipeline.Name)
@@ -79,7 +84,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 	/* ---------- Import pipeline tarball by URL ---------- */
 	time.Sleep(1 * time.Second)
 	argumentUrlPipeline, err := s.pipelineClient.Create(&params.CreatePipelineParams{
-		Body:  &pipeline_model.APIPipeline{URL:&pipeline_model.APIURL{
+		Body: &pipeline_model.APIPipeline{URL: &pipeline_model.APIURL{
 			PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/arguments.tar.gz"}}})
 	assert.Nil(t, err)
 	assert.Equal(t, "arguments.tar.gz", argumentUrlPipeline.Name)
diff --git a/backend/test/run_api_test.go b/backend/test/run_api_test.go
index 219f98de53e..3384bc7761e 100644
--- a/backend/test/run_api_test.go
+++ b/backend/test/run_api_test.go
@@ -28,6 +28,11 @@ type RunApiTestSuite struct {
 
 // Check the namespace have ML pipeline installed and ready
 func (s *RunApiTestSuite) SetupTest() {
+	if runAsUnitTest {
+		s.T().SkipNow()
+		return
+	}
+
 	err := waitForReady(*namespace, *initializeTimeout)
 	if err != nil {
 		glog.Exitf("Failed to initialize test. Error: %s", err.Error())

From 2b6c510320b7f6e4212f6081bda22050e06cb9d3 Mon Sep 17 00:00:00 2001
From: Yasser Elsayed <yasser.ebrahim@gmail.com>
Date: Wed, 12 Dec 2018 14:47:40 -0800
Subject: [PATCH 2/3] use cli arg to run integration tests

---
 backend/test/cli_test.go              | 7 +------
 backend/test/experiment_api_test.go   | 2 +-
 backend/test/job_api_test.go          | 2 +-
 backend/test/pipeline_api_test.go     | 2 +-
 backend/test/run_api_test.go          | 2 +-
 backend/test/test_utils.go            | 1 +
 test/api-integration-test/run_test.sh | 2 +-
 test/backend-unit-test/run_test.sh    | 4 ++--
 8 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/backend/test/cli_test.go b/backend/test/cli_test.go
index 4c542f1a27d..b057e661dfd 100644
--- a/backend/test/cli_test.go
+++ b/backend/test/cli_test.go
@@ -16,11 +16,6 @@ import (
 const (
 	defaultPageSize    = int32(10)
 	myUploadedPipeline = "my-uploaded-pipeline"
-	// If true, run as a unit test using a fake client.
-	// If false, run as an integration test calling the service.
-	// This is useful to test locally before running an e2e test (which takes a while).
-	// IMPORTANT: This should always be set to FALSE in the checked-in code.
-	runAsUnitTest = true
 )
 
 func GetRealRootCommand() (*cmd.RootCommand, cmd.ClientFactoryInterface) {
@@ -37,7 +32,7 @@ type CLIIntegrationTest struct {
 
 // Check the cluster namespace has Kubeflow pipelines installed and ready.
 func (c *CLIIntegrationTest) SetupTest() {
-	if runAsUnitTest {
+	if *runAsUnitTest {
 		c.T().SkipNow()
 		return
 	}
diff --git a/backend/test/experiment_api_test.go b/backend/test/experiment_api_test.go
index 23603b9b520..c2212926be9 100644
--- a/backend/test/experiment_api_test.go
+++ b/backend/test/experiment_api_test.go
@@ -22,7 +22,7 @@ type ExperimentApiTest struct {
 
 // Check the namespace have ML job installed and ready
 func (s *ExperimentApiTest) SetupTest() {
-	if runAsUnitTest {
+	if *runAsUnitTest {
 		s.T().SkipNow()
 		return
 	}
diff --git a/backend/test/job_api_test.go b/backend/test/job_api_test.go
index 7ca00b44a7e..0a927c92fb1 100644
--- a/backend/test/job_api_test.go
+++ b/backend/test/job_api_test.go
@@ -34,7 +34,7 @@ type JobApiTestSuite struct {
 
 // Check the namespace have ML pipeline installed and ready
 func (s *JobApiTestSuite) SetupTest() {
-	if runAsUnitTest {
+	if *runAsUnitTest {
 		s.T().SkipNow()
 		return
 	}
diff --git a/backend/test/pipeline_api_test.go b/backend/test/pipeline_api_test.go
index 737d2e25b95..d079f927cb6 100644
--- a/backend/test/pipeline_api_test.go
+++ b/backend/test/pipeline_api_test.go
@@ -32,7 +32,7 @@ type PipelineApiTest struct {
 
 // Check the namespace have ML job installed and ready
 func (s *PipelineApiTest) SetupTest() {
-	if runAsUnitTest {
+	if *runAsUnitTest {
 		s.T().SkipNow()
 		return
 	}
diff --git a/backend/test/run_api_test.go b/backend/test/run_api_test.go
index 3384bc7761e..2682d2fe7ca 100644
--- a/backend/test/run_api_test.go
+++ b/backend/test/run_api_test.go
@@ -28,7 +28,7 @@ type RunApiTestSuite struct {
 
 // Check the namespace have ML pipeline installed and ready
 func (s *RunApiTestSuite) SetupTest() {
-	if runAsUnitTest {
+	if *runAsUnitTest {
 		s.T().SkipNow()
 		return
 	}
diff --git a/backend/test/test_utils.go b/backend/test/test_utils.go
index 3ed97159361..5b5c9bf739a 100644
--- a/backend/test/test_utils.go
+++ b/backend/test/test_utils.go
@@ -39,6 +39,7 @@ import (
 
 var namespace = flag.String("namespace", "kubeflow", "The namespace ml pipeline deployed to")
 var initializeTimeout = flag.Duration("initializeTimeout", 2*time.Minute, "Duration to wait for test initialization")
+var runAsUnitTest = flag.Bool("unitTestsOnly", true, "Whether to skip integration tests that call the service")
 
 func waitForReady(namespace string, initializeTimeout time.Duration) error {
 	var operation = func() error {
diff --git a/test/api-integration-test/run_test.sh b/test/api-integration-test/run_test.sh
index ca0bcc1d0f4..8357c3b4c6d 100755
--- a/test/api-integration-test/run_test.sh
+++ b/test/api-integration-test/run_test.sh
@@ -61,7 +61,7 @@ TEST_DIR=backend/test
 cd "${BASE_DIR}/${TEST_DIR}"
 
 echo "Run integration test..."
-TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} 2>&1`
+TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} -args -unitTestsOnly=false 2>&1`
 TEST_EXIT_CODE=$?
 
 # Log the test result
diff --git a/test/backend-unit-test/run_test.sh b/test/backend-unit-test/run_test.sh
index e83b43e7edd..a82a2d63272 100755
--- a/test/backend-unit-test/run_test.sh
+++ b/test/backend-unit-test/run_test.sh
@@ -51,7 +51,7 @@ cd "${BASE_DIR}/${TEST_DIR}"
 
 # Run test and store the exit code.
 echo "Run unit test..."
-TEST_RESULT=`go test -v ./... 2>&1`
+TEST_RESULT=`go test -v ./... -args -unitTestsOnly=false 2>&1`
 TEST_EXIT_CODE=$?
 
 # Log the test result
@@ -62,4 +62,4 @@ printf '%s\n' "$TEST_RESULT" | go-junit-report > ${JUNIT_TEST_RESULT}
 echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}"
 gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}
 
-exit $TEST_EXIT_CODE
\ No newline at end of file
+exit $TEST_EXIT_CODE

From 09779eb537a6fb254485aa728933ae43d7cb0ae7 Mon Sep 17 00:00:00 2001
From: Yasser Elsayed <yasser.ebrahim@gmail.com>
Date: Wed, 12 Dec 2018 15:19:15 -0800
Subject: [PATCH 3/3] use runIntegrationTests flag instead

---
 backend/test/cli_test.go              | 2 +-
 backend/test/experiment_api_test.go   | 2 +-
 backend/test/job_api_test.go          | 2 +-
 backend/test/pipeline_api_test.go     | 2 +-
 backend/test/run_api_test.go          | 2 +-
 backend/test/test_utils.go            | 2 +-
 test/api-integration-test/run_test.sh | 2 +-
 test/backend-unit-test/run_test.sh    | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/backend/test/cli_test.go b/backend/test/cli_test.go
index b057e661dfd..ac4d8528615 100644
--- a/backend/test/cli_test.go
+++ b/backend/test/cli_test.go
@@ -32,7 +32,7 @@ type CLIIntegrationTest struct {
 
 // Check the cluster namespace has Kubeflow pipelines installed and ready.
 func (c *CLIIntegrationTest) SetupTest() {
-	if *runAsUnitTest {
+	if !*runIntegrationTests {
 		c.T().SkipNow()
 		return
 	}
diff --git a/backend/test/experiment_api_test.go b/backend/test/experiment_api_test.go
index c2212926be9..475a46326d2 100644
--- a/backend/test/experiment_api_test.go
+++ b/backend/test/experiment_api_test.go
@@ -22,7 +22,7 @@ type ExperimentApiTest struct {
 
 // Check the namespace have ML job installed and ready
 func (s *ExperimentApiTest) SetupTest() {
-	if *runAsUnitTest {
+	if !*runIntegrationTests {
 		s.T().SkipNow()
 		return
 	}
diff --git a/backend/test/job_api_test.go b/backend/test/job_api_test.go
index 0a927c92fb1..f2b094e9380 100644
--- a/backend/test/job_api_test.go
+++ b/backend/test/job_api_test.go
@@ -34,7 +34,7 @@ type JobApiTestSuite struct {
 
 // Check the namespace have ML pipeline installed and ready
 func (s *JobApiTestSuite) SetupTest() {
-	if *runAsUnitTest {
+	if !*runIntegrationTests {
 		s.T().SkipNow()
 		return
 	}
diff --git a/backend/test/pipeline_api_test.go b/backend/test/pipeline_api_test.go
index d079f927cb6..8113fc15b6e 100644
--- a/backend/test/pipeline_api_test.go
+++ b/backend/test/pipeline_api_test.go
@@ -32,7 +32,7 @@ type PipelineApiTest struct {
 
 // Check the namespace have ML job installed and ready
 func (s *PipelineApiTest) SetupTest() {
-	if *runAsUnitTest {
+	if !*runIntegrationTests {
 		s.T().SkipNow()
 		return
 	}
diff --git a/backend/test/run_api_test.go b/backend/test/run_api_test.go
index 2682d2fe7ca..74d231fe1ed 100644
--- a/backend/test/run_api_test.go
+++ b/backend/test/run_api_test.go
@@ -28,7 +28,7 @@ type RunApiTestSuite struct {
 
 // Check the namespace have ML pipeline installed and ready
 func (s *RunApiTestSuite) SetupTest() {
-	if *runAsUnitTest {
+	if !*runIntegrationTests {
 		s.T().SkipNow()
 		return
 	}
diff --git a/backend/test/test_utils.go b/backend/test/test_utils.go
index 5b5c9bf739a..1e25e21154a 100644
--- a/backend/test/test_utils.go
+++ b/backend/test/test_utils.go
@@ -39,7 +39,7 @@ import (
 
 var namespace = flag.String("namespace", "kubeflow", "The namespace ml pipeline deployed to")
 var initializeTimeout = flag.Duration("initializeTimeout", 2*time.Minute, "Duration to wait for test initialization")
-var runAsUnitTest = flag.Bool("unitTestsOnly", true, "Whether to skip integration tests that call the service")
+var runIntegrationTests = flag.Bool("runIntegrationTests", false, "Whether to also run integration tests that call the service")
 
 func waitForReady(namespace string, initializeTimeout time.Duration) error {
 	var operation = func() error {
diff --git a/test/api-integration-test/run_test.sh b/test/api-integration-test/run_test.sh
index 8357c3b4c6d..74682fa28b7 100755
--- a/test/api-integration-test/run_test.sh
+++ b/test/api-integration-test/run_test.sh
@@ -61,7 +61,7 @@ TEST_DIR=backend/test
 cd "${BASE_DIR}/${TEST_DIR}"
 
 echo "Run integration test..."
-TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} -args -unitTestsOnly=false 2>&1`
+TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true 2>&1`
 TEST_EXIT_CODE=$?
 
 # Log the test result
diff --git a/test/backend-unit-test/run_test.sh b/test/backend-unit-test/run_test.sh
index a82a2d63272..9e8b8edc60a 100755
--- a/test/backend-unit-test/run_test.sh
+++ b/test/backend-unit-test/run_test.sh
@@ -51,7 +51,7 @@ cd "${BASE_DIR}/${TEST_DIR}"
 
 # Run test and store the exit code.
 echo "Run unit test..."
-TEST_RESULT=`go test -v ./... -args -unitTestsOnly=false 2>&1`
+TEST_RESULT=`go test -v ./... 2>&1`
 TEST_EXIT_CODE=$?
 
 # Log the test result