move integration test to sub dir (kubeflow#888)

* move integration test to sub dir * revert
alibaba · Mar 1, 2019 · fb16914 · fb16914
1 parent 5d8c3c5
commit fb16914
Show file tree

Hide file tree

Showing 10 changed files with 104 additions and 83 deletions.
diff --git a/backend/test/BUILD.bazel b/backend/test/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
@@ -18,37 +18,3 @@ go_library(
         "@io_k8s_client_go//tools/clientcmd/api:go_default_library",
     ],
 )
-
-go_test(
-    name = "go_default_test",
-    srcs = [
-        "cli_test.go",
-        "experiment_api_test.go",
-        "job_api_test.go",
-        "pipeline_api_test.go",
-        "run_api_test.go",
-    ],
-    embed = [":go_default_library"],
-    deps = [
-        "//backend/api:go_default_library",
-        "//backend/api/go_http_client/experiment_client/experiment_service:go_default_library",
-        "//backend/api/go_http_client/experiment_model:go_default_library",
-        "//backend/api/go_http_client/job_client/job_service:go_default_library",
-        "//backend/api/go_http_client/job_model:go_default_library",
-        "//backend/api/go_http_client/pipeline_client/pipeline_service:go_default_library",
-        "//backend/api/go_http_client/pipeline_model:go_default_library",
-        "//backend/api/go_http_client/pipeline_upload_client/pipeline_upload_service:go_default_library",
-        "//backend/api/go_http_client/run_client/run_service:go_default_library",
-        "//backend/api/go_http_client/run_model:go_default_library",
-        "//backend/src/cmd/ml/cmd:go_default_library",
-        "//backend/src/common/client/api_server:go_default_library",
-        "//backend/src/common/util:go_default_library",
-        "@com_github_argoproj_argo//pkg/apis/workflow/v1alpha1:go_default_library",
-        "@com_github_ghodss_yaml//:go_default_library",
-        "@com_github_golang_glog//:go_default_library",
-        "@com_github_stretchr_testify//assert:go_default_library",
-        "@com_github_stretchr_testify//suite:go_default_library",
-        "@io_k8s_apimachinery//pkg/util/yaml:go_default_library",
-        "@org_golang_google_grpc//:go_default_library",
-    ],
-)
diff --git a/backend/test/integration/BUILD.bazel b/backend/test/integration/BUILD.bazel
@@ -0,0 +1,43 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "cli_test.go",
+        "experiment_api_test.go",
+        "job_api_test.go",
+        "pipeline_api_test.go",
+        "run_api_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//backend/api:go_default_library",
+        "//backend/api/go_http_client/experiment_client/experiment_service:go_default_library",
+        "//backend/api/go_http_client/experiment_model:go_default_library",
+        "//backend/api/go_http_client/job_client/job_service:go_default_library",
+        "//backend/api/go_http_client/job_model:go_default_library",
+        "//backend/api/go_http_client/pipeline_client/pipeline_service:go_default_library",
+        "//backend/api/go_http_client/pipeline_model:go_default_library",
+        "//backend/api/go_http_client/pipeline_upload_client/pipeline_upload_service:go_default_library",
+        "//backend/api/go_http_client/run_client/run_service:go_default_library",
+        "//backend/api/go_http_client/run_model:go_default_library",
+        "//backend/src/cmd/ml/cmd:go_default_library",
+        "//backend/src/common/client/api_server:go_default_library",
+        "//backend/src/common/util:go_default_library",
+        "//backend/test:go_default_library",
+        "@com_github_argoproj_argo//pkg/apis/workflow/v1alpha1:go_default_library",
+        "@com_github_ghodss_yaml//:go_default_library",
+        "@com_github_golang_glog//:go_default_library",
+        "@com_github_stretchr_testify//assert:go_default_library",
+        "@com_github_stretchr_testify//suite:go_default_library",
+        "@io_k8s_apimachinery//pkg/util/yaml:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["flags.go"],
+    importpath = "github.com/kubeflow/pipelines/backend/test/integration",
+    visibility = ["//visibility:public"],
+)
diff --git a/backend/test/cli_test.go → backend/test/integration/cli_test.go b/backend/test/cli_test.go → backend/test/integration/cli_test.go
@@ -1,4 +1,4 @@
-package test
+package integration
 
 import (
 	"testing"
@@ -9,6 +9,7 @@ import (
 	"github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_model"
 	"github.com/kubeflow/pipelines/backend/api/go_http_client/pipeline_model"
 	"github.com/kubeflow/pipelines/backend/src/cmd/ml/cmd"
+	"github.com/kubeflow/pipelines/backend/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 )
@@ -40,7 +41,7 @@ func (c *CLIIntegrationTest) SetupTest() {
 	c.namespace = *namespace
 
 	// Wait for the system to be ready.
-	err := waitForReady(c.namespace, *initializeTimeout)
+	err := test.WaitForReady(c.namespace, *initializeTimeout)
 	if err != nil {
 		glog.Exitf("Cluster namespace '%s' is still not ready after timeout. Error: %s", c.namespace,
 			err.Error())
@@ -94,7 +95,7 @@ func (c *CLIIntegrationTest) TestPipelineUploadSuccess() {
 	t := c.T()
 	rootCmd, _ := GetRealRootCommand()
 	args := []string{"pipeline", "upload", "--name", myUploadedPipeline, "--file",
-		"./resources/hello-world.yaml"}
+		"../resources/hello-world.yaml"}
 	args = addCommonArgs(args, c.namespace)
 	rootCmd.Command().SetArgs(args)
 	_, err := rootCmd.Command().ExecuteC()

diff --git a/backend/test/experiment_api_test.go → ...d/test/integration/experiment_api_test.go b/backend/test/experiment_api_test.go → ...d/test/integration/experiment_api_test.go
@@ -1,4 +1,4 @@
-package test
+package integration
 
 import (
 	"testing"
@@ -10,6 +10,7 @@ import (
 	"github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_model"
 	"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
 	"github.com/kubeflow/pipelines/backend/src/common/util"
+	"github.com/kubeflow/pipelines/backend/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 )
@@ -27,12 +28,12 @@ func (s *ExperimentApiTest) SetupTest() {
 		return
 	}
 
-	err := waitForReady(*namespace, *initializeTimeout)
+	err := test.WaitForReady(*namespace, *initializeTimeout)
 	if err != nil {
 		glog.Exitf("Failed to initialize test. Error: %s", err.Error())
 	}
 	s.namespace = *namespace
-	clientConfig := getClientConfig(*namespace)
+	clientConfig := test.GetClientConfig(*namespace)
 	s.experimentClient, err = api_server.NewExperimentClient(clientConfig, false)
 	if err != nil {
 		glog.Exitf("Failed to get pipeline upload client. Error: %s", err.Error())
@@ -155,7 +156,7 @@ func (s *ExperimentApiTest) TestExperimentAPI() {
 	assert.Equal(t, expectedTrainingExperiment, experiment)
 
 	/* ---------- Clean up ---------- */
-	deleteAllExperiments(s.experimentClient, t)
+	test.DeleteAllExperiments(s.experimentClient, t)
 }
 
 func TestExperimentAPI(t *testing.T) {

diff --git a/backend/test/integration/flags.go b/backend/test/integration/flags.go
@@ -0,0 +1,10 @@
+package integration
+
+import (
+	"flag"
+	"time"
+)
+
+var namespace = flag.String("namespace", "kubeflow", "The namespace ml pipeline deployed to")
+var initializeTimeout = flag.Duration("initializeTimeout", 2*time.Minute, "Duration to wait for test initialization")
+var runIntegrationTests = flag.Bool("runIntegrationTests", false, "Whether to also run integration tests that call the service")
diff --git a/backend/test/job_api_test.go → backend/test/integration/job_api_test.go b/backend/test/job_api_test.go → backend/test/integration/job_api_test.go
@@ -1,10 +1,12 @@
-package test
+package integration
 
 import (
 	"io/ioutil"
 	"testing"
 	"time"
 
+	"github.com/kubeflow/pipelines/backend/test"
+
 	"github.com/golang/glog"
 	experimentparams "github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_client/experiment_service"
 	"github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_model"
@@ -39,12 +41,12 @@ func (s *JobApiTestSuite) SetupTest() {
 		return
 	}
 
-	err := waitForReady(*namespace, *initializeTimeout)
+	err := test.WaitForReady(*namespace, *initializeTimeout)
 	if err != nil {
 		glog.Exitf("Failed to initialize test. Error: %s", err.Error())
 	}
 	s.namespace = *namespace
-	clientConfig := getClientConfig(*namespace)
+	clientConfig := test.GetClientConfig(*namespace)
 	s.experimentClient, err = api_server.NewExperimentClient(clientConfig, false)
 	if err != nil {
 		glog.Exitf("Failed to get pipeline upload client. Error: %s", err.Error())
@@ -71,7 +73,7 @@ func (s *JobApiTestSuite) TestJobApis() {
 	t := s.T()
 
 	/* ---------- Upload pipelines YAML ---------- */
-	helloWorldPipeline, err := s.pipelineUploadClient.UploadFile("resources/hello-world.yaml", uploadParams.NewUploadPipelineParams())
+	helloWorldPipeline, err := s.pipelineUploadClient.UploadFile("../resources/hello-world.yaml", uploadParams.NewUploadPipelineParams())
 	assert.Nil(t, err)
 
 	/* ---------- Create a new hello world experiment ---------- */
@@ -108,7 +110,7 @@ func (s *JobApiTestSuite) TestJobApis() {
 	assert.Nil(t, err)
 
 	/* ---------- Create a new argument parameter job by uploading workflow manifest ---------- */
-	argParamsBytes, err := ioutil.ReadFile("resources/arguments-parameters.yaml")
+	argParamsBytes, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
 	assert.Nil(t, err)
 	argParamsBytes, err = yaml.ToJSON(argParamsBytes)
 	assert.Nil(t, err)
@@ -206,10 +208,10 @@ func (s *JobApiTestSuite) TestJobApis() {
 	s.checkArgParamsRun(t, argParamsRun, argParamsExperiment.ID, argParamsJob.ID)
 
 	/* ---------- Clean up ---------- */
-	deleteAllExperiments(s.experimentClient, t)
-	deleteAllPipelines(s.pipelineClient, t)
-	deleteAllJobs(s.jobClient, t)
-	deleteAllRuns(s.runClient, t)
+	test.DeleteAllExperiments(s.experimentClient, t)
+	test.DeleteAllPipelines(s.pipelineClient, t)
+	test.DeleteAllJobs(s.jobClient, t)
+	test.DeleteAllRuns(s.runClient, t)
 }
 
 func (s *JobApiTestSuite) checkHelloWorldJob(t *testing.T, job *job_model.APIJob, experimentID string, pipelineID string) {
@@ -240,7 +242,7 @@ func (s *JobApiTestSuite) checkHelloWorldJob(t *testing.T, job *job_model.APIJob
 }
 
 func (s *JobApiTestSuite) checkArgParamsJob(t *testing.T, job *job_model.APIJob, experimentID string) {
-	argParamsBytes, err := ioutil.ReadFile("resources/arguments-parameters.yaml")
+	argParamsBytes, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
 	assert.Nil(t, err)
 	argParamsBytes, err = yaml.ToJSON(argParamsBytes)
 	assert.Nil(t, err)

diff --git a/backend/test/pipeline_api_test.go → ...end/test/integration/pipeline_api_test.go b/backend/test/pipeline_api_test.go → ...end/test/integration/pipeline_api_test.go
@@ -1,10 +1,12 @@
-package test
+package integration
 
 import (
 	"io/ioutil"
 	"testing"
 	"time"
 
+	"github.com/kubeflow/pipelines/backend/test"
+
 	"github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
 	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
@@ -37,11 +39,11 @@ func (s *PipelineApiTest) SetupTest() {
 		return
 	}
 
-	err := waitForReady(*namespace, *initializeTimeout)
+	err := test.WaitForReady(*namespace, *initializeTimeout)
 	if err != nil {
 		glog.Exitf("Failed to initialize test. Error: %s", err.Error())
 	}
-	clientConfig := getClientConfig(*namespace)
+	clientConfig := test.GetClientConfig(*namespace)
 	s.pipelineUploadClient, err = api_server.NewPipelineUploadClient(clientConfig, false)
 	if err != nil {
 		glog.Exitf("Failed to get pipeline upload client. Error: %s", err.Error())
@@ -55,15 +57,15 @@ func (s *PipelineApiTest) SetupTest() {
 func (s *PipelineApiTest) TestPipelineAPI() {
 	t := s.T()
 
-	deleteAllPipelines(s.pipelineClient, t)
+	test.DeleteAllPipelines(s.pipelineClient, t)
 
 	/* ---------- Upload pipelines YAML ---------- */
-	argumentYAMLPipeline, err := s.pipelineUploadClient.UploadFile("resources/arguments-parameters.yaml", uploadParams.NewUploadPipelineParams())
+	argumentYAMLPipeline, err := s.pipelineUploadClient.UploadFile("../resources/arguments-parameters.yaml", uploadParams.NewUploadPipelineParams())
 	assert.Nil(t, err)
 	assert.Equal(t, "arguments-parameters.yaml", argumentYAMLPipeline.Name)
 
 	/* ---------- Upload the same pipeline again. Should fail due to name uniqueness ---------- */
-	_, err = s.pipelineUploadClient.UploadFile("resources/arguments-parameters.yaml", uploadParams.NewUploadPipelineParams())
+	_, err = s.pipelineUploadClient.UploadFile("../resources/arguments-parameters.yaml", uploadParams.NewUploadPipelineParams())
 	assert.NotNil(t, err)
 	assert.Contains(t, err.Error(), "Failed to upload pipeline.")
 
@@ -78,7 +80,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 	/* ---------- Upload pipelines zip ---------- */
 	time.Sleep(1 * time.Second)
 	argumentUploadPipeline, err := s.pipelineUploadClient.UploadFile(
-		"resources/zip-arguments.zip", &uploadParams.UploadPipelineParams{Name: util.StringPointer("zip-arguments-parameters")})
+		"../resources/zip-arguments.zip", &uploadParams.UploadPipelineParams{Name: util.StringPointer("zip-arguments-parameters")})
 	assert.Nil(t, err)
 	assert.Equal(t, "zip-arguments-parameters", argumentUploadPipeline.Name)
 
@@ -172,14 +174,14 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 	/* ---------- Verify get template works ---------- */
 	template, err := s.pipelineClient.GetTemplate(&params.GetTemplateParams{ID: argumentYAMLPipeline.ID})
 	assert.Nil(t, err)
-	expected, err := ioutil.ReadFile("resources/arguments-parameters.yaml")
+	expected, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
 	assert.Nil(t, err)
 	var expectedWorkflow v1alpha1.Workflow
 	err = yaml.Unmarshal(expected, &expectedWorkflow)
 	assert.Equal(t, expectedWorkflow, *template)
 
 	/* ---------- Clean up ---------- */
-	deleteAllPipelines(s.pipelineClient, t)
+	test.DeleteAllPipelines(s.pipelineClient, t)
 }
 
 func verifyPipeline(t *testing.T, pipeline *model.APIPipeline) {

diff --git a/backend/test/run_api_test.go → backend/test/integration/run_api_test.go b/backend/test/run_api_test.go → backend/test/integration/run_api_test.go
@@ -1,9 +1,11 @@
-package test
+package integration
 
 import (
 	"io/ioutil"
 	"testing"
 
+	"github.com/kubeflow/pipelines/backend/test"
+
 	"github.com/golang/glog"
 	api "github.com/kubeflow/pipelines/backend/api/go_client"
 	experimentparams "github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_client/experiment_service"
@@ -34,12 +36,12 @@ func (s *RunApiTestSuite) SetupTest() {
 		return
 	}
 
-	err := waitForReady(*namespace, *initializeTimeout)
+	err := test.WaitForReady(*namespace, *initializeTimeout)
 	if err != nil {
 		glog.Exitf("Failed to initialize test. Error: %s", err.Error())
 	}
 	s.namespace = *namespace
-	clientConfig := getClientConfig(*namespace)
+	clientConfig := test.GetClientConfig(*namespace)
 	s.experimentClient, err = api_server.NewExperimentClient(clientConfig, false)
 	if err != nil {
 		glog.Exitf("Failed to get pipeline upload client. Error: %s", err.Error())
@@ -62,7 +64,7 @@ func (s *RunApiTestSuite) TestRunApis() {
 	t := s.T()
 
 	/* ---------- Upload pipelines YAML ---------- */
-	helloWorldPipeline, err := s.pipelineUploadClient.UploadFile("resources/hello-world.yaml", uploadParams.NewUploadPipelineParams())
+	helloWorldPipeline, err := s.pipelineUploadClient.UploadFile("../resources/hello-world.yaml", uploadParams.NewUploadPipelineParams())
 	assert.Nil(t, err)
 
 	/* ---------- Create a new hello world experiment ---------- */
@@ -97,7 +99,7 @@ func (s *RunApiTestSuite) TestRunApis() {
 	assert.Nil(t, err)
 
 	/* ---------- Create a new argument parameter run by uploading workflow manifest ---------- */
-	argParamsBytes, err := ioutil.ReadFile("resources/arguments-parameters.yaml")
+	argParamsBytes, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
 	assert.Nil(t, err)
 	argParamsBytes, err = yaml.ToJSON(argParamsBytes)
 	assert.Nil(t, err)
@@ -183,9 +185,9 @@ func (s *RunApiTestSuite) TestRunApis() {
 	assert.Equal(t, string(runs[0].StorageState), api.Run_STORAGESTATE_ARCHIVED.String())
 
 	/* ---------- Clean up ---------- */
-	deleteAllExperiments(s.experimentClient, t)
-	deleteAllPipelines(s.pipelineClient, t)
-	deleteAllRuns(s.runClient, t)
+	test.DeleteAllExperiments(s.experimentClient, t)
+	test.DeleteAllPipelines(s.pipelineClient, t)
+	test.DeleteAllRuns(s.runClient, t)
 }
 
 func (s *RunApiTestSuite) checkHelloWorldRunDetail(t *testing.T, runDetail *run_model.APIRunDetail, experimentId string, pipelineId string) {
@@ -215,7 +217,7 @@ func (s *RunApiTestSuite) checkHelloWorldRunDetail(t *testing.T, runDetail *run_
 }
 
 func (s *RunApiTestSuite) checkArgParamsRunDetail(t *testing.T, runDetail *run_model.APIRunDetail, experimentId string) {
-	argParamsBytes, err := ioutil.ReadFile("resources/arguments-parameters.yaml")
+	argParamsBytes, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
 	assert.Nil(t, err)
 	argParamsBytes, err = yaml.ToJSON(argParamsBytes)
 	assert.Nil(t, err)