Added retry and extracted isVisualizationServiceAlive logic to function

backend/src/apiserver/main.go

@@ -83,7 +83,7 @@ func startRpcServer(resourceManager *resource.ResourceManager) {
 	api.RegisterRunServiceServer(s, server.NewRunServer(resourceManager))
 	api.RegisterJobServiceServer(s, server.NewJobServer(resourceManager))
 	api.RegisterReportServiceServer(s, server.NewReportServer(resourceManager))
-	api.RegisterVisualizationServiceServer(s, server.NewVisualizationServer(resourceManager, getStringConfig(visualizationServiceHost), getStringConfig(podNamespace)))
+	api.RegisterVisualizationServiceServer(s, server.NewVisualizationServer(resourceManager, getStringConfig(visualizationServiceHost), getStringConfig(podNamespace), getDurationConfig(initConnectionTimeout)))
 
 	// Register reflection service on gRPC server.
 	reflection.Register(s)

backend/src/apiserver/server/BUILD.bazel

@@ -27,6 +27,7 @@ go_library(
         "//backend/src/common/util:go_default_library",
         "//backend/src/crd/pkg/apis/scheduledworkflow/v1beta1:go_default_library",
         "@com_github_argoproj_argo//pkg/apis/workflow/v1alpha1:go_default_library",
+        "@com_github_cenkalti_backoff//:go_default_library",
         "@com_github_golang_glog//:go_default_library",
         "@com_github_golang_protobuf//jsonpb:go_default_library_gen",
         "@com_github_robfig_cron//:go_default_library",

backend/src/apiserver/server/visualization_server.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/cenkalti/backoff"
 	"github.com/kubeflow/pipelines/backend/api/go_client"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/resource"
 	"github.com/kubeflow/pipelines/backend/src/common/util"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 )
 
 type VisualizationServer struct {
@@ -80,12 +82,26 @@ func (s *VisualizationServer) generateVisualizationFromRequest(request *go_clien
 	return body, nil
 }
 
-func NewVisualizationServer(resourceManager *resource.ResourceManager, serviceName string, namespace string) *VisualizationServer {
+func isVisualizationServiceAlive(serviceURL string, initConnectionTimeout time.Duration) bool {
+	var operation = func() error {
+		_, err := http.Get(serviceURL)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	b := backoff.NewExponentialBackOff()
+	b.MaxElapsedTime = initConnectionTimeout
+	err := backoff.Retry(operation, b)
+	return err == nil
+}
+
+func NewVisualizationServer(resourceManager *resource.ResourceManager, serviceName string, namespace string, initConnectionTimeout time.Duration) *VisualizationServer {
 	serviceURL := fmt.Sprintf("http://%s.%s", serviceName, namespace)
-	_, err := http.Get(serviceURL)
+	isServiceAvailable := isVisualizationServiceAlive(serviceURL, initConnectionTimeout)
 	return &VisualizationServer{
 		resourceManager:    resourceManager,
 		serviceURL:         serviceURL,
-		isServiceAvailable: err == nil,
+		isServiceAvailable: isServiceAvailable,
 	}
 }