Merge pull request #325 from bowei/basic-e2e

Many small cleanups to get basic_test.go working

Dockerfile.e2e-test

@@ -15,4 +15,4 @@
 FROM debian:9
 
 ADD bin/ARG_ARCH/ARG_BIN /ARG_BIN
-ENTRYPOINT ["/ARG_BIN"]
+ENTRYPOINT ["/ARG_BIN", "-inCluster"]

Makefile

@@ -21,7 +21,7 @@ PKG := k8s.io/ingress-gce
 
 # List of binaries to build. You must have a matching Dockerfile.BINARY
 # for each BINARY.
-CONTAINER_BINARIES := glbc e2e-test echo
+CONTAINER_BINARIES := glbc e2e-test echo fuzzer
 
 # Latest commit hash for current branch.
 GIT_COMMIT := $(shell git rev-parse HEAD)

build/rules.mk

@@ -122,11 +122,10 @@ $(GO_BINARIES): build-dirs
 	        ARCH=$(ARCH)                                                   \
 	        VERSION=$(VERSION)                                             \
 	        PKG=$(PKG)                                                     \
-			TARGET=$@                                                      \
-		GIT_COMMIT=$(GIT_COMMIT)                                           \
+			TARGET=$@                                              \
+		GIT_COMMIT=$(GIT_COMMIT)                                       \
 	        ./build/build.sh                                               \
 	    "                                                                  \
-		$(VERBOSE_OUTPUT)
 
 # Rules for dockerfiles.
 define DOCKERFILE_RULE

cmd/e2e-test/basic_test.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"testing"
+
+	"github.com/kr/pretty"
+	"k8s.io/api/extensions/v1beta1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/ingress-gce/pkg/e2e"
+	"k8s.io/ingress-gce/pkg/fuzz"
+	"k8s.io/ingress-gce/pkg/fuzz/features"
+)
+
+func TestBasic(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		desc string
+		ing  *v1beta1.Ingress
+
+		numForwardingRules int
+		numBackendServices int
+	}{
+		{
+			desc:               "http default backend",
+			ing:                fuzz.NewIngressBuilder("", "ingress-1", "").DefaultBackend("service-1", intstr.FromInt(80)).I,
+			numForwardingRules: 1,
+			numBackendServices: 1,
+		},
+		{
+			desc:               "http one path",
+			ing:                fuzz.NewIngressBuilder("", "ingress-1", "").AddPath("test.com", "/", "service-1", intstr.FromInt(80)).I,
+			numForwardingRules: 1,
+			numBackendServices: 2,
+		},
+		{
+			desc:               "http multiple paths",
+			ing:                fuzz.NewIngressBuilder("", "ingress-1", "").AddPath("test.com", "/foo", "service-1", intstr.FromInt(80)).AddPath("test.com", "/bar", "service-1", intstr.FromInt(80)).I,
+			numForwardingRules: 1,
+			numBackendServices: 2,
+		},
+	} {
+		tc := tc // Capture tc as we are running this in parallel.
+		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
+			t.Parallel()
+
+			_, _, err := e2e.CreateEchoService(s, "service-1")
+			if err != nil {
+				t.Fatalf("error creating echo service: %v", err)
+			}
+
+			if _, err := Framework.Clientset.Extensions().Ingresses(s.Namespace).Create(tc.ing); err != nil {
+				t.Fatalf("error creating Ingress spec: %v", err)
+			}
+
+			ing, err := e2e.WaitForIngress(s, tc.ing)
+			if err != nil {
+				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
+			}
+
+			// Perform whitebox testing.
+			if len(ing.Status.LoadBalancer.Ingress) < 1 {
+				t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
+			}
+
+			vip := ing.Status.LoadBalancer.Ingress[0].IP
+			gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, vip, fuzz.FeatureValidators(features.All))
+			if err != nil {
+				t.Fatalf("Error getting GCP resources for LB with IP = %q", vip)
+			}
+			// Do some cursory checks on the GCP objects.
+			if len(gclb.ForwardingRule) != tc.numForwardingRules {
+				t.Errorf("got %d fowarding rules, want %d;\n%s", len(gclb.ForwardingRule), tc.numForwardingRules, pretty.Sprint(gclb.ForwardingRule))
+			}
+			if len(gclb.BackendService) != tc.numBackendServices {
+				t.Errorf("got %d backend services, want %d;\n%s", len(gclb.BackendService), tc.numBackendServices, pretty.Sprint(gclb.BackendService))
+			}
+		})
+	}
+}

cmd/e2e-test/e2e-test.yaml

@@ -0,0 +1,71 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# To enable creating RBAC rules on the GKE cluster:
+# $ kubectl create clusterrolebinding cluster-admin-binding \
+#   --clusterrole cluster-admin
+#   --user $(gcloud config get-value account)
+#   clusterrolebinding "cluster-admin-binding"
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: ingress-e2e-test
+  namespace: default
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: default
+  name: ingress-e2e-test
+rules:
+   - apiGroups: [""]
+     resources: ["pods"]
+     verbs: ["list"]
+   - apiGroups: [""]
+     resources: ["nodes"]
+     verbs: ["list"]
+   - apiGroups: [""]
+     resources: ["services"]
+     verbs: ["list"]
+   - apiGroups: [""]
+     resources: ["configmaps"]
+     verbs: ["get", "create"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ingress-e2e-test
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: ingress-e2e-test
+    namespace: default
+roleRef:
+  kind: Role
+  name: ingress-e2e-test
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ingress-e2e
+  namespace: default
+spec:
+  containers:
+  - name: ingress-e2e
+    image: gcr.io/k8s-ingress-image-push/ingress-gce-e2e-test-amd64:v1.1.0-123-g7aaa9456
+    imagePullPolicy: Always
+  restartPolicy: Never
+  serviceAccount: ingress-e2e-test

cmd/e2e-test/main_test.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/golang/glog"
 	"k8s.io/client-go/rest"
@@ -34,8 +35,13 @@ import (
 
 var (
 	flags struct {
-		inCluster  bool
-		kubeconfig string
+		run              bool
+		inCluster        bool
+		kubeconfig       string
+		project          string
+		seed             int64
+		destroySandboxes bool
+		handleSIGINT     bool
 	}
 
 	Framework *e2e.Framework
@@ -48,21 +54,31 @@ func init() {
 	} else {
 		flag.StringVar(&flags.kubeconfig, "kubeconfig", "", "absolute path to the kubeconfig file")
 	}
+	flag.BoolVar(&flags.run, "run", false, "set to true to run tests (suppresses test suite from 'go test ./...')")
 	flag.BoolVar(&flags.inCluster, "inCluster", false, "set to true if running in the cluster")
+	flag.StringVar(&flags.project, "project", "", "GCP project")
+	flag.Int64Var(&flags.seed, "seed", -1, "random seed")
+	flag.BoolVar(&flags.destroySandboxes, "destroySandboxes", true, "set to false to leave sandboxed resources for debugging")
+	flag.BoolVar(&flags.handleSIGINT, "handleSIGINT", true, "catch SIGINT to perform clean")
+
 }
 
 // TestMain is the entrypoint for the end-to-end test suite. This is where
 // global resource setup should be done.
 func TestMain(m *testing.M) {
 	flag.Parse()
 
-	if os.Getenv("RUN_INGRESS_E2E") != "true" {
-		fmt.Fprintln(os.Stderr, "You must set the RUN_INGRESS_E2E environment variable to 'true' run the tests.")
-		return
+	if !flags.inCluster && !flags.run {
+		fmt.Fprintln(os.Stderr, "Set -run to run the tests.")
+		// Return 0 here so 'go test ./...' will succeed.
+		os.Exit(0)
+	}
+	if flags.project == "" {
+		fmt.Fprintln(os.Stderr, "-project must be set to the Google Cloud test project")
+		os.Exit(1)
 	}
 
-	fmt.Printf("Version: %q\n", version.Version)
-	fmt.Printf("Commit: %q\n", version.GitCommit)
+	fmt.Printf("Version: %q, Commit: %q\n", version.Version, version.GitCommit)
 
 	var err error
 	var kubeconfig *rest.Config
@@ -79,7 +95,19 @@ func TestMain(m *testing.M) {
 		}
 	}
 
-	Framework = e2e.NewFramework(kubeconfig)
+	if flags.seed == -1 {
+		flags.seed = time.Now().UnixNano()
+	}
+	glog.Infof("Using random seed = %d", flags.seed)
+
+	Framework = e2e.NewFramework(kubeconfig, e2e.Options{
+		Project:          flags.project,
+		Seed:             flags.seed,
+		DestroySandboxes: flags.destroySandboxes,
+	})
+	if flags.handleSIGINT {
+		Framework.CatchSIGINT()
+	}
 	if err := Framework.SanityCheck(); err != nil {
 		glog.Fatalf("Framework sanity check failed: %v", err)
 	}

cmd/fuzzer/app/validate.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	backendconfig "k8s.io/ingress-gce/pkg/backendconfig/client/clientset/versioned"
+	"k8s.io/ingress-gce/pkg/e2e"
 	"k8s.io/ingress-gce/pkg/fuzz"
 	"k8s.io/ingress-gce/pkg/fuzz/features"
 	// Pull in the auth library for GCP.
@@ -85,8 +86,12 @@ func Validate() {
 		panic(err.Error())
 	}
 
-	k8s := k8sClientSet(config)
-	env, err := fuzz.NewClientsetValidatorEnv(config, validateOptions.ns)
+	gce, err := e2e.NewCloud("bowei-gke")
+	if err != nil {
+		panic(err)
+	}
+
+	env, err := fuzz.NewDefaultValidatorEnv(config, validateOptions.ns, gce)
 
 	if err != nil {
 		panic(err)
@@ -109,6 +114,7 @@ func Validate() {
 	}
 	fmt.Printf("Features = %v\n\n", fsNames)
 
+	k8s := k8sClientSet(config)
 	ing, err := k8s.Extensions().Ingresses(validateOptions.ns).Get(validateOptions.name, metav1.GetOptions{})
 	if err != nil {
 		panic(err)
@@ -127,6 +133,13 @@ func Validate() {
 	if result.Err != nil {
 		os.Exit(1)
 	}
+
+	vip := ing.Status.LoadBalancer.Ingress[0].IP
+	gclb, err := fuzz.GCLBForVIP(context.Background(), gce, vip, []fuzz.FeatureValidator{})
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("GCP resources = \n%s\n", pretty.Sprint(gclb))
 }
 
 func homeDir() string {

pkg/e2e/doc.go

@@ -16,4 +16,21 @@ limitations under the License.
 
 // Package e2e contains supporting infrastructure for end-to-end integration
 // testing driven by the tests in cmd/e2e-test.
+//
+// Test should be written with a Sandbox:
+//   func TestExample(t *testing.T) {
+//     for _, tc := range []struct{
+//       ...
+//     }{
+//       ...
+//     }{
+//       tc := tc // avoid variable capture
+//       Framework.RunWithSandbox(t, func(t *testing.T, s *e2e.Sandbox) {
+//         t.Parallel()
+//         // Test code...
+//       })
+//     }
+//   }
+//
+// The Sandbox will handle resource isolation and reclaimation.
 package e2e