diff --git a/cmd/e2e-test/upgrade/basic_http.go b/cmd/e2e-test/upgrade/basic_http.go index a64fd51d4b..06ab87e109 100644 --- a/cmd/e2e-test/upgrade/basic_http.go +++ b/cmd/e2e-test/upgrade/basic_http.go @@ -97,7 +97,7 @@ func (bh *BasicHTTP) DuringUpgrade() error { // PostUpgrade implements e2e.UpgradeTest.PostUpgrade func (bh *BasicHTTP) PostUpgrade() error { - // force ingress update. only add path once + // Force ingress update by adding a new path. newIng := fuzz.NewIngressBuilderFromExisting(bh.ing). AddPath("bar.com", "/", svcName, port80). Build() @@ -107,18 +107,16 @@ func (bh *BasicHTTP) PostUpgrade() error { bh.t.Fatalf("error updating Ingress %s: %v", ingKey, err) } else { // If Ingress upgrade succeeds, we update the status on this Ingress - // to Unstable. It is set back to Stable after WaitForIngress below - // finishes successfully. + // to Unstable. It is set back to Stable after UpgradeTestWaitForIngress + // below finishes successfully. bh.s.PutStatus(e2e.Unstable) } - // Verify the Ingress has stabilized after the master upgrade and we - // trigger an Ingress update - ing, err := e2e.WaitForIngress(bh.s, bh.ing, &e2e.WaitForIngressOptions{ExpectUnreachable: true}) + // Verify the Ingress has stabilized after the master upgrade. + ing, err := e2e.UpgradeTestWaitForIngress(bh.s, bh.ing, &e2e.WaitForIngressOptions{ExpectUnreachable: true}) if err != nil { bh.t.Fatalf("error waiting for Ingress %s to stabilize: %v", ingKey, err) } - bh.s.PutStatus(e2e.Stable) bh.t.Logf("GCLB is stable (%s)", ingKey) gclb, err := e2e.WhiteboxTest(ing, bh.s, bh.framework.Cloud, "") if err != nil { diff --git a/cmd/e2e-test/upgrade/v2frontendnamer.go b/cmd/e2e-test/upgrade/v2frontendnamer.go new file mode 100644 index 0000000000..1fe5a6d3cd --- /dev/null +++ b/cmd/e2e-test/upgrade/v2frontendnamer.go @@ -0,0 +1,144 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "context" + "testing" + + "k8s.io/api/networking/v1beta1" + "k8s.io/ingress-gce/pkg/e2e" + "k8s.io/ingress-gce/pkg/fuzz" + "k8s.io/ingress-gce/pkg/utils/common" +) + +// V2FrontendNamer implements e2e.UpgradeTest interface. +type V2FrontendNamer struct { + t *testing.T + s *e2e.Sandbox + framework *e2e.Framework + crud e2e.IngressCRUD + ing *v1beta1.Ingress +} + +// NewV2FrontendNamerTest returns upgrade test for v2 frontend namer. +// This test asserts that v1 finalizer is retained/attached and ingress continues +// to use v1 naming naming scheme when master is upgraded to a gke version that use v1.8. +func NewV2FrontendNamerTest() e2e.UpgradeTest { + return &V2FrontendNamer{} +} + +// Name implements e2e.UpgradeTest.Init. +func (vf *V2FrontendNamer) Name() string { + return "V2FrontendNamerUpgrade" +} + +// Init implements e2e.UpgradeTest.Init. +func (vf *V2FrontendNamer) Init(t *testing.T, s *e2e.Sandbox, framework *e2e.Framework) error { + vf.t = t + vf.s = s + vf.framework = framework + return nil +} + +// PreUpgrade implements e2e.UpgradeTest.PreUpgrade. +func (vf *V2FrontendNamer) PreUpgrade() error { + if _, err := e2e.CreateEchoService(vf.s, svcName, nil); err != nil { + vf.t.Fatalf("CreateEchoService(_, %q, nil): %v, want nil", svcName, err) + } + vf.t.Logf("Echo service created (%s/%s)", vf.s.Namespace, svcName) + + vf.ing = fuzz.NewIngressBuilder(vf.s.Namespace, ingName, ""). + AddPath("foo.com", "/", svcName, port80). + SetIngressClass("gce"). + Build() + ingKey := common.NamespacedName(vf.ing) + vf.crud = e2e.IngressCRUD{C: vf.framework.Clientset} + + if _, err := vf.crud.Create(vf.ing); err != nil { + vf.t.Fatalf("Create(%s) = %v, want nil; Ingress: %v", ingKey, err, vf.ing) + } + vf.t.Logf("Ingress created (%s)", ingKey) + ing, err := e2e.UpgradeTestWaitForIngress(vf.s, vf.ing, &e2e.WaitForIngressOptions{ExpectUnreachable: true}) + if err != nil { + vf.t.Fatalf("error waiting for Ingress %s to stabilize: %v", ingKey, err) + } + + // Assert that v1 finalizer is added. + if err := e2e.CheckV1Finalizer(ing); err != nil { + vf.t.Fatalf("CheckV1Finalizer(%s) = %v, want nil", ingKey, err) + } + + // Perform whitebox testing. + if _, err := e2e.WhiteboxTest(ing, vf.s, vf.framework.Cloud, ""); err != nil { + vf.t.Fatalf("e2e.WhiteboxTest(%s, ...) = %v, want nil", ingKey, err) + } + return nil +} + +// DuringUpgrade implements e2e.UpgradeTest.DuringUpgrade. +func (vf *V2FrontendNamer) DuringUpgrade() error { + return nil +} + +// PostUpgrade implements e2e.UpgradeTest.PostUpgrade +func (vf *V2FrontendNamer) PostUpgrade() error { + // Force ingress update by adding a new path and mark sandbox unstable. + newIng := fuzz.NewIngressBuilderFromExisting(vf.ing). + AddPath("bar.com", "/", svcName, port80). + Build() + ingKey := common.NamespacedName(newIng) + if _, err := vf.crud.Update(newIng); err != nil { + vf.t.Fatalf("error updating ingress %s: %v", ingKey, err) + } else { + // If Ingress upgrade succeeds, we update the status on this Ingress + // to Unstable. It is set back to Stable after WaitForFinalizer below + // finishes successfully. + vf.s.PutStatus(e2e.Unstable) + } + // Wait for ingress to stabilize after the master upgrade. + if _, err := e2e.WaitForIngress(vf.s, vf.ing, nil); err != nil { + vf.t.Fatalf("error waiting for Ingress %s to stabilize: %v", ingKey, err) + } + // Wait for an ingress finalizer to be added. + ing, err := e2e.WaitForFinalizer(vf.s, vf.ing) + if err != nil { + vf.t.Fatalf("e2e.WaitForFinalizer(_, %q) = _, %v, want nil", ingKey, err) + } + + // Assert that v1 finalizer is retained. + if err := e2e.CheckV1Finalizer(ing); err != nil { + vf.t.Fatalf("CheckV1Finalizer(%s) = %v, want nil", ingKey, err) + } + + // Perform whitebox testing. + gclb, err := e2e.WhiteboxTest(ing, vf.s, vf.framework.Cloud, "") + if err != nil { + vf.t.Fatalf("e2e.WhiteboxTest(%s, ...) = %v, want nil", ingKey, err) + } + + // If the Master has upgraded and the Ingress is stable, + // we delete the Ingress and exit out of the loop to indicate that + // the test is done. + deleteOptions := &fuzz.GCLBDeleteOptions{ + SkipDefaultBackend: true, + } + if err := e2e.WaitForIngressDeletion(context.Background(), gclb, vf.s, ing, deleteOptions); err != nil { + vf.t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", ing.Name, err) + } + return nil +} diff --git a/cmd/e2e-test/upgrade_test.go b/cmd/e2e-test/upgrade_test.go index 028c601611..d8d528b818 100644 --- a/cmd/e2e-test/upgrade_test.go +++ b/cmd/e2e-test/upgrade_test.go @@ -41,6 +41,12 @@ func TestUpgradeToV1dot7(t *testing.T) { runUpgradeTest(t, upgrade.NewFinalizerUpgradeTest()) } +// TestUpgradeToV1dot8 runs upgrade tests for features that are introduced in v1.8.0. +// Note that this test runs only when an upgrade results in enabling these features. +func TestUpgradeToV1dot8(t *testing.T) { + runUpgradeTest(t, upgrade.NewV2FrontendNamerTest()) +} + func runUpgradeTest(t *testing.T, test e2e.UpgradeTest) { desc := test.Name() Framework.RunWithSandbox(desc, t, func(t *testing.T, s *e2e.Sandbox) { diff --git a/cmd/e2e-test/v2frontendnamer_test.go b/cmd/e2e-test/v2frontendnamer_test.go new file mode 100644 index 0000000000..6b49afcb10 --- /dev/null +++ b/cmd/e2e-test/v2frontendnamer_test.go @@ -0,0 +1,152 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "testing" + + "k8s.io/api/networking/v1beta1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-gce/pkg/e2e" + "k8s.io/ingress-gce/pkg/fuzz" + "k8s.io/ingress-gce/pkg/utils/common" +) + +// TestV2FrontendNamer tests basic lifecycle of an ingress with v1/v2 frontend naming scheme +// when v2 naming policy is enabled. This test adds v1 finalizer manually to generate an ingress +// with v1 naming scheme. +func TestV2FrontendNamer(t *testing.T) { + t.Parallel() + port80 := intstr.FromInt(80) + svcName := "service-1" + v1Ing := fuzz.NewIngressBuilder("", "ing-v1", ""). + AddPath("foo.com", "/", svcName, port80). + Build() + v1Ing.SetFinalizers([]string{common.FinalizerKey}) + v2Ing := fuzz.NewIngressBuilder("", "ing-v2", ""). + AddPath("foo.com", "/", svcName, port80). + Build() + + for _, tc := range []struct { + desc string + ings []*v1beta1.Ingress + }{ + {"v2 only", []*v1beta1.Ingress{v2Ing}}, + {"v1 only", []*v1beta1.Ingress{v1Ing}}, + // The following test cases create ingresses with both naming schemes. These + // test cases assert that GC of v1 naming scheme does not affect ingress with + // v2 naming scheme and vice-versa. + // Note that the first element in the list of ingresses is deleted first, + // which tests that GC of the naming scheme for first ingress does not affect + // other naming scheme. + {"both v1 and v2, delete v1 first", []*v1beta1.Ingress{v1Ing, v2Ing}}, + {"both v1 and v2, delete v2 first", []*v1beta1.Ingress{v2Ing, v1Ing}}, + } { + tc := tc + desc := fmt.Sprintf("v2 frontend namer %s", tc.desc) + Framework.RunWithSandbox(desc, t, func(t *testing.T, s *e2e.Sandbox) { + t.Parallel() + ctx := context.Background() + + if _, err := e2e.CreateEchoService(s, svcName, nil); err != nil { + t.Fatalf("CreateEchoService(_, %q, nil): %v, want nil", svcName, err) + } + t.Logf("Echo service created (%s/%s)", s.Namespace, svcName) + + crud := e2e.IngressCRUD{C: Framework.Clientset} + var gclbs []*fuzz.GCLB + var updatedIngs []*v1beta1.Ingress + + // Create Ingresses. + for _, ing := range tc.ings { + ing = ing.DeepCopy() + ing.Namespace = s.Namespace + if _, err := crud.Create(ing); err != nil { + t.Fatalf("create(%s/%s) = %v, want nil; Ingress: %v", ing.Namespace, ing.Name, err, ing) + } + t.Logf("Ingress created (%s/%s)", ing.Namespace, ing.Name) + } + // Wait for ingress to stabilize and perform whitebox testing. + for _, ing := range tc.ings { + ingKey := fmt.Sprintf(s.Namespace, ing.Name) + // Determine the expected finalizer after ingress creation. + isV1Finalizer := false + if len(ing.GetFinalizers()) > 0 { + isV1Finalizer = true + } + var err error + if ing, err = e2e.WaitForIngress(s, ing, &e2e.WaitForIngressOptions{ExpectUnreachable: true}); err != nil { + t.Fatalf("error waiting for Ingress %s to stabilize: %v", ingKey, err) + } + if isV1Finalizer { + // Assert that v1 finalizer is added. + if err := e2e.CheckV1Finalizer(ing); err != nil { + t.Fatalf("CheckV1Finalizer(%s) = %v, want nil", ingKey, err) + } + } else { + // Assert that v2 finalizer is added. + if err := e2e.CheckV2Finalizer(ing); err != nil { + t.Fatalf("CheckV2Finalizer(%s) = %v, want nil", ingKey, err) + } + } + // Perform whitebox testing. This also tests naming scheme. + gclb, err := e2e.WhiteboxTest(ing, s, Framework.Cloud, "") + if err != nil { + t.Fatalf("e2e.WhiteboxTest(%s, ...) = %v, want nil", ingKey, err) + } + gclbs = append(gclbs, gclb) + updatedIngs = append(updatedIngs, ing) + } + // Return immediately if there are no ingresses. + if updatedIngs == nil || len(updatedIngs) == 0 { + return + } + ingCount := len(updatedIngs) + + // Delete the first ingress. After deleting the first ingress, assert that + // the resources of remaining ingress are unaffected. + deleteOptions := &fuzz.GCLBDeleteOptions{ + SkipDefaultBackend: true, + } + // Skip checking deletion for backends of first ingress as backends are shared. + if ingCount > 1 { + deleteOptions.SkipBackends = true + } + // Wait for Ingress and GCLB resources to be deleted for first ingress. + if err := e2e.WaitForIngressDeletion(ctx, gclbs[0], s, updatedIngs[0], deleteOptions); err != nil { + t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", common.NamespacedName(updatedIngs[0]), err) + } + // Return if there are no more ingresses to check. + if ingCount < 2 { + return + } + ingKey := common.NamespacedName(updatedIngs[1]) + // Verify that GCLB resources of second ingress are intact. + gclb, err := e2e.WhiteboxTest(updatedIngs[1], s, Framework.Cloud, "") + if err != nil { + t.Fatalf("e2e.WhiteboxTest(%s, ...) = %v, want nil", ingKey, err) + } + // Delete the second ingress. + deleteOptions.SkipBackends = false + if err := e2e.WaitForIngressDeletion(ctx, gclb, s, updatedIngs[1], deleteOptions); err != nil { + t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", ingKey, err) + } + }) + } +} diff --git a/cmd/fuzzer/app/validate.go b/cmd/fuzzer/app/validate.go index 24c7cc3ee4..184626b1d8 100644 --- a/cmd/fuzzer/app/validate.go +++ b/cmd/fuzzer/app/validate.go @@ -157,6 +157,10 @@ func Validate() { if err := iv.PerformWhiteboxTests(gclb); err != nil { panic(err) } + + if err := iv.FrontendNamingSchemeTest(gclb); err != nil { + panic(err) + } } func homeDir() string { diff --git a/pkg/e2e/helpers.go b/pkg/e2e/helpers.go index 33a7936285..4a5e57d239 100644 --- a/pkg/e2e/helpers.go +++ b/pkg/e2e/helpers.go @@ -188,7 +188,10 @@ func performWhiteboxTests(s *Sandbox, ing *v1beta1.Ingress, gclb *fuzz.GCLB) err if err != nil { return err } - return validator.PerformWhiteboxTests(gclb) + if err := validator.PerformWhiteboxTests(gclb); err != nil { + return err + } + return validator.FrontendNamingSchemeTest(gclb) } // WaitForIngressDeletion deletes the given ingress and waits for the @@ -488,7 +491,7 @@ func CheckForAnyFinalizer(ing *v1beta1.Ingress) error { return nil } -// CheckV1Finalizer asserts that v1 finalizer exists on Ingress. +// CheckV1Finalizer asserts that only v1 finalizer exists on Ingress. func CheckV1Finalizer(ing *v1beta1.Ingress) error { ingFinalizers := ing.GetFinalizers() if l := len(ingFinalizers); l != 1 { @@ -499,3 +502,15 @@ func CheckV1Finalizer(ing *v1beta1.Ingress) error { } return nil } + +// CheckV2Finalizer asserts that only v2 finalizer exists on Ingress. +func CheckV2Finalizer(ing *v1beta1.Ingress) error { + ingFinalizers := ing.GetFinalizers() + if l := len(ingFinalizers); l != 1 { + return fmt.Errorf("expected 1 Finalizer but got %d", l) + } + if ingFinalizers[0] != common.FinalizerKeyV2 { + return fmt.Errorf("expected Finalizer %q but got %q", common.FinalizerKeyV2, ingFinalizers[0]) + } + return nil +}