[CI] Add Legacy Integration Test Workflow

.github/workflows/integration_test.yaml

@@ -1,6 +1,16 @@
 name: Integration Tests
 
-on: [push, pull_request]
+on:
+  pull_request:
+    branches:
+      - 'main'
+      - 'next'
+  push:
+    branches:
+      - 'main'
+      - 'next'
+    tags:
+      - 'v*.*.*'
 
 jobs:
   test:

.github/workflows/integration_test_legacy.yaml

@@ -0,0 +1,39 @@
+name: Integration Tests (Legacy)
+
+on:
+  pull_request:
+    branches:
+      - 'main'
+      - 'next'
+  push:
+    branches:
+      - 'main'
+      - 'next'
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: setup golang
+      uses: actions/setup-go@v2
+      with:
+        go-version: '^1.16'
+    - name: cache go modules
+      uses: actions/cache@v1
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-build-codegen-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go
+    - name: checkout repository
+      uses: actions/checkout@v2
+    - name: Login to GitHub Packages Docker Registry
+      uses: docker/login-action@v1
+      with:
+        registry: docker.pkg.github.com
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: run railgun integration tests
+      run: ./scripts/railgun-integration-tests.sh LEGACY

railgun/Makefile

@@ -121,3 +121,9 @@ clean.test.proxy:
 test.integration:
 	@./scripts/setup-integration-tests.sh
 	@GOFLAGS="-tags=integration_tests" go test -race -v ./test/integration/
+
+# Our integration tests using the legacy v1 controller manager
+.PHONY: test.integration.legacy
+test.integration.legacy:
+	@./scripts/setup-integration-tests.sh
+	@KONG_LEGACY_CONTROLLER=1 GOFLAGS="-tags=integration_tests" go test -race -v ./test/integration/

railgun/test/integration/ingress_controllers_test.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strings"
 	"testing"
 	"time"
 
@@ -56,17 +57,16 @@ func TestMinimalIngress(t *testing.T) {
 			return false
 		}
 		defer resp.Body.Close()
-		return resp.StatusCode == http.StatusOK
+		if resp.StatusCode == http.StatusOK {
+			// now that the ingress backend is routable, make sure the contents we're getting back are what we expect
+			// Expected: Welcome to nginx!
+			b := new(bytes.Buffer)
+			b.ReadFrom(resp.Body)
+			return strings.Contains(b.String(), "Welcome to nginx!")
+		}
+		return false
 	}, ingressTimeout, ingressTimeoutTick)
 
-	// now that the ingress backend is routable, make sure the contents we're getting back are what we expect
-	// Expected: Welcome to nginx!
-	resp, err := http.Get(fmt.Sprintf("%s/nginx", u.String()))
-	assert.NoError(t, err)
-	b := new(bytes.Buffer)
-	b.ReadFrom(resp.Body)
-	assert.Contains(t, b.String(), "Welcome to nginx!")
-
 	// ensure that a deleted ingress results in the route being torn down
 	assert.NoError(t, cluster.Client().NetworkingV1().Ingresses("default").Delete(ctx, ingress.Name, metav1.DeleteOptions{}))
 	assert.Eventually(t, func() bool {
@@ -76,20 +76,22 @@ func TestMinimalIngress(t *testing.T) {
 			return false
 		}
 		defer resp.Body.Close()
-		return resp.StatusCode == http.StatusNotFound
+		if resp.StatusCode == http.StatusNotFound {
+			// once the route is torn down and returning 404's, ensure that we got the expected response body back from Kong
+			// Expected: {"message":"no Route matched with those values"}
+			b := new(bytes.Buffer)
+			b.ReadFrom(resp.Body)
+			body := struct {
+				Message string `json:"message"`
+			}{}
+			if err := json.Unmarshal(b.Bytes(), &body); err != nil {
+				t.Logf("WARNING: error decoding JSON from proxy while waiting for %s: %v", u.String(), err)
+				return false
+			}
+			return body.Message == "no Route matched with those values"
+		}
+		return false
 	}, ingressTimeout, ingressTimeoutTick)
-
-	// once the route is torn down and returning 404's, ensure that we got the expected response body back from Kong
-	// Expected: {"message":"no Route matched with those values"}
-	resp, err = http.Get(fmt.Sprintf("%s/nginx", u.String()))
-	assert.NoError(t, err)
-	b = new(bytes.Buffer)
-	b.ReadFrom(resp.Body)
-	body := struct {
-		Message string `json:"message"`
-	}{}
-	assert.NoError(t, json.Unmarshal(b.Bytes(), &body))
-	assert.Equal(t, "no Route matched with those values", body.Message)
 }
 
 func TestHTTPSRedirect(t *testing.T) {

railgun/test/integration/suite_test.go

@@ -3,8 +3,10 @@
 package integration
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"os/exec"
@@ -13,13 +15,18 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
 
+	"github.com/kong/kubernetes-testing-framework/pkg/kind"
 	ktfkind "github.com/kong/kubernetes-testing-framework/pkg/kind"
 
 	"github.com/kong/kubernetes-ingress-controller/railgun/controllers"
 )
 
+const (
+	// LegacyControllerEnvVar indicates the environment variable which can be used to trigger tests against the legacy KIC controller-manager
+	LegacyControllerEnvVar = "KONG_LEGACY_CONTROLLER"
+)
+
 var (
 	// cluster is the object which contains a Kubernetes client for the testing cluster
 	cluster ktfkind.Cluster
@@ -43,7 +50,7 @@ func TestMain(m *testing.M) {
 	cluster = newCluster
 
 	// deploy the Kong Kubernetes Ingress Controller (KIC) to the cluster
-	if err := deployControllers(ctx, ready, cluster.Client(), os.Getenv("KONG_CONTROLLER_TEST_IMAGE"), controllers.DefaultNamespace); err != nil {
+	if err := deployControllers(ctx, ready, cluster, os.Getenv("KONG_CONTROLLER_TEST_IMAGE"), controllers.DefaultNamespace); err != nil {
 		newCluster.Cleanup()
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(11)
@@ -55,10 +62,10 @@ func TestMain(m *testing.M) {
 }
 
 // FIXME: this is a total hack for now, in the future we should deploy the controller into the cluster via image or run it as a goroutine.
-func deployControllers(ctx context.Context, ready chan ktfkind.ProxyReadinessEvent, kc *kubernetes.Clientset, containerImage, namespace string) error {
+func deployControllers(ctx context.Context, ready chan ktfkind.ProxyReadinessEvent, cluster kind.Cluster, containerImage, namespace string) error {
 	// ensure the controller namespace is created
 	ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
-	if _, err := kc.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}); err != nil {
+	if _, err := cluster.Client().CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}); err != nil {
 		if !errors.IsAlreadyExists(err) {
 			return err
 		}
@@ -73,9 +80,63 @@ func deployControllers(ctx context.Context, ready chan ktfkind.ProxyReadinessEve
 		u := event.URL
 		proxyReady <- u
 
-		cmd := exec.CommandContext(ctx, "go", "run", "../../main.go", "--kong-url", fmt.Sprintf("http://%s:8001", u.Hostname()))
+		// create a tempfile to hold the cluster kubeconfig that will be used for the controller
+		kubeconfig, err := ioutil.TempFile(os.TempDir(), "kubeconfig-")
+		if err != nil {
+			panic(err)
+		}
+		defer kubeconfig.Close()
+
+		// dump the kubeconfig from kind into the tempfile
+		generateKubeconfig := exec.CommandContext(ctx, "kind", "get", "kubeconfig", "--name", cluster.Name())
+		generateKubeconfig.Stdout = kubeconfig
+		generateKubeconfig.Stderr = os.Stderr
+		if err := generateKubeconfig.Run(); err != nil {
+			panic(err)
+		}
+
+		// set the default command which runs the current controller manager code
+		cmd := exec.CommandContext(ctx, "go", "run", "../../main.go",
+			"--kong-url", fmt.Sprintf("http://%s:8001", u.Hostname()),
+			"--kubeconfig", kubeconfig.Name())
+
+		// if set, allow running the legacy controller for the tests instead
+		// TODO: this will be removed as part of KIC 2.0, where the legacy controller will be replaced.
+		//       for more details see the relevant milestone: https://github.com/Kong/kubernetes-ingress-controller/milestone/12
+		if os.Getenv(LegacyControllerEnvVar) != "" {
+			fmt.Fprintln(os.Stdout, "WARNING: deploying legacy Kong Kubernetes Ingress Controller (KIC)")
+
+			// get the proxy pod
+			podList, err := cluster.Client().CoreV1().Pods("kong-system").List(ctx, metav1.ListOptions{
+				LabelSelector: "app.kubernetes.io/component=app,app.kubernetes.io/instance=ingress-controller,app.kubernetes.io/name=kong",
+			})
+			if err != nil {
+				panic(err)
+			}
+			if len(podList.Items) != 1 {
+				panic(fmt.Errorf("expected 1 result, found %d", len(podList.Items)))
+			}
+			proxyPod := podList.Items[0].Name
+
+			// set the environment according to the legacy controller's needs
+			os.Setenv("POD_NAMESPACE", "kong-system")
+			os.Setenv("POD_NAME", proxyPod)
+
+			// custom command for the legacy controller as there are several differences in flags.
+			cmd = exec.CommandContext(ctx, "go", "run", "../../../cli/ingress-controller/",
+				"--publish-service", "kong-system/ingress-controller-kong-proxy",
+				"--kubeconfig", kubeconfig.Name(),
+				"--kong-admin-url", fmt.Sprintf("http://%s:8001", u.Hostname()))
+		}
+
+		// capture stdout/stderr in case we need to report an error
+		stdout, stderr := new(bytes.Buffer), new(bytes.Buffer)
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+
 		if err := cmd.Run(); err != nil {
-			fmt.Fprintf(os.Stderr, err.Error())
+			fmt.Fprintln(os.Stdout, stdout.String())
+			panic(fmt.Errorf("%s: %w", stderr.String(), err))
 		}
 	}()
 

scripts/railgun-integration-tests.sh

@@ -1,6 +1,14 @@
 #!/bin/bash
 
-set -euox pipefail
-
 cd railgun/
-make test.integration
+
+if [ -z "$1" ]
+then
+    make test.integration
+elif [ "$1" == "LEGACY" ]
+then
+    make test.integration.legacy
+else
+    echo "$1 isn't a valid option (valid options: LEGACY)"
+    exit 1
+fi