[CI] Add Legacy Integration Test Workflow (#1122)

.github/workflows/integration_test.yaml

@@ -1,9 +1,19 @@
 name: Integration Tests
 
-on: [push, pull_request]
+on:
+  pull_request:
+    branches:
+      - 'main'
+      - 'next'
+  push:
+    branches:
+      - 'main'
+      - 'next'
+    tags:
+      - 'v*.*.*'
 
 jobs:
-  test:
+  integration-test:
     runs-on: ubuntu-latest
     steps:
     - name: setup golang
@@ -19,11 +29,25 @@ jobs:
           ${{ runner.os }}-go
     - name: checkout repository
       uses: actions/checkout@v2
-    - name: Login to GitHub Packages Docker Registry
-      uses: docker/login-action@v1
-      with:
-        registry: docker.pkg.github.com
-        username: ${{ github.repository_owner }}
-        password: ${{ secrets.GITHUB_TOKEN }}
     - name: run railgun integration tests
-      run: ./scripts/railgun-integration-tests.sh
+      run: make test.integration
+      working-directory: ./railgun
+  integration-test-legacy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: setup golang
+      uses: actions/setup-go@v2
+      with:
+        go-version: '^1.16'
+    - name: cache go modules
+      uses: actions/cache@v1
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-build-codegen-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go
+    - name: checkout repository
+      uses: actions/checkout@v2
+    - name: run railgun integration tests on legacy KIC
+      run: make test.integration.legacy
+      working-directory: ./railgun

railgun/Makefile

@@ -121,3 +121,9 @@ clean.test.proxy:
 test.integration:
 	@./scripts/setup-integration-tests.sh
 	@GOFLAGS="-tags=integration_tests" go test -race -v ./test/integration/
+
+# Our integration tests using the legacy v1 controller manager
+.PHONY: test.integration.legacy
+test.integration.legacy:
+	@./scripts/setup-integration-tests.sh
+	@KONG_LEGACY_CONTROLLER=1 GOFLAGS="-tags=integration_tests" go test -race -v ./test/integration/

railgun/test/integration/ingress_controllers_test.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strings"
 	"testing"
 	"time"
 
@@ -56,17 +57,16 @@ func TestMinimalIngress(t *testing.T) {
 			return false
 		}
 		defer resp.Body.Close()
-		return resp.StatusCode == http.StatusOK
+		if resp.StatusCode == http.StatusOK {
+			// now that the ingress backend is routable, make sure the contents we're getting back are what we expect
+			// Expected: Welcome to nginx!
+			b := new(bytes.Buffer)
+			b.ReadFrom(resp.Body)
+			return strings.Contains(b.String(), "Welcome to nginx!")
+		}
+		return false
 	}, ingressTimeout, ingressTimeoutTick)
 
-	// now that the ingress backend is routable, make sure the contents we're getting back are what we expect
-	// Expected: Welcome to nginx!
-	resp, err := http.Get(fmt.Sprintf("%s/nginx", u.String()))
-	assert.NoError(t, err)
-	b := new(bytes.Buffer)
-	b.ReadFrom(resp.Body)
-	assert.Contains(t, b.String(), "Welcome to nginx!")
-
 	// ensure that a deleted ingress results in the route being torn down
 	assert.NoError(t, cluster.Client().NetworkingV1().Ingresses("default").Delete(ctx, ingress.Name, metav1.DeleteOptions{}))
 	assert.Eventually(t, func() bool {
@@ -76,20 +76,22 @@ func TestMinimalIngress(t *testing.T) {
 			return false
 		}
 		defer resp.Body.Close()
-		return resp.StatusCode == http.StatusNotFound
+		if resp.StatusCode == http.StatusNotFound {
+			// once the route is torn down and returning 404's, ensure that we got the expected response body back from Kong
+			// Expected: {"message":"no Route matched with those values"}
+			b := new(bytes.Buffer)
+			b.ReadFrom(resp.Body)
+			body := struct {
+				Message string `json:"message"`
+			}{}
+			if err := json.Unmarshal(b.Bytes(), &body); err != nil {
+				t.Logf("WARNING: error decoding JSON from proxy while waiting for %s: %v", u.String(), err)
+				return false
+			}
+			return body.Message == "no Route matched with those values"
+		}
+		return false
 	}, ingressTimeout, ingressTimeoutTick)
-
-	// once the route is torn down and returning 404's, ensure that we got the expected response body back from Kong
-	// Expected: {"message":"no Route matched with those values"}
-	resp, err = http.Get(fmt.Sprintf("%s/nginx", u.String()))
-	assert.NoError(t, err)
-	b = new(bytes.Buffer)
-	b.ReadFrom(resp.Body)
-	body := struct {
-		Message string `json:"message"`
-	}{}
-	assert.NoError(t, json.Unmarshal(b.Bytes(), &body))
-	assert.Equal(t, "no Route matched with those values", body.Message)
 }
 
 func TestHTTPSRedirect(t *testing.T) {

railgun/test/integration/suite_test.go

@@ -3,8 +3,10 @@
 package integration
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"os/exec"
@@ -15,11 +17,17 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 
+	"github.com/kong/kubernetes-testing-framework/pkg/kind"
 	ktfkind "github.com/kong/kubernetes-testing-framework/pkg/kind"
 
 	"github.com/kong/kubernetes-ingress-controller/railgun/controllers"
 )
 
+const (
+	// LegacyControllerEnvVar indicates the environment variable which can be used to trigger tests against the legacy KIC controller-manager
+	LegacyControllerEnvVar = "KONG_LEGACY_CONTROLLER"
+)
+
 var (
 	// cluster is the object which contains a Kubernetes client for the testing cluster
 	cluster ktfkind.Cluster
@@ -43,7 +51,7 @@ func TestMain(m *testing.M) {
 	cluster = newCluster
 
 	// deploy the Kong Kubernetes Ingress Controller (KIC) to the cluster
-	if err := deployControllers(ctx, ready, cluster.Client(), os.Getenv("KONG_CONTROLLER_TEST_IMAGE"), controllers.DefaultNamespace); err != nil {
+	if err := deployControllers(ctx, ready, cluster, os.Getenv("KONG_CONTROLLER_TEST_IMAGE"), controllers.DefaultNamespace); err != nil {
 		newCluster.Cleanup()
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(11)
@@ -55,10 +63,10 @@ func TestMain(m *testing.M) {
 }
 
 // FIXME: this is a total hack for now, in the future we should deploy the controller into the cluster via image or run it as a goroutine.
-func deployControllers(ctx context.Context, ready chan ktfkind.ProxyReadinessEvent, kc *kubernetes.Clientset, containerImage, namespace string) error {
+func deployControllers(ctx context.Context, ready chan ktfkind.ProxyReadinessEvent, cluster kind.Cluster, containerImage, namespace string) error {
 	// ensure the controller namespace is created
 	ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
-	if _, err := kc.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}); err != nil {
+	if _, err := cluster.Client().CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}); err != nil {
 		if !errors.IsAlreadyExists(err) {
 			return err
 		}
@@ -71,13 +79,85 @@ func deployControllers(ctx context.Context, ready chan ktfkind.ProxyReadinessEve
 			panic(event.Err)
 		}
 		u := event.URL
+		adminHost := u.Hostname()
 		proxyReady <- u
 
-		cmd := exec.CommandContext(ctx, "go", "run", "../../main.go", "--kong-url", fmt.Sprintf("http://%s:8001", u.Hostname()))
+		// create a tempfile to hold the cluster kubeconfig that will be used for the controller
+		kubeconfig, err := ioutil.TempFile(os.TempDir(), "kubeconfig-")
+		if err != nil {
+			panic(err)
+		}
+		defer os.Remove(kubeconfig.Name())
+
+		// dump the kubeconfig from kind into the tempfile
+		generateKubeconfig := exec.CommandContext(ctx, "kind", "get", "kubeconfig", "--name", cluster.Name())
+		generateKubeconfig.Stdout = kubeconfig
+		generateKubeconfig.Stderr = os.Stderr
+		if err := generateKubeconfig.Run(); err != nil {
+			panic(err)
+		}
+		kubeconfig.Close()
+
+		// if set, allow running the legacy controller for the tests instead of the current controller
+		var cmd *exec.Cmd
+		if useLegacyKIC() {
+			cmd = buildLegacyCommand(ctx, kubeconfig.Name(), adminHost, cluster.Client())
+		} else {
+			cmd = buildControllerCommand(ctx, kubeconfig.Name(), adminHost)
+		}
+
+		// capture stdout/stderr in case we need to report an error
+		stdout, stderr := new(bytes.Buffer), new(bytes.Buffer)
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+
 		if err := cmd.Run(); err != nil {
-			fmt.Fprintf(os.Stderr, err.Error())
+			fmt.Fprintln(os.Stdout, stdout.String())
+			panic(fmt.Errorf("%s: %w", stderr.String(), err))
 		}
 	}()
 
 	return nil
 }
+
+func useLegacyKIC() bool {
+	return os.Getenv(LegacyControllerEnvVar) != ""
+}
+
+// TODO: this will be removed as part of KIC 2.0, where the legacy controller will be replaced.
+//       for more details see the relevant milestone: https://github.com/Kong/kubernetes-ingress-controller/milestone/12
+func buildLegacyCommand(ctx context.Context, kubeconfigPath, adminHost string, kc *kubernetes.Clientset) *exec.Cmd {
+	fmt.Fprintln(os.Stdout, "WARNING: deploying legacy Kong Kubernetes Ingress Controller (KIC)")
+
+	// get the proxy pod
+	podList, err := kc.CoreV1().Pods("kong-system").List(ctx, metav1.ListOptions{
+		LabelSelector: "app.kubernetes.io/component=app,app.kubernetes.io/instance=ingress-controller,app.kubernetes.io/name=kong",
+	})
+	if err != nil {
+		panic(err)
+	}
+	if len(podList.Items) != 1 {
+		panic(fmt.Errorf("expected 1 result, found %d", len(podList.Items)))
+	}
+	proxyPod := podList.Items[0].Name
+
+	// custom command for the legacy controller as there are several differences in flags.
+	cmd := exec.CommandContext(ctx, "go", "run", "../../../cli/ingress-controller/",
+		"--publish-service", "kong-system/ingress-controller-kong-proxy",
+		"--kubeconfig", kubeconfigPath,
+		"--kong-admin-url", fmt.Sprintf("http://%s:8001", adminHost))
+
+	// set the environment according to the legacy controller's needs
+	cmd.Env = append(os.Environ(),
+		"POD_NAMESPACE=kong-system",
+		fmt.Sprintf("POD_NAME=%s", proxyPod),
+	)
+
+	return cmd
+}
+
+func buildControllerCommand(ctx context.Context, kubeconfigPath, adminHost string) *exec.Cmd {
+	return exec.CommandContext(ctx, "go", "run", "../../main.go",
+		"--kong-url", fmt.Sprintf("http://%s:8001", adminHost),
+		"--kubeconfig", kubeconfigPath)
+}