diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index d2824ed1ae..ddca5f5519 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -1,9 +1,19 @@ name: Integration Tests -on: [push, pull_request] +on: + pull_request: + branches: + - 'main' + - 'next' + push: + branches: + - 'main' + - 'next' + tags: + - 'v*.*.*' jobs: - test: + integration-test: runs-on: ubuntu-latest steps: - name: setup golang @@ -19,11 +29,25 @@ jobs: ${{ runner.os }}-go - name: checkout repository uses: actions/checkout@v2 - - name: Login to GitHub Packages Docker Registry - uses: docker/login-action@v1 - with: - registry: docker.pkg.github.com - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - name: run railgun integration tests - run: ./scripts/railgun-integration-tests.sh + run: make test.integration + working-directory: ./railgun + integration-test-legacy: + runs-on: ubuntu-latest + steps: + - name: setup golang + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + - name: cache go modules + uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-build-codegen-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go + - name: checkout repository + uses: actions/checkout@v2 + - name: run railgun integration tests on legacy KIC + run: make test.integration.legacy + working-directory: ./railgun diff --git a/railgun/Makefile b/railgun/Makefile index a2386e1cb3..53af1a1348 100644 --- a/railgun/Makefile +++ b/railgun/Makefile @@ -121,3 +121,9 @@ clean.test.proxy: test.integration: @./scripts/setup-integration-tests.sh @GOFLAGS="-tags=integration_tests" go test -race -v ./test/integration/ + +# Our integration tests using the legacy v1 controller manager +.PHONY: test.integration.legacy +test.integration.legacy: + @./scripts/setup-integration-tests.sh + @KONG_LEGACY_CONTROLLER=1 GOFLAGS="-tags=integration_tests" go test -race -v ./test/integration/ diff --git a/railgun/test/integration/ingress_controllers_test.go b/railgun/test/integration/ingress_controllers_test.go index 7c9c46e7d3..5af7477b6c 100644 --- a/railgun/test/integration/ingress_controllers_test.go +++ b/railgun/test/integration/ingress_controllers_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "net/http" + "strings" "testing" "time" @@ -56,17 +57,16 @@ func TestMinimalIngress(t *testing.T) { return false } defer resp.Body.Close() - return resp.StatusCode == http.StatusOK + if resp.StatusCode == http.StatusOK { + // now that the ingress backend is routable, make sure the contents we're getting back are what we expect + // Expected: Welcome to nginx! + b := new(bytes.Buffer) + b.ReadFrom(resp.Body) + return strings.Contains(b.String(), "Welcome to nginx!") + } + return false }, ingressTimeout, ingressTimeoutTick) - // now that the ingress backend is routable, make sure the contents we're getting back are what we expect - // Expected: Welcome to nginx! - resp, err := http.Get(fmt.Sprintf("%s/nginx", u.String())) - assert.NoError(t, err) - b := new(bytes.Buffer) - b.ReadFrom(resp.Body) - assert.Contains(t, b.String(), "Welcome to nginx!") - // ensure that a deleted ingress results in the route being torn down assert.NoError(t, cluster.Client().NetworkingV1().Ingresses("default").Delete(ctx, ingress.Name, metav1.DeleteOptions{})) assert.Eventually(t, func() bool { @@ -76,20 +76,22 @@ func TestMinimalIngress(t *testing.T) { return false } defer resp.Body.Close() - return resp.StatusCode == http.StatusNotFound + if resp.StatusCode == http.StatusNotFound { + // once the route is torn down and returning 404's, ensure that we got the expected response body back from Kong + // Expected: {"message":"no Route matched with those values"} + b := new(bytes.Buffer) + b.ReadFrom(resp.Body) + body := struct { + Message string `json:"message"` + }{} + if err := json.Unmarshal(b.Bytes(), &body); err != nil { + t.Logf("WARNING: error decoding JSON from proxy while waiting for %s: %v", u.String(), err) + return false + } + return body.Message == "no Route matched with those values" + } + return false }, ingressTimeout, ingressTimeoutTick) - - // once the route is torn down and returning 404's, ensure that we got the expected response body back from Kong - // Expected: {"message":"no Route matched with those values"} - resp, err = http.Get(fmt.Sprintf("%s/nginx", u.String())) - assert.NoError(t, err) - b = new(bytes.Buffer) - b.ReadFrom(resp.Body) - body := struct { - Message string `json:"message"` - }{} - assert.NoError(t, json.Unmarshal(b.Bytes(), &body)) - assert.Equal(t, "no Route matched with those values", body.Message) } func TestHTTPSRedirect(t *testing.T) { diff --git a/railgun/test/integration/suite_test.go b/railgun/test/integration/suite_test.go index 62ceb05447..bc1c758ba8 100644 --- a/railgun/test/integration/suite_test.go +++ b/railgun/test/integration/suite_test.go @@ -3,8 +3,10 @@ package integration import ( + "bytes" "context" "fmt" + "io/ioutil" "net/url" "os" "os/exec" @@ -15,11 +17,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "github.com/kong/kubernetes-testing-framework/pkg/kind" ktfkind "github.com/kong/kubernetes-testing-framework/pkg/kind" "github.com/kong/kubernetes-ingress-controller/railgun/controllers" ) +const ( + // LegacyControllerEnvVar indicates the environment variable which can be used to trigger tests against the legacy KIC controller-manager + LegacyControllerEnvVar = "KONG_LEGACY_CONTROLLER" +) + var ( // cluster is the object which contains a Kubernetes client for the testing cluster cluster ktfkind.Cluster @@ -43,7 +51,7 @@ func TestMain(m *testing.M) { cluster = newCluster // deploy the Kong Kubernetes Ingress Controller (KIC) to the cluster - if err := deployControllers(ctx, ready, cluster.Client(), os.Getenv("KONG_CONTROLLER_TEST_IMAGE"), controllers.DefaultNamespace); err != nil { + if err := deployControllers(ctx, ready, cluster, os.Getenv("KONG_CONTROLLER_TEST_IMAGE"), controllers.DefaultNamespace); err != nil { newCluster.Cleanup() fmt.Fprintf(os.Stderr, err.Error()) os.Exit(11) @@ -55,10 +63,10 @@ func TestMain(m *testing.M) { } // FIXME: this is a total hack for now, in the future we should deploy the controller into the cluster via image or run it as a goroutine. -func deployControllers(ctx context.Context, ready chan ktfkind.ProxyReadinessEvent, kc *kubernetes.Clientset, containerImage, namespace string) error { +func deployControllers(ctx context.Context, ready chan ktfkind.ProxyReadinessEvent, cluster kind.Cluster, containerImage, namespace string) error { // ensure the controller namespace is created ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} - if _, err := kc.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}); err != nil { + if _, err := cluster.Client().CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}); err != nil { if !errors.IsAlreadyExists(err) { return err } @@ -71,13 +79,85 @@ func deployControllers(ctx context.Context, ready chan ktfkind.ProxyReadinessEve panic(event.Err) } u := event.URL + adminHost := u.Hostname() proxyReady <- u - cmd := exec.CommandContext(ctx, "go", "run", "../../main.go", "--kong-url", fmt.Sprintf("http://%s:8001", u.Hostname())) + // create a tempfile to hold the cluster kubeconfig that will be used for the controller + kubeconfig, err := ioutil.TempFile(os.TempDir(), "kubeconfig-") + if err != nil { + panic(err) + } + defer os.Remove(kubeconfig.Name()) + + // dump the kubeconfig from kind into the tempfile + generateKubeconfig := exec.CommandContext(ctx, "kind", "get", "kubeconfig", "--name", cluster.Name()) + generateKubeconfig.Stdout = kubeconfig + generateKubeconfig.Stderr = os.Stderr + if err := generateKubeconfig.Run(); err != nil { + panic(err) + } + kubeconfig.Close() + + // if set, allow running the legacy controller for the tests instead of the current controller + var cmd *exec.Cmd + if useLegacyKIC() { + cmd = buildLegacyCommand(ctx, kubeconfig.Name(), adminHost, cluster.Client()) + } else { + cmd = buildControllerCommand(ctx, kubeconfig.Name(), adminHost) + } + + // capture stdout/stderr in case we need to report an error + stdout, stderr := new(bytes.Buffer), new(bytes.Buffer) + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { - fmt.Fprintf(os.Stderr, err.Error()) + fmt.Fprintln(os.Stdout, stdout.String()) + panic(fmt.Errorf("%s: %w", stderr.String(), err)) } }() return nil } + +func useLegacyKIC() bool { + return os.Getenv(LegacyControllerEnvVar) != "" +} + +// TODO: this will be removed as part of KIC 2.0, where the legacy controller will be replaced. +// for more details see the relevant milestone: https://github.com/Kong/kubernetes-ingress-controller/milestone/12 +func buildLegacyCommand(ctx context.Context, kubeconfigPath, adminHost string, kc *kubernetes.Clientset) *exec.Cmd { + fmt.Fprintln(os.Stdout, "WARNING: deploying legacy Kong Kubernetes Ingress Controller (KIC)") + + // get the proxy pod + podList, err := kc.CoreV1().Pods("kong-system").List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/component=app,app.kubernetes.io/instance=ingress-controller,app.kubernetes.io/name=kong", + }) + if err != nil { + panic(err) + } + if len(podList.Items) != 1 { + panic(fmt.Errorf("expected 1 result, found %d", len(podList.Items))) + } + proxyPod := podList.Items[0].Name + + // custom command for the legacy controller as there are several differences in flags. + cmd := exec.CommandContext(ctx, "go", "run", "../../../cli/ingress-controller/", + "--publish-service", "kong-system/ingress-controller-kong-proxy", + "--kubeconfig", kubeconfigPath, + "--kong-admin-url", fmt.Sprintf("http://%s:8001", adminHost)) + + // set the environment according to the legacy controller's needs + cmd.Env = append(os.Environ(), + "POD_NAMESPACE=kong-system", + fmt.Sprintf("POD_NAME=%s", proxyPod), + ) + + return cmd +} + +func buildControllerCommand(ctx context.Context, kubeconfigPath, adminHost string) *exec.Cmd { + return exec.CommandContext(ctx, "go", "run", "../../main.go", + "--kong-url", fmt.Sprintf("http://%s:8001", adminHost), + "--kubeconfig", kubeconfigPath) +} diff --git a/scripts/railgun-integration-tests.sh b/scripts/railgun-integration-tests.sh deleted file mode 100755 index 7164422e22..0000000000 --- a/scripts/railgun-integration-tests.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -euox pipefail - -cd railgun/ -make test.integration