[e2e-tests] feat: add k8s integration tests (#5013)

* feat: wip k8s integration tests

* feat: remove kubernetes platform package

* feat: remove Package target from kubernetes

* feat: support provisioning multiple instances for kind

* fix: reinstate go.mod changes from merge

* fix: use supported kind k8s versions

* feat: add agent standalone test

* fix: reinstate go.mod changes from merge

* fix: correct import order and check err with require

* ci: update k8s 1.28 to latest kind supported version

.buildkite/pipeline.yml

@@ -198,6 +198,21 @@ steps:
         retry:
           manual:
             allowed: true
+      - label: "K8s integration tests"
+        key: "k8s-integration-tests"
+        env:
+          K8S_VERSION: "v1.30.2,v1.29.4,v1.28.9"
+          KIND_VERSION: "v0.20.0"
+        command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
+        artifact_paths:
+          - "build/TEST-**"
+          - "build/diagnostics/*"
+        agents:
+          provider: "gcp"
+          image: "family/core-ubuntu-2204"
+        retry:
+          manual:
+            allowed: true
 
   - label: ":sonarqube: Continuous Code Inspection"
     env:

.buildkite/scripts/steps/k8s-extended-tests.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+export PATH=$HOME/bin:${PATH}
+source .buildkite/scripts/install-kubectl.sh
+source .buildkite/scripts/install-kind.sh
+
+# Increase max inotify instances and watches to avoid 'too many open files' errors
+# when spawning more than one k8s cluster
+sudo sysctl fs.inotify.max_user_instances=1280
+sudo sysctl fs.inotify.max_user_watches=655360
+
+# Run k8s integration tests
+set +e
+
+arch_type="$(uname -m)"
+if [ "${arch_type}" == "x86_64" ]; then
+  export PLATFORMS="linux/amd64"
+elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then
+  export PLATFORMS="linux/arm64"
+else
+  echo "Unsupported OS"
+  exit 10
+fi
+
+DEV=true SNAPSHOT=true EXTERNAL=true PACKAGES=docker mage -v package
+TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:kubernetes
+TESTS_EXIT_STATUS=$?
+set -e
+
+# HTML report
+outputXML="build/TEST-go-integration.xml"
+
+if [ -f "$outputXML" ]; then
+  go install github.com/alexec/junit2html@latest
+  junit2html < "$outputXML" > build/TEST-report.html
+else
+    echo "Cannot generate HTML test report: $outputXML not found"
+fi
+
+exit $TESTS_EXIT_STATUS

go.mod

@@ -77,10 +77,13 @@ require (
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/gotestsum v1.9.0
-	k8s.io/api v0.29.5
-	k8s.io/apimachinery v0.29.5
-	k8s.io/client-go v0.29.5
+	k8s.io/api v0.30.1
+	k8s.io/apimachinery v0.30.1
+	k8s.io/client-go v0.30.1
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
+	sigs.k8s.io/e2e-framework v0.4.0
+	sigs.k8s.io/kustomize/api v0.12.1
+	sigs.k8s.io/kustomize/kyaml v0.13.9
 )
 
 require (
@@ -152,9 +155,11 @@ require (
 	github.com/elastic/pkcs8 v1.0.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/expr-lang/expr v1.16.9 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
@@ -171,10 +176,12 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/gnostic v0.7.0 // indirect
+	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/licenseclassifier v0.0.0-20221004142553-c1ed8fcf4bab // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/hashicorp/consul/api v1.29.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -215,6 +222,7 @@ require (
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/mostynb/go-grpc-compression v1.2.3 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
@@ -253,6 +261,7 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/valyala/fastjson v1.6.4 // indirect
+	github.com/xlab/treeprint v1.1.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.elastic.co/apm/module/apmhttp/v2 v2.6.0 // indirect
 	go.elastic.co/apm/module/apmzap/v2 v2.6.0 // indirect
@@ -295,6 +304,7 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect
 	go.opentelemetry.io/otel/trace v1.27.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.2.0 // indirect
+	go.starlark.net v0.0.0-20221205180719-3fd0dac74452 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
 	golang.org/x/mod v0.17.0 // indirect
@@ -306,17 +316,18 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	howett.net/plist v1.0.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/kubelet v0.29.3 // indirect
+	sigs.k8s.io/controller-runtime v0.18.2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )
 
 require (
 	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect; indirecthttps://github.com/elastic/ingest-dev/issues/3253
-	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/klog/v2 v2.120.1 // indirect
 )
 
 replace (