Enable e2e tests with cloud provider emulators for aws,gcp,azure (#743)

* Make e2e tests functional
* Provide support to run them with major cloud providers & their emulators
* Upgrade helm pkg `k8.io/helm/v2` to `helm.sh/helm/v3`

.ci/integration_test

@@ -276,7 +276,7 @@ function run_test_as_processes() {
   setup_test_cluster
 
   echo "Starting integration tests..."
-  cd test/e2e/integration
+  cd test/integration
 
   set +e
   ginkgo -r -mod=vendor
@@ -295,81 +295,6 @@ function run_test_as_processes() {
   fi
 }
 
-function run_test_on_cluster() {
-  if ! [ -x "$(command -v ginkgo)" ]; then
-    setup_ginkgo
-  fi
-
-  export TEST_ID=${STORAGE_CONTAINER}
-  if [ "$STORAGE_CONTAINER" == "" ]; then
-    setup_awscli
-    get_test_id
-    setup-aws-infrastructure
-  fi
-
-  # TODO: change the etcd wrapper version to a newer version which spawns etcd v3.4.34
-  export ETCD_WRAPPER_VERSION=${ETCD_WRAPPER_VERSION:-"v0.2.0"}
-  echo "etcd-wrapper version: ${ETCD_WRAPPER_VERSION}"
-
-  export ETCDBR_VERSION=${ETCDBR_VERSION:-${ETCDBR_VER:-"v0.12.1"}}
-  echo "etcd-backup-restore version: ${ETCDBR_VERSION}"
-
-  echo "Starting integration tests on k8s cluster."
-
-  set +e
-
-  if [ -r "$INTEGRATION_TEST_KUBECONFIG" ]; then
-    KUBECONFIG=$INTEGRATION_TEST_KUBECONFIG STORAGE_CONTAINER=$TEST_ID ginkgo -v -timeout=15m -mod=vendor test/e2e/integrationcluster
-    TEST_RESULT=$?
-    echo "Successfully completed all tests."
-  else
-    echo "Invalid kubeconfig for integration test $INTEGRATION_TEST_KUBECONFIG"
-    TEST_RESULT=255
-  fi
-
-  set -e
-
-  echo "Done with integration test on k8s cluster."
-
-  if [ "$STORAGE_CONTAINER" == "" ]; then
-    echo "Deleting test bucket..."
-    cleanup-aws-infrastructure
-  fi
-}
-
-function run_test_on_tm() {
-  if [ "$ACCESS_KEY_ID" == "" ] || [ "$SECRET_ACCESS_KEY_B64" == "" ] || [ "$AWS_REGION" == "" ] ; then
-    echo "AWS S3 credentials unavailable. Exiting."
-    exit 1
-  fi
-  export SECRET_ACCESS_KEY=`echo $SECRET_ACCESS_KEY_B64 | base64 -d`
-  export REGION=$AWS_REGION
-
-  get_tm_test_id
-  export STORAGE_CONTAINER=$TEST_ID
-  export ETCDBR_VER=$EFFECTIVE_VERSION
-
-  setup_awscli
-  write_aws_secret "${ACCESS_KEY_ID}" "${SECRET_ACCESS_KEY}" "${REGION}"
-  create_s3_bucket
-
-  export INTEGRATION_TEST_KUBECONFIG=$TM_KUBECONFIG_PATH/shoot.config
-  echo "Starting integration tests on TM cluster $PROJECT_NAMESPACE/$SHOOT_NAME."
-  run_test_on_cluster
-  echo "Done with integration test on TM cluster."
-  cleanup-aws-infrastructure
-}
-
-case $1 in
-  tm)
-    run_test_on_tm
-    ;;
-  cluster)
-    run_test_on_cluster
-    ;;
-  *)
-    run_test_as_processes
-    ;;
-esac
+run_test_as_processes
 
 exit $TEST_RESULT

.gitignore

@@ -18,6 +18,7 @@
 
 # Build Binaries
 bin
+hack/tools/bin
 test/output
 test/e2e_test_data
 default.bkp*
@@ -28,6 +29,9 @@ compctedsnap.bkp*
 .vscode
 .idea/
 
+# kubeconfig
+hack/e2e-test/infrastructure/kind/kubeconfig
+
 # developers workspace
 tmp
 dev

Makefile

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 VERSION             ?= $(shell cat VERSION)
+REPO_ROOT           := $(shell dirname "$(realpath $(lastword $(MAKEFILE_LIST)))")
 REGISTRY            ?= europe-docker.pkg.dev/gardener-project/snapshots
 IMAGE_REPOSITORY    := $(REGISTRY)/gardener/etcdbrctl
 IMAGE_TAG           := $(VERSION)
@@ -11,10 +12,11 @@ PLATFORM            ?= $(shell docker info --format '{{.OSType}}/{{.Architecture
 BIN_DIR             := bin
 COVERPROFILE        := test/output/coverprofile.out
 IMG                 ?= ${IMAGE_REPOSITORY}:${IMAGE_TAG}
+KUBECONFIG_PATH     :=$(REPO_ROOT)/hack/e2e-test/infrastructure/kind/kubeconfig
 
 .DEFAULT_GOAL := build-local
 
-include hack/tools.mk
+include $(REPO_ROOT)/hack/tools.mk
 
 .PHONY: revendor
 revendor:
@@ -26,6 +28,8 @@ update-dependencies:
 	@env go get -u
 	@make revendor
 
+kind-up kind-down ci-e2e-kind ci-e2e-kind-aws ci-e2e-kind-azure ci-e2e-kind-gcp: export KUBECONFIG = $(KUBECONFIG_PATH)
+
 .PHONY: build
 build:
 	@.ci/build
@@ -74,11 +78,47 @@ perf-regression-test:
 integration-test:
 	@.ci/integration_test
 
-.PHONY: integration-test-cluster
-integration-test-cluster:
-	@.ci/integration_test cluster
-
 .PHONY: show-coverage
 show-coverage:
 	@if [ ! -f $(COVERPROFILE) ]; then echo "$(COVERPROFILE) is not yet built. Please run 'COVER=true make test'"; false; fi
 	@go tool cover -html $(COVERPROFILE)
+
+.PHONY: test-e2e
+test-e2e: $(KIND) $(HELM) $(GINKGO) $(KUBECTL)
+	@"$(REPO_ROOT)/hack/e2e-test/run-e2e-test.sh" $(PROVIDERS) $(KUBECONFIG)
+
+.PHONY: kind-up
+kind-up: $(KIND)
+	./hack/kind-up.sh
+
+.PHONY: kind-down
+kind-down: $(KIND)
+	kind delete cluster --name etcdbr-e2e
+
+.PHONY: deploy-localstack
+deploy-localstack: $(KUBECTL)
+	./hack/deploy-localstack.sh $(KUBECONFIG)
+
+.PHONY: deploy-fakegcs
+deploy-fakegcs: $(KUBECTL)
+	./hack/deploy-fakegcs.sh $(KUBECONFIG)
+
+.PHONY: deploy-azurite
+deploy-azurite: $(KUBECTL)
+	./hack/deploy-azurite.sh $(KUBECONFIG)
+
+.PHONY: ci-e2e-kind
+ci-e2e-kind:
+	./hack/ci-e2e-kind.sh $(PROVIDERS)
+
+.PHONY: ci-e2e-kind-aws
+ci-e2e-kind-aws:
+	./hack/ci-e2e-kind.sh aws
+
+.PHONY: ci-e2e-kind-azure
+ci-e2e-kind-azure:
+	./hack/ci-e2e-kind.sh azure
+
+.PHONY: ci-e2e-kind-gcp
+ci-e2e-kind-gcp:
+	./hack/ci-e2e-kind.sh gcp

chart/etcd-backup-restore/templates/etcd-backup-secret.yaml

@@ -14,7 +14,9 @@ data:
   region: {{ .Values.backup.s3.region | b64enc }}
   secretAccessKey: {{ .Values.backup.s3.secretAccessKey | b64enc  }}
   accessKeyID: {{ .Values.backup.s3.accessKeyID | b64enc }}
+  {{- if .Values.backup.s3.s3ForcePathStyle }}
   s3ForcePathStyle: {{ .Values.backup.s3.s3ForcePathStyle | b64enc}}
+  {{- end }}
   {{- if .Values.backup.s3.endpoint }}
   endpoint: {{ .Values.backup.s3.endpoint | b64enc }}
   {{- end }}

chart/etcd-backup-restore/templates/etcd-statefulset.yaml

@@ -350,7 +350,7 @@ spec:
       - name: local-backup
         hostPath:
           path: {{ .Values.backup.local.path }}/{{ .Values.backup.storageContainer }}
-          type: Directory
+          type: DirectoryOrCreate
   {{- else }}
       - name: etcd-backup
         secret:
@@ -366,3 +366,4 @@ spec:
       resources:
         requests:
           storage: {{ .Values.storageCapacity }}
+

docs/development/testing_and_dependencies.md

@@ -33,24 +33,15 @@ By default, we run tests without computing code coverage. To get the code covera
 
 ### Integration tests
 
-You can also run integration tests for etcd-backup-restore on any given Kubernetes cluster. The test creates namespace `integration-test` on the cluster and deploys the [etcd-backup-restore helm chart](../../chart/etcd-backup-restore) which in turn deploys the required secrets, configmap, services and finally the statefulset which contains the pod that runs etcd and backup-restore as a sidecar.
+Integration tests are process based tests i.e spins up `etcd` and `etcdbr` as processes and runs the tests to check the functionality. The tests expect that the AWS credentials are present in the `$HOME/.aws` directory. Make sure to provide the correct credentials before running the tests.
 
 ```sh
-make integration-test-cluster
+make integration-test
 ```
 
-:warning: Prerequisite for this command is to set the following environment variables:
-
-- INTEGRATION_TEST_KUBECONFIG: kubeconfig to the cluster on which you wish to run the test
-<!-- TODO: change the etcd wrapper version to a newer version which spawns etcd v3.4.34 -->
-- ETCD_WRAPPER_VERSION: optional, defaults to `v0.2.0`
-- ETCDBR_VERSION: optional, defaults to `v0.12.1`
-- ACCESS_KEY_ID: S3 credentials
-- SECRET_ACCESS_KEY: S3 credentials
-- REGION: S3 credentials
-- STORAGE_CONTAINER: S3 bucket name
+### E2E tests
 
-If you have a working setup of [TestMachinery](https://github.com/gardener/test-infra), you can run the integration tests on a TM-generated cluster as well.
+The e2e tests for etcd-backup-restore are cluster-based tests located in the test/e2e package, run on Kubernetes clusters using both emulators and real cloud providers (AWS, GCP, Azure). These tests deploy the etcd-backup-restore helm chart to verify full functionality in various environments.
 
 ### Performance regression tests
 

docs/development/tests.md

@@ -18,10 +18,94 @@ Integration tests include the basic working of:
 - **data validation**: corrupted etcd data should be marked for deletion and restoration should be triggered
 - **restoration**: etcd data should be restored correctly from latest set of snapshots (full + deltas)
 
+**Note**: The tests expects that the aws credentials are present in the `$HOME/.aws` directory. Make sure to provide the correct credentials before running the tests.
+
+These tests can be run locally with both `etcd` and `etcdbr` running as processes.  To execute the tests, run the following command:
+
+```sh
+make integration-test
+```
+
 ### Unit tests
 
 Each package within this repo contains its own set of unit tests to test the functionality of the methods contained within the packages.
 
 ### Performance regression tests
 
 These tests help check any regression in performance in terms of memory consumption and CPU utilization.
+
+### End-to-end tests
+
+The e2e tests for etcd-backup-restore are cluster based tests located in the `test/e2e` package. These tests are run on a Kubernetes cluster and test the full functionality of etcd-backup-restore. The tests create a provider namespace on the cluster and deploy the [etcd-backup-restore helm chart](../../chart/etcd-backup-restore) which in turn deploys the required secrets, configmap, services and finally the statefulset which deploys the pod that runs etcd and backup-restore as a sidecar.
+
+These tests are setup to be run with both emulators and real cloud providers. The emulators can be used for local development and testing as well as running jobs to test code changes when a PR is raised. The real cloud providers can be used for testing in a real cloud environment to ensure that the changes work as expected in an actual environment.
+
+Currently, the tests can be run using the following cloud providers:
+
+- AWS
+- GCP
+- Azure
+
+#### Running the e2e tests with the emulators
+
+##### On a Kind Cluster
+
+To run the e2e tests with the emulators, run the following command:
+
+```sh
+make ci-e2e-kind PROVIDERS="{providers}"
+```
+
+By default, when no provider is specified, the tests are run using AWS emulator i.e Localstack as storage provider. The provider can be specified as comma separated values of the cloud providers mentioned above in small case. For example, to run the tests on AWS and GCP, run the following command:
+
+```sh
+make ci-e2e-kind PROVIDERS="aws,gcp"
+```
+
+##### On any Kubernetes Cluster
+
+The e2e tests can also be run on any other cluster by running the following command:
+
+> **_NOTE:_** If using emulators for e2e tests, make sure to port-forward the snapstore service to the local machine before running the tests.
+
+- For AWS: `kubectl port-forward service/localstack 4566:4566`
+- For GCP: `kubectl port-forward service/fake-gcs 4443:4443 8000:8000`
+- For Azure: `kubectl port-forward service/azurite 10000:10000`
+
+```sh
+make test-e2e PROVIDERS="{providers}" KUBECONFIG="{path-to-kubeconfig}"
+```
+
+#### Running the e2e tests with real cloud providers
+
+To run the tests with real cloud providers, the required credentials need to be set as environment variables before running the tests. See the below sections for the required environment variables for each cloud provider. To test with multiple providers, set the required environment variables for each provider.
+
+##### Set the required environment variables
+
+- AWS:
+  - `AWS_ACCESS_KEY_ID`
+  - `AWS_SECRET_ACCESS_KEY`
+  - `AWS_DEFAULT_REGION`
+
+- GCP:
+  - `GOOGLE_APPLICATION_CREDENTIALS`
+  - `GCP_PROJECT_ID`
+
+- Azure:
+  - `STORAGE_ACCOUNT`
+  - `STORAGE_KEY`
+
+##### Run the tests
+
+To run the tests with a kind cluster, run the following command:
+
+```sh
+make ci-e2e-kind PROVIDERS="{providers}"
+```
+
+To run the tests on any other cluster, run the following command:
+
+```sh
+make test-e2e PROVIDERS="{providers}" KUBECONFIG="{path-to-kubeconfig}"
+```
+