diff --git a/.github/workflows/master-e2e.yaml b/.github/workflows/master-e2e.yaml index 9d3bc1049..e9bd6b1df 100644 --- a/.github/workflows/master-e2e.yaml +++ b/.github/workflows/master-e2e.yaml @@ -455,6 +455,9 @@ jobs: else cd tests && VM_INDEX=${VM_START} VM_NUMBERS=${VM_END} make e2e-bootstrap-node fi + - name: Install a simple application + if: inputs.test_type == 'cli' && contains(inputs.upstream_cluster_version, 'k3s') + run: cd tests && make e2e-install-app && make e2e-check-app - name: Upgrade Elemental Operator if: inputs.test_type == 'cli' && inputs.operator_upgrade != '' id: operator_upgrade @@ -462,6 +465,9 @@ jobs: OPERATOR_UPGRADE: ${{ inputs.operator_upgrade }} run: | cd tests && make e2e-upgrade-operator + if ${{ contains(inputs.upstream_cluster_version, 'k3s') }}; then + make e2e-check-app + fi # Extract elemental-operator version OPERATOR_VERSION=$(kubectl get pod \ --namespace cattle-elemental-system \ @@ -481,6 +487,9 @@ jobs: RANCHER_UPGRADE: ${{ inputs.rancher_upgrade }} run: | cd tests && make e2e-upgrade-rancher-manager + if ${{ contains(inputs.upstream_cluster_version, 'k3s') }}; then + make e2e-check-app + fi # Extract Rancher Manager version RM_VERSION=$(kubectl get pod \ --namespace cattle-system \ @@ -494,7 +503,11 @@ jobs: UPGRADE_IMAGE: ${{ inputs.upgrade_image }} UPGRADE_TYPE: osImage VM_INDEX: 1 - run: cd tests && make e2e-upgrade-node + run: | + cd tests && make e2e-upgrade-node + if ${{ contains(inputs.upstream_cluster_version, 'k3s') }}; then + make e2e-check-app + fi - name: Upgrade other nodes to specified OS version with managedOSVersionName if: inputs.test_type == 'cli' && inputs.upgrade_os_channel != '' env: @@ -503,7 +516,11 @@ jobs: UPGRADE_TYPE: managedOSVersionName VM_INDEX: 2 VM_NUMBERS: 3 - run: cd tests && make e2e-upgrade-node + run: | + cd tests && make e2e-upgrade-node + if ${{ contains(inputs.upstream_cluster_version, 'k3s') }}; then + make e2e-check-app + fi - name: Bootstrap additional nodes in pool "worker" (total of ${{ inputs.node_number }}) if: inputs.test_type == 'cli' && inputs.node_number > 3 env: @@ -526,6 +543,10 @@ jobs: else cd tests && VM_INDEX=${VM_START} VM_NUMBERS=${VM_END} make e2e-bootstrap-node fi + # Check the installed application + if ${{ contains(inputs.upstream_cluster_version, 'k3s') }}; then + make e2e-check-app + fi - name: List installed nodes if: inputs.test_type == 'cli' run: sudo virsh list @@ -533,7 +554,11 @@ jobs: if: inputs.test_type == 'cli' && contains(inputs.upstream_cluster_version, 'k3s') env: BACKUP_RESTORE_VERSION: ${{ inputs.backup_restore_version }} - run: cd tests && make e2e-backup-restore + run: | + cd tests && make e2e-backup-restore + if ${{ contains(inputs.upstream_cluster_version, 'k3s') }}; then + make e2e-check-app + fi - name: Uninstall Elemental Operator env: OPERATOR_REPO: ${{ inputs.operator_repo }} diff --git a/tests/Makefile b/tests/Makefile index d1832341a..9d07fa4b2 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -69,12 +69,16 @@ e2e-bootstrap-node: deps ginkgo --timeout $(GINKGO_TIMEOUT)s --label-filter bootstrap -r -v ./e2e e2e-backup-restore: deps ginkgo --label-filter test-backup-restore -r -v ./e2e +e2e-check-app: deps + ginkgo --label-filter check-app -r -v ./e2e e2e-configure-rancher: deps ginkgo --label-filter configure -r -v ./e2e e2e-get-logs: deps ginkgo --label-filter logs -r -v ./e2e e2e-install-rancher: deps ginkgo --label-filter install -r -v ./e2e +e2e-install-app: deps + ginkgo --label-filter install-app -r -v ./e2e e2e-install-backup-restore: deps ginkgo --label-filter install-backup-restore -r -v ./e2e e2e-ui-rancher: deps diff --git a/tests/assets/hello-world_app.yaml b/tests/assets/hello-world_app.yaml new file mode 100644 index 000000000..40c6c3b8f --- /dev/null +++ b/tests/assets/hello-world_app.yaml @@ -0,0 +1,99 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + labels: + workload.user.cattle.io/workloadselector: apps.deployment-default-hello-world + app: hello-world + name: hello-world + namespace: default +spec: + progressDeadlineSeconds: 600 + replicas: 3 + revisionHistoryLimit: 10 + selector: + matchLabels: + workload.user.cattle.io/workloadselector: apps.deployment-default-hello-world + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + workload.user.cattle.io/workloadselector: apps.deployment-default-hello-world + namespace: default + spec: + containers: + - image: rancher/hello-world + imagePullPolicy: Always + name: container-0 + ports: + - containerPort: 80 + name: 80tcp8080 + protocol: TCP + resources: {} + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: false + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + field.cattle.io/targetWorkloadIds: '["default/hello-world"]' + management.cattle.io/ui-managed: "true" + name: hello-world + namespace: default +spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: 80tcp8080 + port: 80 + protocol: TCP + targetPort: 80 + selector: + workload.user.cattle.io/workloadselector: apps.deployment-default-hello-world + sessionAffinity: None + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + cloudprovider.harvesterhci.io/ipam: dhcp + field.cattle.io/targetWorkloadIds: '["default/hello-world"]' + management.cattle.io/ui-managed: "true" + finalizers: + - service.kubernetes.io/load-balancer-cleanup + name: hello-world-loadbalancer + namespace: default +spec: + allocateLoadBalancerNodePorts: true + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: 80tcp8080 + port: 8080 + protocol: TCP + targetPort: 80 + selector: + workload.user.cattle.io/workloadselector: apps.deployment-default-hello-world + sessionAffinity: None + type: LoadBalancer diff --git a/tests/e2e/app_test.go b/tests/e2e/app_test.go new file mode 100644 index 000000000..56462ca38 --- /dev/null +++ b/tests/e2e/app_test.go @@ -0,0 +1,86 @@ +/* +Copyright © 2023 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_test + +import ( + "os" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/rancher-sandbox/ele-testhelpers/kubectl" + "github.com/rancher/elemental/tests/e2e/helpers/misc" +) + +var _ = Describe("E2E - Install a simple application", Label("install-app"), func() { + It("Install HelloWorld application", func() { + kubeConfig, err := misc.SetClientKubeConfig(clusterNS, clusterName) + defer os.Remove(kubeConfig) + Expect(err).To(Not(HaveOccurred())) + + By("Installing application", func() { + err := kubectl.Apply("default", appYaml) + Expect(err).To(Not(HaveOccurred())) + }) + }) +}) + +var _ = Describe("E2E - Checking a simple application", Label("check-app"), func() { + It("Check HelloWorld application", func() { + appName := "hello-world" + + // File where to host client cluster kubeconfig + kubeConfig, err := misc.SetClientKubeConfig(clusterNS, clusterName) + defer os.Remove(kubeConfig) + Expect(err).To(Not(HaveOccurred())) + + By("Waiting for deployment to be rollout", func() { + // Wait for application to be started + status, err := kubectl.Run("rollout", "status", "deployment/"+appName) + Expect(err).To(Not(HaveOccurred())) + Expect(status).To(ContainSubstring("successfully rolled out")) + }) + + By("Checking application", func() { + // Get load balancer IPs + appIPs, err := kubectl.Run("get", "svc", + appName+"-loadbalancer", + "-o", "jsonpath={.status.loadBalancer.ingress[*].ip}") + Expect(err).To(Not(HaveOccurred())) + Expect(appIPs).To(Not(BeEmpty())) + + // Loop on each IP to check the application + for _, ip := range strings.Fields(appIPs) { + GinkgoWriter.Printf("Checking node with IP %s...\n", ip) + + // Retry if needed, could take some times if a pod is restarted for example + var htmlPage []byte + Eventually(func() error { + htmlPage, err = exec.Command("curl", "http://"+ip+":8080").CombinedOutput() + return err + }, misc.SetTimeout(2*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) + + // Check HTML page content + Expect(string(htmlPage)).To(And( + ContainSubstring("Hello world!"), + ContainSubstring("My hostname is hello-world-"), + ContainSubstring(ip+":8080"), + ), string(htmlPage)) + } + }) + }) +}) diff --git a/tests/e2e/helpers/misc/misc.go b/tests/e2e/helpers/misc/misc.go index c249b3b33..3bc875847 100644 --- a/tests/e2e/helpers/misc/misc.go +++ b/tests/e2e/helpers/misc/misc.go @@ -15,6 +15,7 @@ limitations under the License. package misc import ( + "encoding/base64" "errors" "fmt" "io" @@ -382,7 +383,22 @@ func ConcateFiles(srcfile, dstfile string, data []byte) error { return err } - // All good! + return nil +} + +func WriteFile(dstfile string, data []byte) error { + // Open/create destination file + d, err := os.OpenFile(dstfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer d.Close() + + // Add data to dest + if _, err = d.Write([]byte(data)); err != nil { + return err + } + return nil } @@ -395,6 +411,7 @@ func TrimStringFromChar(s, c string) string { if idx := strings.Index(s, c); idx != -1 { return s[:idx] } + return s } @@ -441,7 +458,6 @@ func AddNode(file, name string, index int) error { return err } - // All good! return nil } @@ -459,9 +475,11 @@ func SetHostname(baseName string, index int) string { if baseName == "" { baseName = "emtpy" } + if index < 0 { index = 0 } + return baseName + "-" + fmt.Sprintf("%03d", index) } @@ -470,6 +488,7 @@ func CreateTemp(baseName string) (string, error) { if err != nil { return "", err } + return t.Name(), nil } @@ -482,3 +501,40 @@ func CheckPod(k *kubectl.Kubectl, checkList [][]string) error { return nil } + +func SetClientKubeConfig(ns, name string) (string, error) { + clientKubeConfig, err := CreateTemp("clientKubeConfig") + if err != nil { + return "", err + } + + // Get Kubeconfig of client cluster + out, err := kubectl.Run("get", "secret", + "--namespace", ns, + name+"-kubeconfig", "-o", "jsonpath={.data.value}") + if err != nil { + os.Remove(clientKubeConfig) + return "", err + } + + // Decode Kubeconfig data and write into file + data, err := base64.StdEncoding.DecodeString(out) + if err != nil { + os.Remove(clientKubeConfig) + return "", err + } + err = WriteFile(clientKubeConfig, data) + if err != nil { + os.Remove(clientKubeConfig) + return "", err + } + + // Export KUBECONFIG envar + err = os.Setenv("KUBECONFIG", clientKubeConfig) + if err != nil { + os.Remove(clientKubeConfig) + return "", err + } + + return clientKubeConfig, nil +} diff --git a/tests/e2e/install_test.go b/tests/e2e/install_test.go index eb510fa4e..6ce29d1a0 100644 --- a/tests/e2e/install_test.go +++ b/tests/e2e/install_test.go @@ -222,7 +222,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { // Check issuer for Private CA if caType == "private" { Eventually(func() error { - out, err := exec.Command("bash", "-c", "curl -vk https://"+rancherHostname).CombinedOutput() + out, err := exec.Command("curl", "-vk", "https://"+rancherHostname).CombinedOutput() if err != nil { // Show only if there's no error GinkgoWriter.Printf("%s\n", out) @@ -248,14 +248,12 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { // Getting Rancher Manager local cluster CA // NOTE: loop until the cmd return something, it could take some time var rancherCA string - cmd := []string{ - "get", "secret", - "--namespace", "cattle-system", - "tls-rancher-ingress", - "-o", "jsonpath={.data.tls\\.crt}", - } Eventually(func() error { - rancherCA, err = kubectl.Run(cmd...) + rancherCA, err = kubectl.Run("get", "secret", + "--namespace", "cattle-system", + "tls-rancher-ingress", + "-o", "jsonpath={.data.tls\\.crt}", + ) return err }, misc.SetTimeout(2*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index a625bedeb..4061d0f1f 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -29,6 +29,7 @@ import ( ) const ( + appYaml = "../assets/hello-world_app.yaml" clusterYaml = "../assets/cluster.yaml" backupYaml = "../assets/backup.yaml" emulateTPMYaml = "../assets/emulateTPM.yaml"