diff --git a/.github/workflows/cli-k3s-os-upgrade-rancher_latest.yaml b/.github/workflows/cli-k3s-os-upgrade-rancher_latest.yaml index 1550f9b10..69e678543 100644 --- a/.github/workflows/cli-k3s-os-upgrade-rancher_latest.yaml +++ b/.github/workflows/cli-k3s-os-upgrade-rancher_latest.yaml @@ -20,8 +20,7 @@ jobs: cluster_name: cluster-k3s iso_to_test: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso k8s_version_to_provision: v1.25.7+k3s1 - # Force to only deploy the first 3 nodes - node_number: 3 + node_number: 5 operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher rancher_version: latest/devel diff --git a/.github/workflows/cli-k3s-os-upgrade-rancher_stable.yaml b/.github/workflows/cli-k3s-os-upgrade-rancher_stable.yaml index a8099acaf..a3b57c899 100644 --- a/.github/workflows/cli-k3s-os-upgrade-rancher_stable.yaml +++ b/.github/workflows/cli-k3s-os-upgrade-rancher_stable.yaml @@ -20,10 +20,10 @@ jobs: cluster_name: cluster-k3s iso_to_test: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso k8s_version_to_provision: v1.25.7+k3s1 - # Force to only deploy the first 3 nodes - node_number: 3 + node_number: 5 operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher + rancher_upgrade: latest/devel rancher_version: stable/latest upgrade_channel_list: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal-dev-channel/5.3:latest upgrade_image: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal/5.3:latest diff --git a/.github/workflows/cli-k3s-os-upgrade.yaml b/.github/workflows/cli-k3s-os-upgrade.yaml index 789ff076d..c768bae46 100644 --- a/.github/workflows/cli-k3s-os-upgrade.yaml +++ b/.github/workflows/cli-k3s-os-upgrade.yaml @@ -12,6 +12,10 @@ on: description: ISO to test default: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso type: string + rancher_upgrade: + description: Rancher Manager channel/version to upgrade to + default: latest/devel + type: string rancher_version: description: Rancher Manager channel/version to use for installation default: stable/latest @@ -38,10 +42,10 @@ jobs: destroy_runner: ${{ inputs.destroy_runner }} iso_to_test: ${{ inputs.iso_to_test }} k8s_version_to_provision: v1.25.7+k3s1 - # Force to only deploy the first 3 nodes - node_number: 3 + node_number: 5 operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/${{ inputs.upgrade_os_channel }}/charts/rancher operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher + rancher_upgrade: ${{ inputs.rancher_upgrade }} rancher_version: ${{ inputs.rancher_version }} upgrade_channel_list: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal-dev-channel/5.3:latest upgrade_image: registry.opensuse.org/isv/rancher/elemental/${{ inputs.upgrade_os_channel }}/teal53/15.4/rancher/elemental-teal/5.3:latest diff --git a/.github/workflows/cli-rke2-os-upgrade-rancher_latest.yaml b/.github/workflows/cli-rke2-os-upgrade-rancher_latest.yaml index 3af30a962..2bb1b56e5 100644 --- a/.github/workflows/cli-rke2-os-upgrade-rancher_latest.yaml +++ b/.github/workflows/cli-rke2-os-upgrade-rancher_latest.yaml @@ -21,8 +21,7 @@ jobs: cluster_name: cluster-rke2 iso_to_test: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso k8s_version_to_provision: v1.25.7+rke2r1 - # Force to only deploy the first 3 nodes - node_number: 3 + node_number: 5 operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher rancher_version: latest/devel diff --git a/.github/workflows/cli-rke2-os-upgrade-rancher_stable.yaml b/.github/workflows/cli-rke2-os-upgrade-rancher_stable.yaml index eb1b3caae..c41ae33c6 100644 --- a/.github/workflows/cli-rke2-os-upgrade-rancher_stable.yaml +++ b/.github/workflows/cli-rke2-os-upgrade-rancher_stable.yaml @@ -21,10 +21,10 @@ jobs: cluster_name: cluster-rke2 iso_to_test: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso k8s_version_to_provision: v1.25.7+rke2r1 - # Force to only deploy the first 3 nodes - node_number: 3 + node_number: 5 operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher + rancher_upgrade: latest/devel rancher_version: stable/latest upgrade_channel_list: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal-dev-channel/5.3:latest upgrade_image: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal/5.3:latest diff --git a/.github/workflows/cli-rke2-os-upgrade.yaml b/.github/workflows/cli-rke2-os-upgrade.yaml index 5c015aa10..f4b881614 100644 --- a/.github/workflows/cli-rke2-os-upgrade.yaml +++ b/.github/workflows/cli-rke2-os-upgrade.yaml @@ -12,6 +12,10 @@ on: description: ISO to test default: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso type: string + rancher_upgrade: + description: Rancher Manager channel/version to upgrade to + default: latest/devel + type: string rancher_version: description: Rancher Manager channel/version to use for installation default: stable/latest @@ -39,10 +43,10 @@ jobs: destroy_runner: ${{ inputs.destroy_runner }} iso_to_test: ${{ inputs.iso_to_test }} k8s_version_to_provision: v1.25.7+rke2r1 - # Force to only deploy the first 3 nodes - node_number: 3 + node_number: 5 operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/${{ inputs.upgrade_os_channel }}/charts/rancher operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher + rancher_upgrade: ${{ inputs.rancher_upgrade }} rancher_version: ${{ inputs.rancher_version }} upgrade_channel_list: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal-dev-channel/5.3:latest upgrade_image: registry.opensuse.org/isv/rancher/elemental/${{ inputs.upgrade_os_channel }}/teal53/15.4/rancher/elemental-teal/5.3:latest diff --git a/.github/workflows/master-e2e.yaml b/.github/workflows/master-e2e.yaml index b3a9007a7..df195e380 100644 --- a/.github/workflows/master-e2e.yaml +++ b/.github/workflows/master-e2e.yaml @@ -89,6 +89,9 @@ on: description: Rancher Manager channel/version to use for installation default: stable/latest type: string + rancher_upgrade: + description: Rancher Manager channel/version to upgrade to + type: string runner_template: description: Runner template to use default: elemental-e2e-ci-runner-spot-x86-64-template-n2-standard-16-v4 @@ -265,9 +268,10 @@ jobs: CA_TYPE: ${{ inputs.ca_type }} OPERATOR_REPO: ${{ inputs.operator_repo }} PROXY: ${{ inputs.proxy }} + PUBLIC_DNS: ${{ needs.create-runner.outputs.public_dns }} PUBLIC_DOMAIN: bc.googleusercontent.com TEST_TYPE: ${{ inputs.test_type }} - run: cd tests && PUBLIC_DNS=${{ needs.create-runner.outputs.public_dns }} make e2e-install-rancher + run: cd tests && make e2e-install-rancher - name: Workaround for DynamicSchemas (if needed) run: | # Check if DynamicSchemas for MachineInventorySelectorTemplate exists @@ -459,6 +463,24 @@ jobs: # Export values echo "operator_upgrade=${OPERATOR_UPGRADE}" >> ${GITHUB_OUTPUT} echo "operator_version=${OPERATOR_VERSION}" >> ${GITHUB_OUTPUT} + - name: Upgrade Rancher Manager + if: inputs.test_type == 'cli' && inputs.rancher_upgrade != '' + id: rancher_upgrade + env: + CA_TYPE: ${{ inputs.ca_type }} + PROXY: ${{ inputs.proxy }} + PUBLIC_DNS: ${{ needs.create-runner.outputs.public_dns }} + PUBLIC_DOMAIN: bc.googleusercontent.com + RANCHER_UPGRADE: ${{ inputs.rancher_upgrade }} + run: | + cd tests && make e2e-upgrade-rancher-manager + # Extract Rancher Manager version + RM_VERSION=$(kubectl get pod \ + --namespace cattle-system \ + -l app=rancher \ + -o jsonpath={.items[*].status.containerStatuses[*].image} 2> /dev/null || true) + # Export values + echo "rm_version=${RM_VERSION}" >> ${GITHUB_OUTPUT} - name: Upgrade node 1 to specified OS version with osImage if: inputs.test_type == 'cli' && inputs.upgrade_image != '' env: @@ -577,6 +599,7 @@ jobs: echo "## Upgrade details" >> ${GITHUB_STEP_SUMMARY} echo "Elemental Operator Upgrade: ${{ steps.operator_upgrade.outputs.operator_upgrade }}" >> ${GITHUB_STEP_SUMMARY} echo "Elemental Operator Image: ${{ steps.operator_upgrade.outputs.operator_version }}" >> ${GITHUB_STEP_SUMMARY} + echo "Rancher Manager Image: ${{ steps.rancher_upgrade.outputs.rm_version }}" >> ${GITHUB_STEP_SUMMARY} echo "Channel list: ${{ inputs.upgrade_channel_list }}" >> ${GITHUB_STEP_SUMMARY} echo "Channel used: ${{ inputs.upgrade_os_channel }}" >> ${GITHUB_STEP_SUMMARY} echo "Upgrade image: ${{ inputs.upgrade_image }}" >> ${GITHUB_STEP_SUMMARY} diff --git a/tests/Makefile b/tests/Makefile index dd92c8f06..d1832341a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -82,8 +82,10 @@ e2e-ui-rancher: deps e2e-uninstall-operator: ginkgo --label-filter uninstall-operator -r -v ./e2e e2e-upgrade-node: deps - ginkgo --label-filter upgrade -r -v ./e2e + ginkgo --label-filter upgrade-node -r -v ./e2e e2e-upgrade-operator: deps ginkgo --label-filter upgrade-operator -r -v ./e2e +e2e-upgrade-rancher-manager: deps + ginkgo --label-filter upgrade-rancher-manager -r -v ./e2e start-cypress-tests: @./scripts/start-cypress-tests diff --git a/tests/e2e/helpers/install/install.go b/tests/e2e/helpers/install/install.go new file mode 100644 index 000000000..c16e5a5c1 --- /dev/null +++ b/tests/e2e/helpers/install/install.go @@ -0,0 +1,86 @@ +/* +Copyright © 2023 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package install + +import ( + "strings" + + . "github.com/onsi/gomega" + "github.com/rancher-sandbox/ele-testhelpers/kubectl" +) + +// Install or upgrade Rancher Manager +func DeployRancherManager(hostname, channel, version, ca, proxy string) { + channelName := "rancher-" + channel + + // Add Helm repository + err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", channelName, + "https://releases.rancher.com/server-charts/"+channel, + ) + Expect(err).To(Not(HaveOccurred())) + + err = kubectl.RunHelmBinaryWithCustomErr("repo", "update") + Expect(err).To(Not(HaveOccurred())) + + // Set flags for Rancher Manager installation + flags := []string{ + "upgrade", "--install", "rancher", channelName + "/rancher", + "--namespace", "cattle-system", + "--create-namespace", + "--set", "hostname=" + hostname, + "--set", "extraEnv[0].name=CATTLE_SERVER_URL", + "--set", "extraEnv[0].value=https://" + hostname, + "--set", "extraEnv[1].name=CATTLE_BOOTSTRAP_PASSWORD", + "--set", "extraEnv[1].value=rancherpassword", + "--set", "replicas=1", + "--set", "global.cattle.psp.enabled=false", + } + + // Set specified version if needed + if version != "" && version != "latest" { + if version == "devel" { + flags = append(flags, + "--devel", + "--set", "rancherImageTag=v2.7-head", + ) + } else if strings.Contains(version, "-rc") { + flags = append(flags, + "--devel", + "--version", version, + ) + } else { + flags = append(flags, "--version", version) + } + } + + // For Private CA + if ca == "private" { + flags = append(flags, + "--set", "ingress.tls.source=secret", + "--set", "privateCA=true", + ) + } + + // Use Rancher Manager behind proxy + if proxy == "rancher" { + flags = append(flags, + "--set", "proxy=http://172.17.0.1:3128", + "--set", "noProxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local", + ) + } + + err = kubectl.RunHelmBinaryWithCustomErr(flags...) + Expect(err).To(Not(HaveOccurred())) +} diff --git a/tests/e2e/install_test.go b/tests/e2e/install_test.go index c434d81b6..2ff4a1592 100644 --- a/tests/e2e/install_test.go +++ b/tests/e2e/install_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" "github.com/rancher-sandbox/ele-testhelpers/tools" + "github.com/rancher/elemental/tests/e2e/helpers/install" "github.com/rancher/elemental/tests/e2e/helpers/misc" ) @@ -190,63 +191,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { } By("Installing Rancher", func() { - err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", "rancher", - "https://releases.rancher.com/server-charts/"+rancherChannel, - ) - Expect(err).To(Not(HaveOccurred())) - - err = kubectl.RunHelmBinaryWithCustomErr("repo", "update") - Expect(err).To(Not(HaveOccurred())) - - // Set flags for Rancher Manager installation - flags := []string{ - "upgrade", "--install", "rancher", "rancher/rancher", - "--namespace", "cattle-system", - "--create-namespace", - "--set", "hostname=" + rancherHostname, - "--set", "extraEnv[0].name=CATTLE_SERVER_URL", - "--set", "extraEnv[0].value=https://" + rancherHostname, - "--set", "extraEnv[1].name=CATTLE_BOOTSTRAP_PASSWORD", - "--set", "extraEnv[1].value=rancherpassword", - "--set", "replicas=1", - "--set", "global.cattle.psp.enabled=false", - } - - // Set specified version if needed - if rancherVersion != "" && rancherVersion != "latest" { - if rancherVersion == "devel" { - flags = append(flags, - "--devel", - "--set", "rancherImageTag=v2.7-head", - ) - } else if strings.Contains(rancherVersion, "-rc") { - flags = append(flags, - "--devel", - "--version", rancherVersion, - ) - } else { - flags = append(flags, "--version", rancherVersion) - } - } - - // For Private CA - if caType == "private" { - flags = append(flags, - "--set", "ingress.tls.source=secret", - "--set", "privateCA=true", - ) - } - - // Use Rancher Manager behind proxy - if proxy == "rancher" { - flags = append(flags, - "--set", "proxy=http://172.17.0.1:3128", - "--set", "noProxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local", - ) - } - - err = kubectl.RunHelmBinaryWithCustomErr(flags...) - Expect(err).To(Not(HaveOccurred())) + install.DeployRancherManager(rancherHostname, rancherChannel, rancherVersion, caType, proxy) // Inject secret for Private CA if caType == "private" { diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index c97c5cf8c..8a5a7fc4c 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -52,38 +52,41 @@ const ( ) var ( - arch string - backupRestoreVersion string - caType string - CertManagerVersion string - clusterName string - clusterNS string - clusterType string - elementalSupport string - emulateTPM bool - rancherHostname string - imageVersion string - isoBoot string - k8sUpstreamVersion string - k8sVersion string - numberOfVMs int - operatorUpgrade string - operatorRepo string - osImage string - poolType string - proxy string - rancherChannel string - rancherLogCollector string - rancherVersion string - sequential bool - testType string - upgradeChannelList string - upgradeImage string - upgradeOsChannel string - upgradeType string - usedNodes int - vmIndex int - vmName string + arch string + backupRestoreVersion string + caType string + CertManagerVersion string + clusterName string + clusterNS string + clusterType string + elementalSupport string + emulateTPM bool + rancherHostname string + imageVersion string + isoBoot string + k8sUpstreamVersion string + k8sVersion string + numberOfVMs int + operatorUpgrade string + operatorRepo string + osImage string + poolType string + proxy string + rancherChannel string + rancherLogCollector string + rancherVersion string + rancherUpgrade string + rancherUpgradeChannel string + rancherUpgradeVersion string + sequential bool + testType string + upgradeChannelList string + upgradeImage string + upgradeOsChannel string + upgradeType string + usedNodes int + vmIndex int + vmName string ) func CheckClusterState(ns, cluster string) { @@ -156,6 +159,7 @@ var _ = BeforeSuite(func() { proxy = os.Getenv("PROXY") rancherLogCollector = os.Getenv("RANCHER_LOG_COLLECTOR") rancherVersion = os.Getenv("RANCHER_VERSION") + rancherUpgrade = os.Getenv("RANCHER_UPGRADE") seqString := os.Getenv("SEQUENTIAL") testType = os.Getenv("TEST_TYPE") upgradeChannelList = os.Getenv("UPGRADE_CHANNEL_LIST") @@ -213,6 +217,13 @@ var _ = BeforeSuite(func() { rancherVersion = s[1] } + // Extract Rancher Manager channel/version to upgrade + if rancherUpgrade != "" { + s := strings.Split(rancherUpgrade, "/") + rancherUpgradeChannel = s[0] + rancherUpgradeVersion = s[1] + } + // Start HTTP server misc.FileShare("../..", ":8000") }) diff --git a/tests/e2e/upgrade_test.go b/tests/e2e/upgrade_test.go index 3961369d7..bcba86260 100644 --- a/tests/e2e/upgrade_test.go +++ b/tests/e2e/upgrade_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" "github.com/rancher-sandbox/ele-testhelpers/tools" + "github.com/rancher/elemental/tests/e2e/helpers/install" "github.com/rancher/elemental/tests/e2e/helpers/misc" ) @@ -64,12 +65,57 @@ var _ = Describe("E2E - Upgrading Elemental Operator", Label("upgrade-operator") // Delay few seconds before checking, needed because we may have 2 pods at the same time time.Sleep(misc.SetTimeout(30 * time.Second)) - err = k.WaitForNamespaceWithPod("cattle-elemental-system", "app=elemental-operator") + // Wait for all pods to be started + misc.CheckPod(k, [][]string{{"cattle-elemental-system", "app=elemental-operator"}}) + }) +}) + +var _ = Describe("E2E - Upgrading Rancher Manager", Label("upgrade-rancher-manager"), func() { + // Create kubectl context + // Default timeout is too small, so New() cannot be used + k := &kubectl.Kubectl{ + Namespace: "", + PollTimeout: misc.SetTimeout(300 * time.Second), + PollInterval: 500 * time.Millisecond, + } + + It("Upgrade Rancher Manager", func() { + // Get before-upgrade Rancher Manager version + getImageVersion := []string{ + "get", "pod", + "--namespace", "cattle-system", + "-l", "app=rancher", + "-o", "jsonpath={.items[*].status.containerStatuses[*].image}", + } + versionBeforeUpgrade, err := kubectl.Run(getImageVersion...) + Expect(err).To(Not(HaveOccurred())) + + // Upgrade Rancher Manager + install.DeployRancherManager(rancherHostname, rancherUpgradeChannel, rancherUpgradeVersion, caType, proxy) + + // Wait for Rancher Manager to be running + checkList := [][]string{ + {"cattle-system", "app=rancher"}, + {"cattle-fleet-local-system", "app=fleet-agent"}, + {"cattle-system", "app=rancher-webhook"}, + } + misc.CheckPod(k, checkList) + + // Check that all pods are using the same version + Eventually(func() int { + out, _ := kubectl.Run(getImageVersion...) + return len(strings.Fields(out)) + }, misc.SetTimeout(3*time.Minute), 5*time.Second).Should(Equal(1)) + + // Get after-upgrade Rancher Manager version + // and check that it's different to the before-upgrade version + versionAfterUpgrade, err := kubectl.Run(getImageVersion...) Expect(err).To(Not(HaveOccurred())) + Expect(versionAfterUpgrade).To(Not(Equal(versionBeforeUpgrade))) }) }) -var _ = Describe("E2E - Upgrading node", Label("upgrade"), func() { +var _ = Describe("E2E - Upgrading node", Label("upgrade-node"), func() { var ( wg sync.WaitGroup value string