From 6a72c3469089da583fec1282169a966f2049b797 Mon Sep 17 00:00:00 2001 From: Chandan Pinjani Date: Wed, 8 Jan 2025 15:18:16 +0530 Subject: [PATCH] p0upgradeK8sVersionChecks fix (#233) Signed-off-by: Chandan Pinjani --- hosted/eks/helper/helper_cluster.go | 51 ++++++++++++------- .../k8s_chart_support_upgrade_suite_test.go | 14 ++--- hosted/eks/p0/p0_suite_test.go | 13 ++--- hosted/eks/p1/p1_suite_test.go | 2 +- 4 files changed, 48 insertions(+), 32 deletions(-) diff --git a/hosted/eks/helper/helper_cluster.go b/hosted/eks/helper/helper_cluster.go index e408a90e..2a011bc3 100644 --- a/hosted/eks/helper/helper_cluster.go +++ b/hosted/eks/helper/helper_cluster.go @@ -84,12 +84,14 @@ func UpgradeClusterKubernetesVersion(cluster *management.Cluster, upgradeToVersi } // Check if the desired config has been applied in Rancher - Eventually(func() string { - ginkgo.GinkgoLogr.Info("Waiting for k8s upgrade to appear in EKSStatus.UpstreamSpec ...") + // Check if EKSConfig has correct KubernetesVersion after upgrade (Ref: eks-operator/issues/668) + Eventually(func() bool { + ginkgo.GinkgoLogr.Info("Waiting for k8s upgrade to appear in EKSStatus.UpstreamSpec & EKSConfig ...") cluster, err = client.Management.Cluster.ByID(cluster.ID) Expect(err).To(BeNil()) - return *cluster.EKSStatus.UpstreamSpec.KubernetesVersion - }, tools.SetTimeout(15*time.Minute), 30*time.Second).Should(Equal(upgradeToVersion)) + return *cluster.EKSStatus.UpstreamSpec.KubernetesVersion == upgradeToVersion && *cluster.EKSConfig.KubernetesVersion == upgradeToVersion + }, tools.SetTimeout(15*time.Minute), 30*time.Second).Should(BeTrue()) + // ensure nodegroup version is same in Rancher for _, ng := range cluster.EKSStatus.UpstreamSpec.NodeGroups { Expect(*ng.Version).To(Equal(currentVersion)) @@ -101,29 +103,34 @@ func UpgradeClusterKubernetesVersion(cluster *management.Cluster, upgradeToVersi // UpgradeNodeKubernetesVersion upgrades the k8s version of nodegroup to the value defined by upgradeToVersion. // if wait is set to true, it will wait until the cluster finishes upgrading; // if checkClusterConfig is set to true, it will validate that nodegroup has been upgraded successfully -func UpgradeNodeKubernetesVersion(cluster *management.Cluster, upgradeToVersion string, client *rancher.Client, wait, checkClusterConfig bool) (*management.Cluster, error) { - upgradedCluster := cluster - for i := range upgradedCluster.EKSConfig.NodeGroups { - upgradedCluster.EKSConfig.NodeGroups[i].Version = &upgradeToVersion - } - +// if useEksctl is set to true, nodegroup will be upgraded using eksctl utility instead of updating it from Rancher +func UpgradeNodeKubernetesVersion(cluster *management.Cluster, upgradeToVersion string, client *rancher.Client, wait, checkClusterConfig, useEksctl bool) (*management.Cluster, error) { var err error - cluster, err = client.Management.Cluster.Update(cluster, &upgradedCluster) - Expect(err).To(BeNil()) - // Check if the desired config is set correctly - for _, ng := range cluster.EKSConfig.NodeGroups { - Expect(*ng.Version).To(Equal(upgradeToVersion)) - } + if !useEksctl { + upgradedCluster := cluster + for i := range upgradedCluster.EKSConfig.NodeGroups { + upgradedCluster.EKSConfig.NodeGroups[i].Version = &upgradeToVersion + } - if wait { - err = clusters.WaitClusterToBeUpgraded(client, cluster.ID) + cluster, err = client.Management.Cluster.Update(cluster, &upgradedCluster) Expect(err).To(BeNil()) + + if wait { + err = clusters.WaitClusterToBeUpgraded(client, cluster.ID) + Expect(err).To(BeNil()) + } + } else { + // Upgrade Nodegroup using eksctl due to custom Launch template + for _, ng := range cluster.EKSConfig.NodeGroups { + err = UpgradeEKSNodegroupOnAWS(helpers.GetEKSRegion(), cluster.EKSConfig.DisplayName, *ng.NodegroupName, upgradeToVersion) + Expect(err).To(BeNil()) + } } if checkClusterConfig { Eventually(func() bool { - // Check if the desired config has been applied in + // Check if the desired config has been applied cluster, err = client.Management.Cluster.ByID(cluster.ID) Expect(err).To(BeNil()) ginkgo.GinkgoLogr.Info("waiting for the nodegroup upgrade to appear in EKSStatus.UpstreamSpec ...") @@ -135,6 +142,12 @@ func UpgradeNodeKubernetesVersion(cluster *management.Cluster, upgradeToVersion return true }, tools.SetTimeout(15*time.Minute), 30*time.Second).Should(BeTrue()) } + + // Ensure nodegroup version is correct in Rancher after upgrade + for _, ng := range cluster.EKSConfig.NodeGroups { + Expect(*ng.Version).To(Equal(upgradeToVersion)) + } + return cluster, nil } diff --git a/hosted/eks/k8s_chart_support/upgrade/k8s_chart_support_upgrade_suite_test.go b/hosted/eks/k8s_chart_support/upgrade/k8s_chart_support_upgrade_suite_test.go index 5189df1c..4de44723 100644 --- a/hosted/eks/k8s_chart_support/upgrade/k8s_chart_support_upgrade_suite_test.go +++ b/hosted/eks/k8s_chart_support/upgrade/k8s_chart_support_upgrade_suite_test.go @@ -168,13 +168,15 @@ func commonchecks(ctx *helpers.Context, cluster *management.Cluster, clusterName cluster, err = helper.UpgradeClusterKubernetesVersion(cluster, *latestVersion, ctx.RancherAdminClient, true) Expect(err).To(BeNil()) - // Does not upgrade noodegroup version since using custom LT, skip for imported cluster - if !helpers.IsImport { - By("upgrading the NodeGroups", func() { - cluster, err = helper.UpgradeNodeKubernetesVersion(cluster, *latestVersion, ctx.RancherAdminClient, true, true) - Expect(err).To(BeNil()) - }) + + var useEksctl bool + if helpers.IsImport { + useEksctl = true } + By("upgrading the NodeGroups", func() { + cluster, err = helper.UpgradeNodeKubernetesVersion(cluster, *latestVersion, ctx.RancherAdminClient, true, true, useEksctl) + Expect(err).To(BeNil()) + }) }) var downgradeVersion string diff --git a/hosted/eks/p0/p0_suite_test.go b/hosted/eks/p0/p0_suite_test.go index a1a04523..8985aab9 100644 --- a/hosted/eks/p0/p0_suite_test.go +++ b/hosted/eks/p0/p0_suite_test.go @@ -80,13 +80,14 @@ func p0upgradeK8sVersionChecks(cluster *management.Cluster, client *rancher.Clie Expect(err).To(BeNil()) }) - // Does not upgrades version since using custom LT, skip for imported cluster - if !helpers.IsImport { - By("upgrading the NodeGroups", func() { - cluster, err = helper.UpgradeNodeKubernetesVersion(cluster, upgradeToVersion, client, true, true) - Expect(err).To(BeNil()) - }) + var useEksctl bool + if helpers.IsImport { + useEksctl = true } + By("upgrading the NodeGroups", func() { + cluster, err = helper.UpgradeNodeKubernetesVersion(cluster, upgradeToVersion, client, true, true, useEksctl) + Expect(err).To(BeNil()) + }) } func p0NodesChecks(cluster *management.Cluster, client *rancher.Client, clusterName string) { diff --git a/hosted/eks/p1/p1_suite_test.go b/hosted/eks/p1/p1_suite_test.go index ba33a383..7ce5bdd2 100644 --- a/hosted/eks/p1/p1_suite_test.go +++ b/hosted/eks/p1/p1_suite_test.go @@ -526,7 +526,7 @@ func syncRancherToAWSCheck(cluster *management.Cluster, client *rancher.Client, func upgradeNodeKubernetesVersionGTCPCheck(cluster *management.Cluster, client *rancher.Client, upgradeToVersion string) { GinkgoLogr.Info("Upgrading only Nodegroup's EKS version to: " + upgradeToVersion) var err error - cluster, err = helper.UpgradeNodeKubernetesVersion(cluster, upgradeToVersion, client, false, false) + cluster, err = helper.UpgradeNodeKubernetesVersion(cluster, upgradeToVersion, client, false, false, false) Expect(err).To(BeNil()) // wait until the error is visible on the cluster