Fix GKE version and add fallback mechanism (#7453)

### Description of the change

Since GKE updates their `stable` and `regular` channels often, our CI
eventually breaks because of it. This PR adds a fallback mechanism using
the gcloud sdk to auto fetch the default one, adding a warning in the CI
logs.

### Benefits

CI will still work even if GKE performs an update.

### Possible drawbacks

In some cases, we won't have an entire control of which version we are
running our tests on... but it is a tradeoff between this or having a
failing pipeline until we manually fix it.

### Applicable issues

N/A

### Additional information

N/A

Signed-off-by: Antonio Gamez Diaz <agamez@vmware.com>

.github/workflows/gke_e2e_tests.yaml

@@ -52,6 +52,10 @@ on:
         type: boolean
         required: false
         default: false
+      ALLOW_GKE_VERSION_FALLBACK:
+        type: boolean
+        required: false
+        default: true
     secrets:
       GKE_ADMIN:
         required: true
@@ -83,6 +87,7 @@ jobs:
       TEST_LATEST_RELEASE: ${{ inputs.TEST_LATEST_RELEASE }}
       TEST_TIMEOUT_MINUTES: ${{ inputs.TEST_TIMEOUT_MINUTES }}
       USE_MULTICLUSTER_OIDC_ENV: ${{ inputs.USE_MULTICLUSTER_OIDC_ENV }}
+      ALLOW_GKE_VERSION_FALLBACK: ${{ inputs.ALLOW_GKE_VERSION_FALLBACK }}
     steps:
       - uses: actions/checkout@v4
       - run: |
@@ -97,7 +102,7 @@ jobs:
       - name: Start GKE environment
         run: |
           set -euo pipefail
-          ./script/start-gke-env.sh "${ESCAPED_GKE_CLUSTER}" "${GKE_ZONE}" "${GKE_VERSION}" "${GKE_ADMIN}" > /dev/null
+          ./script/start-gke-env.sh "${ESCAPED_GKE_CLUSTER}" "${GKE_ZONE}" "${GKE_VERSION}" "${GKE_ADMIN}" "${GKE_RELEASE_CHANNEL} "${ALLOW_GKE_VERSION_FALLBACK}" > /dev/null
       - # TODO(castelblanque) Unify shared resources with kubeapps-local-dev-users-rbac.yaml that only applies to Kind clusters
         name: Apply customizations to GKE cluster
         run: |

.github/workflows/kubeapps-general.yaml

@@ -53,17 +53,17 @@ env:
   IMG_PLATFORMS: "linux/amd64"
   KAPP_CONTROLLER_VERSION: "v0.50.0"
   FLUX_VERSION: "v2.2.2"
-  K8S_KIND_VERSION: "v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72"
   KIND_VERSION: "v0.20.0"
-  KUBECTL_VERSION: "v1.27.9"
   MKCERT_VERSION: "v1.4.4"
   NODE_VERSION: "20.11.0"
   OLM_VERSION: "v0.26.0"
   POSTGRESQL_VERSION: "16.1.0-debian-11-r25"
   RUST_VERSION: "1.75.0"
   SEMVER_VERSION: "3.4.0"
-  GKE_STABLE_VERSION: "1.27.3"
-  GKE_REGULAR_VERSION: "1.27.3"
+  K8S_KIND_VERSION: "v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72"
+  KUBECTL_VERSION: "v1.27.9"
+  GKE_REGULAR_VERSION: "1.27.9"
+  GKE_STABLE_VERSION: "1.27.8"
   GKE_ZONE: "us-east1-c"
   GKE_PROJECT: "vmware-kubeapps-ci"
   GKE_CLUSTER: "kubeapps-test"

script/start-gke-env.sh

@@ -11,6 +11,8 @@ CLUSTER=${1:?}
 ZONE=${2:?}
 GKE_VERSION=${3:?}
 ADMIN=${4:?}
+GKE_RELEASE_CHANNEL=${5:?}
+ALLOW_GKE_VERSION_FALLBACK=${6:?}
 DEBUG_MODE=${DEBUG_MODE:-"false"}
 
 [[ "${DEBUG_MODE}" == "true" ]] && set -x
@@ -34,6 +36,31 @@ if [[ $(gcloud container clusters list --filter="name:${CLUSTER}") ]]; then
     fi
 fi
 
+# Check if the version is allowed in the current channel
+gcloud container get-server-config --location "${ZONE}" >list.yaml
+for i in {0..2}; do
+    channel=$(yq .channels[$i].channel <list.yaml)
+    if [[ "${channel,,}" == "${GKE_RELEASE_CHANNEL,,}" ]]; then
+        defaultVersion=$(yq .channels[$i].defaultVersion <list.yaml)
+        valid_versions=$(yq -o=j -I=0 .channels[$i].validVersions <list.yaml)
+        if [[ $valid_versions =~ $GKE_VERSION ]]; then
+            echo "[INFO] The version '${GKE_VERSION}' is one of the currently allowed versions in GKE channel '${channel}'"
+            echo "[INFO] The default version for GKE channel '${channel}' is: '${defaultVersion}'"
+            break
+        else
+            echo "[WARNING] The version '${GKE_VERSION}' is not allowed in GKE channel '${channel}'"
+            echo "[INFO] The allowed versions for GKE channel '${channel}' (default: ${defaultVersion}) are: ${valid_versions}"
+            if ! ${ALLOW_GKE_VERSION_FALLBACK}; then
+                echo "[ERROR] Please, change the 'GKE_VERSION' variable in the script to one of the allowed versions, or set the 'ALLOW_GKE_VERSION_FALLBACK' variable to 'true'"
+                exit 1
+            else
+                echo "[WARNING] Falling back to default version '${defaultVersion}, to change this behavior, set the 'ALLOW_GKE_VERSION_FALLBACK' variable to 'false'"
+                GKE_VERSION=$defaultVersion
+            fi
+        fi
+    fi
+done
+
 echo "Creating cluster ${CLUSTER} in ${ZONE} (v$GKE_VERSION)"
 
 # TODO(bjesus) Removing the use of --labels=team=kubeapps due to a bug in gcloud cli: https://issuetracker.google.com/issues/255708239

site/content/docs/latest/reference/developer/release-process.md

@@ -59,6 +59,7 @@ The versions used there _must_ match the ones used for building the container im
 - `HUGO_VERSION` should be updated with the [latest stable version from the Hugo releases](https://github.com/gohugoio/hugo/releases).
 
 Besides, the `GKE_STABLE_VERSION` and the `GKE_REGULAR_VERSION` might have to be updated if the _Stable_ and _Regular_ Kubernetes versions in GKE have changed. Check this information on [this GKE release notes website](https://cloud.google.com/kubernetes-engine/docs/release-notes).
+Note that, if `ALLOW_GKE_VERSION_FALLBACK` is set, when no matching versions are found GKE, it will fall back to the current default one.
 
 When updating the `GOLANG_VERSION`, the Go version used in the [CodeQL Github Action](https://github.com/vmware-tanzu/kubeapps/blob/main/.github/workflows/codeql-analysis.yml) might be updated as well.