Add upgrade tests to test main against previous release

This commit adds the upgrade tests for testing main against the e2e
tests from the previous release to prevent regression. Incompatible
changes made in the current release cycle are expected to be catched.

test/e2e-common.sh

@@ -169,3 +169,9 @@ function delete_pipeline_resources() {
 function delete_resolvers_resources() {
   kubectl delete --ignore-not-found=true resolutionrequests.resolution.tekton.dev --all
 }
+
+function get_tests_from_release() {
+  echo ">> Retrieving tests from release-$1"
+  RELEASE_BRANCH="release-$1"
+  git checkout $RELEASE_BRANCH
+}

test/e2e-tests-upgrade.sh

@@ -16,19 +16,35 @@
 
 # This script calls out to scripts in tektoncd/plumbing to setup a cluster
 # and deploy Tekton Pipelines to it for running upgrading tests. There are
-# two scenarios we need to cover in this script:
+# three scenarios we need to cover in this script:
 
 # Scenario 1: install the previous release, upgrade to the current release, and
 # validate whether the Tekton pipeline works.
 
 # Scenario 2: install the previous release, create the pipelines and tasks, upgrade
 # to the current release, and validate whether the Tekton pipeline works.
 
+# Scenario 3: install the current pipelines, use the integration tests from the previous release(s)
+# to prevent regression. Incompatible changes made in the current release cycle are expected to fail
+# and will be skipped after failing once.
+
 source $(git rev-parse --show-toplevel)/test/e2e-common.sh
 RELEASES='https://github.com/tektoncd/pipeline/releases/latest'
 PREVIOUS_PIPELINE_VERSION=$(curl -L -s -H 'Accept: application/json' $RELEASES |
   sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
 
+# BACKWARD_INCOMPATIBLE_TESTS is a manually maintained list of e2e tests that contains known breaking changes
+# made in the current devel cycle. It prevents the upgrade test job from keeping failing after it failed once.
+# TODO: #5782 this struct could be changed to a map once we are testing against more releases
+BACKWARD_INCOMPATIBLE_TESTS=()
+
+function skip_breaking_tests {
+  for TEST in "${BACKWARD_INCOMPATIBLE_TESTS[@]}"; do
+    echo "Remove $TEST since it is a breaking change at devel"
+    rm ./test/"$TEST".go
+  done
+}
+
 # Script entry point.
 
 if [ "${SKIP_INITIALIZE}" != "true" ]; then
@@ -85,6 +101,19 @@ go_test_e2e -timeout=20m ./test || failed=1
 # the upgrade.
 go_test_e2e -tags=examples -timeout=20m ./test/ || failed=1
 
+# Testing main against the latest release to detect regression.
+# TODO: #5782 add previous LTS and supported releases
+header "Testing main against the latest release to detect regression"
+get_tests_from_release $PREVIOUS_PIPELINE_VERSION
+
+skip_breaking_tests
+
+# Run the integration tests.
+go_test_e2e -timeout=20m ./test || failed=1
+
+# Run the post-integration tests.
+go_test_e2e -tags=examples -timeout=20m ./test/ || failed=1
+
 (( failed )) && fail_test
 
 success