diff --git a/.github/workflows/kind_e2e.yaml b/.github/workflows/kind_e2e.yaml index eda30b41d..d3eac6785 100644 --- a/.github/workflows/kind_e2e.yaml +++ b/.github/workflows/kind_e2e.yaml @@ -173,6 +173,7 @@ jobs: export TEST_OUTPUT_FILE=$GITHUB_WORKSPACE/test-e2e-kind.json echo "TEST_OUTPUT_FILE=$TEST_OUTPUT_FILE" >> $GITHUB_ENV export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + export TEST_DAPR_HA_MODE=${{ matrix.mode }} make e2e-build-run-k8s shell: bash - name: Run tests with Docker hub @@ -181,6 +182,7 @@ jobs: export TEST_OUTPUT_FILE=$GITHUB_WORKSPACE/test-e2e-kind.json echo "TEST_OUTPUT_FILE=$TEST_OUTPUT_FILE" >> $GITHUB_ENV export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + export TEST_DAPR_HA_MODE=${{ matrix.mode }} make e2e-build-run-k8s shell: bash - name: Upload test results diff --git a/.github/workflows/upgrade_e2e.yaml b/.github/workflows/upgrade_e2e.yaml index 5c34bcf9d..83ebfb87b 100644 --- a/.github/workflows/upgrade_e2e.yaml +++ b/.github/workflows/upgrade_e2e.yaml @@ -137,6 +137,7 @@ jobs: run: | export TEST_OUTPUT_FILE=$GITHUB_WORKSPACE/test-e2e-upgrade-kind.json echo "TEST_OUTPUT_FILE=$TEST_OUTPUT_FILE" >> $GITHUB_ENV + export TEST_DAPR_HA_MODE=${{ matrix.mode }} make e2e-build-run-upgrade - name: Run tests with Docker hub @@ -144,6 +145,7 @@ jobs: run: | export TEST_OUTPUT_FILE=$GITHUB_WORKSPACE/test-e2e-upgrade-kind.json echo "TEST_OUTPUT_FILE=$TEST_OUTPUT_FILE" >> $GITHUB_ENV + export TEST_DAPR_HA_MODE=${{ matrix.mode }} make e2e-build-run-upgrade - name: Upload test results diff --git a/Makefile b/Makefile index 298c35d2d..3bc4b4005 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,7 @@ e2e-build-run-k8s: build test-e2e-k8s ################################################################################ .PHONY: test-e2e-upgrade test-e2e-upgrade: test-deps - gotestsum --jsonfile $(TEST_OUTPUT_FILE) --format standard-verbose -- -timeout 40m -count=1 -tags=e2e ./tests/e2e/upgrade/... + gotestsum --jsonfile $(TEST_OUTPUT_FILE) --format standard-verbose -- -timeout 60m -count=1 -tags=e2e ./tests/e2e/upgrade/... ################################################################################ # Build, E2E Tests for Kubernetes Upgrade # diff --git a/pkg/kubernetes/upgrade.go b/pkg/kubernetes/upgrade.go index a1505c67c..f65a5c36a 100644 --- a/pkg/kubernetes/upgrade.go +++ b/pkg/kubernetes/upgrade.go @@ -14,6 +14,8 @@ limitations under the License. package kubernetes import ( + "context" + "errors" "fmt" "net/http" "os" @@ -23,8 +25,11 @@ import ( helm "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/release" + core_v1 "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/helm/pkg/strvals" + "github.com/Masterminds/semver/v3" "github.com/hashicorp/go-version" "github.com/dapr/cli/pkg/print" @@ -49,6 +54,8 @@ var crdsFullResources = []string{ "httpendpoints.dapr.io", } +var versionWithHAScheduler = semver.MustParse("1.15.0-rc.1") + type UpgradeConfig struct { RuntimeVersion string DashboardVersion string @@ -157,6 +164,10 @@ func Upgrade(conf UpgradeConfig) error { return err } + // used to signal the deletion of the scheduler pods only when downgrading from 1.15 to previous versions to handle incompatible changes + // in other cases the channel should be nil + var downgradeDeletionChan chan error + if !isDowngrade(conf.RuntimeVersion, daprVersion) { err = applyCRDs("v" + conf.RuntimeVersion) if err != nil { @@ -164,6 +175,29 @@ func Upgrade(conf UpgradeConfig) error { } } else { print.InfoStatusEvent(os.Stdout, "Downgrade detected, skipping CRDs.") + + targetVersion, errVersion := semver.NewVersion(conf.RuntimeVersion) + if errVersion != nil { + return fmt.Errorf("unable to parse dapr target version: %w", errVersion) + } + + currentVersion, errVersion := semver.NewVersion(daprVersion) + if errVersion != nil { + return fmt.Errorf("unable to parse dapr current version: %w", errVersion) + } + + if currentVersion.GreaterThanEqual(versionWithHAScheduler) && targetVersion.LessThan(versionWithHAScheduler) { + downgradeDeletionChan = make(chan error) + // Must delete all scheduler pods from cluster due to incompatible changes in version 1.15 with older versions. + go func() { + errDeletion := deleteSchedulerPods(status[0].Namespace, currentVersion, targetVersion) + if errDeletion != nil { + downgradeDeletionChan <- fmt.Errorf("failed to delete scheduler pods: %w", errDeletion) + print.FailureStatusEvent(os.Stderr, "Failed to delete scheduler pods: "+errDeletion.Error()) + } + close(downgradeDeletionChan) + }() + } } chart, err := GetDaprHelmChartName(helmConf) @@ -180,6 +214,15 @@ func Upgrade(conf UpgradeConfig) error { return fmt.Errorf("failure while running upgrade: %w", err) } + // wait for the deletion of the scheduler pods to finish + if downgradeDeletionChan != nil { + select { + case <-downgradeDeletionChan: + case <-time.After(3 * time.Minute): + return errors.New("timed out waiting for downgrade deletion") + } + } + if dashboardChart != nil { if dashboardExists { if _, err = upgradeClient.Run(dashboardReleaseName, dashboardChart, vals); err != nil { @@ -202,6 +245,73 @@ func Upgrade(conf UpgradeConfig) error { return nil } +func deleteSchedulerPods(namespace string, currentVersion *semver.Version, targetVersion *semver.Version) error { + ctxWithTimeout, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + var pods *core_v1.PodList + + // wait for at least one pod of the target version to be in the list before deleting the rest + // check the label app.kubernetes.io/version to determine the version of the pod + foundTargetVersion := false + for { + if foundTargetVersion { + break + } + k8sClient, err := Client() + if err != nil { + return err + } + + pods, err = k8sClient.CoreV1().Pods(namespace).List(ctxWithTimeout, meta_v1.ListOptions{ + LabelSelector: "app=dapr-scheduler-server", + }) + if err != nil && !errors.Is(err, context.DeadlineExceeded) { + return err + } + + if len(pods.Items) == 0 { + return nil + } + + for _, pod := range pods.Items { + pv, ok := pod.Labels["app.kubernetes.io/version"] + if ok { + podVersion, err := semver.NewVersion(pv) + if err == nil && podVersion.Equal(targetVersion) { + foundTargetVersion = true + break + } + } + } + time.Sleep(5 * time.Second) + } + + if pods == nil { + return errors.New("no scheduler pods found") + } + + // get a fresh client to ensure we have the latest state of the cluster + k8sClient, err := Client() + if err != nil { + return err + } + + // delete scheduler pods of the current version, i.e. >1.15.0 + for _, pod := range pods.Items { + if pv, ok := pod.Labels["app.kubernetes.io/version"]; ok { + podVersion, err := semver.NewVersion(pv) + if err == nil && podVersion.Equal(currentVersion) { + err = k8sClient.CoreV1().Pods(namespace).Delete(ctxWithTimeout, pod.Name, meta_v1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete pod %s during downgrade: %w", pod.Name, err) + } + } + } + } + return nil +} + // WithRetry enables retry with the specified max retries and retry interval. func WithRetry(maxRetries int, retryInterval time.Duration) UpgradeOption { return func(o *UpgradeOptions) { diff --git a/tests/e2e/common/common.go b/tests/e2e/common/common.go index 1dccc090a..f94af9c3f 100644 --- a/tests/e2e/common/common.go +++ b/tests/e2e/common/common.go @@ -56,6 +56,9 @@ const ( thirdPartyDevNamespace = "default" devRedisReleaseName = "dapr-dev-redis" devZipkinReleaseName = "dapr-dev-zipkin" + + DaprModeHA = "ha" + DaprModeNonHA = "non-ha" ) var ( @@ -120,6 +123,22 @@ func GetRuntimeVersion(t *testing.T, latest bool) *semver.Version { return runtimeVersion } +func GetDaprTestHaMode() string { + daprHaMode := os.Getenv("TEST_DAPR_HA_MODE") + if daprHaMode != "" { + return daprHaMode + } + return "" +} + +func ShouldSkipTest(mode string) bool { + envDaprHaMode := GetDaprTestHaMode() + if envDaprHaMode != "" { + return envDaprHaMode != mode + } + return false +} + func UpgradeTest(details VersionDetails, opts TestOptions) func(t *testing.T) { return func(t *testing.T) { daprPath := GetDaprPath() @@ -213,7 +232,7 @@ func GetTestsOnInstall(details VersionDetails, opts TestOptions) []TestCase { {"clusterroles exist " + details.RuntimeVersion, ClusterRolesTest(details, opts)}, {"clusterrolebindings exist " + details.RuntimeVersion, ClusterRoleBindingsTest(details, opts)}, {"apply and check components exist " + details.RuntimeVersion, ComponentsTestOnInstallUpgrade(opts)}, - {"apply and check httpendpoints exist " + details.RuntimeVersion, HTTPEndpointsTestOnInstallUpgrade(opts)}, + {"apply and check httpendpoints exist " + details.RuntimeVersion, HTTPEndpointsTestOnInstallUpgrade(opts, TestOptions{})}, {"check mtls " + details.RuntimeVersion, MTLSTestOnInstallUpgrade(opts)}, {"status check " + details.RuntimeVersion, StatusTestOnInstallUpgrade(details, opts)}, } @@ -341,10 +360,10 @@ func ComponentsTestOnInstallUpgrade(opts TestOptions) func(t *testing.T) { } } -func HTTPEndpointsTestOnInstallUpgrade(opts TestOptions) func(t *testing.T) { +func HTTPEndpointsTestOnInstallUpgrade(installOpts TestOptions, upgradeOpts TestOptions) func(t *testing.T) { return func(t *testing.T) { // if dapr is installed with httpendpoints. - if opts.ApplyHTTPEndpointChanges { + if installOpts.ApplyHTTPEndpointChanges { // apply any changes to the httpendpoint. t.Log("apply httpendpoint changes") output, err := spawn.Command("kubectl", "apply", "-f", "../testdata/namespace.yaml") @@ -353,12 +372,17 @@ func HTTPEndpointsTestOnInstallUpgrade(opts TestOptions) func(t *testing.T) { output, err = spawn.Command("kubectl", "apply", "-f", "../testdata/httpendpoint.yaml") t.Log(output) require.NoError(t, err, "expected no error on kubectl apply") - require.Equal(t, "httpendpoints.dapr.io/httpendpoint created\nhttpendpoints.dapr.io/httpendpoint created\n", output, "expected output to match") - httpEndpointOutputCheck(t, output) + + if installOpts.ApplyHTTPEndpointChanges && upgradeOpts.ApplyHTTPEndpointChanges { + require.Equal(t, "httpendpoint.dapr.io/httpendpoint unchanged\n", output, "expected output to match") + } else { + require.Equal(t, "httpendpoint.dapr.io/httpendpoint created\n", output, "expected output to match") + } t.Log("check applied httpendpoint exists") - _, err = spawn.Command("kubectl", "get", "httpendpoint") + output, err = spawn.Command("kubectl", "get", "httpendpoint") require.NoError(t, err, "expected no error on calling to retrieve httpendpoints") + httpEndpointOutputCheck(t, output) } } } @@ -984,7 +1008,7 @@ func componentOutputCheck(t *testing.T, opts TestOptions, output string) { return } - lines = strings.Split(output, "\n")[2:] // remove header and warning message. + lines = lines[2:] // remove header and warning message. if opts.DevEnabled { // default, test statestore. @@ -1152,6 +1176,8 @@ func waitPodDeletionDev(t *testing.T, done, podsDeleted chan struct{}) { devRedisReleaseName: "dapr-dev-redis-master-", devZipkinReleaseName: "dapr-dev-zipkin-", } + + t.Logf("dev pods waiting to be deleted: %d", len(list.Items)) for _, pod := range list.Items { t.Log(pod.ObjectMeta.Name) for component, prefix := range prefixes { @@ -1169,7 +1195,7 @@ func waitPodDeletionDev(t *testing.T, done, podsDeleted chan struct{}) { if len(found) == 2 { podsDeleted <- struct{}{} } - time.Sleep(15 * time.Second) + time.Sleep(10 * time.Second) } } @@ -1181,19 +1207,28 @@ func waitPodDeletion(t *testing.T, done, podsDeleted chan struct{}) { default: break } + ctx := context.Background() ctxt, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() + k8sClient, err := getClient() require.NoError(t, err, "error getting k8s client for pods check") + list, err := k8sClient.CoreV1().Pods(DaprTestNamespace).List(ctxt, v1.ListOptions{ Limit: 100, }) require.NoError(t, err, "error getting pods list from k8s") + if len(list.Items) == 0 { podsDeleted <- struct{}{} + } else { + t.Logf("pods waiting to be deleted: %d", len(list.Items)) + for _, pod := range list.Items { + t.Log(pod.ObjectMeta.Name) + } } - time.Sleep(15 * time.Second) + time.Sleep(5 * time.Second) } } @@ -1214,7 +1249,8 @@ func waitAllPodsRunning(t *testing.T, namespace string, haEnabled bool, done, po Limit: 100, }) require.NoError(t, err, "error getting pods list from k8s") - t.Logf("items %d", len(list.Items)) + + t.Logf("waiting for pods to be running, current count: %d", len(list.Items)) countOfReadyPods := 0 for _, item := range list.Items { t.Log(item.ObjectMeta.Name) @@ -1235,11 +1271,11 @@ func waitAllPodsRunning(t *testing.T, namespace string, haEnabled bool, done, po if err != nil { t.Error(err) } - if len(list.Items) == countOfReadyPods && ((haEnabled && countOfReadyPods == pods) || (!haEnabled && countOfReadyPods == pods)) { + if len(list.Items) == countOfReadyPods && countOfReadyPods == pods { podsRunning <- struct{}{} } - time.Sleep(15 * time.Second) + time.Sleep(5 * time.Second) } } diff --git a/tests/e2e/kubernetes/kubernetes_test.go b/tests/e2e/kubernetes/kubernetes_test.go index d080eba4c..5f0f362b9 100644 --- a/tests/e2e/kubernetes/kubernetes_test.go +++ b/tests/e2e/kubernetes/kubernetes_test.go @@ -17,12 +17,17 @@ limitations under the License. package kubernetes_test import ( + "fmt" "testing" "github.com/dapr/cli/tests/e2e/common" ) func TestKubernetesNonHAModeMTLSDisabled(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -54,6 +59,10 @@ func TestKubernetesNonHAModeMTLSDisabled(t *testing.T) { } func TestKubernetesHAModeMTLSDisabled(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -85,6 +94,10 @@ func TestKubernetesHAModeMTLSDisabled(t *testing.T) { } func TestKubernetesDev(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -119,6 +132,10 @@ func TestKubernetesDev(t *testing.T) { } func TestKubernetesNonHAModeMTLSEnabled(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -150,6 +167,10 @@ func TestKubernetesNonHAModeMTLSEnabled(t *testing.T) { } func TestKubernetesHAModeMTLSEnabled(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -182,6 +203,10 @@ func TestKubernetesHAModeMTLSEnabled(t *testing.T) { } func TestKubernetesInitWithCustomCert(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -216,6 +241,10 @@ func TestKubernetesInitWithCustomCert(t *testing.T) { // Test for certificate renewal func TestRenewCertificateMTLSEnabled(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -251,6 +280,10 @@ func TestRenewCertificateMTLSEnabled(t *testing.T) { } func TestRenewCertificateMTLSDisabled(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -286,6 +319,10 @@ func TestRenewCertificateMTLSDisabled(t *testing.T) { } func TestRenewCertWithPrivateKey(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -328,6 +365,10 @@ func TestRenewCertWithPrivateKey(t *testing.T) { } func TestKubernetesUninstall(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -359,6 +400,10 @@ func TestKubernetesUninstall(t *testing.T) { } func TestRenewCertWithIncorrectFlags(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + common.EnsureUninstall(true, true) tests := []common.TestCase{} @@ -397,6 +442,10 @@ func TestRenewCertWithIncorrectFlags(t *testing.T) { // install dapr control plane with mariner docker images. // Renew the certificate of this control plane. func TestK8sInstallwithMarinerImagesAndRenewCertificate(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, false) @@ -435,6 +484,10 @@ func TestK8sInstallwithMarinerImagesAndRenewCertificate(t *testing.T) { } func TestKubernetesInstallwithoutRuntimeVersionFlag(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, true) @@ -467,6 +520,10 @@ func TestKubernetesInstallwithoutRuntimeVersionFlag(t *testing.T) { } func TestK8sInstallwithoutRuntimeVersionwithMarinerImagesFlag(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // ensure clean env for test ensureCleanEnv(t, true) diff --git a/tests/e2e/kubernetes/run_template_test.go b/tests/e2e/kubernetes/run_template_test.go index 5d050f4cc..10dc27a11 100644 --- a/tests/e2e/kubernetes/run_template_test.go +++ b/tests/e2e/kubernetes/run_template_test.go @@ -47,6 +47,10 @@ var ( ) func TestKubernetesRunFile(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + ensureCleanEnv(t, false) // setup tests diff --git a/tests/e2e/upgrade/upgrade_test.go b/tests/e2e/upgrade/upgrade_test.go index 0f5a8e0c7..5998fdeff 100644 --- a/tests/e2e/upgrade/upgrade_test.go +++ b/tests/e2e/upgrade/upgrade_test.go @@ -33,48 +33,46 @@ var supportedUpgradePaths = []upgradePath{ { // test upgrade on mariner images. previous: common.VersionDetails{ - RuntimeVersion: "1.8.0", - DashboardVersion: "0.10.0", - ImageVariant: "mariner", - ClusterRoles: []string{"dapr-operator-admin", "dashboard-reader"}, - ClusterRoleBindings: []string{"dapr-operator", "dapr-role-tokenreview-binding", "dashboard-reader-global"}, - CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io"}, + RuntimeVersion: "1.14.4", + DashboardVersion: "0.15.0", + ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, + ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, + CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"}, }, next: common.VersionDetails{ - RuntimeVersion: "1.8.7", - DashboardVersion: "0.10.0", - ImageVariant: "mariner", - ClusterRoles: []string{"dapr-operator-admin", "dashboard-reader"}, - ClusterRoleBindings: []string{"dapr-operator", "dapr-role-tokenreview-binding", "dashboard-reader-global"}, - CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io"}, + RuntimeVersion: "1.15.2", + DashboardVersion: "0.15.0", + ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, + ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, + CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"}, }, }, { previous: common.VersionDetails{ - RuntimeVersion: "1.9.5", - DashboardVersion: "0.11.0", - ClusterRoles: []string{"dapr-operator-admin", "dashboard-reader"}, - ClusterRoleBindings: []string{"dapr-operator", "dapr-role-tokenreview-binding", "dashboard-reader-global"}, - CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io"}, + RuntimeVersion: "1.13.6", + DashboardVersion: "0.14.0", + ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, + ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, + CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"}, }, next: common.VersionDetails{ - RuntimeVersion: "1.10.7", - DashboardVersion: "0.12.0", + RuntimeVersion: "1.14.4", + DashboardVersion: "0.14.0", ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, - CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io"}, + CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"}, }, }, { previous: common.VersionDetails{ - RuntimeVersion: "1.11.0", + RuntimeVersion: "1.13.6", DashboardVersion: "0.14.0", ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"}, }, next: common.VersionDetails{ - RuntimeVersion: "1.12.0", + RuntimeVersion: "1.15.2", DashboardVersion: "0.14.0", ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, @@ -83,14 +81,14 @@ var supportedUpgradePaths = []upgradePath{ }, { previous: common.VersionDetails{ - RuntimeVersion: "1.12.0", + RuntimeVersion: "1.14.4", DashboardVersion: "0.14.0", ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"}, }, next: common.VersionDetails{ - RuntimeVersion: "1.13.0-rc.2", + RuntimeVersion: "1.15.2", DashboardVersion: "0.14.0", ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, @@ -100,14 +98,30 @@ var supportedUpgradePaths = []upgradePath{ // test downgrade. { previous: common.VersionDetails{ - RuntimeVersion: "1.13.0-rc.2", + RuntimeVersion: "1.15.2", + DashboardVersion: "0.14.0", + ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, + ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, + CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"}, + }, + next: common.VersionDetails{ + RuntimeVersion: "1.14.4", + DashboardVersion: "0.14.0", + ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, + ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, + CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"}, + }, + }, + { + previous: common.VersionDetails{ + RuntimeVersion: "1.15.2", DashboardVersion: "0.14.0", ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"}, }, next: common.VersionDetails{ - RuntimeVersion: "1.12.0", + RuntimeVersion: "1.13.6", DashboardVersion: "0.14.0", ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, @@ -116,14 +130,14 @@ var supportedUpgradePaths = []upgradePath{ }, { previous: common.VersionDetails{ - RuntimeVersion: "1.12.0", + RuntimeVersion: "1.14.4", DashboardVersion: "0.14.0", ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, CustomResourceDefs: []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"}, }, next: common.VersionDetails{ - RuntimeVersion: "1.11.0", + RuntimeVersion: "1.13.6", DashboardVersion: "0.14.0", ClusterRoles: []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}, ClusterRoleBindings: []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}, @@ -146,7 +160,7 @@ func getTestsOnUpgrade(p upgradePath, installOpts, upgradeOpts common.TestOption {Name: "clusterroles exist " + details.RuntimeVersion, Callable: common.ClusterRolesTest(details, upgradeOpts)}, {Name: "clusterrolebindings exist " + details.RuntimeVersion, Callable: common.ClusterRoleBindingsTest(details, upgradeOpts)}, {Name: "previously applied components exist " + details.RuntimeVersion, Callable: common.ComponentsTestOnInstallUpgrade(upgradeOpts)}, - {Name: "previously applied http endpoints exist " + details.RuntimeVersion, Callable: common.HTTPEndpointsTestOnInstallUpgrade(upgradeOpts)}, + {Name: "previously applied http endpoints exist " + details.RuntimeVersion, Callable: common.HTTPEndpointsTestOnInstallUpgrade(installOpts, upgradeOpts)}, {Name: "check mtls " + details.RuntimeVersion, Callable: common.MTLSTestOnInstallUpgrade(upgradeOpts)}, {Name: "status check " + details.RuntimeVersion, Callable: common.StatusTestOnInstallUpgrade(details, upgradeOpts)}, }...) @@ -172,16 +186,19 @@ func getTestsOnUpgrade(p upgradePath, installOpts, upgradeOpts common.TestOption // Upgrade path tests. func TestUpgradePathNonHAModeMTLSDisabled(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // Ensure a clean environment. common.EnsureUninstall(false, false) // does not wait for pod deletion. + for _, p := range supportedUpgradePaths { t.Run(fmt.Sprintf("setup v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { t.Run(deleteCRDs+p.previous.RuntimeVersion, common.DeleteCRD(p.previous.CustomResourceDefs)) t.Run(deleteCRDs+p.next.RuntimeVersion, common.DeleteCRD(p.next.CustomResourceDefs)) }) - } - for _, p := range supportedUpgradePaths { t.Run(fmt.Sprintf("v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { installOpts := common.TestOptions{ HAEnabled: false, @@ -217,16 +234,19 @@ func TestUpgradePathNonHAModeMTLSDisabled(t *testing.T) { } func TestUpgradePathNonHAModeMTLSEnabled(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeNonHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeNonHA)) + } + // Ensure a clean environment. common.EnsureUninstall(false, false) // does not wait for pod deletion. + for _, p := range supportedUpgradePaths { t.Run(fmt.Sprintf("setup v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { t.Run(deleteCRDs+p.previous.RuntimeVersion, common.DeleteCRD(p.previous.CustomResourceDefs)) t.Run(deleteCRDs+p.next.RuntimeVersion, common.DeleteCRD(p.next.CustomResourceDefs)) }) - } - for _, p := range supportedUpgradePaths { t.Run(fmt.Sprintf("v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { installOpts := common.TestOptions{ HAEnabled: false, @@ -262,16 +282,18 @@ func TestUpgradePathNonHAModeMTLSEnabled(t *testing.T) { } func TestUpgradePathHAModeMTLSDisabled(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeHA)) + } + // Ensure a clean environment. common.EnsureUninstall(false, false) // does not wait for pod deletion. + for _, p := range supportedUpgradePaths { t.Run(fmt.Sprintf("setup v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { t.Run(deleteCRDs+p.previous.RuntimeVersion, common.DeleteCRD(p.previous.CustomResourceDefs)) t.Run(deleteCRDs+p.next.RuntimeVersion, common.DeleteCRD(p.next.CustomResourceDefs)) }) - } - - for _, p := range supportedUpgradePaths { t.Run(fmt.Sprintf("v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { installOpts := common.TestOptions{ HAEnabled: true, @@ -307,16 +329,19 @@ func TestUpgradePathHAModeMTLSDisabled(t *testing.T) { } func TestUpgradePathHAModeMTLSEnabled(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeHA)) + } + // Ensure a clean environment. common.EnsureUninstall(false, false) // does not wait for pod deletion. + for _, p := range supportedUpgradePaths { t.Run(fmt.Sprintf("setup v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { t.Run(deleteCRDs+p.previous.RuntimeVersion, common.DeleteCRD(p.previous.CustomResourceDefs)) t.Run(deleteCRDs+p.next.RuntimeVersion, common.DeleteCRD(p.next.CustomResourceDefs)) }) - } - for _, p := range supportedUpgradePaths { t.Run(fmt.Sprintf("v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { installOpts := common.TestOptions{ HAEnabled: true, @@ -354,14 +379,12 @@ func TestUpgradePathHAModeMTLSEnabled(t *testing.T) { // HTTPEndpoint Dapr resource is a new type as of v1.11. // This test verifies install/upgrade functionality with this additional resource. func TestUpgradeWithHTTPEndpoint(t *testing.T) { + if common.ShouldSkipTest(common.DaprModeHA) { + t.Skip(fmt.Sprintf("Skipping %s mode test", common.DaprModeHA)) + } + // Ensure a clean environment. common.EnsureUninstall(false, false) // does not wait for pod deletion. - for _, p := range supportedUpgradePaths { - t.Run(fmt.Sprintf("setup v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { - t.Run(deleteCRDs+p.previous.RuntimeVersion, common.DeleteCRD(p.previous.CustomResourceDefs)) - t.Run(deleteCRDs+p.next.RuntimeVersion, common.DeleteCRD(p.next.CustomResourceDefs)) - }) - } for _, p := range supportedUpgradePaths { ver, err := semver.NewVersion(p.previous.RuntimeVersion) @@ -373,11 +396,17 @@ func TestUpgradeWithHTTPEndpoint(t *testing.T) { if ver.Major() != 1 || ver.Minor() < 11 { return } + + t.Run(fmt.Sprintf("setup v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { + t.Run(deleteCRDs+p.previous.RuntimeVersion, common.DeleteCRD(p.previous.CustomResourceDefs)) + t.Run(deleteCRDs+p.next.RuntimeVersion, common.DeleteCRD(p.next.CustomResourceDefs)) + }) + t.Run(fmt.Sprintf("v%s to v%s", p.previous.RuntimeVersion, p.next.RuntimeVersion), func(t *testing.T) { installOpts := common.TestOptions{ HAEnabled: true, MTLSEnabled: true, - ApplyComponentChanges: false, + ApplyComponentChanges: true, ApplyHTTPEndpointChanges: true, CheckResourceExists: map[common.Resource]bool{ common.CustomResourceDefs: true,