Run Spark JAR task test on multiple DBR versions

internal/bundle/helpers.go

@@ -12,6 +12,7 @@ import (
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/internal"
 	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/flags"
 	"github.com/databricks/cli/libs/template"
 	"github.com/databricks/databricks-sdk-go"
@@ -56,21 +57,21 @@ func writeConfigFile(t *testing.T, config map[string]any) (string, error) {
 }
 
 func validateBundle(t *testing.T, ctx context.Context, path string) ([]byte, error) {
-	t.Setenv("BUNDLE_ROOT", path)
+	ctx = env.Set(ctx, "BUNDLE_ROOT", path)
 	c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "validate", "--output", "json")
 	stdout, _, err := c.Run()
 	return stdout.Bytes(), err
 }
 
 func deployBundle(t *testing.T, ctx context.Context, path string) error {
-	t.Setenv("BUNDLE_ROOT", path)
+	ctx = env.Set(ctx, "BUNDLE_ROOT", path)
 	c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock", "--auto-approve")
 	_, _, err := c.Run()
 	return err
 }
 
 func deployBundleWithFlags(t *testing.T, ctx context.Context, path string, flags []string) error {
-	t.Setenv("BUNDLE_ROOT", path)
+	ctx = env.Set(ctx, "BUNDLE_ROOT", path)
 	args := []string{"bundle", "deploy", "--force-lock"}
 	args = append(args, flags...)
 	c := internal.NewCobraTestRunnerWithContext(t, ctx, args...)
@@ -79,6 +80,7 @@ func deployBundleWithFlags(t *testing.T, ctx context.Context, path string, flags
 }
 
 func runResource(t *testing.T, ctx context.Context, path string, key string) (string, error) {
+	ctx = env.Set(ctx, "BUNDLE_ROOT", path)
 	ctx = cmdio.NewContext(ctx, cmdio.Default())
 
 	c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "run", key)
@@ -87,6 +89,7 @@ func runResource(t *testing.T, ctx context.Context, path string, key string) (st
 }
 
 func runResourceWithParams(t *testing.T, ctx context.Context, path string, key string, params ...string) (string, error) {
+	ctx = env.Set(ctx, "BUNDLE_ROOT", path)
 	ctx = cmdio.NewContext(ctx, cmdio.Default())
 
 	args := make([]string, 0)
@@ -98,7 +101,7 @@ func runResourceWithParams(t *testing.T, ctx context.Context, path string, key s
 }
 
 func destroyBundle(t *testing.T, ctx context.Context, path string) error {
-	t.Setenv("BUNDLE_ROOT", path)
+	ctx = env.Set(ctx, "BUNDLE_ROOT", path)
 	c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "destroy", "--auto-approve")
 	_, _, err := c.Run()
 	return err

internal/bundle/spark_jar_test.go

@@ -1,28 +1,19 @@
 package bundle
 
 import (
-	"os"
+	"context"
 	"testing"
 
 	"github.com/databricks/cli/internal"
 	"github.com/databricks/cli/internal/acc"
+	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 )
 
-func runSparkJarTest(t *testing.T, sparkVersion string) {
+func runSparkJarTestCommon(t *testing.T, ctx context.Context, sparkVersion string, artifactPath string) {
 	cloudEnv := internal.GetEnvOrSkipTest(t, "CLOUD_ENV")
-	t.Log(cloudEnv)
-
-	if os.Getenv("TEST_METASTORE_ID") == "" {
-		t.Skip("Skipping tests that require a UC Volume when metastore id is not set.")
-	}
-
-	ctx, wt := acc.WorkspaceTest(t)
-	w := wt.W
-	volumePath := internal.TemporaryUcVolume(t, w)
-
 	nodeTypeId := internal.GetNodeTypeId(cloudEnv)
 	tmpDir := t.TempDir()
 	instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID")
@@ -31,7 +22,7 @@ func runSparkJarTest(t *testing.T, sparkVersion string) {
 		"unique_id":        uuid.New().String(),
 		"spark_version":    sparkVersion,
 		"root":             tmpDir,
-		"artifact_path":    volumePath,
+		"artifact_path":    artifactPath,
 		"instance_pool_id": instancePoolId,
 	}, tmpDir)
 	require.NoError(t, err)
@@ -48,6 +39,51 @@ func runSparkJarTest(t *testing.T, sparkVersion string) {
 	require.Contains(t, out, "Hello from Jar!")
 }
 
+func runSparkJarTestFromVolume(t *testing.T, sparkVersion string) {
+	ctx, wt := acc.UcWorkspaceTest(t)
+	volumePath := internal.TemporaryUcVolume(t, wt.W)
+	runSparkJarTestCommon(t, ctx, sparkVersion, volumePath)
+}
+
+func runSparkJarTestFromWorkspace(t *testing.T, sparkVersion string) {
+	ctx, _ := acc.WorkspaceTest(t)
+	runSparkJarTestCommon(t, ctx, sparkVersion, "${workspace.root_path}/artifacts")
+}
+
 func TestAccSparkJarTaskDeployAndRunOnVolumes(t *testing.T) {
-	runSparkJarTest(t, "14.3.x-scala2.12")
+	testutil.RequireJDK(t, context.Background(), "1.8.0")
+
+	versions := []string{
+		// Fails on 12.2 because we require DBR 13 to load JARs from a workspace path
+		// "12.2.x-scala2.12", // 12.2 LTS (includes Apache Spark 3.3.2, Scala 2.12)
+		"13.3.x-scala2.12", // 13.3 LTS (includes Apache Spark 3.4.1, Scala 2.12)
+		"14.3.x-scala2.12", // 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12)
+		"15.4.x-scala2.12", // 15.4 LTS Beta (includes Apache Spark 3.5.0, Scala 2.12)
+	}
+
+	for _, version := range versions {
+		t.Run(version, func(t *testing.T) {
+			t.Parallel()
+			runSparkJarTestFromVolume(t, version)
+		})
+	}
+}
+
+func TestAccSparkJarTaskDeployAndRunOnWorkspace(t *testing.T) {
+	testutil.RequireJDK(t, context.Background(), "1.8.0")
+
+	versions := []string{
+		// Fails on 12.2 because we require DBR 13 to load JARs from a workspace path
+		// "12.2.x-scala2.12", // 12.2 LTS (includes Apache Spark 3.3.2, Scala 2.12)
+		"13.3.x-scala2.12", // 13.3 LTS (includes Apache Spark 3.4.1, Scala 2.12)
+		"14.3.x-scala2.12", // 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12)
+		"15.4.x-scala2.12", // 15.4 LTS Beta (includes Apache Spark 3.5.0, Scala 2.12)
+	}
+
+	for _, version := range versions {
+		t.Run(version, func(t *testing.T) {
+			t.Parallel()
+			runSparkJarTestFromWorkspace(t, version)
+		})
+	}
 }

internal/testutil/jdk.go

@@ -0,0 +1,21 @@
+package testutil
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/databricks/cli/libs/process"
+	"github.com/stretchr/testify/require"
+)
+
+func RequireJDK(t *testing.T, ctx context.Context, version string) {
+	var stderr bytes.Buffer
+	err := process.Forwarded(ctx, []string{"javac", "-version"}, nil, nil, &stderr)
+	require.NoError(t, err, "Unable to run javac -version")
+
+	// Get the first line of the output
+	line := strings.Split(stderr.String(), "\n")[0]
+	require.Contains(t, line, version, "Expected JDK version %s, got %s", version, line)
+}