test: use workload identity for e2e test

Makefile

@@ -233,8 +233,8 @@ create-metrics-svc:
 
 .PHONY: install-smb-provisioner
 install-smb-provisioner:
-	kubectl delete secret smbcreds --ignore-not-found
-	kubectl create secret generic smbcreds --from-literal azurestorageaccountname=USERNAME --from-literal azurestorageaccountkey="PASSWORD"
+	kubectl delete secret smbcreds -n default --ignore-not-found
+	kubectl create secret generic smbcreds -n default --from-literal azurestorageaccountname=USERNAME --from-literal azurestorageaccountkey="PASSWORD"
 ifdef TEST_WINDOWS
 	kubectl apply -f deploy/example/smb-provisioner/smb-server-lb.yaml
 else

deploy/example/smb-provisioner/smb-server-lb.yaml

@@ -3,6 +3,7 @@ kind: Service
 apiVersion: v1
 metadata:
   name: smb-server
+  namespace: default
   labels:
     app: smb-server
 spec:
@@ -17,6 +18,7 @@ kind: Deployment
 apiVersion: apps/v1
 metadata:
   name: smb-server
+  namespace: default
 spec:
   replicas: 1
   selector:

deploy/example/smb-provisioner/smb-server.yaml

@@ -3,6 +3,7 @@ kind: Service
 apiVersion: v1
 metadata:
   name: smb-server
+  namespace: default
   labels:
     app: smb-server
 spec:
@@ -17,6 +18,7 @@ kind: Deployment
 apiVersion: apps/v1
 metadata:
   name: smb-server
+  namespace: default
 spec:
   replicas: 1
   selector:

go.mod

@@ -21,7 +21,6 @@ require (
 	github.com/onsi/ginkgo/v2 v2.19.1
 	github.com/onsi/gomega v1.34.1
 	github.com/pborman/uuid v1.2.0
-	github.com/pelletier/go-toml v1.9.4
 	github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021
 	github.com/stretchr/testify v1.9.0
 	go.uber.org/mock v0.4.0

go.sum

@@ -359,8 +359,6 @@ github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaL
 github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
-github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
-github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=

hack/verify-examples.sh

@@ -22,6 +22,8 @@ fi
 
 echo "begin to create deployment examples with parameter ", $1
 
+kubectl config set-context --current --namespace=default
+
 kubectl apply -f deploy/example/storageclass-azurefile-csi.yaml
 
 rollout_and_wait() {

pkg/azurefile/azure.go

@@ -58,20 +58,26 @@ func getCloudProvider(ctx context.Context, kubeconfig, nodeID, secretName, secre
 	)
 
 	az := &azure.Cloud{}
+	var err error
 
-	kubeCfg, err := getKubeConfig(kubeconfig, enableWindowsHostProcess)
-	if err == nil && kubeCfg != nil {
-		klog.V(2).Infof("set QPS(%f) and QPS Burst(%d) for driver kubeClient", float32(kubeAPIQPS), kubeAPIBurst)
-		kubeCfg.QPS = float32(kubeAPIQPS)
-		kubeCfg.Burst = kubeAPIBurst
-		kubeClient, err = clientset.NewForConfig(kubeCfg)
-		if err != nil {
-			klog.Warningf("NewForConfig failed with error: %v", err)
-		}
+	// for sanity test: if kubeconfig is set as "no-need-kubeconfig", kubeClient will be nil
+	if kubeconfig == "no-need-kubeconfig" {
+		klog.V(2).Infof("kubeconfig is set as no-need-kubeconfig, kubeClient will be nil")
 	} else {
-		klog.Warningf("get kubeconfig(%s) failed with error: %v", kubeconfig, err)
-		if !os.IsNotExist(err) && !errors.Is(err, rest.ErrNotInCluster) {
-			return az, fmt.Errorf("failed to get KubeClient: %v", err)
+		kubeCfg, err := getKubeConfig(kubeconfig, enableWindowsHostProcess)
+		if err == nil && kubeCfg != nil {
+			klog.V(2).Infof("set QPS(%f) and QPS Burst(%d) for driver kubeClient", float32(kubeAPIQPS), kubeAPIBurst)
+			kubeCfg.QPS = float32(kubeAPIQPS)
+			kubeCfg.Burst = kubeAPIBurst
+			kubeClient, err = clientset.NewForConfig(kubeCfg)
+			if err != nil {
+				klog.Warningf("NewForConfig failed with error: %v", err)
+			}
+		} else {
+			klog.Warningf("get kubeconfig(%s) failed with error: %v", kubeconfig, err)
+			if !os.IsNotExist(err) && !errors.Is(err, rest.ErrNotInCluster) {
+				return az, fmt.Errorf("failed to get KubeClient: %v", err)
+			}
 		}
 	}
 

test/e2e/suite_test.go

@@ -38,7 +38,6 @@ import (
 	"sigs.k8s.io/azurefile-csi-driver/test/e2e/driver"
 	"sigs.k8s.io/azurefile-csi-driver/test/utils/azure"
 	"sigs.k8s.io/azurefile-csi-driver/test/utils/credentials"
-	"sigs.k8s.io/azurefile-csi-driver/test/utils/testutil"
 )
 
 const (
@@ -89,10 +88,10 @@ var _ = ginkgo.BeforeSuite(func(ctx ginkgo.SpecContext) {
 
 	// Default storage driver configuration is CSI. Freshly built
 	// CSI driver is installed for that case.
-	if testutil.IsRunningInProw() && (isTestingMigration || !isUsingInTreeVolumePlugin) {
+	if isTestingMigration || !isUsingInTreeVolumePlugin {
 		creds, err := credentials.CreateAzureCredentialFile(false)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		azureClient, err := azure.GetAzureClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)
+		azureClient, err := azure.GetAzureClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret, creds.AADFederatedTokenFile)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 		_, err = azureClient.EnsureResourceGroup(ctx, creds.ResourceGroup, creds.Location, nil)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
@@ -157,87 +156,85 @@ var _ = ginkgo.BeforeSuite(func(ctx ginkgo.SpecContext) {
 })
 
 var _ = ginkgo.AfterSuite(func(ctx ginkgo.SpecContext) {
-	if testutil.IsRunningInProw() {
-		if isTestingMigration || isUsingInTreeVolumePlugin {
-			cmLog := testCmd{
-				command:  "bash",
-				args:     []string{"test/utils/controller-manager-log.sh"},
-				startLog: "===================controller-manager log=======",
-				endLog:   "===================================================",
+	if isTestingMigration || isUsingInTreeVolumePlugin {
+		cmLog := testCmd{
+			command:  "bash",
+			args:     []string{"test/utils/controller-manager-log.sh"},
+			startLog: "===================controller-manager log=======",
+			endLog:   "===================================================",
+		}
+		execTestCmd([]testCmd{cmLog})
+	}
+	if isTestingMigration || !isUsingInTreeVolumePlugin {
+		checkPodsRestart := testCmd{
+			command:  "bash",
+			args:     []string{"test/utils/check_driver_pods_restart.sh", "log"},
+			startLog: "Check driver pods if restarts ...",
+			endLog:   "Check successfully",
+		}
+		execTestCmd([]testCmd{checkPodsRestart})
+
+		os := "linux"
+		if isWindowsCluster {
+			os = "windows"
+			if winServerVer == "windows-2022" {
+				os = winServerVer
 			}
-			execTestCmd([]testCmd{cmLog})
 		}
-		if isTestingMigration || !isUsingInTreeVolumePlugin {
-			checkPodsRestart := testCmd{
+		createExampleDeployment := testCmd{
+			command:  "bash",
+			args:     []string{"hack/verify-examples.sh", os},
+			startLog: "create example deployments",
+			endLog:   "example deployments created",
+		}
+		execTestCmd([]testCmd{createExampleDeployment})
+
+		azurefileLog := testCmd{
+			command:     "bash",
+			args:        []string{"test/utils/azurefile_log.sh"},
+			startLog:    "===================azurefile log===================",
+			endLog:      "===================================================",
+			ignoreError: true,
+		}
+		e2eTeardown := testCmd{
+			command:  "make",
+			args:     []string{"e2e-teardown"},
+			startLog: "Uninstalling Azure File CSI Driver...",
+			endLog:   "Azure File CSI Driver uninstalled",
+		}
+		execTestCmd([]testCmd{azurefileLog, e2eTeardown})
+
+		if !isTestingMigration {
+			// install CSI Driver deployment scripts test
+			installDriver := testCmd{
 				command:  "bash",
-				args:     []string{"test/utils/check_driver_pods_restart.sh", "log"},
-				startLog: "Check driver pods if restarts ...",
-				endLog:   "Check successfully",
-			}
-			execTestCmd([]testCmd{checkPodsRestart})
-
-			os := "linux"
-			if isWindowsCluster {
-				os = "windows"
-				if winServerVer == "windows-2022" {
-					os = winServerVer
-				}
+				args:     []string{"deploy/install-driver.sh", "master", "windows,local"},
+				startLog: "===================install CSI Driver deployment scripts test===================",
+				endLog:   "===================================================",
 			}
+
 			createExampleDeployment := testCmd{
 				command:  "bash",
 				args:     []string{"hack/verify-examples.sh", os},
-				startLog: "create example deployments",
-				endLog:   "example deployments created",
+				startLog: "create example deployments#2",
+				endLog:   "example deployments#2 created",
 			}
 			execTestCmd([]testCmd{createExampleDeployment})
 
-			azurefileLog := testCmd{
-				command:     "bash",
-				args:        []string{"test/utils/azurefile_log.sh"},
-				startLog:    "===================azurefile log===================",
-				endLog:      "===================================================",
-				ignoreError: true,
-			}
-			e2eTeardown := testCmd{
-				command:  "make",
-				args:     []string{"e2e-teardown"},
-				startLog: "Uninstalling Azure File CSI Driver...",
-				endLog:   "Azure File CSI Driver uninstalled",
-			}
-			execTestCmd([]testCmd{azurefileLog, e2eTeardown})
-
-			if !isTestingMigration {
-				// install CSI Driver deployment scripts test
-				installDriver := testCmd{
-					command:  "bash",
-					args:     []string{"deploy/install-driver.sh", "master", "windows,local"},
-					startLog: "===================install CSI Driver deployment scripts test===================",
-					endLog:   "===================================================",
-				}
-
-				createExampleDeployment := testCmd{
-					command:  "bash",
-					args:     []string{"hack/verify-examples.sh", os},
-					startLog: "create example deployments#2",
-					endLog:   "example deployments#2 created",
-				}
-				execTestCmd([]testCmd{createExampleDeployment})
-
-				// uninstall CSI Driver deployment scripts test
-				uninstallDriver := testCmd{
-					command:  "bash",
-					args:     []string{"deploy/uninstall-driver.sh", "master", "windows,local"},
-					startLog: "===================uninstall CSI Driver deployment scripts test===================",
-					endLog:   "===================================================",
-				}
-				execTestCmd([]testCmd{installDriver, uninstallDriver})
+			// uninstall CSI Driver deployment scripts test
+			uninstallDriver := testCmd{
+				command:  "bash",
+				args:     []string{"deploy/uninstall-driver.sh", "master", "windows,local"},
+				startLog: "===================uninstall CSI Driver deployment scripts test===================",
+				endLog:   "===================================================",
 			}
+			execTestCmd([]testCmd{installDriver, uninstallDriver})
+		}
 
-			checkAccountCreationLeak(ctx)
+		checkAccountCreationLeak(ctx)
 
-			err := credentials.DeleteAzureCredentialFile()
-			gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		}
+		err := credentials.DeleteAzureCredentialFile()
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	}
 })
 
@@ -283,7 +280,7 @@ func execTestCmd(cmds []testCmd) {
 func checkAccountCreationLeak(ctx context.Context) {
 	creds, err := credentials.CreateAzureCredentialFile(false)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
-	azureClient, err := azure.GetAzureClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)
+	azureClient, err := azure.GetAzureClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret, creds.AADFederatedTokenFile)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 	accountNum, err := azureClient.GetAccountNumByResourceGroup(ctx, creds.ResourceGroup)

test/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go

@@ -117,7 +117,7 @@ func (t *DynamicallyProvisionedResizeVolumeTest) Run(ctx context.Context, client
 
 		creds, err := credentials.CreateAzureCredentialFile(false)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		azureClient, err := azure.GetAzureClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)
+		azureClient, err := azure.GetAzureClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret, creds.AADFederatedTokenFile)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 		var resourceGroup string

test/e2e/testsuites/dynamically_provisioned_tags_tester.go

@@ -60,7 +60,7 @@ func (t *DynamicallyProvisionedAccountWithTags) Run(ctx context.Context, client
 
 		creds, err := credentials.CreateAzureCredentialFile(false)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		azureClient, err := azureUtils.GetAzureClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)
+		azureClient, err := azureUtils.GetAzureClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret, creds.AADFederatedTokenFile)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 		account, err := azureClient.GetStorageAccount(ctx, resourceGroupName, accountName)

test/sanity/run-test.sh

@@ -47,7 +47,7 @@ if [ ! -f "$azcopyPath" ]; then
   chmod +x /usr/local/bin/azcopy
 fi
 
-_output/${ARCH}/azurefileplugin --endpoint "$endpoint" --nodeid "$nodeid" -v=5 &
+_output/${ARCH}/azurefileplugin --endpoint "$endpoint" --nodeid "$nodeid" -v=5 --kubeconfig "no-need-kubeconfig" &
 
 # sleep a while waiting for azurefileplugin start up
 sleep 1

test/sanity/sanity_test.go

@@ -46,7 +46,7 @@ func TestSanity(t *testing.T) {
 	os.Setenv("AZURE_CREDENTIAL_FILE", credentials.TempAzureCredentialFilePath)
 	os.Setenv("nodeid", nodeid)
 
-	azureClient, err := azure.GetAzureClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)
+	azureClient, err := azure.GetAzureClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret, creds.AADFederatedTokenFile)
 	assert.NoError(t, err)
 
 	ctx := context.Background()

test/utils/azure/azure_helpers.go

@@ -50,14 +50,20 @@ type Client struct {
 	sshPublicKeysClient sshpublickeyresourceclient.Interface
 }
 
-func GetAzureClient(cloud, subscriptionID, clientID, tenantID, clientSecret string) (*Client, error) {
+func GetAzureClient(cloud, subscriptionID, clientID, tenantID, clientSecret, aadFederatedTokenFile string) (*Client, error) {
 	armConfig := &azclient.ARMClientConfig{
 		Cloud:    cloud,
 		TenantID: tenantID,
 	}
+	useFederatedWorkloadIdentityExtension := false
+	if aadFederatedTokenFile != "" {
+		useFederatedWorkloadIdentityExtension = true
+	}
 	credProvider, err := azclient.NewAuthProvider(armConfig, &azclient.AzureAuthConfig{
-		AADClientID:     clientID,
-		AADClientSecret: clientSecret,
+		AADClientID:                           clientID,
+		AADClientSecret:                       clientSecret,
+		AADFederatedTokenFile:                 aadFederatedTokenFile,
+		UseFederatedWorkloadIdentityExtension: useFederatedWorkloadIdentityExtension,
 	})
 	if err != nil {
 		return nil, err