feat: support workload identity setting in static PV mount (kubernete…

…s-sigs#1204) * feat: support workload identity setting in static PV mount * fix
cvvz · Jan 8, 2024 · 78b9a9e · 78b9a9e
1 parent cf3db2e
commit 78b9a9e
Show file tree

Hide file tree

Showing 5 changed files with 225 additions and 0 deletions.
diff --git a/charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml b/charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml
@@ -12,3 +12,5 @@ spec:
   volumeLifecycleModes:
     - Persistent
     - Ephemeral
+  tokenRequests:
+    - audience: api://AzureADTokenExchange
diff --git a/docs/workload-identity.md → docs/workload-identity-deploy-csi-driver.md b/docs/workload-identity.md → docs/workload-identity-deploy-csi-driver.md
diff --git a/docs/workload-identity-static-pv-mount.md b/docs/workload-identity-static-pv-mount.md
@@ -0,0 +1,178 @@
+# Example of static PV mount with workload identity
+
+> Note: 
+> - Available kubernetes version >= v1.20
+
+## prerequisite
+
+
+### 1. Create a cluster with oidc-issuer enabled and get the credential
+
+Following the [documentation](https://learn.microsoft.com/en-us/azure/aks/use-oidc-issuer#create-an-aks-cluster-with-oidc-issuer) to create an AKS cluster with the `--enable-oidc-issuer` parameter and get the AKS credentials. And export following environment variables:
+```
+export RESOURCE_GROUP=<your resource group name>
+export CLUSTER_NAME=<your cluster name>
+export REGION=<your region>
+```
+
+
+### 2. Create a new storage account and container
+
+Following the [documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-cli) to create a new storage account and container or use your own. And export following environment variables:
+```
+export STORAGE_RESOURCE_GROUP=<your storage account resource group>
+export ACCOUNT=<your storage account name>
+export CONTAINER=<your container name>
+```
+
+### 3. Create managed identity and role assignment
+```
+export UAMI=<your managed identity name>
+az identity create --name $UAMI --resource-group $RESOURCE_GROUP
+
+export USER_ASSIGNED_CLIENT_ID="$(az identity show -g $RESOURCE_GROUP --name $UAMI --query 'clientId' -o tsv)"
+export IDENTITY_TENANT=$(az aks show --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --query identity.tenantId -o tsv)
+export ACCOUNT_SCOPE=$(az storage account show --name $ACCOUNT --query id -o tsv)
+
+# please retry if you meet `Cannot find user or service principal in graph database` error, it may take a while for the identity to propagate
+az role assignment create --role "Storage Account Contributor" --assignee $USER_ASSIGNED_CLIENT_ID --scope $ACCOUNT_SCOPE
+```
+
+### 4. Create service account on AKS
+```
+export SERVICE_ACCOUNT_NAME=<your sa name>
+export SERVICE_ACCOUNT_NAMESPACE=<your sa namespace>
+
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ${SERVICE_ACCOUNT_NAME}
+  namespace: ${SERVICE_ACCOUNT_NAMESPACE}
+EOF
+```
+
+### 5. Create the federated identity credential between the managed identity, service account issuer, and subject using the `az identity federated-credential create` command.
+```
+export FEDERATED_IDENTITY_NAME=<your federated identity name>
+export AKS_OIDC_ISSUER="$(az aks show --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --query "oidcIssuerProfile.issuerUrl" -o tsv)"
+
+az identity federated-credential create --name $FEDERATED_IDENTITY_NAME \
+--identity-name $UAMI \
+--resource-group $RESOURCE_GROUP \
+--issuer $AKS_OIDC_ISSUER \
+--subject system:serviceaccount:${SERVICE_ACCOUNT_NAMESPACE}:${SERVICE_ACCOUNT_NAME}
+```
+
+## option#1: static provision with PV
+```
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  annotations:
+    pv.kubernetes.io/provisioned-by: blob.csi.azure.com
+  name: pv-blob
+spec:
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: blob-fuse
+  mountOptions:
+    - -o allow_other
+    - --file-cache-timeout-in-seconds=120
+  csi:
+    driver: blob.csi.azure.com
+    # make sure volumeid is unique for every storage blob container in the cluster
+    # the # character is reserved for internal use, the / character is not allowed
+    volumeHandle: unique_volume_id
+    volumeAttributes:
+      storageaccount: $ACCOUNT # required
+      containerName: $CONTAINER  # required
+      clientID: $USER_ASSIGNED_CLIENT_ID # required
+      resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_")
+      # tenantID: $IDENTITY_TENANT  #optional, only specified when workload identity and AKS cluster are in different tenant
+      # subscriptionid: $SUBSCRIPTION #optional, only specified when workload identity and AKS cluster are in different subscription
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: statefulset-blob
+  labels:
+    app: nginx
+spec:
+  serviceName: statefulset-blob
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      serviceAccountName: $SERVICE_ACCOUNT_NAME  #required, Pod does not use this service account has no permission to mount the volume
+      nodeSelector:
+        "kubernetes.io/os": linux
+      containers:
+        - name: statefulset-blob
+          image: mcr.microsoft.com/oss/nginx/nginx:1.19.5
+          command:
+            - "/bin/bash"
+            - "-c"
+            - set -euo pipefail; while true; do echo $(date) >> /mnt/blob/outfile; sleep 1; done
+          volumeMounts:
+            - name: persistent-storage
+              mountPath: /mnt/blob
+              readOnly: false
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: nginx
+  volumeClaimTemplates:
+    - metadata:
+        name: persistent-storage
+      spec:
+        storageClassName: blob-fuse
+        accessModes: ["ReadWriteMany"]
+        resources:
+          requests:
+            storage: 10Gi
+EOF
+```
+
+## option#2: Pod with ephemeral inline volume
+```
+cat <<EOF | kubectl apply -f -
+kind: Pod
+apiVersion: v1
+metadata:
+  name: nginx-blobfuse-inline-volume
+spec:
+  serviceAccountName: $SERVICE_ACCOUNT_NAME  #required, Pod does not use this service account has no permission to mount the volume
+  nodeSelector:
+    "kubernetes.io/os": linux
+  containers:
+    - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5
+      name: nginx-blobfuse
+      command:
+        - "/bin/bash"
+        - "-c"
+        - set -euo pipefail; while true; do echo $(date) >> /mnt/blobfuse/outfile; sleep 1; done
+      volumeMounts:
+        - name: persistent-storage
+          mountPath: "/mnt/blobfuse"
+          readOnly: false
+  volumes:
+    - name: persistent-storage
+      csi:
+        driver: blob.csi.azure.com
+        volumeAttributes:
+          storageaccount: $ACCOUNT # required
+          containerName: $CONTAINER  # required
+          clientID: $USER_ASSIGNED_CLIENT_ID # required
+          resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_")
+          # tenantID: $IDENTITY_TENANT  # optional, only specified when workload identity and AKS cluster are in different tenant
+          # subscriptionid: $SUBSCRIPTION # optional, only specified when workload identity and AKS cluster are in different subscription
+EOF
+```
diff --git a/pkg/blob/blob.go b/pkg/blob/blob.go
@@ -96,6 +96,9 @@ const (
 	requireInfraEncryptionField    = "requireinfraencryption"
 	ephemeralField                 = "csi.storage.k8s.io/ephemeral"
 	podNamespaceField              = "csi.storage.k8s.io/pod.namespace"
+	serviceAccountTokenField       = "csi.storage.k8s.io/serviceAccount.tokens"
+	clientIDField                  = "clientID"
+	tenantIDField                  = "tenantID"
 	mountOptionsField              = "mountoptions"
 	falseValue                     = "false"
 	trueValue                      = "true"
@@ -431,6 +434,9 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
 		authEnv                 []string
 		getAccountKeyFromSecret bool
 		getLatestAccountKey     bool
+		clientID                string
+		tenantID                string
+		serviceAccountToken     string
 	)
 
 	for k, v := range attrib {
@@ -480,6 +486,12 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
 			if getLatestAccountKey, err = strconv.ParseBool(v); err != nil {
 				return rgName, accountName, accountKey, containerName, authEnv, fmt.Errorf("invalid %s: %s in volume context", getLatestAccountKeyField, v)
 			}
+		case strings.ToLower(clientIDField):
+			clientID = v
+		case strings.ToLower(tenantIDField):
+			tenantID = v
+		case strings.ToLower(serviceAccountTokenField):
+			serviceAccountToken = v
 		}
 	}
 	klog.V(2).Infof("volumeID(%s) authEnv: %s", volumeID, authEnv)
@@ -501,6 +513,21 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
 		rgName = d.cloud.ResourceGroup
 	}
 
+	if tenantID == "" {
+		tenantID = d.cloud.TenantID
+	}
+
+	// if client id is specified, we only use service account token to get account key
+	if clientID != "" {
+		klog.V(2).Infof("clientID(%s) is specified, use service account token to get account key", clientID)
+		if subsID == "" {
+			subsID = d.cloud.SubscriptionID
+		}
+		accountKey, err := d.cloud.GetStorageAccesskeyFromServiceAccountToken(ctx, subsID, accountName, rgName, clientID, tenantID, serviceAccountToken)
+		authEnv = append(authEnv, "AZURE_STORAGE_ACCESS_KEY="+accountKey)
+		return rgName, accountName, accountKey, containerName, authEnv, err
+	}
+
 	// 1. If keyVaultURL is not nil, preferentially use the key stored in key vault.
 	// 2. Then if secrets map is not nil, use the key stored in the secrets map.
 	// 3. Finally if both keyVaultURL and secrets map are nil, get the key from Azure.

diff --git a/pkg/blob/nodeserver.go b/pkg/blob/nodeserver.go
@@ -80,6 +80,19 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu
 	mountPermissions := d.mountPermissions
 	context := req.GetVolumeContext()
 	if context != nil {
+		// token request
+		if context[serviceAccountTokenField] != "" && getValueInMap(context, clientIDField) != "" {
+			klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with service account token, clientID: %s", volumeID, target, getValueInMap(context, clientIDField))
+			_, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{
+				StagingTargetPath: target,
+				VolumeContext:     context,
+				VolumeCapability:  volCap,
+				VolumeId:          volumeID,
+			})
+			return &csi.NodePublishVolumeResponse{}, err
+		}
+
+		// ephemeral volume
 		if strings.EqualFold(context[ephemeralField], trueValue) {
 			setKeyValueInMap(context, secretNamespaceField, context[podNamespaceField])
 			if !d.allowInlineVolumeKeyAccessWithIdentity {
@@ -239,6 +252,11 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 	attrib := req.GetVolumeContext()
 	secrets := req.GetSecrets()
 
+	if getValueInMap(attrib, clientIDField) != "" && attrib[serviceAccountTokenField] == "" {
+		klog.V(2).Infof("Skip NodeStageVolume for volume(%s) since clientID %s is provided but service account token is empty", volumeID, getValueInMap(attrib, clientIDField))
+		return &csi.NodeStageVolumeResponse{}, nil
+	}
+
 	var serverAddress, storageEndpointSuffix, protocol, ephemeralVolMountOptions string
 	var ephemeralVol, isHnsEnabled bool