From 26f06265350a0adcf1a04dcb0530a3db18c48309 Mon Sep 17 00:00:00 2001 From: weizhichen Date: Wed, 3 Jan 2024 06:57:50 +0000 Subject: [PATCH 1/2] feat: support workload identity setting in static PV mount --- .../templates/csi-blob-driver.yaml | 2 + ...=> workload-identity-deploy-csi-driver.md} | 0 docs/workload-identity-static-pv-mount.md | 178 ++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- pkg/blob/blob.go | 27 +++ pkg/blob/nodeserver.go | 27 +++ pkg/blob/nodeserver_test.go | 53 ++++++ vendor/modules.txt | 2 +- .../pkg/provider/azure.go | 20 +- .../pkg/provider/azure_controller_common.go | 17 +- .../pkg/provider/azure_loadbalancer.go | 2 + .../azure_loadbalancer_healthprobe.go | 39 ++++ .../pkg/provider/azure_storageaccount.go | 60 ++++++ 14 files changed, 402 insertions(+), 31 deletions(-) rename docs/{workload-identity.md => workload-identity-deploy-csi-driver.md} (100%) create mode 100644 docs/workload-identity-static-pv-mount.md diff --git a/charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml b/charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml index 9a6aea64a..9c5de5b91 100644 --- a/charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml +++ b/charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml @@ -12,3 +12,5 @@ spec: volumeLifecycleModes: - Persistent - Ephemeral + tokenRequests: + - audience: api://AzureADTokenExchange diff --git a/docs/workload-identity.md b/docs/workload-identity-deploy-csi-driver.md similarity index 100% rename from docs/workload-identity.md rename to docs/workload-identity-deploy-csi-driver.md diff --git a/docs/workload-identity-static-pv-mount.md b/docs/workload-identity-static-pv-mount.md new file mode 100644 index 000000000..fbb7426a8 --- /dev/null +++ b/docs/workload-identity-static-pv-mount.md @@ -0,0 +1,178 @@ +# Example of static PV mount with workload identity + +> Note: +> - Available kubernetes version >= v1.20 + +## prerequisite + + +### 1. Create a cluster with oidc-issuer enabled and get the credential + +Following the [documentation](https://learn.microsoft.com/en-us/azure/aks/use-oidc-issuer#create-an-aks-cluster-with-oidc-issuer) to create an AKS cluster with the `--enable-oidc-issuer` parameter and get the AKS credentials. And export following environment variables: +``` +export RESOURCE_GROUP= +export CLUSTER_NAME= +export REGION= +``` + + +### 2. Create a new storage account and fileshare + +Following the [documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-cli) to create a new storage account and container or use your own. And export following environment variables: +``` +export STORAGE_RESOURCE_GROUP= +export ACCOUNT= +export CONTAINER= +``` + +### 3. Create managed identity and role assignment +``` +export UAMI= +az identity create --name $UAMI --resource-group $RESOURCE_GROUP + +export USER_ASSIGNED_CLIENT_ID="$(az identity show -g $RESOURCE_GROUP --name $UAMI --query 'clientId' -o tsv)" +export IDENTITY_TENANT=$(az aks show --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --query identity.tenantId -o tsv) +export ACCOUNT_SCOPE=$(az storage account show --name $ACCOUNT --query id -o tsv) + +# please retry if you meet `Cannot find user or service principal in graph database` error, it may take a while for the identity to propagate +az role assignment create --role "Storage Account Contributor" --assignee $USER_ASSIGNED_CLIENT_ID --scope $ACCOUNT_SCOPE +``` + +### 4. Create service account on AKS +``` +export SERVICE_ACCOUNT_NAME= +export SERVICE_ACCOUNT_NAMESPACE= + +cat < +export AKS_OIDC_ISSUER="$(az aks show --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --query "oidcIssuerProfile.issuerUrl" -o tsv)" + +az identity federated-credential create --name $FEDERATED_IDENTITY_NAME \ +--identity-name $UAMI \ +--resource-group $RESOURCE_GROUP \ +--issuer $AKS_OIDC_ISSUER \ +--subject system:serviceaccount:${SERVICE_ACCOUNT_NAMESPACE}:${SERVICE_ACCOUNT_NAME} +``` + +## option#1: static provision with PV +``` +cat <> /mnt/blob/outfile; sleep 1; done + volumeMounts: + - name: persistent-storage + mountPath: /mnt/blob + readOnly: false + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: nginx + volumeClaimTemplates: + - metadata: + name: persistent-storage + spec: + storageClassName: blob-fuse + accessModes: ["ReadWriteMany"] + resources: + requests: + storage: 10Gi +EOF +``` + +## option#2: Pod with ephemeral inline volume +``` +cat <> /mnt/blobfuse/outfile; sleep 1; done + volumeMounts: + - name: persistent-storage + mountPath: "/mnt/blobfuse" + readOnly: false + volumes: + - name: persistent-storage + csi: + driver: blob.csi.azure.com + volumeAttributes: + storageaccount: $ACCOUNT # required + containerName: $CONTAINER # required + clientID: $USER_ASSIGNED_CLIENT_ID # required + resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_") + # tenantID: $IDENTITY_TENANT # optional, only specified when workload identity and AKS cluster are in different tenant + # subscriptionid: $SUBSCRIPTION # optional, only specified when workload identity and AKS cluster are in different subscription +EOF +``` \ No newline at end of file diff --git a/go.mod b/go.mod index a56afdd05..aee1261cd 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( k8s.io/kubernetes v1.28.5 k8s.io/mount-utils v0.28.4 k8s.io/utils v0.0.0-20231127182322-b307cd553661 - sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231208022044-b9ede3fc98e9 + sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231213062409-f1ce7de3fdcb sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.0-20231208022044-b9ede3fc98e9 sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index a6fcad5ed..a1f1b1e8a 100644 --- a/go.sum +++ b/go.sum @@ -473,8 +473,8 @@ k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6R k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= -sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231208022044-b9ede3fc98e9 h1:UybRilKUwfcg3CZh51++O/e6ppBRdT/UY0TjGJfWkPw= -sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231208022044-b9ede3fc98e9/go.mod h1:BsbV0DptIzi3NdbPXIxruq9TRI4RSp49eV4CFXBssy4= +sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231213062409-f1ce7de3fdcb h1:YApm24ngCVkpQTUxu0/wYV/oiccfqWEPZnX1BbpadKY= +sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231213062409-f1ce7de3fdcb/go.mod h1:UkVMiNELbKLa07K/ubQ+vg8AK3XFyd2FMr5vCIYk0Pg= sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.0-20231205023417-1ba5a224ab0e h1:U001A7jnOOi8eiYceYeCLK2S3rTX4K2atR8uNDw+SL8= sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.0-20231205023417-1ba5a224ab0e/go.mod h1:dckGAqm0wUQNqqvCEeWhfXKL7DB/r9zchDq9xdcF/Qk= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.0-20231208022044-b9ede3fc98e9 h1:0XsdZlKjVI0UZYhvg3VbXCPFRRQS5VL1idrTKgzJjnc= diff --git a/pkg/blob/blob.go b/pkg/blob/blob.go index 9f3fabf8b..0effdccd6 100644 --- a/pkg/blob/blob.go +++ b/pkg/blob/blob.go @@ -96,6 +96,9 @@ const ( requireInfraEncryptionField = "requireinfraencryption" ephemeralField = "csi.storage.k8s.io/ephemeral" podNamespaceField = "csi.storage.k8s.io/pod.namespace" + serviceAccountTokenField = "csi.storage.k8s.io/serviceAccount.tokens" + clientIDField = "clientID" + tenantIDField = "tenantID" mountOptionsField = "mountoptions" falseValue = "false" trueValue = "true" @@ -431,6 +434,9 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr authEnv []string getAccountKeyFromSecret bool getLatestAccountKey bool + clientID string + tenantID string + serviceAccountToken string ) for k, v := range attrib { @@ -480,6 +486,12 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr if getLatestAccountKey, err = strconv.ParseBool(v); err != nil { return rgName, accountName, accountKey, containerName, authEnv, fmt.Errorf("invalid %s: %s in volume context", getLatestAccountKeyField, v) } + case strings.ToLower(clientIDField): + clientID = v + case strings.ToLower(tenantIDField): + tenantID = v + case strings.ToLower(serviceAccountTokenField): + serviceAccountToken = v } } klog.V(2).Infof("volumeID(%s) authEnv: %s", volumeID, authEnv) @@ -501,6 +513,21 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr rgName = d.cloud.ResourceGroup } + if tenantID == "" { + tenantID = d.cloud.TenantID + } + + // if client id is specified, we only use service account token to get account key + if clientID != "" { + klog.V(2).Infof("clientID(%s) is specified, use service account token to get account key", clientID) + if subsID == "" { + subsID = d.cloud.SubscriptionID + } + accountKey, err := d.cloud.GetStorageAccesskeyFromServiceAccountToken(ctx, subsID, accountName, rgName, clientID, tenantID, serviceAccountToken) + authEnv = append(authEnv, "AZURE_STORAGE_ACCESS_KEY="+accountKey) + return rgName, accountName, accountKey, containerName, authEnv, err + } + // 1. If keyVaultURL is not nil, preferentially use the key stored in key vault. // 2. Then if secrets map is not nil, use the key stored in the secrets map. // 3. Finally if both keyVaultURL and secrets map are nil, get the key from Azure. diff --git a/pkg/blob/nodeserver.go b/pkg/blob/nodeserver.go index 891da8c23..fda3b2fe6 100644 --- a/pkg/blob/nodeserver.go +++ b/pkg/blob/nodeserver.go @@ -81,6 +81,19 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu mountPermissions := d.mountPermissions context := req.GetVolumeContext() if context != nil { + // token request + if context[serviceAccountTokenField] != "" && getClientID(context) != "" { + klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with service account token, clientID: %s", volumeID, target, getClientID(context)) + _, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{ + StagingTargetPath: target, + VolumeContext: context, + VolumeCapability: volCap, + VolumeId: volumeID, + }) + return &csi.NodePublishVolumeResponse{}, err + } + + // ephemeral volume if strings.EqualFold(context[ephemeralField], trueValue) { setKeyValueInMap(context, secretNamespaceField, context[podNamespaceField]) if !d.allowInlineVolumeKeyAccessWithIdentity { @@ -240,6 +253,11 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe attrib := req.GetVolumeContext() secrets := req.GetSecrets() + if getClientID(attrib) != "" && attrib[serviceAccountTokenField] == "" { + klog.V(2).Infof("Skip NodeStageVolume for volume(%s) since clientID %s is provided but service account token is empty", volumeID, getClientID(attrib)) + return &csi.NodeStageVolumeResponse{}, nil + } + mc := metrics.NewMetricContext(blobCSIDriverName, "node_stage_volume", d.cloud.ResourceGroup, "", d.Name) isOperationSucceeded := false defer func() { @@ -653,3 +671,12 @@ func waitForMount(path string, intervel, timeout time.Duration) error { } } } + +func getClientID(context map[string]string) string { + for k, v := range context { + if strings.EqualFold(k, clientIDField) && v != "" { + return v + } + } + return "" +} diff --git a/pkg/blob/nodeserver_test.go b/pkg/blob/nodeserver_test.go index 6a4cec9b1..a0ac3e8df 100644 --- a/pkg/blob/nodeserver_test.go +++ b/pkg/blob/nodeserver_test.go @@ -812,3 +812,56 @@ func Test_waitForMount(t *testing.T) { }) } } + +func Test_getClientID(t *testing.T) { + type args struct { + context map[string]string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "get client id", + args: args{ + context: map[string]string{ + clientIDField: "test-client-id", + }, + }, + want: "test-client-id", + }, + { + name: "case not sensitive client id", + args: args{ + context: map[string]string{ + "ClientId": "test-client-id", + }, + }, + want: "test-client-id", + }, + { + name: "no client id", + args: args{ + context: map[string]string{}, + }, + want: "", + }, + { + name: "client id empty", + args: args{ + context: map[string]string{ + clientIDField: "", + }, + }, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getClientID(tt.args.context); got != tt.want { + t.Errorf("getClientID() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index bee5b5a1c..67c66faaf 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1541,7 +1541,7 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client -# sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231208022044-b9ede3fc98e9 +# sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231213062409-f1ce7de3fdcb ## explicit; go 1.21 sigs.k8s.io/cloud-provider-azure/pkg/azureclients sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armclient diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go index 5d1ed7351..8009029eb 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go @@ -41,7 +41,6 @@ import ( corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" - "k8s.io/client-go/util/flowcontrol" cloudprovider "k8s.io/cloud-provider" cloudproviderapi "k8s.io/cloud-provider/api" cloudnodeutil "k8s.io/cloud-provider/node/helpers" @@ -49,6 +48,7 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/yaml" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient" "sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader" azclients "sigs.k8s.io/cloud-provider-azure/pkg/azureclients" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/blobclient" @@ -279,7 +279,7 @@ type Config struct { // ClusterServiceLoadBalancerHealthProbeMode determines the health probe mode for cluster service load balancer. // Supported values are `shared` and `servicenodeport`. - // `unshared`: the health probe will be created against each port of each service by watching the backend application (default). + // `servicenodeport`: the health probe will be created against each port of each service by watching the backend application (default). // `shared`: all cluster services shares one HTTP probe targeting the kube-proxy on the node (/healthz:10256). ClusterServiceLoadBalancerHealthProbeMode string `json:"clusterServiceLoadBalancerHealthProbeMode,omitempty" yaml:"clusterServiceLoadBalancerHealthProbeMode,omitempty"` // ClusterServiceSharedLoadBalancerHealthProbePort defines the target port of the shared health probe. Default to 10256. @@ -382,6 +382,8 @@ type Cloud struct { PrivateLinkServiceClient privatelinkserviceclient.Interface containerServiceClient containerserviceclient.Interface deploymentClient deploymentclient.Interface + ComputeClientFactory azclient.ClientFactory + NetworkClientFactory azclient.ClientFactory ResourceRequestBackoff wait.Backoff Metadata *InstanceMetadataService @@ -601,13 +603,11 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *Config, return fmt.Errorf("clusterServiceLoadBalancerHealthProbeMode %s is not supported, supported values are %v", config.ClusterServiceLoadBalancerHealthProbeMode, supportedClusterServiceLoadBalancerHealthProbeModes.UnsortedList()) } } - if strings.EqualFold(config.ClusterServiceLoadBalancerHealthProbeMode, consts.ClusterServiceLoadBalancerHealthProbeModeShared) { - if config.ClusterServiceSharedLoadBalancerHealthProbePort == 0 { - config.ClusterServiceSharedLoadBalancerHealthProbePort = consts.ClusterServiceLoadBalancerHealthProbeDefaultPort - } - if config.ClusterServiceSharedLoadBalancerHealthProbePath == "" { - config.ClusterServiceSharedLoadBalancerHealthProbePath = consts.ClusterServiceLoadBalancerHealthProbeDefaultPath - } + if config.ClusterServiceSharedLoadBalancerHealthProbePort == 0 { + config.ClusterServiceSharedLoadBalancerHealthProbePort = consts.ClusterServiceLoadBalancerHealthProbeDefaultPort + } + if config.ClusterServiceSharedLoadBalancerHealthProbePath == "" { + config.ClusterServiceSharedLoadBalancerHealthProbePath = consts.ClusterServiceLoadBalancerHealthProbeDefaultPath } env, err := ratelimitconfig.ParseAzureEnvironment(config.Cloud, config.ResourceManagerEndpoint, config.IdentitySystem) @@ -719,7 +719,6 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *Config, common := &controllerCommon{ cloud: az, lockMap: newLockMap(), - diskOpRateLimiter: flowcontrol.NewTokenBucketRateLimiter(qps, bucket), AttachDetachInitialDelayInMs: defaultAttachDetachInitialDelayInMs, } @@ -1168,7 +1167,6 @@ func InitDiskControllers(az *Cloud) error { common := &controllerCommon{ cloud: az, lockMap: newLockMap(), - diskOpRateLimiter: flowcontrol.NewTokenBucketRateLimiter(qps, bucket), AttachDetachInitialDelayInMs: defaultAttachDetachInitialDelayInMs, } diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_common.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_common.go index 1248a57b1..48c9491ba 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_common.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_common.go @@ -32,7 +32,6 @@ import ( "k8s.io/apimachinery/pkg/types" kwait "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/flowcontrol" cloudprovider "k8s.io/cloud-provider" volerr "k8s.io/cloud-provider/volume/errors" "k8s.io/klog/v2" @@ -102,8 +101,6 @@ type controllerCommon struct { // > attachDiskMap sync.Map detachDiskMap sync.Map - // attach/detach disk rate limiter - diskOpRateLimiter flowcontrol.RateLimiter // DisableUpdateCache whether disable update cache in disk attach/detach DisableUpdateCache bool // DisableDiskLunCheck whether disable disk lun check after disk attach/detach @@ -167,10 +164,9 @@ func (c *controllerCommon) getNodeVMSet(nodeName types.NodeName, crt azcache.Azu } // AttachDisk attaches a disk to vm -// parameter async indicates whether allow multiple batch disk attach on one node in parallel // occupiedLuns is used to avoid conflict with other disk attach in k8s VolumeAttachments // return (lun, error) -func (c *controllerCommon) AttachDisk(ctx context.Context, async bool, diskName, diskURI string, nodeName types.NodeName, +func (c *controllerCommon) AttachDisk(ctx context.Context, diskName, diskURI string, nodeName types.NodeName, cachingMode compute.CachingTypes, disk *compute.Disk, occupiedLuns []int) (int32, error) { diskEncryptionSetID := "" writeAcceleratorEnabled := false @@ -300,17 +296,6 @@ func (c *controllerCommon) AttachDisk(ctx context.Context, async bool, diskName, return -1, err } // err will be handled by waitForUpdateResult below - - if async && c.diskOpRateLimiter.TryAccept() { - // unlock and wait for attach disk complete - unlock = true - c.lockMap.UnlockEntry(node) - } else { - if async { - klog.Warningf("azureDisk - switch to batch operation due to rate limited, QPS: %f", c.diskOpRateLimiter.QPS()) - } - } - if err = c.waitForUpdateResult(ctx, vmset, nodeName, future, err); err != nil { return -1, err } diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go index ad2e808f4..f7817d418 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go @@ -2152,6 +2152,8 @@ func (az *Cloud) reconcileMultipleStandardLoadBalancerConfigurationStatus(wantLb } func (az *Cloud) reconcileLBProbes(lb *network.LoadBalancer, service *v1.Service, serviceName string, wantLb bool, expectedProbes []network.Probe) bool { + expectedProbes, _ = az.keepSharedProbe(service, *lb, expectedProbes, wantLb) + // remove unwanted probes dirtyProbes := false var updatedProbes []network.Probe diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer_healthprobe.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer_healthprobe.go index 9cf7bce13..b754e0667 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer_healthprobe.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer_healthprobe.go @@ -291,3 +291,42 @@ func findProbe(probes []network.Probe, probe network.Probe) bool { } return false } + +// keepSharedProbe ensures the shared probe will not be removed if there are more than 1 service referencing it. +func (az *Cloud) keepSharedProbe( + service *v1.Service, + lb network.LoadBalancer, + expectedProbes []network.Probe, + wantLB bool, +) ([]network.Probe, error) { + var shouldConsiderRemoveSharedProbe bool + if !wantLB { + shouldConsiderRemoveSharedProbe = true + } + + if lb.LoadBalancerPropertiesFormat != nil && lb.Probes != nil { + for _, probe := range *lb.Probes { + if strings.EqualFold(pointer.StringDeref(probe.Name, ""), consts.SharedProbeName) { + if !az.useSharedLoadBalancerHealthProbeMode() { + shouldConsiderRemoveSharedProbe = true + } + if probe.ProbePropertiesFormat != nil && probe.LoadBalancingRules != nil { + for _, rule := range *probe.LoadBalancingRules { + ruleName, err := getLastSegment(*rule.ID, "/") + if err != nil { + klog.Errorf("failed to parse load balancing rule name %s attached to health probe %s", *rule.ID, *probe.ID) + return []network.Probe{}, err + } + if !az.serviceOwnsRule(service, ruleName) && shouldConsiderRemoveSharedProbe { + klog.V(4).Infof("there are load balancing rule %s of another service referencing the health probe %s, so the health probe should not be removed", *rule.ID, *probe.ID) + sharedProbe := az.buildClusterServiceSharedProbe() + expectedProbes = append(expectedProbes, *sharedProbe) + return expectedProbes, nil + } + } + } + } + } + } + return expectedProbes, nil +} diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go index d7602be72..52d296292 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go @@ -19,11 +19,13 @@ package provider import ( "context" "crypto/rand" + "encoding/json" "fmt" "math/big" "strings" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network" "github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2021-09-01/storage" @@ -31,6 +33,7 @@ import ( "k8s.io/klog/v2" "k8s.io/utils/pointer" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient/accountclient" "sigs.k8s.io/cloud-provider-azure/pkg/cache" azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache" "sigs.k8s.io/cloud-provider-azure/pkg/consts" @@ -41,6 +44,7 @@ import ( const SkipMatchingTag = "skip-matching" const LocationGlobal = "global" const privateDNSZoneNameFmt = "privatelink.%s.%s" +const DefaultTokenAudience = "api://AzureADTokenExchange" //nolint:gosec // G101 ignore this! type StorageType string @@ -129,6 +133,62 @@ func (az *Cloud) getStorageAccounts(ctx context.Context, accountOptions *Account return accounts, nil } +// serviceAccountToken represents the service account token sent from NodePublishVolume Request. +// ref: https://kubernetes-csi.github.io/docs/token-requests.html +type serviceAccountToken struct { + APIAzureADTokenExchange struct { + Token string `json:"token"` + ExpirationTimestamp time.Time `json:"expirationTimestamp"` + } `json:"api://AzureADTokenExchange"` +} + +// parseServiceAccountToken parses the bound service account token from the token passed from NodePublishVolume Request. +func parseServiceAccountToken(tokenStr string) (string, error) { + if len(tokenStr) == 0 { + return "", fmt.Errorf("service account token is empty") + } + token := serviceAccountToken{} + if err := json.Unmarshal([]byte(tokenStr), &token); err != nil { + return "", fmt.Errorf("failed to unmarshal service account tokens, error: %w", err) + } + if token.APIAzureADTokenExchange.Token == "" { + return "", fmt.Errorf("token for audience %s not found", DefaultTokenAudience) + } + return token.APIAzureADTokenExchange.Token, nil +} + +func (az *Cloud) GetStorageAccesskeyFromServiceAccountToken(ctx context.Context, subsID, accountName, rgName, clientID, tenantID, serviceAccountToken string) (string, error) { + cred, err := azidentity.NewClientAssertionCredential(tenantID, clientID, func(context.Context) (string, error) { + return parseServiceAccountToken(serviceAccountToken) + }, &azidentity.ClientAssertionCredentialOptions{}) + if err != nil { + return "", fmt.Errorf("failed to create client assertion credential, error: %w", err) + } + + client, err := accountclient.New(subsID, cred, nil) + if err != nil { + return "", fmt.Errorf("failed to create storage account client, error: %w", err) + } + + keys, err := client.ListKeys(ctx, rgName, accountName) + if err != nil { + return "", fmt.Errorf("failed to list keys, error: %w", err) + } + + for _, k := range keys { + if k != nil && k.Value != nil && *k.Value != "" { + v := *k.Value + if ind := strings.LastIndex(v, " "); ind >= 0 { + v = v[(ind + 1):] + } + // get first key + return v, nil + } + } + + return "", fmt.Errorf("failed to list keys, found no key") +} + // GetStorageAccesskey gets the storage account access key // getLatestAccountKey: get the latest account key per CreationTime if true, otherwise get the first account key func (az *Cloud) GetStorageAccesskey(ctx context.Context, subsID, account, resourceGroup string, getLatestAccountKey bool) (string, error) { From 67cc5d2c67768e6bbc0c313034bef20408bb3cc1 Mon Sep 17 00:00:00 2001 From: weizhichen Date: Fri, 5 Jan 2024 06:38:16 +0000 Subject: [PATCH 2/2] fix --- charts/latest/blob-csi-driver-v0.0.0.tgz | Bin 5933 -> 5969 bytes pkg/blob/nodeserver.go | 17 ++------ pkg/blob/nodeserver_test.go | 53 ----------------------- 3 files changed, 4 insertions(+), 66 deletions(-) diff --git a/charts/latest/blob-csi-driver-v0.0.0.tgz b/charts/latest/blob-csi-driver-v0.0.0.tgz index 414366591599fb669e0be26d04db7daa972b6eee..217a742e8016d55e189495e94ef47f7735c1beb1 100644 GIT binary patch delta 5946 zcmV-A7scqUF3~QKOn)BylvXo-;aEzli6d(yC%yZ3Iv$8zNjQ@L3xJAM*Wdj&07&s! zqGZL+^@KNx$;IvhTr3v*08Lyrv1~yshm!^H=H(o7*;r!F{cdlq)oQiQPEPc{tyU}h zZ>xQF{9XIx^yKu-QM>)-?7LR`r2VG#9ct~5h4K~%iTQV}Cx6q*4(`9CVV`^kE(oI? zv}o6`@2~!Ju>>J&4hWkQU+QNUe*_%RTbE7HSTc@hfG)>Ek zHCnZo9`d0jT6X;}up5BbuL4-V{@bc$*Z;{;`>_6>qY)3!pi@I=f(4AgXC36(yx|d> zGr^{^VKYy?b${7JK_SG!XM#w^mmSo+`%yG)L7I^iX$XmDE9Ig7_5;@)GMCuP4(d+_ zOb$5+pz5V9$dI!IQThRR0RbjN^DdYGryxNn5{jPX$DLx5i{r*w<7g8}mn?vS5JS!; zP>o{L#aCSt^o|g`XPj3bQ-{LH^ig zhPmmGAk}XCv}ty0w#x!1+$Dy=#3lBndQz*sdWA+X1rF2(BpG595aY;W9FPE;gB0rV z1W+KrL0{%TH5o!@NQFXy*l^Q9zy4Nxg)C(DG;bMxyyXX28ju0@pjH#W z7sQ4O+hzfkYJP1FSV9L$9sqiUeo;eF|9_1_ScDC*KwpUKsxg}`4w^EKmMmzXF-SEX zQ|t;rWU3ybFPQ3wfdKV4N9+r2D0+-a&RiFGM}K{3%dExKLrY-v3Mpn0o1$^ZLdb78 z=MjCz_4J2An?R?Y#_mJQ^2Iz+H#oN&}eA$g1Hj^+Ka>6)~%S#svjUT8`)|CF3iYX)lz90)g90Uuv4M@xxmsO?T zX4D2>#tT5tPE#!cKO}AW9Kh4IT(jmU`5EXhB9=meXW4EbByh{-L=qdjiuxGrKtXGH zgL{c)ic>Y8%=a0W0aY(0Lp-2pwdfEQh<`H58Ix$TL;|E#OB$<1uhCr^Y>L8UiQpc4 zz6+?M{?34O(1XH#AwCHu>g`smjUH=xpvPJ*u_RW)R zv)4G~(89+ybddTWmYKgAQnJRr^gLh9?yAp9{O3%_0mv_m->vk7oD;E{ADacPqkk5Z zLj<0<;3y>!GmabvYF;IBzz6CeM$yM{Kk8mXN@6*&#gK8?K}SEG9iJ3$eRIIBoX22F z!E^9kVq?)k`mcaq(~nrmL&qqj=;5W|XXs)I@hg|jl+3#wbgFR~Nfw4&(SEP^@`x1C zUy6B8?@gzz)<+UPQyk_a$@Z#5{oUQavDcZq>6EDD9I7~_{l}nGE92oK$z0%}_SIM{bO*~0s_UJCGZz|Z4drPGs)rPruX-zN_1--Z zQbaFcIt5#HP<_D0_8gqRg?c6Y;o|U>Fn)C^))UCo^wyeJOX~59DJ1>2Pk&uP;WKjs z4?e*1!nQ$(Ux=Ld9iUQ+L$t^}eE*5!1tzZ2P94-PQaPv9RL*fuX#iDN-@0rv4Fn81 zyI-z2our;>goFx-T{nVXfyTfqmqxwaIBA^Li@F^Z^gD_>9yQvHBhyiU{D|+beW34% z3xmK*^z>OYy|py?TE^M6migC%s|<_2|%?KupwoOe*u)22K2m43e=9*iaS{65tn zU6Rnnp0dA&z&#>_v4&c2-n_}{-e=W@rH7suR=`gsg8!gX@dZnJ?y%X{9nnroxKJ0F z^>m?F|ENzXM(%1;z%t-k7N+7oXMrD`>PCQ_UpSF)O>Iyr;NAFU>3_ot*L#pgPmCp2 zXC)(xHDmX7QU-p*7NnuoSLjMGHncM0E@q&ZsVg3X=*y6)=^$hrSL*Q(lqrQG{_z@hdv8Cze~tQsch|4cFBhXhfAIb_>WxO% zBV~puTRK!PnhEtF{(rmX5*pmA(I{2wg6yE?0`sOzCe83kO|NOmSqC{VA(+-`%BXW0 zc2-JLt7!sM_PZgOSLj!!ZE)=VhWb-&L|Rcv)szV|c`cY`bYl2dpLWMe2YLI>xeIatL!GE@+8j|RDmHdUc^srvq zk#%T0wXFYFg6C`BU2|}K(;GMBy?io_P4S}~?D@k3 zYK%a`HCDDQdVhRWEE!~$CBj|typDZ8W3VesIV9-uv5qWDZ{z>#daHc>-_F7Hfs2Op zpBM_-u>OyZ&rUMy|LpV-|9zhJ@X-7xS`e?Jt%E6XK`wnb_tdo5bLgOdHXk2rH4WCg z_c3*}(Lh5tu(mUF%7BJ@c(9NIQ$hjNwehFm>d1;;;eWuDs9~}U^jd6uHW5b!dA$J6 zi38|QWua0VogM4{63Qg!MO3PY=NSKRha;Kehi%X{q}EtE2jDeV4&7&tuA7Gm1@=F&P0E zhqS>~tbYSL=-~lHYdiIA8mO?7J{?orbj)hca+Zzryo_Hr!KZpjsO<($*=R~Qvzg~J zs!t9HUwaLo(_@b!9ue=;=*wTMawduquJRugwp^D#(eDA1sabt| ztf_-fL(H3642y;k%4Z|VPPmeiK??I4N%n8-xqoXQR)kYu)+~EJaNSjqZpwJ<-503% zJNt1QE%C8q4zs~FMK~9L)15e`Gg#XXtzvX)dP_!=9@tCk+;6Fs@Bak>lO5AS*6;sE zt)sKk^!|U;ZnX~f|Fg6^LY)rkYQx?GDr?DvFzoxHxoFp-_-E|dQEM$yCb4+XaC|=~XQtbq zC`mOlAR~b4p?0I)N?5kaCQ#K785)W1@q!H@8gClIkE1cqasDeOxni?55_XT&lCgl7E5p z3M5lO$P&uygf2N@8T)XPpcSNyBtBN|ZPK`nK6g4@W5v1vTN(C4&eRz+evXhjk9N=~ zpptmdH7Cfm<4c4^V#29d$rdAyZQYnBD-pX*W+H~4d5*=9gFfLk>*;0HNGCuv;!M1x z(hApU2h|gkYx+l7n&|N{_ii2T&VMFoiD&5Xv2G^76;P3%{n1SC0|;S06G<^p(_Xh0 z=STJ2Lv6oe$KfEv`S(9ubi1S8c>Iq7Ko_tB+=4JoY%@d!S7ZSaC8;cs=k23bOBB3^ zd&#k7a^kFLk3Uaujw@gt$HzXge)q-sX=@$CB;Z1xw_8O(VX)7dDdYb9XMaW+AATNz z?PB7YuwRY=po{DmFoGY>aOxd05x5R&hAE^C!R))N3=I>hqnpGdPBTqQ#zIFPbtE$< zAto4ZBIr>*a@-8}iZ76sLwMqK94to#c1Frt3;{FjCM=NlQIg?k(xUjgWzvW%D2r6Y ze!se64eI=|O#|Q~EmV%jC zEB|RcyNLf90avYjb?hMiDQ-7Pzs~ioAoKcUvR6^hW6aB&hS^cDcf((8yR#N$A{VAX z6z@Mk9iI^@&%giSa@@b|4Z6eY{@|u)M+~#JR*OZKXf;}$VzP-GuYY37#*4Iyx`i0b z)96TAw1I1fq-AmsW%;qayk>28hn*v%vG^GDWWhePME72D8$CISE}^$0nWcDe4ie_Gbg}N9LW+nPfI| zJLu+exCWB_HZlz$t4Q}h-MPMGX9Wb2A*@4oBDj`gbw?ycSE~%^)q@ zUp8=9g~9Fl+lIZgP`$st9d|$6UR}Sxz3P4LT~(!;PZTVf2KYVJ5lZKwx=vgfOKjh1 zC0IHYhQZ$8bANwyJ^0ug+-R{MUfld#avlO{C(3o2nnj`LXu^&E2u3$g&}Qp#6?uEM zGgG#6W}tanAN_;hDilQ^(i=v zKG!j_pYvi-#DaDRqSGQ08UkSs^7=3)zrWcKb} zE5uvq=87P9cdI>-R<-j*RmWMO+wD1=X@9rom^v<4MuAy`1_CVLobhv=bpH33^;bo6 z|E(X^V^&L+n_;xT$z8~wK3V80l!yF!EmZMyCd^>cX6NRO!?^#ugzI?4@|v4WuNBNr zJ4dl9-haqrUN4u2Ft^zq_Qp>6F0c{VhAm-J%rO(#DUbLRiEOrWVK6tr!AmDi{DoSD z{GZUY-u+6LP5J+)XJ?uGpH{p5=8*sM9L=(C-8@rhmzTwp)#p{oNhVQL3=dLO5^;=)_sFM!d{Xa(w^Zzh-jE-T4^gk^f`%L`H zAb;FRZ3Mc)dMr7XFbm7Xl}$|O%oJZoW>B_}JeZ*0il9EKYOXgs1q%V>)Sir?0 zTkyrQ1q%oE5(#}F3yh})rQmLcB7nHqJe?vq>~~Y{r1g1(2|$O6XjhMNz@qcTHVK4^ z#a&Yeb2(>b{@Yk zeJmx3<)G(1nL1mr*3&SX*SmS}?dCV?@$5CTQO(Tdn(W5#eiSRZoW%0gVn5_?lz;nJ za~{LYJ&aMandf2ZBA5;(RcS>%!w4W|BJd~|? z@CX}X5s}-^z0tUTJ=lp@=6aw~Vt?6@9qnUiM;pg>w3ophZ9BSkVcs0gB6FaO(71PV z`|;vmx7WkoK*vmP2cJIfG`8FgOrMJvi&WW*?3)yGQ!f#1Ez3+v^QaiLDps3XL4s_e1Ff8RJmn* z{E*oD-<{a27iQI*HHBw!epER)r;+cPLtA_qbOm`UxQ%VUQn!&nUJ{DB>?na>gZlQ; z7{||ZVBvXY_0e=N+{bc${{0W#QU7yqbUW(3za3v+ez+Z8-29x!ke0DEnTK#6dl8~m zXm1+K8!j2&??56f9&_Nl7=Kkpz3-UYcv{8s-6<&`vEvsEw-{GK_)H`?96EY@Z2E36 zBUCg?LnuW|e3hhvxO3`@tVfXq-Bkw`EX6}L!8247n0;brQgMJXiqe9Ol69F&I4vs!j-Z!u=A zQY}Be!#9U8d&%#*sU-Ry=V;(}u(Ytg%GvrtMr)1ft z0t36%Bbf;)twh?gP=94BbAPuIl|!M(p-?1;s$^T-j5WPG9x72P@wZDQO4@3uF0NAo zu2UkXnV;Ls1L)4ix)wgzT0QAs8Y<%OP%HB_YGrnfu{=kO&APK>;W>>nAJ4G)@y55r z(P`9L%N4Sq`)I@c>y`SX5qYV|Slrp}={^=t{E|38Jz}bFb4FT{3BU%n4k`&Sn&Xm{`T{_gg2G@s1Q$NNqcRR&pYu zU#~d$Ps;r#WfA(+1lC`E%h^hlQp}gS*sAYidtZakySe2)d}#?(L0olhIZYCDV{CO3 zrc8`=O?-7ziGNQ5$2!m_xvSGh$ql7jYYd;=S=@<(61(Tg6LlN^!00Y)m3rdWGxnr< zx;55Y<>^S}HmrWsDlHzRX5OKo?_~=5wl~W*NMWV@p}IKc_77{VWww^JGGnhwvFTr{ z3bMTZhflC=?d1~i4fTI#$7jdc`@i3`5BGmROUnqb`hVyRb1|j9xIZlZGe7e)VlJc# zgfuh`;9_7FSGzdpat-W2AF}YzeaVJ2>q`%q1AuPJgNUVYTpOlZwG?<4$jI{ zO>I*Vt$%dwW)bBO!gjF`6~|d|Ot1hdcbK_wG_O(JFC({kShgT)v)b!`o`le5j5~xT zmqZZ_T^qNTNvm0wRm-o|w}d)`&c4Q?&0KzLO+r$;m;45k|9~ySRcnSaPF3&JVsl`d zE5fUkBr0tay6p2R?Xru_)`UqC5kCZe)J+N(ZNU!?SFttKZ*cX~dh6#lGLiI9gf{aeX| z>_LLJo0tEAJ$5HSsUp)44~N@>3R>0k|5Gjj*>L`EA0HoO@BcYDIXj&HpQCL!|3B+S zpnvNSzNIbTg%p2s9=;WopS(EXS(JT&FdWqU{#&{J4+Q`l0eFfhc$5Bb<$nM1q;>HB zK2OU9Fw%xzyc1+H7Yh~1+7sFa8*=-Lqqb0j+ulu7?OvkPMbBv%~NIKTCTt{T~~Y2hD%b c{D*dEhjwU(_RQ`70ssL2|7YSXOaQ0=0E4l&rT_o{ delta 5893 zcmV+g7y9VYF0C$*On;vIkXAE(;aEzli6d()C%yZ3Iv$8zNjQ@P3xJAM*Wdj&07&r( zlqgxTb3NfrVsf#&02hnJK0p(fPAr=f%VA^zta&lVOf;6*bHCeLYqeUf)8k|HZ>!bH z|J!Px9(~t7J~`^NJMEJiyR5(xF|{Ev_Udh5~&8Vkzs4A8}RfVA+`eC$CNo0DcNMG8XT*-ClHzk|?qN7NA=oqwaHX|cF_hJ~LR?FzGe4V?V3 z%?vZ$Aw{a)_-WJZ)@+xCPP9vm!ih`l3;Cp0d-V!k!xR|!HXuO}^#M_aJjwuZuo(z0 zA5Q><931pz4!$Bo{25Xr$|VX0(=M6|5%6v^#dL$vrbBJs6qw)frcM1RnS~6jfI5~< zeSwJ&tbZx-SqhmIFtWe`;wfD=YwGzZbltJ$ri*_4t@a98NbhOhGWvMS53tlAL+nAV z#(^z}4d=E^Ltn`GwHaUmT_jit=oR`!4n_X|HHu&n*1#NnA+9UOth+d9N*P+xuz|)P zwg04$}er{wYYj{ag1Ie$t%w^ z_>JZ~rcarg{zzz3=#&`lfT+eM*v?OiM87f=lxSQ?PHkynNYvQ(slf6tB@@_j)IY9^ z#2ipS9bOBlbDltT(ZgdAjf*GXuETJt0CZ<$S?D6RAAByYDfKZWQ;3IbK^A}*a2hfj5T8>f zs!G32eH#LiEC4k-O}Pkyh_n?j08iO+&03J=XQ;jiUvdGSWxF94z$}{)L2T?w>SM4& z39aM}?gg4jPUU>kAfQZyzI-VtVxf;#i+>Jbj(qAfN(GuM5eFgUlE!k;D|8nco1`#V zBDlw1-~#H(zjGj6^dNDc^G{rgdb`zXqsLki=&@EyEs2$|8J32WOR1hN)O;nYx{I(> zV3vBR_Zp`hng`g1E|MR_((_kCQr75~z8A>ZUG-Us|D19$1o4HkyOo|0bHZ2iV}H}o zb>xC_2*(o_9H|5n#*sro&Z|HU1mHV}`sm|$5O=R3DY1;$d_Y|Ohz{T2FG{zok3p{o^A{P05aGjg#H$t#!6q|CcrbfR!+N#>fIX}_0z zc|`E|FUh>8_NJ3o>m!MtGWb0No_~h2gHTF=SLoJ>dl|-CPOSYzVu^)RPNVZg8u^X< zYfT-VPKjL3k&2Vre+*K!QWiZD)CCr4Uxmd&cd+cDy4ooib)k{fP@a;YdPI>1vbV%m z@81(Hc>DsUQ?NxB)rWL!&%p^@s8_-tEsj76<5#C*J%LP3ZLLMMBp<(+LVr?k2fj;u z_)OiVCshpE)D(9%6G{Bcw-@0@% z4LOV$y0!@HdE{%G-aojkmo4R!j{W?jVKY*HvbejSU~S>VNe&?p2G->c^5T3WxC@)>h~Ms!B}7~ z82AdLM^d`jllIpLm`6C*)==xsn>R(>`>fiq^w9Id3izo+@E`t^f5F0@J9PGSN3@d? zHtHh1p3WufAJr*E%Ux{>ScFW;!j!*fGz{WX-8Eq67e)kJ`8G%u@PBT6vkYK`>pcjq zC&mKHvyzs@nznnp83VszbJ9@iD{>`h8(JE1=QEJZ)FqE${P#GVPRV^2)vv~5YYwg_ z->Uxa>A3gda-}Kib@U3YbP&>xEBW{bpGt+o|M432`foqIe~kvicUP~`FXz|8!SMZS z)W5#Ix|U{`w522UqJNl>50bxYF7d;AIU1=-T@YQ=TwvC8$)p)Qsi`$B80{hlCItJn znl$QMik+3xRBD=lF9yAc%q#RORW>+we?x<*GNRAVdvdW$!%H7v)Cr1Nm~%4o!9gDO z@k|J1xS3=pUZQZ0AC&wF;|ZwD_#*4@)XMDsZxOqdVlB(5pxB$9o2}$peN-o z!lj1w(vGY{+o|RKzXH5K`R5xB9JliR-;>jm-f`e*a; zu~t)H{rdp>jxrkj$PKLQ43#pV;2s_<c)c+N4wF~87*1Tx$B&$*ASz1IYt}t09 z9BW5!qh@AYfh5ENMtm_v^}liJZ@iuzfc}4UR78I&|C!WM_5W8#_1pR`gQ1>BrM;#V z8!uxr27k~FX^qXS1H0(q0mW-O^KBZ)*h!s^eOq_TYtM3)we!4;UpK|4d`W!U4V|*l zlyGKKFQC3UIi!5;MYsSnVbST`fu#HYamvHGhbFO`yh1PRgi8< zS>oO2$o!pyB#xH)*a?UEV4Ead2*9aM4Er-!+YhZ`baHx2Mw1=bOY7Wksg>{lhJeYA zX(8+P|4yrOdXnA$JFPcw4)*`Ev^(NEUDQ*Cy$8Oir4zz%5b)-rU5n$NiDyTywOE-X z;(y^s+k=LR{Lp14d<{jrh4x*k2&sHOL zYo+^WPhSk$i2X)Q<(RJv7Oui38h@mnOMghNqV%x_&uwJM`!z#IZ2(!+sT6{wPb&MM zYuGw--Tp*rs<{Da0hAAwv(504VjpjceC+j2(_T-eFl***QZ~~jn;&7Jcb0g`r^|%2 z%@UacN&02;XNGy@l6{@RmSD&@##bLcYl2bUMar97CTMAOkOaI_g0w!a898<{bAMAV z)p=s+Kzap|nIL2d<#j@rp0J#KxJl3qDI=+mRd|~=ZsX6LP1jhl&cT+3{fJR{22GwL zB+sK=bRGJFc+k@)$hG53g+&s=nOMmdBTj7HgeNN%yG>^zMxS|(`G|o!;Wg{oW!1;8EB4+EeJ*a2oilqR+rqJk^ZfJ90v4aHfz z(`s?Ud$<=2TRJDsiud@l?B=)vR&jhB5bO7VpPjVUK}CYp`8RJRSBcFaZ7{{{^(*M>Cvxhjaw4i<(giX@gVyE-yo)RO;v^@kr84vy!n; zkw+EDEJ%nBhU*A=oR6F|&AsFcgyj&HdYuHzk-^T$Sc?&0j@^_6((a@gjwdaSzgs$u zxPr3CMC|vgE7qVcF54{dj(?hBUlgn8iL6E_6G#oIzQ$_B9`OQz&R4Ml%EX!3u*q_p zu4;SOXZGYGsT@8jerX4ZL+FC6ZRif2sb$`udQ#oE{%G!&x zin>J@^waoAYTCe+L(NEjbC{+6D@f?dI@alYPf?F(wLfc!KXUH^ zMg_I0+eJ4Qqg~j@%73nras*P5vN54fw<#uW$lzwqfX}Jx=)A_H&4mcwgXrcdY1LdZ z0cvcfcf==xV7CV@UXH=0zQeodsFi_ExTv47;@&;X~hcIE6D-o>d_ zsVvR>K+;OmY}AH6x$icc<)%cmwfT_OHc65hs@L;^7c=nl&CO_hJG#F5*T2p5i&`v& zH;2@?zii;J3WMA8w+(w)p?ZIHJMMkBy}WvVd)fcozpP5Nm?&5>4e)!SBc#qlb(Od> z7TCU1O0aY)G=GEr;pf5i)$n6~c%#IAbbj-5$$1EbohsL9Di*nPY{f;L-^ ztH|54otg5Ta|1OZO4jmzzY6E>8e>;-#<|-R<-j*RmYjp?e-kbw7*+(>^m-4T7g-_4>?%C7-eTF>HP06 z>#vIE{#!q+C#;q%H_fQQ$!+9Moh(!p%0qFz8ddz9ay^)=*|~Y+(C+^pVJcp+yka`j zYX!5@&VON6#T$7n>gDnf<~F;--qRGW$>;yHTb)Dx&vP`(vN9<@T<7dJ?tav{i|v}u5V{x-bUokt zWT334FPGx8L+%b10^9Rz_&p>HoTTV6pw7Ub(tj&IUC>r2V@oj>g})jZ2+gA2`Yy>7 z?4|9(N9w?;o^B-Pg$sR;ihR%y=HLOt?q5U+RapN%b>K-CLakr_N3GL0`TO7AoE#t4 z|Fg6x{||#l{W0v2{->m4pNW4tggdE?P*qru1;YYnQJJ{3iHSec#n-VJlrLn@^O1cY zbAJ@2q@)q5JqH8n`%Q3%*#kpmAU?kMD$(95`! z*5?tX03DylyLy}h7N0M+Ng$Lg?wUGS$T`z1XZLKnB-b-0Es>oUp6c5Jqyf|m8S!V) zWg^6%4Q4)FJ?`Iw9ja?OGiSUsgEzoDo_`OtSXx*0?*j(W6%vLpN^~`f(ANMITXOTj zkcxqCgo$eOjqfCd1z{q@u9Z~RCsLAH4r<=hsk4T)o`u=G-t~iTH@{JhXRn!!a%L{p zWH(0lqge6fBvv37`yq#;+{aq*80PL_jGIk85BpBR9ox)KbyI$!2A`*WqNVZ`Nq_Hx zxG<9krJT#Y)HT}cza5+pZ{J;C4R89xp4mlO_o+{O0Sw!MEDA}A8q;7nzB#|VynTCl z_4eJTu`v*woqHwInG1ayxl4Hr9$`%^CUX0^e?1;t4R<1zyB?^NSbk)keGIL$acrHv z46d{7=+=dK3p7pUKpCNN|K|4N`G3D|uSWf$ikaRHKYiS3Y=s+`N@+@ZPa@@2qthFV z&);75Z$FKC=QsV{+ns3Tu7}DI+Z;-ewv!d|b%6v4(#(V8`&%xx@wymgjHOuw%-{lH zZ^Dv*tjn~|<<2Jfo+GJp%lP;qvG>0_u~#k3syS;C&*H38IXI_~@0vqvzJCn5g1i}S zW81IPZ6r{X1XGtCCGcxd-(DJ%_;~>=I?t>=nhJ&oSj^A9|Dkt1_}ss~z3#ui9ba91 zxE-C}{9MG4m9aIMhj5>G5#naFH#O!BmyGXsATbt?I&fx2l~eCK<~E*Gv3z$*3P|nv zhT)oVrG!sMf}^3M$H!*ihJQ2S^JZxX#l*x{Nh(M>XTHdK6lu_1b)aD>9;ykRp_)MN zbNi|z$R1tOJ+tSEwMg`7B8RKMbG0)^-BNmuIA`Dg@Zr+9l9yCtO{RtA|svY%_s@-RhC_gp^hyZCR+YmASuLiOQi+aS7VC4XG~t4wJ5A3aa)T#SSwUjpj9~2W8|o56k96ZdQ98(323_lro3- z=_O&Jp=#syGHEr-vTDWE`j+?(@n>IS(dI5cwk8p&-AjJM$$!8W(W=!$8E2|@YKb|p z&3_f)RZ0@2wuxN!MV0py{S}MFnt3n#cwbf3Mi=V*H>lYPDq_!C_DK|)`seNPgmxT- zS+71#p+38IlJfoBx@kWG{ufY6xrwN(koKyNzTM<<8#|8ukr=l^Hj z2vi-yx3mSkkm66z!?&XH(-$W^i?TNe!$HmOzm@C%NC2=FfTwtZH|hUY;rAbpTL=H| z^Rz+$BWvg-J3%gU(Wpq)p3pYfklSAzwS^Mg_HLqT_Y!3;|5?{fqqm5YnUJ+yY);}> zfRC$itl>72PNRF+^!vF7)%w?I+w1@BPXZhD|H+%2{_mU|AAbM;S=x)~|HPm?X#Rud bKeR(Tv_m_zXKw!&00960bk(