From 43112fb47312f59aeef5d6b5e721b526b1e45cc8 Mon Sep 17 00:00:00 2001 From: Jialun Cai Date: Sat, 18 May 2024 18:33:51 +0800 Subject: [PATCH] Enable multi-tenant authentication with auxiliary token provider --- go.mod | 2 +- go.sum | 161 +--- .../armauth/multi_tenant_token_provider.go | 70 ++ pkg/azureclients/armauth/token_provider.go | 47 ++ pkg/provider/azure.go | 32 +- pkg/provider/azure_test.go | 8 + pkg/provider/config/azure_auth.go | 80 +- pkg/provider/config/azure_auth_test.go | 761 ++++++++++-------- pkg/provider/config/fixture_test.go | 24 + vendor/modules.txt | 2 +- .../azclient/armauth/keyvault_credential.go | 123 ++- .../cloud-provider-azure/pkg/azclient/auth.go | 13 +- .../pkg/azclient/auth_conf.go | 16 +- 13 files changed, 754 insertions(+), 585 deletions(-) create mode 100644 pkg/azureclients/armauth/multi_tenant_token_provider.go create mode 100644 pkg/azureclients/armauth/token_provider.go create mode 100644 pkg/provider/config/fixture_test.go diff --git a/go.mod b/go.mod index 7745dd9513..4d50e1621a 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( k8s.io/klog/v2 v2.120.1 k8s.io/kubelet v0.30.1 k8s.io/utils v0.0.0-20231127182322-b307cd553661 - sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.19 + sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.21 sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.11 sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 16d3603139..94496dee9b 100644 --- a/go.sum +++ b/go.sum @@ -1,123 +1,8 @@ cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= -cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= -cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= -cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= -cloud.google.com/go/aiplatform v1.48.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= -cloud.google.com/go/analytics v0.21.3/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= -cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= -cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= -cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= -cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= -cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= -cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= -cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= -cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= -cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= -cloud.google.com/go/baremetalsolution v1.1.1/go.mod h1:D1AV6xwOksJMV4OSlWHtWuFNZZYujJknMAP4Qa27QIA= -cloud.google.com/go/batch v1.3.1/go.mod h1:VguXeQKXIYaeeIYbuozUmBR13AfL4SJP7IltNPS+A4A= -cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= -cloud.google.com/go/bigquery v1.53.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= -cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= -cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= -cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= -cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= -cloud.google.com/go/cloudbuild v1.13.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= -cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= -cloud.google.com/go/cloudtasks v1.12.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/contactcenterinsights v1.10.0/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= -cloud.google.com/go/container v1.24.0/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= -cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= -cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= -cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= -cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= -cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= -cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= -cloud.google.com/go/dataplex v1.9.0/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= -cloud.google.com/go/dataproc/v2 v2.0.1/go.mod h1:7Ez3KRHdFGcfY7GcevBbvozX+zyWGcwLJvvAMwCaoZ4= -cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= -cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= -cloud.google.com/go/datastream v1.10.0/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= -cloud.google.com/go/deploy v1.13.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= -cloud.google.com/go/dialogflow v1.40.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= -cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= -cloud.google.com/go/documentai v1.22.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= -cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= -cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= -cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= -cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= -cloud.google.com/go/eventarc v1.13.0/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= -cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= -cloud.google.com/go/firestore v1.12.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= -cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= -cloud.google.com/go/gkebackup v1.3.0/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= -cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= -cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= -cloud.google.com/go/gkemulticloud v1.0.0/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= -cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= -cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= -cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= -cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= -cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= -cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= -cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= -cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= -cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= -cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= -cloud.google.com/go/maps v1.4.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= -cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= -cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= -cloud.google.com/go/metastore v1.12.0/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= -cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= -cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= -cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= -cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= -cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= -cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= -cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= -cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= -cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= -cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= -cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= -cloud.google.com/go/policytroubleshooter v1.8.0/go.mod h1:tmn5Ir5EToWe384EuboTcVQT7nTag2+DuH3uHmKd1HU= -cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= -cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= -cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= -cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= -cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= -cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= -cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= -cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= -cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= -cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= -cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo= -cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= -cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= -cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= -cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= -cloud.google.com/go/servicedirectory v1.11.0/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= -cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= -cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= -cloud.google.com/go/speech v1.19.0/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= -cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= -cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= -cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= -cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= -cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= -cloud.google.com/go/translate v1.8.2/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= -cloud.google.com/go/video v1.19.0/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= -cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= -cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= -cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= -cloud.google.com/go/vmwareengine v1.0.0/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= -cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= -cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= -cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= -cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= github.com/Azure/azure-kusto-go v0.15.2 h1:OlABJilic9TythSgWW6i8Fd0SgNTg0t9jBu6WVsaixM= github.com/Azure/azure-kusto-go v0.15.2/go.mod h1:9F2zvXH8B6eWzgI1S4k1ZXAIufnBZ1bv1cW1kB1n3D0= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= @@ -185,12 +70,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mx github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -199,17 +80,10 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= -github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= -github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= -github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= -github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -227,7 +101,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= @@ -236,9 +109,6 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -253,18 +123,13 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= -github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -296,7 +161,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= @@ -305,7 +169,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= -github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -314,10 +177,8 @@ github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9q github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -336,7 +197,6 @@ github.com/mattn/go-ieproxy v0.0.11 h1:MQ/5BuGSgDAHZOJe6YY80IF2UVCfGkwfo6AeD7HtH github.com/mattn/go-ieproxy v0.0.11/go.mod h1:/NsJd+kxZBmjMc5hrJCKMbP57B84rvq9BiDRbtO9AS0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -344,24 +204,18 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.17.3 h1:oJcvKpIb7/8uLpDDtnQuf18xVnwKp8DTD7DQ6gTd/MU= github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= @@ -370,7 +224,6 @@ github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdO github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -391,7 +244,6 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -404,8 +256,6 @@ github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -445,7 +295,6 @@ go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= @@ -467,7 +316,6 @@ golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqR golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -503,7 +351,6 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -530,7 +377,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= @@ -553,7 +399,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -577,8 +422,6 @@ k8s.io/component-helpers v0.30.1 h1:/UcxSLzZ0owluTE2WMDrFfZl2L+WVXKdYYYm68qnH7U= k8s.io/component-helpers v0.30.1/go.mod h1:b1Xk27UJ3p/AmPqDx7khrnSxrdwQy9gTP7O1y6MZ6rg= k8s.io/controller-manager v0.30.1 h1:vrpfinHQWGf40U08Zmrt+QxK/2yTgjJl/9DKtjaB1gI= k8s.io/controller-manager v0.30.1/go.mod h1:8rTEPbn8LRKC/vS+If+JAKBfsftCfTMaF8/n4SJC+PQ= -k8s.io/cri-api v0.30.1/go.mod h1://4/umPJSW1ISNSNng4OwjpkvswJOQwU8rnkvO8P+xg= -k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kms v0.30.1 h1:gEIbEeCbFiaN2tNfp/EUhFdGr5/CSj8Eyq6Mkr7cCiY= @@ -591,8 +434,8 @@ k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6R k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4= -sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.19 h1:G0PillnfWBYB26Yqa50ce0oU29aUkP95c3g3kis3CRk= -sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.19/go.mod h1:d5hbS2VV55CcaYd2WTKNjpRFgURgjNR18hkxXyo13OQ= +sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.21 h1:At0mwzuQFt99g6ZQS9W1ZE6IlkbDTKxhVxZv+IL8qVw= +sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.21/go.mod h1:d5hbS2VV55CcaYd2WTKNjpRFgURgjNR18hkxXyo13OQ= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.11 h1:go/utmfXRoIoj1UXk1PWztK3d0q2P1wOmVpxq/9/62g= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.11/go.mod h1:e/uOdKMWxnqs6OotausM//0RSoer8kXzy8YI3yVV0mA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/pkg/azureclients/armauth/multi_tenant_token_provider.go b/pkg/azureclients/armauth/multi_tenant_token_provider.go new file mode 100644 index 0000000000..727f34ca5a --- /dev/null +++ b/pkg/azureclients/armauth/multi_tenant_token_provider.go @@ -0,0 +1,70 @@ +package armauth + +import ( + "context" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/go-logr/logr" +) + +// MultiTenantTokenProvider is the track1 multi-tenant token provider wrapper for track2 implementation. +type MultiTenantTokenProvider struct { + logger logr.Logger + primaryCredential azcore.TokenCredential + auxiliaryCredentials []azcore.TokenCredential + timeout time.Duration + scope string +} + +func NewMultiTenantTokenProvider( + logger logr.Logger, + primaryCredential azcore.TokenCredential, + auxiliaryCredentials []azcore.TokenCredential, + scope string, +) (*MultiTenantTokenProvider, error) { + return &MultiTenantTokenProvider{ + logger: logger, + primaryCredential: primaryCredential, + auxiliaryCredentials: auxiliaryCredentials, + timeout: 10 * time.Second, + scope: scope, + }, nil +} + +func (p *MultiTenantTokenProvider) PrimaryOAuthToken() string { + p.logger.V(4).Info("Fetching primary oauth token") + ctx, cancel := context.WithTimeout(context.Background(), p.timeout) + defer cancel() + + token, err := p.primaryCredential.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{p.scope}, + }) + if err != nil { + p.logger.Error(err, "Failed to fetch primary OAuth token") + return "" + } + return token.Token +} + +func (p *MultiTenantTokenProvider) AuxiliaryOAuthTokens() []string { + p.logger.V(4).Info("Fetching auxiliary oauth token", "num-credentials", len(p.auxiliaryCredentials)) + ctx, cancel := context.WithTimeout(context.Background(), p.timeout) + defer cancel() + + var tokens []string + for _, cred := range p.auxiliaryCredentials { + token, err := cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{p.scope}, + }) + if err != nil { + p.logger.Error(err, "Failed to fetch auxiliary OAuth token") + return nil + } + + tokens = append(tokens, token.Token) + } + + return tokens +} diff --git a/pkg/azureclients/armauth/token_provider.go b/pkg/azureclients/armauth/token_provider.go new file mode 100644 index 0000000000..faf06aa7f8 --- /dev/null +++ b/pkg/azureclients/armauth/token_provider.go @@ -0,0 +1,47 @@ +package armauth + +import ( + "context" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/go-logr/logr" +) + +// TokenProvider is the track1 token provider wrapper for track2 implementation. +type TokenProvider struct { + logger logr.Logger + credential azcore.TokenCredential + timeout time.Duration + scope string +} + +func NewTokenProvider( + logger logr.Logger, + credential azcore.TokenCredential, + scope string, +) (*TokenProvider, error) { + return &TokenProvider{ + logger: logger, + credential: credential, + timeout: 10 * time.Second, + scope: scope, + }, nil +} + +func (p *TokenProvider) OAuthToken() string { + p.logger.V(4).Info("Fetching OAuth token") + ctx, cancel := context.WithTimeout(context.Background(), p.timeout) + defer cancel() + + token, err := p.credential.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{p.scope}, + }) + if err != nil { + p.logger.Error(err, "Failed to fetch OAuth token") + return "" + } + p.logger.V(4).Info("Fetched OAuth token successfully", "token", token.Token) + return token.Token +} diff --git a/pkg/provider/azure.go b/pkg/provider/azure.go index a44fccfbe9..8006fe1cb9 100644 --- a/pkg/provider/azure.go +++ b/pkg/provider/azure.go @@ -693,18 +693,19 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *Config, return nil } + var authProvider *azclient.AuthProvider + authProvider, err = azclient.NewAuthProvider(&az.ARMClientConfig, &az.AzureAuthConfig.AzureAuthConfig) + if err != nil { + return err + } // If uses network resources in different AAD Tenant, then prepare corresponding Service Principal Token for VM/VMSS client and network resources client - multiTenantServicePrincipalToken, networkResourceServicePrincipalToken, err := az.getAuthTokenInMultiTenantEnv(servicePrincipalToken) + multiTenantServicePrincipalToken, networkResourceServicePrincipalToken, err := az.getAuthTokenInMultiTenantEnv(servicePrincipalToken, authProvider) if err != nil { return err } az.configAzureClients(servicePrincipalToken, multiTenantServicePrincipalToken, networkResourceServicePrincipalToken) if az.ComputeClientFactory == nil { - authProvider, err := azclient.NewAuthProvider(&az.ARMClientConfig, &az.AzureAuthConfig.AzureAuthConfig) - if err != nil { - return err - } var cred azcore.TokenCredential if authProvider.IsMultiTenantModeEnabled() { multiTenantCred := authProvider.GetMultiTenantIdentity() @@ -888,21 +889,21 @@ func (az *Cloud) setLBDefaults(config *Config) error { return nil } -func (az *Cloud) getAuthTokenInMultiTenantEnv(_ *adal.ServicePrincipalToken) (*adal.MultiTenantServicePrincipalToken, *adal.ServicePrincipalToken, error) { +func (az *Cloud) getAuthTokenInMultiTenantEnv(_ *adal.ServicePrincipalToken, authProvider *azclient.AuthProvider) (adal.MultitenantOAuthTokenProvider, adal.OAuthTokenProvider, error) { var err error - var multiTenantServicePrincipalToken *adal.MultiTenantServicePrincipalToken - var networkResourceServicePrincipalToken *adal.ServicePrincipalToken + var multiTenantOAuthToken adal.MultitenantOAuthTokenProvider + var networkResourceServicePrincipalToken adal.OAuthTokenProvider if az.Config.UsesNetworkResourceInDifferentTenant() { - multiTenantServicePrincipalToken, err = ratelimitconfig.GetMultiTenantServicePrincipalToken(&az.Config.AzureAuthConfig, &az.Environment) + multiTenantOAuthToken, err = ratelimitconfig.GetMultiTenantServicePrincipalToken(&az.Config.AzureAuthConfig, &az.Environment, authProvider) if err != nil { return nil, nil, err } - networkResourceServicePrincipalToken, err = ratelimitconfig.GetNetworkResourceServicePrincipalToken(&az.Config.AzureAuthConfig, &az.Environment) + networkResourceServicePrincipalToken, err = ratelimitconfig.GetNetworkResourceServicePrincipalToken(&az.Config.AzureAuthConfig, &az.Environment, authProvider) if err != nil { return nil, nil, err } } - return multiTenantServicePrincipalToken, networkResourceServicePrincipalToken, nil + return multiTenantOAuthToken, networkResourceServicePrincipalToken, nil } func (az *Cloud) setCloudProviderBackoffDefaults(config *Config) wait.Backoff { @@ -947,8 +948,8 @@ func (az *Cloud) setCloudProviderBackoffDefaults(config *Config) wait.Backoff { func (az *Cloud) configAzureClients( servicePrincipalToken *adal.ServicePrincipalToken, - multiTenantServicePrincipalToken *adal.MultiTenantServicePrincipalToken, - networkResourceServicePrincipalToken *adal.ServicePrincipalToken) { + multiTenantOAuthTokenProvider adal.MultitenantOAuthTokenProvider, + networkResourceServicePrincipalToken adal.OAuthTokenProvider) { azClientConfig := az.getAzureClientConfig(servicePrincipalToken) // Prepare AzureClientConfig for all azure clients @@ -981,8 +982,9 @@ func (az *Cloud) configAzureClients( zoneClientConfig := azClientConfig.WithRateLimiter(nil) // If uses network resources in different AAD Tenant, update Authorizer for VM/VMSS/VMAS client config - if multiTenantServicePrincipalToken != nil { - multiTenantServicePrincipalTokenAuthorizer := autorest.NewMultiTenantServicePrincipalTokenAuthorizer(multiTenantServicePrincipalToken) + if multiTenantOAuthTokenProvider != nil { + multiTenantServicePrincipalTokenAuthorizer := autorest.NewMultiTenantServicePrincipalTokenAuthorizer(multiTenantOAuthTokenProvider) + vmClientConfig.Authorizer = multiTenantServicePrincipalTokenAuthorizer vmssClientConfig.Authorizer = multiTenantServicePrincipalTokenAuthorizer vmssVMClientConfig.Authorizer = multiTenantServicePrincipalTokenAuthorizer diff --git a/pkg/provider/azure_test.go b/pkg/provider/azure_test.go index e26e560437..c7b3e95b04 100644 --- a/pkg/provider/azure_test.go +++ b/pkg/provider/azure_test.go @@ -2089,6 +2089,7 @@ func TestNewCloudFromJSON(t *testing.T) { // Test Backoff and Rate Limit defaults (json) func TestCloudDefaultConfigFromJSON(t *testing.T) { config := `{ + "tenantId": "--tenant-id--", "aadClientId": "--aad-client-id--", "aadClientSecret": "--aad-client-secret--" }` @@ -2099,6 +2100,7 @@ func TestCloudDefaultConfigFromJSON(t *testing.T) { // Test Backoff and Rate Limit defaults (yaml) func TestCloudDefaultConfigFromYAML(t *testing.T) { config := ` +tenantId: --tenant-id-- aadClientId: --aad-client-id-- aadClientSecret: --aad-client-secret-- ` @@ -2294,9 +2296,15 @@ func getCloudFromConfig(t *testing.T, config string) *Cloud { mockZoneClient := az.ZoneClient.(*mockzoneclient.MockInterface) mockZoneClient.EXPECT().GetZones(gomock.Any(), gomock.Any()).Return(map[string][]string{"eastus": {"1", "2", "3"}}, nil) + // Skip AAD client cert path validation since it will read the file from the path + aadCertPath := c.AADClientCertPath + c.AADClientCertPath = "" + err = az.InitializeCloudFromConfig(context.Background(), c, false, true) assert.NoError(t, err) + az.AADClientCertPath = aadCertPath + return az } diff --git a/pkg/provider/config/azure_auth.go b/pkg/provider/config/azure_auth.go index 58741f9e48..0f3d5f4f66 100644 --- a/pkg/provider/config/azure_auth.go +++ b/pkg/provider/config/azure_auth.go @@ -23,14 +23,15 @@ import ( "os" "strings" + "k8s.io/klog/v2" "sigs.k8s.io/yaml" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" - "k8s.io/klog/v2" - "sigs.k8s.io/cloud-provider-azure/pkg/azclient" + "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armauth" "sigs.k8s.io/cloud-provider-azure/pkg/consts" ) @@ -67,6 +68,7 @@ type AzureAuthConfig struct { // than only azure clients except VM/VMSS and network resource ones use this method to fetch Token. // For tokens for VM/VMSS and network resource ones, please check GetMultiTenantServicePrincipalToken and GetNetworkResourceServicePrincipalToken. func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment, resource string) (*adal.ServicePrincipalToken, error) { + logger := klog.Background().WithName("GetServicePrincipalToken") var tenantID string if strings.EqualFold(config.IdentitySystem, consts.ADFSIdentitySystem) { tenantID = consts.ADFSIdentitySystem @@ -79,7 +81,7 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment, r } if config.UseFederatedWorkloadIdentityExtension { - klog.V(2).Infoln("azure: using workload identity extension to retrieve access token") + logger.V(2).Info("Setup ARM general resource token provider", "method", "workload_identity") oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.TenantID, nil) if err != nil { return nil, fmt.Errorf("failed to create the OAuth config: %w", err) @@ -101,29 +103,29 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment, r } if config.UseManagedIdentityExtension { - klog.V(2).Infoln("azure: using managed identity extension to retrieve access token") + logger.V(2).Info("Setup ARM general resource token provider", "method", "msi") msiEndpoint, err := adal.GetMSIVMEndpoint() if err != nil { return nil, fmt.Errorf("error getting the managed service identity endpoint: %w", err) } if len(config.UserAssignedIdentityID) > 0 { - klog.V(4).Info("azure: using User Assigned MSI ID to retrieve access token") + logger.V(2).Info("Parsing user assigned managed identity") resourceID, err := azure.ParseResourceID(config.UserAssignedIdentityID) if err == nil && strings.EqualFold(resourceID.Provider, "Microsoft.ManagedIdentity") && strings.EqualFold(resourceID.ResourceType, "userAssignedIdentities") { - klog.V(4).Info("azure: User Assigned MSI ID is resource ID") + logger.V(2).Info("Setup with user assigned managed identity", "id-type", "resource_id") return adal.NewServicePrincipalTokenFromMSIWithIdentityResourceID(msiEndpoint, resource, config.UserAssignedIdentityID) } - klog.V(4).Info("azure: User Assigned MSI ID is client ID") + logger.V(2).Info("Setup with user assigned managed identity", "id-type", "client_id") return adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, resource, config.UserAssignedIdentityID) } - klog.V(4).Info("azure: using System Assigned MSI to retrieve access token") + logger.V(2).Info("Setup with system assigned managed identity") return adal.NewServicePrincipalTokenFromMSI( msiEndpoint, resource) @@ -135,7 +137,7 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment, r } if len(config.AADClientSecret) > 0 { - klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token") + logger.V(2).Info("Setup ARM general resource token provider", "method", "sp_with_password") return adal.NewServicePrincipalToken( *oauthConfig, config.AADClientID, @@ -144,7 +146,7 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment, r } if len(config.AADClientCertPath) > 0 { - klog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token") + logger.V(2).Info("Setup ARM general resource token provider", "method", "sp_with_certificate") certData, err := os.ReadFile(config.AADClientCertPath) if err != nil { return nil, fmt.Errorf("reading the client certificate from file %s: %w", config.AADClientCertPath, err) @@ -161,6 +163,8 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment, r resource) } + logger.V(2).Info("No valid auth method found") + return nil, ErrorNoAuth } @@ -172,8 +176,10 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment, r // PrimaryToken of the returned multi-tenant token is for the AAD Tenant specified by TenantID, and AuxiliaryToken of the returned multi-tenant token is for the AAD Tenant specified by NetworkResourceTenantID. // // Azure VM/VMSS clients use this multi-tenant token, in order to operate those VM/VMSS in AAD Tenant specified by TenantID, and meanwhile in their payload they are referencing network resources (e.g. Load Balancer, Network Security Group, etc.) in AAD Tenant specified by NetworkResourceTenantID. -func GetMultiTenantServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.MultiTenantServicePrincipalToken, error) { - err := config.checkConfigWhenNetworkResourceInDifferentTenant() +func GetMultiTenantServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment, authProvider *azclient.AuthProvider) (adal.MultitenantOAuthTokenProvider, error) { + logger := klog.Background().WithName("GetMultiTenantServicePrincipalToken") + + err := config.ValidateForMultiTenant() if err != nil { return nil, fmt.Errorf("got error getting multi-tenant service principal token: %w", err) } @@ -184,8 +190,8 @@ func GetMultiTenantServicePrincipalToken(config *AzureAuthConfig, env *azure.Env return nil, fmt.Errorf("creating the multi-tenant OAuth config: %w", err) } - if len(config.AADClientSecret) > 0 { - klog.V(2).Infoln("azure: using client_id+client_secret to retrieve multi-tenant access token") + if len(config.AADClientSecret) > 0 && !strings.EqualFold(config.AADClientSecret, "msi") { + logger.V(2).Info("Setup ARM multi-tenant token provider", "method", "sp_with_password") return adal.NewMultiTenantServicePrincipalToken( multiTenantOAuthConfig, config.AADClientID, @@ -194,7 +200,7 @@ func GetMultiTenantServicePrincipalToken(config *AzureAuthConfig, env *azure.Env } if len(config.AADClientCertPath) > 0 { - klog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve multi-tenant access token") + logger.V(2).Info("Setup ARM multi-tenant token provider", "method", "sp_with_certificate") certData, err := os.ReadFile(config.AADClientCertPath) if err != nil { return nil, fmt.Errorf("reading the client certificate from file %s: %w", config.AADClientCertPath, err) @@ -211,6 +217,18 @@ func GetMultiTenantServicePrincipalToken(config *AzureAuthConfig, env *azure.Env env.ServiceManagementEndpoint) } + if authProvider.ManagedIdentityCredential != nil && authProvider.NetworkTokenCredential != nil { + logger.V(2).Info("Setup ARM multi-tenant token provider", "method", "msi_with_auxiliary_token") + return armauth.NewMultiTenantTokenProvider( + klog.Background().WithName("multi-tenant-resource-token-provider"), + authProvider.ManagedIdentityCredential, + []azcore.TokenCredential{authProvider.NetworkTokenCredential}, + authProvider.TokenScope(), + ) + } + + logger.V(2).Info("No valid auth method found") + return nil, ErrorNoAuth } @@ -220,8 +238,10 @@ func GetMultiTenantServicePrincipalToken(config *AzureAuthConfig, env *azure.Env // and this method creates a new service principal token for network resources tenant based on the configuration. // // Azure network resource (Load Balancer, Public IP, Route Table, Network Security Group and their sub level resources) clients use this multi-tenant token, in order to operate resources in AAD Tenant specified by NetworkResourceTenantID. -func GetNetworkResourceServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) { - err := config.checkConfigWhenNetworkResourceInDifferentTenant() +func GetNetworkResourceServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment, authProvider *azclient.AuthProvider) (adal.OAuthTokenProvider, error) { + logger := klog.Background().WithName("GetNetworkResourceServicePrincipalToken") + + err := config.ValidateForMultiTenant() if err != nil { return nil, fmt.Errorf("got error(%w) in getting network resources service principal token", err) } @@ -231,8 +251,8 @@ func GetNetworkResourceServicePrincipalToken(config *AzureAuthConfig, env *azure return nil, fmt.Errorf("creating the OAuth config for network resources tenant: %w", err) } - if len(config.AADClientSecret) > 0 { - klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token for network resources tenant") + if len(config.AADClientSecret) > 0 && !strings.EqualFold(config.AADClientSecret, "msi") { + logger.V(2).Info("Setup ARM network resource token provider", "method", "sp_with_password") return adal.NewServicePrincipalToken( *oauthConfig, config.AADClientID, @@ -241,7 +261,7 @@ func GetNetworkResourceServicePrincipalToken(config *AzureAuthConfig, env *azure } if len(config.AADClientCertPath) > 0 { - klog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token for network resources tenant") + logger.V(2).Info("Setup ARM network resource token provider", "method", "sp_with_certificate") certData, err := os.ReadFile(config.AADClientCertPath) if err != nil { return nil, fmt.Errorf("reading the client certificate from file %s: %w", config.AADClientCertPath, err) @@ -258,6 +278,18 @@ func GetNetworkResourceServicePrincipalToken(config *AzureAuthConfig, env *azure env.ServiceManagementEndpoint) } + if authProvider.ManagedIdentityCredential != nil && authProvider.NetworkTokenCredential != nil { + logger.V(2).Info("Setup ARM network resource token provider", "method", "msi_with_auxiliary_token") + + return armauth.NewTokenProvider( + klog.Background().WithName("network-resource-token-provider"), + authProvider.NetworkTokenCredential, + authProvider.TokenScope(), + ) + } + + logger.V(2).Info("No valid auth method found") + return nil, ErrorNoAuth } @@ -339,8 +371,8 @@ func azureStackOverrides(env *azure.Environment, resourceManagerEndpoint, identi } } -// checkConfigWhenNetworkResourceInDifferentTenant checks configuration for the scenario of using network resource in different tenant -func (config *AzureAuthConfig) checkConfigWhenNetworkResourceInDifferentTenant() error { +// ValidateForMultiTenant checks configuration for the scenario of using network resource in different tenant +func (config *AzureAuthConfig) ValidateForMultiTenant() error { if !config.UsesNetworkResourceInDifferentTenant() { return fmt.Errorf("NetworkResourceTenantID must be configured") } @@ -349,9 +381,5 @@ func (config *AzureAuthConfig) checkConfigWhenNetworkResourceInDifferentTenant() return fmt.Errorf("ADFS identity system is not supported") } - if config.UseManagedIdentityExtension { - return fmt.Errorf("managed identity is not supported") - } - return nil } diff --git a/pkg/provider/config/azure_auth_test.go b/pkg/provider/config/azure_auth_test.go index 74440b9f1f..8d1f90cf57 100644 --- a/pkg/provider/config/azure_auth_test.go +++ b/pkg/provider/config/azure_auth_test.go @@ -23,6 +23,8 @@ import ( "os" "testing" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" "github.com/stretchr/testify/assert" @@ -34,6 +36,7 @@ import ( var ( CrossTenantNetworkResourceNegativeConfig = []*AzureAuthConfig{ { + // missing NetworkResourceTenantID ARMClientConfig: azclient.ARMClientConfig{ TenantID: "TenantID", }, @@ -52,428 +55,488 @@ var ( AADClientSecret: "AADClientSecret", }, NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID", - IdentitySystem: consts.ADFSIdentitySystem, - }, - { - ARMClientConfig: azclient.ARMClientConfig{ - TenantID: "TenantID", - NetworkResourceTenantID: "NetworkResourceTenantID", - }, - AzureAuthConfig: azclient.AzureAuthConfig{ - AADClientID: "AADClientID", - AADClientSecret: "AADClientSecret", - UseManagedIdentityExtension: true, - }, - NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID", + IdentitySystem: consts.ADFSIdentitySystem, // multi-tenant not supported with ADFS }, } +) - // msiEndpointEnv is the environment variable used to store the endpoint in go-autorest/adal library. - msiEndpointEnv = "MSI_ENDPOINT" - // msiSecretEnv is the environment variable used to store the request secret in go-autorest/adal library. - msiSecretEnv = "MSI_SECRET" +const ( + // envMSIEndpoint is the environment variable used to store the endpoint in go-autorest/adal library. + envMSIEndpoint = "MSI_ENDPOINT" + // envMSISecret is the environment variable used to store the request secret in go-autorest/adal library. + envMSISecret = "MSI_SECRET" ) -func TestGetServicePrincipalTokenFromMSIWithUserAssignedID(t *testing.T) { - configs := []*AzureAuthConfig{ - { - AzureAuthConfig: azclient.AzureAuthConfig{ - UseManagedIdentityExtension: true, - UserAssignedIdentityID: "00000000-0000-0000-0000-000000000000", +func TestGetServicePrincipalToken(t *testing.T) { + env := &azure.PublicCloud + setupLocalMSIServer := func(t *testing.T) (*httptest.Server, func()) { + t.Helper() + var ( + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodGet, r.Method) + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte("{}")) + assert.NoError(t, err) + })) + originalEnv = os.Getenv(envMSIEndpoint) + originalSecret = os.Getenv(envMSISecret) + + cleanUp = func() { + server.Close() + _ = os.Setenv(envMSIEndpoint, originalEnv) + _ = os.Setenv(envMSISecret, originalSecret) + } + ) + + _ = os.Setenv(envMSIEndpoint, server.URL) + _ = os.Setenv(envMSISecret, "secret") + + return server, cleanUp + } + + t.Run("setup with MSI (user assigned managed identity)", func(t *testing.T) { + type IDType int + const ( + ResourceID IDType = iota + ClientID + ) + + tests := []struct { + Name string + Type IDType + Config *AzureAuthConfig + }{ + { + Name: "client id", + Type: ClientID, + Config: &AzureAuthConfig{ + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "00000000-0000-0000-0000-000000000000", + }, + }, }, - }, - // The Azure service principal is ignored when - // UseManagedIdentityExtension is set to true - { + { + Name: "client id with SP (SP config should be ignored)", + Type: ClientID, + Config: &AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "00000000-0000-0000-0000-000000000000", + AADClientID: "AADClientID", + AADClientSecret: "AADClientSecret", + }, + }, + }, + { + Name: "resource id", + Type: ResourceID, + Config: &AzureAuthConfig{ + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ua", + }, + }, + }, + { + Name: "resource id with SP (SP config should be ignored)", + Type: ResourceID, + Config: &AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ua", + AADClientID: "AADClientID", + AADClientSecret: "AADClientSecret", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + _, cleanUp := setupLocalMSIServer(t) + defer cleanUp() + + token, err := GetServicePrincipalToken(tt.Config, env, "") + assert.NoError(t, err) + + msiEndpoint, err := adal.GetMSIVMEndpoint() + assert.NoError(t, err) + + switch tt.Type { + case ClientID: + spt, err := adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, + env.ServiceManagementEndpoint, tt.Config.UserAssignedIdentityID) + assert.NoError(t, err) + assert.Equal(t, token, spt) + case ResourceID: + spt, err := adal.NewServicePrincipalTokenFromMSIWithIdentityResourceID(msiEndpoint, + env.ServiceManagementEndpoint, tt.Config.UserAssignedIdentityID) + assert.NoError(t, err) + assert.Equal(t, token, spt) + } + }) + } + }) + + t.Run("setup with MSI (system managed identity)", func(t *testing.T) { + tests := []struct { + Name string + Config *AzureAuthConfig + }{ + { + Name: "default", + Config: &AzureAuthConfig{ + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + }, + }, + }, + { + Name: "with SP (SP config should be ignored", + Config: &AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + AADClientID: "AADClientID", + AADClientSecret: "AADClientSecret", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + _, cleanUp := setupLocalMSIServer(t) + defer cleanUp() + + token, err := GetServicePrincipalToken(tt.Config, env, "") + assert.NoError(t, err) + + msiEndpoint, err := adal.GetMSIVMEndpoint() + assert.NoError(t, err) + + spt, err := adal.NewServicePrincipalTokenFromMSI(msiEndpoint, env.ServiceManagementEndpoint) + assert.NoError(t, err) + assert.Equal(t, token, spt) + + }) + } + }) + + t.Run("setup with workload identity", func(t *testing.T) { + config := &AzureAuthConfig{ ARMClientConfig: azclient.ARMClientConfig{ TenantID: "TenantID", }, AzureAuthConfig: azclient.AzureAuthConfig{ - UseManagedIdentityExtension: true, - UserAssignedIdentityID: "00000000-0000-0000-0000-000000000000", - AADClientID: "AADClientID", - AADClientSecret: "AADClientSecret", + AADClientID: "AADClientID", + AADFederatedTokenFile: "/tmp/federated-token", + UseFederatedWorkloadIdentityExtension: true, }, - }, - } - env := &azure.PublicCloud + } + env := &azure.PublicCloud - // msiEndpointEnv and msiSecretEnv are required because autorest/adal library requires IMDS endpoint to be available. - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "GET", r.Method) - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte("{}")) - assert.NoError(t, err) - })) - originalEnv := os.Getenv(msiEndpointEnv) - originalSecret := os.Getenv(msiSecretEnv) - os.Setenv(msiEndpointEnv, server.URL) - os.Setenv(msiSecretEnv, "secret") - defer func() { - server.Close() - os.Setenv(msiEndpointEnv, originalEnv) - os.Setenv(msiSecretEnv, originalSecret) - }() - - for _, config := range configs { token, err := GetServicePrincipalToken(config, env, "") assert.NoError(t, err) + marshalToken, _ := token.MarshalJSON() - msiEndpoint, err := adal.GetMSIVMEndpoint() + oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.TenantID, nil) assert.NoError(t, err) - spt, err := adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, - env.ServiceManagementEndpoint, config.UserAssignedIdentityID) + jwtCallback := func() (string, error) { + jwt, err := os.ReadFile(config.AADFederatedTokenFile) + if err != nil { + return "", fmt.Errorf("failed to read a file with a federated token: %w", err) + } + return string(jwt), nil + } + + spt, err := adal.NewServicePrincipalTokenFromFederatedTokenCallback(*oauthConfig, config.AADClientID, jwtCallback, env.ResourceManagerEndpoint) assert.NoError(t, err) - assert.Equal(t, token, spt) - } -} -func TestGetServicePrincipalTokenFromMSIWithIdentityResourceID(t *testing.T) { - configs := []*AzureAuthConfig{ - { - AzureAuthConfig: azclient.AzureAuthConfig{ - UseManagedIdentityExtension: true, - UserAssignedIdentityID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ua", - }, - }, - // The Azure service principal is ignored when - // UseManagedIdentityExtension is set to true - { + marshalSpt, _ := spt.MarshalJSON() + + assert.Equal(t, marshalToken, marshalSpt) + }) + + t.Run("setup with SP with password", func(t *testing.T) { + config := &AzureAuthConfig{ ARMClientConfig: azclient.ARMClientConfig{ TenantID: "TenantID", }, AzureAuthConfig: azclient.AzureAuthConfig{ - UseManagedIdentityExtension: true, - UserAssignedIdentityID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ua", - AADClientID: "AADClientID", - AADClientSecret: "AADClientSecret", + AADClientID: "AADClientID", + AADClientSecret: "AADClientSecret", }, - }, - } - env := &azure.PublicCloud + } + env := &azure.PublicCloud - // msiEndpointEnv and msiSecretEnv are required because autorest/adal library requires IMDS endpoint to be available. - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "GET", r.Method) - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte("{}")) - assert.NoError(t, err) - })) - originalEnv := os.Getenv(msiEndpointEnv) - originalSecret := os.Getenv(msiSecretEnv) - os.Setenv(msiEndpointEnv, server.URL) - os.Setenv(msiSecretEnv, "secret") - defer func() { - server.Close() - os.Setenv(msiEndpointEnv, originalEnv) - os.Setenv(msiSecretEnv, originalSecret) - }() - - for _, config := range configs { token, err := GetServicePrincipalToken(config, env, "") assert.NoError(t, err) - msiEndpoint, err := adal.GetMSIVMEndpoint() + oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.TenantID, nil) assert.NoError(t, err) - spt, err := adal.NewServicePrincipalTokenFromMSIWithIdentityResourceID(msiEndpoint, - env.ServiceManagementEndpoint, config.UserAssignedIdentityID) + spt, err := adal.NewServicePrincipalToken(*oauthConfig, config.AADClientID, config.AADClientSecret, env.ServiceManagementEndpoint) assert.NoError(t, err) + assert.Equal(t, token, spt) - } -} + }) -func TestGetServicePrincipalTokenFromMSI(t *testing.T) { - configs := []*AzureAuthConfig{ - { - AzureAuthConfig: azclient.AzureAuthConfig{ - UseManagedIdentityExtension: true, - }, - }, - // The Azure service principal is ignored when - // UseManagedIdentityExtension is set to true - { + t.Run("setup with SP with certificate", func(t *testing.T) { + config := &AzureAuthConfig{ ARMClientConfig: azclient.ARMClientConfig{ TenantID: "TenantID", }, AzureAuthConfig: azclient.AzureAuthConfig{ - UseManagedIdentityExtension: true, - AADClientID: "AADClientID", - AADClientSecret: "AADClientSecret", + AADClientID: "AADClientID", + AADClientCertPath: "./testdata/test.pfx", + AADClientCertPassword: "id", }, - }, - } - env := &azure.PublicCloud - - // msiEndpointEnv and msiSecretEnv are required because autorest/adal library requires IMDS endpoint to be available. - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "GET", r.Method) - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte("{}")) - assert.NoError(t, err) - })) - originalEnv := os.Getenv(msiEndpointEnv) - originalSecret := os.Getenv(msiSecretEnv) - os.Setenv(msiEndpointEnv, server.URL) - os.Setenv(msiSecretEnv, "secret") - defer func() { - server.Close() - os.Setenv(msiEndpointEnv, originalEnv) - os.Setenv(msiSecretEnv, originalSecret) - }() - - for _, config := range configs { + } + env := &azure.PublicCloud token, err := GetServicePrincipalToken(config, env, "") assert.NoError(t, err) - msiEndpoint, err := adal.GetMSIVMEndpoint() + oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.TenantID, nil) assert.NoError(t, err) - - spt, err := adal.NewServicePrincipalTokenFromMSI(msiEndpoint, env.ServiceManagementEndpoint) + pfxContent, err := os.ReadFile("./testdata/test.pfx") + assert.NoError(t, err) + certificate, privateKey, err := adal.DecodePfxCertificateData(pfxContent, "id") + assert.NoError(t, err) + spt, err := adal.NewServicePrincipalTokenFromCertificate(*oauthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) assert.NoError(t, err) assert.Equal(t, token, spt) - } - -} - -func TestGetServicePrincipalTokenFromWorkloadIdentity(t *testing.T) { - config := &AzureAuthConfig{ - ARMClientConfig: azclient.ARMClientConfig{ - TenantID: "TenantID", - }, - AzureAuthConfig: azclient.AzureAuthConfig{ - AADClientID: "AADClientID", - AADFederatedTokenFile: "/tmp/federated-token", - UseFederatedWorkloadIdentityExtension: true, - }, - } - env := &azure.PublicCloud + }) - token, err := GetServicePrincipalToken(config, env, "") - assert.NoError(t, err) - marshalToken, _ := token.MarshalJSON() - - oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.TenantID, nil) - assert.NoError(t, err) - - jwtCallback := func() (string, error) { - jwt, err := os.ReadFile(config.AADFederatedTokenFile) - if err != nil { - return "", fmt.Errorf("failed to read a file with a federated token: %w", err) + t.Run("setup with SP and certificate (no certificate password)", func(t *testing.T) { + config := &AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "AADClientID", + AADClientCertPath: "./testdata/testnopassword.pfx", + }, } - return string(jwt), nil - } - - spt, err := adal.NewServicePrincipalTokenFromFederatedTokenCallback(*oauthConfig, config.AADClientID, jwtCallback, env.ResourceManagerEndpoint) - assert.NoError(t, err) - - marshalSpt, _ := spt.MarshalJSON() - - assert.Equal(t, marshalToken, marshalSpt) -} - -func TestGetServicePrincipalToken(t *testing.T) { - config := &AzureAuthConfig{ - ARMClientConfig: azclient.ARMClientConfig{ - TenantID: "TenantID", - }, - AzureAuthConfig: azclient.AzureAuthConfig{ - AADClientID: "AADClientID", - AADClientSecret: "AADClientSecret", - }, - } - env := &azure.PublicCloud - - token, err := GetServicePrincipalToken(config, env, "") - assert.NoError(t, err) - - oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.TenantID, nil) - assert.NoError(t, err) - - spt, err := adal.NewServicePrincipalToken(*oauthConfig, config.AADClientID, config.AADClientSecret, env.ServiceManagementEndpoint) - assert.NoError(t, err) + env := &azure.PublicCloud + token, err := GetServicePrincipalToken(config, env, "") + assert.NoError(t, err) - assert.Equal(t, token, spt) + oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.TenantID, nil) + assert.NoError(t, err) + pfxContent, err := os.ReadFile("./testdata/testnopassword.pfx") + assert.NoError(t, err) + certificate, privateKey, err := adal.DecodePfxCertificateData(pfxContent, "") + assert.NoError(t, err) + spt, err := adal.NewServicePrincipalTokenFromCertificate(*oauthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) + assert.NoError(t, err) + assert.Equal(t, token, spt) + }) } func TestGetMultiTenantServicePrincipalToken(t *testing.T) { - config := &AzureAuthConfig{ - ARMClientConfig: azclient.ARMClientConfig{ - TenantID: "TenantID", - NetworkResourceTenantID: "NetworkResourceTenantID", - }, - AzureAuthConfig: azclient.AzureAuthConfig{ - AADClientID: "AADClientID", - AADClientSecret: "AADClientSecret", - }, - NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID", - } - env := &azure.PublicCloud - - multiTenantToken, err := GetMultiTenantServicePrincipalToken(config, env) - assert.NoError(t, err) - - multiTenantOAuthConfig, err := adal.NewMultiTenantOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{}) - assert.NoError(t, err) - - spt, err := adal.NewMultiTenantServicePrincipalToken(multiTenantOAuthConfig, config.AADClientID, config.AADClientSecret, env.ServiceManagementEndpoint) - assert.NoError(t, err) - - assert.Equal(t, multiTenantToken, spt) -} + t.Run("setup with SP with password", func(t *testing.T) { + config := &AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + NetworkResourceTenantID: "NetworkResourceTenantID", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "AADClientID", + AADClientSecret: "AADClientSecret", + }, + NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID", + } + env := &azure.PublicCloud -func TestGetServicePrincipalTokenFromCertificate(t *testing.T) { - config := &AzureAuthConfig{ - ARMClientConfig: azclient.ARMClientConfig{ - TenantID: "TenantID", - }, - AzureAuthConfig: azclient.AzureAuthConfig{ - AADClientID: "AADClientID", - AADClientCertPath: "./testdata/test.pfx", - AADClientCertPassword: "id", - }, - } - env := &azure.PublicCloud - token, err := GetServicePrincipalToken(config, env, "") - assert.NoError(t, err) + multiTenantToken, err := GetMultiTenantServicePrincipalToken(config, env, nil) + assert.NoError(t, err) - oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.TenantID, nil) - assert.NoError(t, err) - pfxContent, err := os.ReadFile("./testdata/test.pfx") - assert.NoError(t, err) - certificate, privateKey, err := adal.DecodePfxCertificateData(pfxContent, "id") - assert.NoError(t, err) - spt, err := adal.NewServicePrincipalTokenFromCertificate(*oauthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) - assert.NoError(t, err) - assert.Equal(t, token, spt) -} + multiTenantOAuthConfig, err := adal.NewMultiTenantOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{}) + assert.NoError(t, err) -func TestGetServicePrincipalTokenFromCertificateWithoutPassword(t *testing.T) { - config := &AzureAuthConfig{ - ARMClientConfig: azclient.ARMClientConfig{ - TenantID: "TenantID", - }, - AzureAuthConfig: azclient.AzureAuthConfig{ - AADClientID: "AADClientID", - AADClientCertPath: "./testdata/testnopassword.pfx", - }, - } - env := &azure.PublicCloud - token, err := GetServicePrincipalToken(config, env, "") - assert.NoError(t, err) + spt, err := adal.NewMultiTenantServicePrincipalToken(multiTenantOAuthConfig, config.AADClientID, config.AADClientSecret, env.ServiceManagementEndpoint) + assert.NoError(t, err) - oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.TenantID, nil) - assert.NoError(t, err) - pfxContent, err := os.ReadFile("./testdata/testnopassword.pfx") - assert.NoError(t, err) - certificate, privateKey, err := adal.DecodePfxCertificateData(pfxContent, "") - assert.NoError(t, err) - spt, err := adal.NewServicePrincipalTokenFromCertificate(*oauthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) - assert.NoError(t, err) - assert.Equal(t, token, spt) -} + assert.Equal(t, multiTenantToken, spt) + }) -func TestGetMultiTenantServicePrincipalTokenFromCertificate(t *testing.T) { - config := &AzureAuthConfig{ - ARMClientConfig: azclient.ARMClientConfig{ - TenantID: "TenantID", - NetworkResourceTenantID: "NetworkResourceTenantID", - }, - AzureAuthConfig: azclient.AzureAuthConfig{ - AADClientID: "AADClientID", - AADClientCertPath: "./testdata/testnopassword.pfx", - }, - NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID", - } - env := &azure.PublicCloud + t.Run("setup with SP with certificate", func(t *testing.T) { + config := &AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + NetworkResourceTenantID: "NetworkResourceTenantID", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "AADClientID", + AADClientCertPath: "./testdata/testnopassword.pfx", + }, + NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID", + } + env := &azure.PublicCloud - multiTenantToken, err := GetMultiTenantServicePrincipalToken(config, env) - assert.NoError(t, err) + multiTenantToken, err := GetMultiTenantServicePrincipalToken(config, env, nil) + assert.NoError(t, err) - multiTenantOAuthConfig, err := adal.NewMultiTenantOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{}) - assert.NoError(t, err) + multiTenantOAuthConfig, err := adal.NewMultiTenantOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{}) + assert.NoError(t, err) - pfxContent, err := os.ReadFile("./testdata/testnopassword.pfx") - assert.NoError(t, err) - certificate, privateKey, err := adal.DecodePfxCertificateData(pfxContent, "") - assert.NoError(t, err) - spt, err := adal.NewMultiTenantServicePrincipalTokenFromCertificate(multiTenantOAuthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) - assert.NoError(t, err) + pfxContent, err := os.ReadFile("./testdata/testnopassword.pfx") + assert.NoError(t, err) + certificate, privateKey, err := adal.DecodePfxCertificateData(pfxContent, "") + assert.NoError(t, err) + spt, err := adal.NewMultiTenantServicePrincipalTokenFromCertificate(multiTenantOAuthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) + assert.NoError(t, err) - assert.Equal(t, multiTenantToken, spt) -} + assert.Equal(t, multiTenantToken, spt) + }) + + t.Run("setup with MSI and auxiliary token provider", func(t *testing.T) { + const ( + managedIdentityToken = "managed-identity-token" + networkToken = "network-token" + ) + var ( + cfg = &AzureAuthConfig{ + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + }, + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + NetworkResourceTenantID: "NetworkResourceTenantID", + }, + } + authProvider = &azclient.AuthProvider{ + ManagedIdentityCredential: NewDummyTokenCredential(managedIdentityToken), + NetworkTokenCredential: NewDummyTokenCredential(networkToken), + ClientOptions: &policy.ClientOptions{ + Cloud: cloud.AzurePublic, + }, + } + ) + + token, err := GetMultiTenantServicePrincipalToken(cfg, &azure.PublicCloud, authProvider) + assert.NoError(t, err) -func TestGetMultiTenantServicePrincipalTokenNegative(t *testing.T) { - env := &azure.PublicCloud - for _, config := range CrossTenantNetworkResourceNegativeConfig { - _, err := GetMultiTenantServicePrincipalToken(config, env) - assert.Error(t, err) - } + assert.Equal(t, managedIdentityToken, token.PrimaryOAuthToken()) + auxTokens := token.AuxiliaryOAuthTokens() + assert.Len(t, auxTokens, 1) + assert.Equal(t, networkToken, auxTokens[0]) + }) + + t.Run("invalid config", func(t *testing.T) { + env := &azure.PublicCloud + for _, config := range CrossTenantNetworkResourceNegativeConfig { + _, err := GetMultiTenantServicePrincipalToken(config, env, nil) + assert.Error(t, err) + } + }) } func TestGetNetworkResourceServicePrincipalToken(t *testing.T) { - config := &AzureAuthConfig{ - ARMClientConfig: azclient.ARMClientConfig{ - TenantID: "TenantID", - NetworkResourceTenantID: "NetworkResourceTenantID", - }, - AzureAuthConfig: azclient.AzureAuthConfig{ - AADClientID: "AADClientID", - AADClientSecret: "AADClientSecret", - }, - NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID", - } - env := &azure.PublicCloud - token, err := GetNetworkResourceServicePrincipalToken(config, env) - assert.NoError(t, err) + t.Run("setup with SP with password", func(t *testing.T) { + config := &AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + NetworkResourceTenantID: "NetworkResourceTenantID", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "AADClientID", + AADClientSecret: "AADClientSecret", + }, + NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID", + } + env := &azure.PublicCloud - oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.NetworkResourceTenantID, nil) - assert.NoError(t, err) + token, err := GetNetworkResourceServicePrincipalToken(config, env, nil) + assert.NoError(t, err) - spt, err := adal.NewServicePrincipalToken(*oauthConfig, config.AADClientID, config.AADClientSecret, env.ServiceManagementEndpoint) - assert.NoError(t, err) + oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.NetworkResourceTenantID, nil) + assert.NoError(t, err) - assert.Equal(t, token, spt) -} + spt, err := adal.NewServicePrincipalToken(*oauthConfig, config.AADClientID, config.AADClientSecret, env.ServiceManagementEndpoint) + assert.NoError(t, err) -func TestGetNetworkResourceServicePrincipalTokenFromCertificate(t *testing.T) { - config := &AzureAuthConfig{ - ARMClientConfig: azclient.ARMClientConfig{ - TenantID: "TenantID", - NetworkResourceTenantID: "NetworkResourceTenantID", - }, - AzureAuthConfig: azclient.AzureAuthConfig{ - AADClientID: "AADClientID", - AADClientCertPath: "./testdata/testnopassword.pfx", - }, - NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID", - } - env := &azure.PublicCloud + assert.Equal(t, token, spt) + }) - token, err := GetNetworkResourceServicePrincipalToken(config, env) - assert.NoError(t, err) + t.Run("setup with SP with certificate", func(t *testing.T) { + config := &AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + NetworkResourceTenantID: "NetworkResourceTenantID", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "AADClientID", + AADClientCertPath: "./testdata/testnopassword.pfx", + }, + NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID", + } + env := &azure.PublicCloud - oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.NetworkResourceTenantID, nil) - assert.NoError(t, err) + token, err := GetNetworkResourceServicePrincipalToken(config, env, nil) + assert.NoError(t, err) - pfxContent, err := os.ReadFile("./testdata/testnopassword.pfx") - assert.NoError(t, err) - certificate, privateKey, err := adal.DecodePfxCertificateData(pfxContent, "") - assert.NoError(t, err) - spt, err := adal.NewServicePrincipalTokenFromCertificate(*oauthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) - assert.NoError(t, err) + oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.NetworkResourceTenantID, nil) + assert.NoError(t, err) - assert.Equal(t, token, spt) -} + pfxContent, err := os.ReadFile("./testdata/testnopassword.pfx") + assert.NoError(t, err) + certificate, privateKey, err := adal.DecodePfxCertificateData(pfxContent, "") + assert.NoError(t, err) + spt, err := adal.NewServicePrincipalTokenFromCertificate(*oauthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) + assert.NoError(t, err) -func TestGetNetworkResourceServicePrincipalTokenNegative(t *testing.T) { - env := &azure.PublicCloud - for _, config := range CrossTenantNetworkResourceNegativeConfig { - _, err := GetNetworkResourceServicePrincipalToken(config, env) - assert.Error(t, err) - } + assert.Equal(t, token, spt) + }) + + t.Run("setup with MSI and auxiliary token provider", func(t *testing.T) { + const ( + managedIdentityToken = "managed-identity-token" + networkToken = "network-token" + ) + var ( + cfg = &AzureAuthConfig{ + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + }, + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + NetworkResourceTenantID: "NetworkResourceTenantID", + }, + } + authProvider = &azclient.AuthProvider{ + ManagedIdentityCredential: NewDummyTokenCredential(managedIdentityToken), + NetworkTokenCredential: NewDummyTokenCredential(networkToken), + ClientOptions: &policy.ClientOptions{ + Cloud: cloud.AzurePublic, + }, + } + ) + + token, err := GetNetworkResourceServicePrincipalToken(cfg, &azure.PublicCloud, authProvider) + assert.NoError(t, err) + assert.Equal(t, networkToken, token.OAuthToken()) + }) + + t.Run("invalid config", func(t *testing.T) { + env := &azure.PublicCloud + for _, config := range CrossTenantNetworkResourceNegativeConfig { + _, err := GetNetworkResourceServicePrincipalToken(config, env, nil) + assert.Error(t, err) + } + }) } func TestParseAzureEnvironment(t *testing.T) { diff --git a/pkg/provider/config/fixture_test.go b/pkg/provider/config/fixture_test.go new file mode 100644 index 0000000000..ef96f341a1 --- /dev/null +++ b/pkg/provider/config/fixture_test.go @@ -0,0 +1,24 @@ +package config + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" +) + +type DummyTokenCredential struct { + token string +} + +func NewDummyTokenCredential(token string) *DummyTokenCredential { + return &DummyTokenCredential{ + token: token, + } +} + +func (d *DummyTokenCredential) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{ + Token: d.token, + }, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 328f43de94..45f6d63168 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1482,7 +1482,7 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client -# sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.19 +# sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.21 ## explicit; go 1.20 sigs.k8s.io/cloud-provider-azure/pkg/azclient sigs.k8s.io/cloud-provider-azure/pkg/azclient/accountclient diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/armauth/keyvault_credential.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/armauth/keyvault_credential.go index 13352d729b..5e9ab48fa4 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/armauth/keyvault_credential.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/armauth/keyvault_credential.go @@ -20,17 +20,34 @@ import ( "context" "encoding/json" "fmt" + "sync" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets" + + "sigs.k8s.io/cloud-provider-azure/pkg/azclient/utils" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient/vaultclient" ) +type SecretResourceID struct { + SubscriptionID string + ResourceGroup string + VaultName string + SecretName string +} + +func (s SecretResourceID) String() string { + return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.KeyVault/vaults/%s/secrets/%s", s.SubscriptionID, s.ResourceGroup, s.VaultName, s.SecretName) +} + type KeyVaultCredential struct { - secretClient *azsecrets.Client - secretPath string + secretClient *azsecrets.Client + vaultURI string + secretResourceID SecretResourceID + mtx sync.RWMutex token *azcore.AccessToken } @@ -41,42 +58,92 @@ type KeyVaultCredentialSecret struct { func NewKeyVaultCredential( msiCredential azcore.TokenCredential, - keyVaultURL string, - secretName string, + secretResourceID SecretResourceID, ) (*KeyVaultCredential, error) { - cli, err := azsecrets.NewClient(keyVaultURL, msiCredential, nil) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Get KeyVault URI + var vaultURI string + { + vaultCli, err := vaultclient.New(secretResourceID.SubscriptionID, msiCredential, utils.GetDefaultOption()) + if err != nil { + return nil, fmt.Errorf("create KeyVault client: %w", err) + } + + vault, err := vaultCli.Get(ctx, secretResourceID.ResourceGroup, secretResourceID.VaultName) + if err != nil { + return nil, fmt.Errorf("get vault %s: %w", secretResourceID.VaultName, err) + } + + if vault.Properties == nil || vault.Properties.VaultURI == nil { + return nil, fmt.Errorf("vault uri is nil") + } + vaultURI = *vault.Properties.VaultURI + } + + cli, err := azsecrets.NewClient(vaultURI, msiCredential, nil) if err != nil { - return nil, fmt.Errorf("create KeyVault client: %w", err) + return nil, fmt.Errorf("create secret client: %w", err) } rv := &KeyVaultCredential{ - secretClient: cli, - secretPath: secretName, + secretClient: cli, + mtx: sync.RWMutex{}, + secretResourceID: secretResourceID, } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if err := rv.refreshToken(ctx); err != nil { - return nil, fmt.Errorf("refresh token: %w", err) + if _, err := rv.refreshToken(ctx); err != nil { + return nil, fmt.Errorf("refresh token from %s: %w", secretResourceID, err) } return rv, nil } -func (c *KeyVaultCredential) refreshToken(ctx context.Context) error { - const LatestVersion = "" +func (c *KeyVaultCredential) refreshToken(ctx context.Context) (*azcore.AccessToken, error) { + const ( + LatestVersion = "" + RefreshTokenOffset = 5 * time.Minute + ) + + cloneAccessToken := func(token *azcore.AccessToken) *azcore.AccessToken { + return &azcore.AccessToken{ + Token: token.Token, + ExpiresOn: token.ExpiresOn, + } + } - resp, err := c.secretClient.GetSecret(ctx, c.secretPath, LatestVersion, nil) - if err != nil { - return err + { + c.mtx.RLock() + if c.token != nil && c.token.ExpiresOn.Add(RefreshTokenOffset).Before(time.Now()) { + c.mtx.RUnlock() + return cloneAccessToken(c.token), nil + } + c.mtx.RUnlock() } - if resp.Value == nil { - return fmt.Errorf("secret value is nil") + + c.mtx.Lock() + defer c.mtx.Unlock() + + if c.token != nil && c.token.ExpiresOn.Add(RefreshTokenOffset).Before(time.Now()) { + return cloneAccessToken(c.token), nil } var secret KeyVaultCredentialSecret - if err := json.Unmarshal([]byte(*resp.Value), &secret); err != nil { - return fmt.Errorf("unmarshal secret value `%s`: %w", *resp.Value, err) + { + resp, err := c.secretClient.GetSecret(ctx, c.secretResourceID.SecretName, LatestVersion, nil) + if err != nil { + return nil, err + } else if resp.Value == nil { + return nil, fmt.Errorf("secret value is nil") + } + + // Parse secret value + if err := json.Unmarshal([]byte(*resp.Value), &secret); err != nil { + return nil, fmt.Errorf("unmarshal secret value `%s`: %w", *resp.Value, err) + } else if secret.AccessToken == "" { + return nil, fmt.Errorf("access token is empty") + } } c.token = &azcore.AccessToken{ @@ -84,19 +151,15 @@ func (c *KeyVaultCredential) refreshToken(ctx context.Context) error { ExpiresOn: secret.ExpiresOn, } - return nil + // Return a copy of the token to avoid concurrent modification + return cloneAccessToken(c.token), nil } func (c *KeyVaultCredential) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) { - const RefreshTokenOffset = 5 * time.Minute - - if c.token != nil && c.token.ExpiresOn.Add(RefreshTokenOffset).Before(time.Now()) { - return *c.token, nil - } - - if err := c.refreshToken(ctx); err != nil { + token, err := c.refreshToken(ctx) + if err != nil { return azcore.AccessToken{}, fmt.Errorf("refresh token: %w", err) } - return *c.token, nil + return *token, nil } diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/auth.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/auth.go index df6316a16e..f95464ab28 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/auth.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/auth.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" @@ -39,6 +40,8 @@ type AuthProvider struct { NetworkClientSecretCredential azcore.TokenCredential MultiTenantCredential azcore.TokenCredential + + ClientOptions *policy.ClientOptions } func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, clientOptionsMutFn ...func(option *policy.ClientOptions)) (*AuthProvider, error) { @@ -88,8 +91,7 @@ func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, client if config.UseManagedIdentityExtension && config.AuxiliaryTokenProvider != nil && IsMultiTenant(armConfig) { networkTokenCredential, err = armauth.NewKeyVaultCredential( managedIdentityCredential, - config.AuxiliaryTokenProvider.KeyVaultURL, - config.AuxiliaryTokenProvider.SecretName, + config.AuxiliaryTokenProvider.SecretResourceID(), ) if err != nil { return nil, fmt.Errorf("create KeyVaultCredential for auxiliary token provider: %w", err) @@ -172,6 +174,8 @@ func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, client NetworkClientSecretCredential: networkClientSecretCredential, NetworkTokenCredential: networkTokenCredential, MultiTenantCredential: multiTenantCredential, + + ClientOptions: clientOption, }, nil } @@ -210,3 +214,8 @@ func (factory *AuthProvider) GetMultiTenantIdentity() azcore.TokenCredential { func (factory *AuthProvider) IsMultiTenantModeEnabled() bool { return factory.MultiTenantCredential != nil } + +func (factory *AuthProvider) TokenScope() string { + audience := factory.ClientOptions.Cloud.Services[cloud.ResourceManager].Audience + return fmt.Sprintf("https://%s/.default", audience) +} diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/auth_conf.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/auth_conf.go index aa7e464f2e..09979f4c44 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/auth_conf.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/azclient/auth_conf.go @@ -19,6 +19,7 @@ package azclient import ( "os" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient/armauth" "sigs.k8s.io/cloud-provider-azure/pkg/azclient/utils" ) @@ -48,8 +49,10 @@ type AzureAuthConfig struct { } type AzureAuthAuxiliaryTokenProvider struct { - KeyVaultURL string `json:"keyVaultURL,omitempty" yaml:"keyVaultURL,omitempty"` - SecretName string `json:"secretName" yaml:"secretName"` + SubscriptionID string `json:"subscriptionID,omitempty"` + ResourceGroup string `json:"resourceGroup,omitempty"` + VaultName string `json:"vaultName,omitempty"` + SecretName string `json:"secretName,omitempty"` } func (config *AzureAuthConfig) GetAADClientID() string { @@ -75,3 +78,12 @@ func (config *AzureAuthConfig) GetAzureFederatedTokenFile() (string, bool) { } return config.AADFederatedTokenFile, config.UseFederatedWorkloadIdentityExtension } + +func (config *AzureAuthAuxiliaryTokenProvider) SecretResourceID() armauth.SecretResourceID { + return armauth.SecretResourceID{ + SubscriptionID: config.SubscriptionID, + ResourceGroup: config.ResourceGroup, + VaultName: config.VaultName, + SecretName: config.SecretName, + } +}