From 4c7e8baba11a716120a94130a2c3cf7c675a4dfc Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 11 Jun 2024 12:10:56 -0700 Subject: [PATCH 1/5] Merge subscription configuration file values with the supplied subscription config --- .../TestResources/build-test-resource-config.yml | 16 ++++++++++++++++ .../TestResources/deploy-test-resources.yml | 16 ++-------------- .../sub-config/AzurePublicMsft.json | 2 ++ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/eng/common/TestResources/build-test-resource-config.yml b/eng/common/TestResources/build-test-resource-config.yml index 2f158c4754c..f8b47ea1a69 100644 --- a/eng/common/TestResources/build-test-resource-config.yml +++ b/eng/common/TestResources/build-test-resource-config.yml @@ -9,6 +9,9 @@ parameters: - name: EnvVars type: object default: null + - name: SubscriptionConfigurationFilePath + type: string + default: '' steps: - ${{ if parameters.SubscriptionConfiguration }}: @@ -44,3 +47,16 @@ steps: displayName: Merge Test Resource Configurations ${{ if parameters.EnvVars }}: env: ${{ parameters.EnvVars }} + + - ${{ if ne(parameters.SubscriptionConfigurationFilePath, '') }}: + - pwsh: | + $configBase = @' + $(SubscriptionConfiguration) + '@ | ConvertFrom-Json -AsHashtable + $config = Get-Content '${{ parameters.SubscriptionConfigurationFilePath }}' ` + | ConvertFrom-Json -AsHashtable + . ./eng/common/TestResources/SubConfig-Helpers.ps1 + UpdateSubscriptionConfiguration $configBase $config + displayName: Merge in Subscription Configuration File Path + ${{ if parameters.EnvVars }}: + env: ${{ parameters.EnvVars }} diff --git a/eng/common/TestResources/deploy-test-resources.yml b/eng/common/TestResources/deploy-test-resources.yml index 724dfd8addb..8976ad269c6 100644 --- a/eng/common/TestResources/deploy-test-resources.yml +++ b/eng/common/TestResources/deploy-test-resources.yml @@ -8,8 +8,6 @@ parameters: ServiceConnection: not-specified ResourceType: test UseFederatedAuth: false - SubscriptionConfigurationFilePath: '' - # SubscriptionConfiguration will be splatted into the parameters of the test # resources script. It should be JSON in the form: @@ -56,18 +54,9 @@ steps: ScriptType: InlineScript Inline: | eng/common/scripts/Import-AzModules.ps1 - - if ('${{ parameters.SubscriptionConfigurationFilePath }}' -ne '') { - $subscriptionConfiguration = ` - Get-Content '${{ parameters.SubscriptionConfigurationFilePath }}' ` - | ConvertFrom-Json -AsHashtable; - } else { - # Multiline string termination ('@) needs to be at the beginning - # of the line - $subscriptionConfiguration = @' - ${{ parameters.SubscriptionConfiguration }} + $subscriptionConfiguration = @' + ${{ parameters.SubscriptionConfiguration }} '@ | ConvertFrom-Json -AsHashtable; - } # The subscriptionConfiguration may have ArmTemplateParameters defined, so # pass those in via the ArmTemplateParameters flag, and handle any @@ -86,7 +75,6 @@ steps: - ${{ else }}: - pwsh: | eng/common/scripts/Import-AzModules.ps1 - $subscriptionConfiguration = @' ${{ parameters.SubscriptionConfiguration }} '@ | ConvertFrom-Json -AsHashtable; diff --git a/eng/common/TestResources/sub-config/AzurePublicMsft.json b/eng/common/TestResources/sub-config/AzurePublicMsft.json index 78d8a782c26..3a1a085e13b 100644 --- a/eng/common/TestResources/sub-config/AzurePublicMsft.json +++ b/eng/common/TestResources/sub-config/AzurePublicMsft.json @@ -2,8 +2,10 @@ "SubscriptionId": "2cd617ea-1866-46b1-90e3-fffb087ebf9b", "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47", "TestApplicationId": "f850650c-1fcf-4489-b46f-71af2e30d360", + "TestApplicationSecret": "", "TestApplicationOid": "30511c9d-ba1a-4c7b-b422-5b543da11b3f", "ProvisionerApplicationId": "f850650c-1fcf-4489-b46f-71af2e30d360", + "ProvisionerApplicationSecret": "", "ProvisionerApplicationOid": "30511c9d-ba1a-4c7b-b422-5b543da11b3f", "Environment": "AzureCloud", "AzureSubscription": "Azure SDK Test Resources" From 7d233b3feee0ce759220dd9b1088694be031dff9 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 14 Jun 2024 11:06:45 -0700 Subject: [PATCH 2/5] Bring forward working parts --- .../TestResources/SubConfig-Helpers.ps1 | 42 ++++++++++--------- .../build-test-resource-config.yml | 42 +++++++++++++------ 2 files changed, 53 insertions(+), 31 deletions(-) diff --git a/eng/common/TestResources/SubConfig-Helpers.ps1 b/eng/common/TestResources/SubConfig-Helpers.ps1 index 9b69c516bd0..8495033d2a9 100644 --- a/eng/common/TestResources/SubConfig-Helpers.ps1 +++ b/eng/common/TestResources/SubConfig-Helpers.ps1 @@ -131,31 +131,35 @@ function SetSubscriptionConfiguration([object]$subscriptionConfiguration) Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" } -function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration) +function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration, $devOpsOutput = $true) { - foreach ($pair in $subscriptionConfiguration.GetEnumerator()) { - if ($pair.Value -is [Hashtable]) { - if (!$subscriptionConfigurationBase.ContainsKey($pair.Name)) { - $subscriptionConfigurationBase[$pair.Name] = @{} - } - foreach($nestedPair in $pair.Value.GetEnumerator()) { - # Mark values as secret so we don't print json blobs containing secrets in the logs. - # Prepend underscore to the variable name, so we can still access the variable names via environment - # variables if they get set subsequently. - if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) { - Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)" - } - $subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value - } - } else { - if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) { - Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)" + foreach ($pair in $subscriptionConfiguration.GetEnumerator()) { + if ($pair.Value -is [Hashtable]) { + if (!$subscriptionConfigurationBase.ContainsKey($pair.Name)) { + $subscriptionConfigurationBase[$pair.Name] = @{} + } + foreach($nestedPair in $pair.Value.GetEnumerator()) { + # Mark values as secret so we don't print json blobs containing secrets in the logs. + # Prepend underscore to the variable name, so we can still access the variable names via environment + # variables if they get set subsequently. + if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) { + Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)" } - $subscriptionConfigurationBase[$pair.Name] = $pair.Value + $subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value + } + } else { + if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) { + Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)" } + $subscriptionConfigurationBase[$pair.Name] = $pair.Value } + } + if ($devOpsOutput) { $serialized = $subscriptionConfigurationBase | ConvertTo-Json -Compress Write-Host ($subscriptionConfigurationBase | ConvertTo-Json) Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" + } + + return $subscriptionConfigurationBase } diff --git a/eng/common/TestResources/build-test-resource-config.yml b/eng/common/TestResources/build-test-resource-config.yml index f8b47ea1a69..e0336effd9f 100644 --- a/eng/common/TestResources/build-test-resource-config.yml +++ b/eng/common/TestResources/build-test-resource-config.yml @@ -9,9 +9,9 @@ parameters: - name: EnvVars type: object default: null - - name: SubscriptionConfigurationFilePath - type: string - default: '' + - name: SubscriptionConfigurationFilePaths + type: object + default: null steps: - ${{ if parameters.SubscriptionConfiguration }}: @@ -22,6 +22,10 @@ steps: . ./eng/common/TestResources/SubConfig-Helpers.ps1 SetSubscriptionConfiguration $config + + Write-Host ($finalConfig | ConvertTo-Json) + $serialized = $finalConfig | ConvertTo-Json -Compress + Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" displayName: Initialize SubscriptionConfiguration variable ${{ if parameters.EnvVars }}: env: ${{ parameters.EnvVars }} @@ -48,15 +52,29 @@ steps: ${{ if parameters.EnvVars }}: env: ${{ parameters.EnvVars }} - - ${{ if ne(parameters.SubscriptionConfigurationFilePath, '') }}: + - ${{ if parameters.SubscriptionConfigurationFilePaths }}: - pwsh: | - $configBase = @' - $(SubscriptionConfiguration) - '@ | ConvertFrom-Json -AsHashtable - $config = Get-Content '${{ parameters.SubscriptionConfigurationFilePath }}' ` - | ConvertFrom-Json -AsHashtable . ./eng/common/TestResources/SubConfig-Helpers.ps1 - UpdateSubscriptionConfiguration $configBase $config - displayName: Merge in Subscription Configuration File Path + $configBase = @{} + if ('$(SubscriptionConfiguration)'.Trim() -ne '') { + # Tabbing of this sort is required for the here-string to work + $configBase = @' + $(SubscriptionConfiguration) + '@ | ConvertFrom-Json -AsHashtable + } + + $configFiles = @' + ${{ convertToJson(parameters.SubscriptionConfigurationFilePaths) }} + '@ | ConvertFrom-Json -AsHashtable + + foreach ($file in $configFiles) { + $config = Get-Content $file | ConvertFrom-Json -AsHashtable + $configBase = UpdateSubscriptionConfiguration $configBase $config -devOpsOutput $false + } + + $serialized = $configBase | ConvertTo-Json -Compress + Write-Host ($configBase | ConvertTo-Json) + Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" + displayName: Merge in Subscription Configuration File Paths ${{ if parameters.EnvVars }}: - env: ${{ parameters.EnvVars }} + env: ${{ parameters.EnvVars }} \ No newline at end of file From dd2d4a256a578435ec1464989c185ea1ff46e84e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 17 Jun 2024 10:16:13 -0700 Subject: [PATCH 3/5] Collapse into a single task --- .../build-test-resource-config.yml | 102 +++++++----------- 1 file changed, 41 insertions(+), 61 deletions(-) diff --git a/eng/common/TestResources/build-test-resource-config.yml b/eng/common/TestResources/build-test-resource-config.yml index e0336effd9f..2042ec344fe 100644 --- a/eng/common/TestResources/build-test-resource-config.yml +++ b/eng/common/TestResources/build-test-resource-config.yml @@ -14,67 +14,47 @@ parameters: default: null steps: - - ${{ if parameters.SubscriptionConfiguration }}: - - pwsh: | - $config = @' - ${{ parameters.SubscriptionConfiguration }} - '@ | ConvertFrom-Json -AsHashtable - - . ./eng/common/TestResources/SubConfig-Helpers.ps1 - SetSubscriptionConfiguration $config - - Write-Host ($finalConfig | ConvertTo-Json) - $serialized = $finalConfig | ConvertTo-Json -Compress - Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" - displayName: Initialize SubscriptionConfiguration variable - ${{ if parameters.EnvVars }}: - env: ${{ parameters.EnvVars }} - - - ${{ if parameters.SubscriptionConfigurations }}: - - pwsh: | - Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]{}" - displayName: Initialize SubscriptionConfiguration variable for merging - condition: eq(variables['SubscriptionConfiguration'], '') - - - ${{ each config in parameters.SubscriptionConfigurations }}: - - pwsh: | - $configBase = @' - $(SubscriptionConfiguration) - '@ | ConvertFrom-Json -AsHashtable - $config = @' - ${{ config }} - '@ | ConvertFrom-Json -AsHashtable - - . ./eng/common/TestResources/SubConfig-Helpers.ps1 - UpdateSubscriptionConfiguration $configBase $config - - displayName: Merge Test Resource Configurations - ${{ if parameters.EnvVars }}: - env: ${{ parameters.EnvVars }} - - - ${{ if parameters.SubscriptionConfigurationFilePaths }}: - - pwsh: | - . ./eng/common/TestResources/SubConfig-Helpers.ps1 - $configBase = @{} - if ('$(SubscriptionConfiguration)'.Trim() -ne '') { - # Tabbing of this sort is required for the here-string to work - $configBase = @' - $(SubscriptionConfiguration) - '@ | ConvertFrom-Json -AsHashtable + - pwsh: | + . ./eng/common/TestResources/SubConfig-Helpers.ps1 + + $finalConfig = @{} + $baseSubConfigRaw = @' + ${{ parameters.SubscriptionConfiguration }} + '@.Trim() + if ($baseSubConfigRaw) { + $baseSubConfig = $baseSubConfigRaw | ConvertFrom-Json -AsHashtable + + Write-Host "Setting base sub config" + $finalConfig = SetSubscriptionConfiguration $baseSubConfig + } + + $subConfigJsonsRaw = @' + ${{ convertToJson(parameters.SubscriptionConfigurations) }} + '@.Trim() -replace '"{', '{' -replace '}"', '}' + + if ($subConfigJsonsRaw) { + $subConfigs = $subConfigJsonsRaw | ConvertFrom-Json -AsHashtable + + foreach ($subConfig in $subConfigs) { + Write-Host "Merging sub config from list" + $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig } - - $configFiles = @' - ${{ convertToJson(parameters.SubscriptionConfigurationFilePaths) }} - '@ | ConvertFrom-Json -AsHashtable - - foreach ($file in $configFiles) { - $config = Get-Content $file | ConvertFrom-Json -AsHashtable - $configBase = UpdateSubscriptionConfiguration $configBase $config -devOpsOutput $false + } + + $subConfigFilesRaw = @' + ${{ convertToJson(parameters.SubscriptionConfigurationFilePaths) }} + '@.Trim() + + if ($subConfigFilesRaw) { + $subConfigFiles = $subConfigFilesRaw | ConvertFrom-Json -AsHashtable + foreach ($file in $subConfigFiles) { + Write-Host "Merging sub config from file: $file" + $subConfig = Get-Content $file | ConvertFrom-Json -AsHashtable + $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig } + } - $serialized = $configBase | ConvertTo-Json -Compress - Write-Host ($configBase | ConvertTo-Json) - Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" - displayName: Merge in Subscription Configuration File Paths - ${{ if parameters.EnvVars }}: - env: ${{ parameters.EnvVars }} \ No newline at end of file + Write-Host ($finalConfig | ConvertTo-Json) + $serialized = $finalConfig | ConvertTo-Json -Compress + Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" + displayName: Merge subscription configurations From 8338c8ed42f70fb004e0179ba09a90ba5c7fad88 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 17 Jun 2024 10:18:38 -0700 Subject: [PATCH 4/5] SubConfig-Helpers.ps1 changes --- eng/common/TestResources/SubConfig-Helpers.ps1 | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/eng/common/TestResources/SubConfig-Helpers.ps1 b/eng/common/TestResources/SubConfig-Helpers.ps1 index 8495033d2a9..b2ee24e6d66 100644 --- a/eng/common/TestResources/SubConfig-Helpers.ps1 +++ b/eng/common/TestResources/SubConfig-Helpers.ps1 @@ -103,7 +103,7 @@ function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$va function SetSubscriptionConfiguration([object]$subscriptionConfiguration) { - foreach($pair in $subscriptionConfiguration.GetEnumerator()) { + foreach ($pair in $subscriptionConfiguration.GetEnumerator()) { if ($pair.Value -is [Hashtable]) { foreach($nestedPair in $pair.Value.GetEnumerator()) { # Mark values as secret so we don't print json blobs containing secrets in the logs. @@ -126,12 +126,10 @@ function SetSubscriptionConfiguration([object]$subscriptionConfiguration) } } - Write-Host ($subscriptionConfiguration | ConvertTo-Json) - $serialized = $subscriptionConfiguration | ConvertTo-Json -Compress - Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" + return $subscriptionConfiguration } -function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration, $devOpsOutput = $true) +function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration) { foreach ($pair in $subscriptionConfiguration.GetEnumerator()) { if ($pair.Value -is [Hashtable]) { @@ -155,11 +153,5 @@ function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, } } - if ($devOpsOutput) { - $serialized = $subscriptionConfigurationBase | ConvertTo-Json -Compress - Write-Host ($subscriptionConfigurationBase | ConvertTo-Json) - Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" - } - return $subscriptionConfigurationBase } From 8d22d3a08a6d1ac52d164f724192f01c364f3e99 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 17 Jun 2024 10:21:37 -0700 Subject: [PATCH 5/5] Update remove-test-resources.yml --- .../TestResources/remove-test-resources.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index 5009eb53f2a..b877d72139a 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -8,7 +8,6 @@ parameters: ResourceType: test EnvVars: {} UseFederatedAuth: false - SubscriptionConfigurationFilePath: '' # SubscriptionConfiguration will be splat into the parameters of the test # resources script. It should be JSON in the form: @@ -39,20 +38,10 @@ steps: Inline: | eng/common/scripts/Import-AzModules.ps1 - - if ('${{ parameters.SubscriptionConfigurationFilePath }}' -ne '') { - $subscriptionConfiguration = ` - Get-Content '${{ parameters.SubscriptionConfigurationFilePath }}' ` - | ConvertFrom-Json -AsHashtable; - } else { - # Multiline string termination ("@) needs to be at the beginning - # of the line - $subscriptionConfiguration = @" - ${{ parameters.SubscriptionConfiguration }} + $subscriptionConfiguration = @" + ${{ parameters.SubscriptionConfiguration }} "@ | ConvertFrom-Json -AsHashtable; - } - eng/common/TestResources/Remove-TestResources.ps1 ` @subscriptionConfiguration ` -ResourceType '${{ parameters.ResourceType }}' `