From 73e656e09d7c251dd2ba38db09619991bcf8cd07 Mon Sep 17 00:00:00 2001 From: Will Johnson Date: Tue, 3 Jan 2023 10:19:19 -0600 Subject: [PATCH] Resolving race condition based on app settings being deployed while ms deploy is going and event hub authorization rules deploying at the same time. Co-authored-by: hmoazam --- deployment/infra/newdeploymenttemp.json | 161 ++++++++++-------- tests/deployment/compare-app-settings.py | 5 +- .../test_arm_mapping_matches_json.py | 8 +- 3 files changed, 97 insertions(+), 77 deletions(-) diff --git a/deployment/infra/newdeploymenttemp.json b/deployment/infra/newdeploymenttemp.json index 074d98e..5aa070e 100644 --- a/deployment/infra/newdeploymenttemp.json +++ b/deployment/infra/newdeploymenttemp.json @@ -138,78 +138,7 @@ "properties": { "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]", "httpsOnly": true, - "siteConfig": { - "appSettings": [ - { - "name": "AzureWebJobsStorage", - "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccountName'), ';EndpointSuffix=', environment().suffixes.storage, ';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2019-06-01').keys[0].value)]" - }, - { - "name": "FunctionStorage", - "value": "[concat('@Microsoft.KeyVault(VaultName=', variables('openlineageKeyVaultName'),';SecretName=',variables('functionStorageSecret'),')')]" - }, - { - "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", - "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccountName'), ';EndpointSuffix=', environment().suffixes.storage, ';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2019-06-01').keys[0].value)]" - }, - { - "name": "WEBSITE_CONTENTSHARE", - "value": "[toLower(variables('functionAppName'))]" - }, - { - "name": "FUNCTIONS_EXTENSION_VERSION", - "value": "~4" - }, - { - "name": "WEBSITE_NODE_DEFAULT_VERSION", - "value": "~10" - }, - { - "name": "APPINSIGHTS_INSTRUMENTATIONKEY", - "value": "[reference(resourceId('microsoft.insights/components', variables('applicationInsightsName')), '2020-02-02-preview').InstrumentationKey]" - }, - { - "name": "FUNCTIONS_WORKER_RUNTIME", - "value": "[variables('functionWorkerRuntime')]" - }, - { - "name": "EventHubName", - "value": "[variables('openlineageNameEventHubName')]" - }, - { - "name": "ListenToMessagesFromEventHub", - "value": "[concat('@Microsoft.KeyVault(VaultName=', variables('openlineageKeyVaultName'),';SecretName=',variables('EventHubConnectionSecretNameListen'),')')]" - }, - { - "name": "SendMessagesToEventHub", - "value": "[concat('@Microsoft.KeyVault(VaultName=', variables('openlineageKeyVaultName'),';SecretName=',variables('EventHubConnectionSecretNameSend'),')')]" - }, - { - "name": "EventHubConsumerGroup", - "value": "read" - }, - { - "name": "OlToPurviewMappings", - "value": "{\"olToPurviewMappings\":[{\"name\":\"wasbs\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"wasbs\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_blob_path\",\"purviewPrefix\":\"https\"},{\"name\":\"wasb\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"wasb\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_blob_path\",\"purviewPrefix\":\"https\"},{\"name\":\"abfsBlobRootFS\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfs\"},{\"op1\":\"nameSpcBodyParts[1]\",\"compare\":\"contains\",\"op2\":\"blob\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"=\",\"op2\":\"\"}],\"qualifiedName\":\"https://{nameSpcConParts[0]}.dfs.{nameSpcConParts[2]}.{nameSpcConParts[3]}.{nameSpcConParts[4]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_filesystem\",\"purviewPrefix\":\"https\"},{\"name\":\"abfsRootFS\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfs\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"=\",\"op2\":\"\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_filesystem\",\"purviewPrefix\":\"https\"},{\"name\":\"abfssBlobRootFS\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfss\"},{\"op1\":\"nameSpcBodyParts[1]\",\"compare\":\"contains\",\"op2\":\"blob\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"=\",\"op2\":\"\"}],\"qualifiedName\":\"https://{nameSpcConParts[0]}.dfs.{nameSpcConParts[2]}.{nameSpcConParts[3]}.{nameSpcConParts[4]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_filesystem\",\"purviewPrefix\":\"https\"},{\"name\":\"abfssRootFS\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfss\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"=\",\"op2\":\"\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_filesystem\",\"purviewPrefix\":\"https\"},{\"name\":\"abfsBlob\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfs\"},{\"op1\":\"nameSpcBodyParts[1]\",\"compare\":\"contains\",\"op2\":\"blob\"}],\"qualifiedName\":\"https://{nameSpcConParts[0]}.dfs.{nameSpcConParts[2]}.{nameSpcConParts[3]}.{nameSpcConParts[4]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_path\",\"purviewPrefix\":\"https\"},{\"name\":\"abfs\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfs\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_path\",\"purviewPrefix\":\"https\"},{\"name\":\"abfssBlob\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfss\"},{\"op1\":\"nameSpcBodyParts[1]\",\"compare\":\"contains\",\"op2\":\"blob\"}],\"qualifiedName\":\"https://{nameSpcConParts[0]}.dfs.{nameSpcConParts[2]}.{nameSpcConParts[3]}.{nameSpcConParts[4]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_path\",\"purviewPrefix\":\"https\"},{\"name\":\"abfss\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfss\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_path\",\"purviewPrefix\":\"https\"},{\"name\":\"synapseSqlNonDbo\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"sqlserver\"},{\"op1\":\"nameSpcBodyParts[0]\",\"compare\":\"contains\",\"op2\":\"azuresynapse\"},{\"op1\":\"nameGroups[0].parts\",\"compare\":\">\",\"op2\":\"1\"}],\"qualifiedName\":\"mssql://{nameSpcBodyParts[0]}/{nameSpcNameVals['database']}/{nameGroups[0].parts[0]}/{nameGroups[0].parts[1]}\",\"purviewDataType\":\"azure_synapse_dedicated_sql_table\",\"purviewPrefix\":\"mssql\"},{\"name\":\"synapseSql\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"sqlserver\"},{\"op1\":\"nameSpcBodyParts[0]\",\"compare\":\"contains\",\"op2\":\"azuresynapse\"}],\"qualifiedName\":\"mssql://{nameSpcBodyParts[0]}/{nameSpcNameVals['database']}/dbo/{nameGroups[0].parts[0]}\",\"purviewDataType\":\"azure_synapse_dedicated_sql_table\",\"purviewPrefix\":\"mssql\"},{\"name\":\"azureSQLNonDbo\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"sqlserver\"},{\"op1\":\"nameGroups\",\"compare\":\">\",\"op2\":\"1\"}],\"qualifiedName\":\"mssql://{nameSpcBodyParts[0]}/{nameSpcNameVals['database']}/{nameGroups[0]}/{nameGroups[1]}\",\"purviewDataType\":\"azure_sql_table\",\"purviewPrefix\":\"mssql\"},{\"name\":\"azureSQLNonDboNoDotsInNames\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"sqlserver\"},{\"op1\":\"nameGroups[0].parts\",\"compare\":\">\",\"op2\":\"1\"}],\"qualifiedName\":\"mssql://{nameSpcBodyParts[0]}/{nameSpcNameVals['database']}/{nameGroups[0].parts[0]}/{nameGroups[0].parts[1]}\",\"purviewDataType\":\"azure_sql_table\",\"purviewPrefix\":\"mssql\"},{\"name\":\"azureSQL\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"sqlserver\"}],\"qualifiedName\":\"mssql://{nameSpcBodyParts[0]}/{nameSpcNameVals['database']}/dbo/{nameGroups[0]}\",\"purviewDataType\":\"azure_sql_table\",\"purviewPrefix\":\"mssql\"},{\"name\":\"azurePostgresNonPublic\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"postgresql\"},{\"op1\":\"nameGroups[0].parts\",\"compare\":\">\",\"op2\":\"1\"},{\"op1\":\"nameSpcConParts\",\"compare\":\">\",\"op2\":\"4\"},{\"op1\":\"nameSpcConParts[3]\",\"compare\":\"=\",\"op2\":\"azure\"}],\"qualifiedName\":\"postgresql://{nameSpcBodyParts[0]}/{nameSpcBodyParts[2]}/{nameGroups[0].parts[0]}/{nameGroups[0].parts[1]}\",\"purviewDataType\":\"azure_postgresql_table\",\"purviewPrefix\":\"postgresql\"},{\"name\":\"azurePostgres\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"postgresql\"},{\"op1\":\"nameSpcConParts\",\"compare\":\">\",\"op2\":\"4\"},{\"op1\":\"nameSpcConParts[3]\",\"compare\":\"=\",\"op2\":\"azure\"}],\"qualifiedName\":\"postgresql://{nameSpcBodyParts[0]}/{nameSpcBodyParts[2]}/public/{nameGroups[0]}\",\"purviewDataType\":\"azure_postgresql_table\",\"purviewPrefix\":\"postgresql\"},{\"name\":\"postgresNonPublic\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"postgresql\"},{\"op1\":\"nameGroups[0].parts\",\"compare\":\">\",\"op2\":\"1\"}],\"qualifiedName\":\"postgresql://servers/{nameSpcBodyParts[0]}:{nameSpcBodyParts[1]}/dbs/{nameSpcBodyParts[2]}/schemas/{nameGroups[0].parts[0]}/tables/{nameGroups[0].parts[1]}\",\"purviewDataType\":\"postgresql_table\",\"purviewPrefix\":\"postgresql\"},{\"name\":\"postgres\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"postgresql\"}],\"qualifiedName\":\"postgresql://servers/{nameSpcBodyParts[0]}:{nameSpcBodyParts[1]}/dbs/{nameSpcBodyParts[2]}/schemas/public/tables/{nameGroups[0]}\",\"purviewDataType\":\"postgresql_table\",\"purviewPrefix\":\"postgresql\"},{\"name\":\"hiveManagedTableNotDefault\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"dbfs\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"contains\",\"op2\":\"hive/warehouse\"},{\"op1\":\"nameGroups[0].parts\",\"compare\":\">\",\"op2\":\"4\"}],\"qualifiedName\":\"{nameGroups[0].parts[3]}.{nameGroups[0].parts[5]}@{AdbWorkspaceUrl}\",\"purviewDataType\":\"hive_table\",\"purviewPrefix\":\"hive\"},{\"name\":\"hiveManagedTableDefault\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"dbfs\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"contains\",\"op2\":\"hive/warehouse\"}],\"qualifiedName\":\"default.{nameGroups[0].parts[3]}@{AdbWorkspaceUrl}\",\"purviewDataType\":\"hive_table\",\"purviewPrefix\":\"hive\"},{\"name\":\"azureMySql\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"mysql\"}],\"qualifiedName\":\"mysql://{nameSpcBodyParts[0]}/{nameSpcBodyParts[2]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_mysql_table\",\"purviewPrefix\":\"mysql\"}]}" - }, - { - "name": "PurviewAccountName", - "value": "[variables('purviewAccountName')]" - }, - { - "name": "ClientID", - "value": "[concat('@Microsoft.KeyVault(VaultName=', variables('openlineageKeyVaultName'),';SecretName=',variables('clientidkey'),')')]" - }, - { - "name": "ClientSecret", - "value": "[concat('@Microsoft.KeyVault(VaultName=', variables('openlineageKeyVaultName'),';SecretName=',variables('clientsecretkey'),')')]" - }, - { - "name": "TenantId", - "value": "[subscription().tenantId]" - } - ] - } + "siteConfig": {} }, "resources": [ { @@ -218,12 +147,93 @@ "location": "[resourceGroup().location]", "apiVersion": "2020-06-01", "dependsOn": [ - "[concat('Microsoft.Web/sites/', variables('functionAppName'))]" + "[concat('Microsoft.Web/sites/', variables('functionAppName'))]", + "[concat('Microsoft.Web/sites/', variables('functionAppName'), '/config/web')]" ], "properties": { "packageUri": "http://aka.ms/APFunctions2-2" } - } + }, + { + "apiVersion": "2020-06-01", + "type": "config", + "name": "web", + "dependsOn": [ + "[concat('Microsoft.Web/sites/', variables('functionAppName'))]" + ], + "properties": { + "appSettings": [ + { + "name": "AzureWebJobsStorage", + "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccountName'), ';EndpointSuffix=', environment().suffixes.storage, ';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2019-06-01').keys[0].value)]" + }, + { + "name": "FunctionStorage", + "value": "[concat('@Microsoft.KeyVault(VaultName=', variables('openlineageKeyVaultName'),';SecretName=',variables('functionStorageSecret'),')')]" + }, + { + "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", + "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccountName'), ';EndpointSuffix=', environment().suffixes.storage, ';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2019-06-01').keys[0].value)]" + }, + { + "name": "WEBSITE_CONTENTSHARE", + "value": "[toLower(variables('functionAppName'))]" + }, + { + "name": "FUNCTIONS_EXTENSION_VERSION", + "value": "~4" + }, + { + "name": "WEBSITE_NODE_DEFAULT_VERSION", + "value": "~10" + }, + { + "name": "APPINSIGHTS_INSTRUMENTATIONKEY", + "value": "[reference(resourceId('microsoft.insights/components', variables('applicationInsightsName')), '2020-02-02-preview').InstrumentationKey]" + }, + { + "name": "FUNCTIONS_WORKER_RUNTIME", + "value": "[variables('functionWorkerRuntime')]" + }, + { + "name": "EventHubName", + "value": "[variables('openlineageNameEventHubName')]" + }, + { + "name": "ListenToMessagesFromEventHub", + "value": "[concat('@Microsoft.KeyVault(VaultName=', variables('openlineageKeyVaultName'),';SecretName=',variables('EventHubConnectionSecretNameListen'),')')]" + }, + { + "name": "SendMessagesToEventHub", + "value": "[concat('@Microsoft.KeyVault(VaultName=', variables('openlineageKeyVaultName'),';SecretName=',variables('EventHubConnectionSecretNameSend'),')')]" + }, + { + "name": "EventHubConsumerGroup", + "value": "read" + }, + { + "name": "OlToPurviewMappings", + "value": "{\"olToPurviewMappings\":[{\"name\":\"wasbs\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"wasbs\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_blob_path\",\"purviewPrefix\":\"https\"},{\"name\":\"wasb\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"wasb\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_blob_path\",\"purviewPrefix\":\"https\"},{\"name\":\"abfsBlobRootFS\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfs\"},{\"op1\":\"nameSpcBodyParts[1]\",\"compare\":\"contains\",\"op2\":\"blob\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"=\",\"op2\":\"\"}],\"qualifiedName\":\"https://{nameSpcConParts[0]}.dfs.{nameSpcConParts[2]}.{nameSpcConParts[3]}.{nameSpcConParts[4]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_filesystem\",\"purviewPrefix\":\"https\"},{\"name\":\"abfsRootFS\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfs\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"=\",\"op2\":\"\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_filesystem\",\"purviewPrefix\":\"https\"},{\"name\":\"abfssBlobRootFS\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfss\"},{\"op1\":\"nameSpcBodyParts[1]\",\"compare\":\"contains\",\"op2\":\"blob\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"=\",\"op2\":\"\"}],\"qualifiedName\":\"https://{nameSpcConParts[0]}.dfs.{nameSpcConParts[2]}.{nameSpcConParts[3]}.{nameSpcConParts[4]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_filesystem\",\"purviewPrefix\":\"https\"},{\"name\":\"abfssRootFS\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfss\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"=\",\"op2\":\"\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_filesystem\",\"purviewPrefix\":\"https\"},{\"name\":\"abfsBlob\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfs\"},{\"op1\":\"nameSpcBodyParts[1]\",\"compare\":\"contains\",\"op2\":\"blob\"}],\"qualifiedName\":\"https://{nameSpcConParts[0]}.dfs.{nameSpcConParts[2]}.{nameSpcConParts[3]}.{nameSpcConParts[4]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_path\",\"purviewPrefix\":\"https\"},{\"name\":\"abfs\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfs\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_path\",\"purviewPrefix\":\"https\"},{\"name\":\"abfssBlob\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfss\"},{\"op1\":\"nameSpcBodyParts[1]\",\"compare\":\"contains\",\"op2\":\"blob\"}],\"qualifiedName\":\"https://{nameSpcConParts[0]}.dfs.{nameSpcConParts[2]}.{nameSpcConParts[3]}.{nameSpcConParts[4]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_path\",\"purviewPrefix\":\"https\"},{\"name\":\"abfss\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"abfss\"}],\"qualifiedName\":\"https://{nameSpcBodyParts[1]}/{nameSpcBodyParts[0]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_datalake_gen2_path\",\"purviewPrefix\":\"https\"},{\"name\":\"synapseSqlNonDbo\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"sqlserver\"},{\"op1\":\"nameSpcBodyParts[0]\",\"compare\":\"contains\",\"op2\":\"azuresynapse\"},{\"op1\":\"nameGroups[0].parts\",\"compare\":\">\",\"op2\":\"1\"}],\"qualifiedName\":\"mssql://{nameSpcBodyParts[0]}/{nameSpcNameVals['database']}/{nameGroups[0].parts[0]}/{nameGroups[0].parts[1]}\",\"purviewDataType\":\"azure_synapse_dedicated_sql_table\",\"purviewPrefix\":\"mssql\"},{\"name\":\"synapseSql\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"sqlserver\"},{\"op1\":\"nameSpcBodyParts[0]\",\"compare\":\"contains\",\"op2\":\"azuresynapse\"}],\"qualifiedName\":\"mssql://{nameSpcBodyParts[0]}/{nameSpcNameVals['database']}/dbo/{nameGroups[0].parts[0]}\",\"purviewDataType\":\"azure_synapse_dedicated_sql_table\",\"purviewPrefix\":\"mssql\"},{\"name\":\"azureSQLNonDbo\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"sqlserver\"},{\"op1\":\"nameGroups\",\"compare\":\">\",\"op2\":\"1\"}],\"qualifiedName\":\"mssql://{nameSpcBodyParts[0]}/{nameSpcNameVals['database']}/{nameGroups[0]}/{nameGroups[1]}\",\"purviewDataType\":\"azure_sql_table\",\"purviewPrefix\":\"mssql\"},{\"name\":\"azureSQLNonDboNoDotsInNames\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"sqlserver\"},{\"op1\":\"nameGroups[0].parts\",\"compare\":\">\",\"op2\":\"1\"}],\"qualifiedName\":\"mssql://{nameSpcBodyParts[0]}/{nameSpcNameVals['database']}/{nameGroups[0].parts[0]}/{nameGroups[0].parts[1]}\",\"purviewDataType\":\"azure_sql_table\",\"purviewPrefix\":\"mssql\"},{\"name\":\"azureSQL\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"sqlserver\"}],\"qualifiedName\":\"mssql://{nameSpcBodyParts[0]}/{nameSpcNameVals['database']}/dbo/{nameGroups[0]}\",\"purviewDataType\":\"azure_sql_table\",\"purviewPrefix\":\"mssql\"},{\"name\":\"azurePostgresNonPublic\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"postgresql\"},{\"op1\":\"nameGroups[0].parts\",\"compare\":\">\",\"op2\":\"1\"},{\"op1\":\"nameSpcConParts\",\"compare\":\">\",\"op2\":\"4\"},{\"op1\":\"nameSpcConParts[3]\",\"compare\":\"=\",\"op2\":\"azure\"}],\"qualifiedName\":\"postgresql://{nameSpcBodyParts[0]}/{nameSpcBodyParts[2]}/{nameGroups[0].parts[0]}/{nameGroups[0].parts[1]}\",\"purviewDataType\":\"azure_postgresql_table\",\"purviewPrefix\":\"postgresql\"},{\"name\":\"azurePostgres\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"postgresql\"},{\"op1\":\"nameSpcConParts\",\"compare\":\">\",\"op2\":\"4\"},{\"op1\":\"nameSpcConParts[3]\",\"compare\":\"=\",\"op2\":\"azure\"}],\"qualifiedName\":\"postgresql://{nameSpcBodyParts[0]}/{nameSpcBodyParts[2]}/public/{nameGroups[0]}\",\"purviewDataType\":\"azure_postgresql_table\",\"purviewPrefix\":\"postgresql\"},{\"name\":\"postgresNonPublic\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"postgresql\"},{\"op1\":\"nameGroups[0].parts\",\"compare\":\">\",\"op2\":\"1\"}],\"qualifiedName\":\"postgresql://servers/{nameSpcBodyParts[0]}:{nameSpcBodyParts[1]}/dbs/{nameSpcBodyParts[2]}/schemas/{nameGroups[0].parts[0]}/tables/{nameGroups[0].parts[1]}\",\"purviewDataType\":\"postgresql_table\",\"purviewPrefix\":\"postgresql\"},{\"name\":\"postgres\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"postgresql\"}],\"qualifiedName\":\"postgresql://servers/{nameSpcBodyParts[0]}:{nameSpcBodyParts[1]}/dbs/{nameSpcBodyParts[2]}/schemas/public/tables/{nameGroups[0]}\",\"purviewDataType\":\"postgresql_table\",\"purviewPrefix\":\"postgresql\"},{\"name\":\"hiveManagedTableNotDefault\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"dbfs\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"contains\",\"op2\":\"hive/warehouse\"},{\"op1\":\"nameGroups[0].parts\",\"compare\":\">\",\"op2\":\"4\"}],\"qualifiedName\":\"{nameGroups[0].parts[3]}.{nameGroups[0].parts[5]}@{AdbWorkspaceUrl}\",\"purviewDataType\":\"hive_table\",\"purviewPrefix\":\"hive\"},{\"name\":\"hiveManagedTableDefault\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"dbfs\"},{\"op1\":\"nameGroups[0]\",\"compare\":\"contains\",\"op2\":\"hive/warehouse\"}],\"qualifiedName\":\"default.{nameGroups[0].parts[3]}@{AdbWorkspaceUrl}\",\"purviewDataType\":\"hive_table\",\"purviewPrefix\":\"hive\"},{\"name\":\"azureMySql\",\"parserConditions\":[{\"op1\":\"prefix\",\"compare\":\"=\",\"op2\":\"mysql\"}],\"qualifiedName\":\"mysql://{nameSpcBodyParts[0]}/{nameSpcBodyParts[2]}/{nameGroups[0]}\",\"purviewDataType\":\"azure_mysql_table\",\"purviewPrefix\":\"mysql\"}]}" + }, + { + "name": "PurviewAccountName", + "value": "[variables('purviewAccountName')]" + }, + { + "name": "ClientID", + "value": "[concat('@Microsoft.KeyVault(VaultName=', variables('openlineageKeyVaultName'),';SecretName=',variables('clientidkey'),')')]" + }, + { + "name": "ClientSecret", + "value": "[concat('@Microsoft.KeyVault(VaultName=', variables('openlineageKeyVaultName'),';SecretName=',variables('clientsecretkey'),')')]" + }, + { + "name": "TenantId", + "value": "[subscription().tenantId]" + } + ] + } + } ] }, { @@ -447,7 +457,8 @@ "name": "[concat(variables('openlineageEventHubNameSpaceName'), '/ListenMessages')]", "tags": "[parameters('resourceTagValues')]", "dependsOn": [ - "[resourceId('Microsoft.EventHub/namespaces', variables('openlineageEventHubNameSpaceName'))]" + "[resourceId('Microsoft.EventHub/namespaces', variables('openlineageEventHubNameSpaceName'))]", + "[resourceId('Microsoft.EventHub/namespaces/AuthorizationRules', variables('openlineageEventHubNameSpaceName'), 'SendMessages')]" ], "properties": { "rights": [ diff --git a/tests/deployment/compare-app-settings.py b/tests/deployment/compare-app-settings.py index 88fc6e0..35c2f8f 100644 --- a/tests/deployment/compare-app-settings.py +++ b/tests/deployment/compare-app-settings.py @@ -56,7 +56,10 @@ app_settings_in_template = None for resource in arm_template["resources"]: if resource["type"] == "Microsoft.Web/sites": - app_settings_in_template = resource["properties"]["siteConfig"]["appSettings"] + for child_resource in resource.get("resources", []): + if child_resource.get("type") == "config" and child_resource.get("name") == "web": + app_settings_in_template = child_resource.get("properties", {}).get("appSettings", []) + break if app_settings_in_template is None: raise ValueError("Unable to extract the Microsoft.web/sites resources") diff --git a/tests/deployment/test_arm_mapping_matches_json.py b/tests/deployment/test_arm_mapping_matches_json.py index 274dd02..9d56c36 100644 --- a/tests/deployment/test_arm_mapping_matches_json.py +++ b/tests/deployment/test_arm_mapping_matches_json.py @@ -22,7 +22,13 @@ if resource["name"] != "[variables('functionAppName')]": continue - app_settings = resource.get("properties", {}).get("siteConfig", {}).get("appSettings", []) + web_config={} + for child_resource in resource.get("resources", []): + if child_resource.get("type") == "config" and child_resource.get("name") == "web": + web_config = child_resource + break + + app_settings = web_config.get("properties", {}).get("appSettings", []) for setting in app_settings: if setting["name"] != "OlToPurviewMappings": continue