Update tests to only check extensionconfig creation (Azure#61)

* Update tests to only check extensionconfig creation * Single set of CRUD for AzureML * Debug logs for connectedk8s
trstringer · Jul 9, 2021 · a5ef607 · a5ef607
1 parent 5bfabdb
commit a5ef607
Show file tree

Hide file tree

Showing 6 changed files with 122 additions and 109 deletions.
diff --git a/testing/Cleanup.ps1 b/testing/Cleanup.ps1
@@ -12,6 +12,11 @@ az account set --subscription $ENVCONFIG.subscriptionId
 $Env:KUBECONFIG="$PSScriptRoot/tmp/KUBECONFIG"
 Write-Host "Removing the connectedk8s arc agents from the cluster..."
 az connectedk8s delete -g $ENVCONFIG.resourceGroup -n $ENVCONFIG.arcClusterName
+if (!$?)
+{
+    kubectl get pods -A
+    Exit 1
+}
 
 # Skip deleting the AKS Cluster if this is CI
 if (!$CI) {

diff --git a/testing/test/extensions/public/AzureDefender.Tests.ps1 b/testing/test/extensions/public/AzureDefender.Tests.ps1
@@ -23,7 +23,7 @@ Describe 'Azure Defender Testing' {
         do 
         {
             # Only check the extension config, not the pod since this doesn't bring up pods
-            if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) {
+            if (Has-ExtensionData $extensionName) {
                 break
             }
             Start-Sleep -Seconds 10

diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1
@@ -10,8 +10,10 @@ Describe 'AzureML Kubernetes Testing' {
         . $PSScriptRoot/../../helper/Helper.ps1
     }
 
-    It 'Creates the extension and checks that it onboards correctly with training enabled' {
-        Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableTraining=true" -ErrorVariable badOut
+    It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' {
+        $sslKeyPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_key.pem"
+        $sslCertPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_cert.pem"
+        Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net inferenceLoadBalancerHA=False --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile" -ErrorVariable badOut
         $badOut | Should -BeNullOrEmpty        
 
         $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
@@ -24,7 +26,7 @@ Describe 'AzureML Kubernetes Testing' {
         $n = 0
         do 
         {
-            if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) {
+            if (Has-ExtensionData $extensionName) {
                 break
             }
             Start-Sleep -Seconds 20
@@ -37,12 +39,6 @@ Describe 'AzureML Kubernetes Testing' {
         $relayResourceID | Should -Not -BeNullOrEmpty
     }
 
-    It "Performs a show on the extension" {
-        $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
-        $badOut | Should -BeNullOrEmpty
-        $output | Should -Not -BeNullOrEmpty
-    }
-
     It "Runs an update on the extension on the cluster" {
         Set-ItResult -Skipped -Because "Update is not a valid scenario for now"
         az k8s-extension update --cluster-name $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters --name $extensionName --auto-upgrade-minor-version false
@@ -70,6 +66,13 @@ Describe 'AzureML Kubernetes Testing' {
         $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS
     }
 
+    It "Performs a show on the extension" {
+        $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
+        $badOut | Should -BeNullOrEmpty
+        $output | Should -Not -BeNullOrEmpty
+    }
+
+
     It "Lists the extensions on the cluster" {
         $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut
         $badOut | Should -BeNullOrEmpty
@@ -79,7 +82,7 @@ Describe 'AzureML Kubernetes Testing' {
         $extensionExists | Should -Not -BeNullOrEmpty
     }
 
-    It "Deletes the extension from the cluster" {
+    It "Deletes the extension from the cluster with inference enabled" {
         # cleanup the relay and servicebus
         $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey
         $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey
@@ -96,7 +99,7 @@ Describe 'AzureML Kubernetes Testing' {
         $badOut | Should -Not -BeNullOrEmpty
         $output | Should -BeNullOrEmpty
     }
-
+    
     It "Performs another list after the delete" {
         $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut
         $badOut | Should -BeNullOrEmpty
@@ -106,95 +109,93 @@ Describe 'AzureML Kubernetes Testing' {
         $extensionExists | Should -BeNullOrEmpty
     }
 
-    It 'Creates the extension and checks that it onboards correctly with inference enabled' {
-        Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True inferenceLoadBalancerHA=false" -ErrorVariable badOut
-        $badOut | Should -BeNullOrEmpty        
-
-        $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
-        $badOut | Should -BeNullOrEmpty
-
-        $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion 
-        $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue
-
-        # Loop and retry until the extension installs
-        $n = 0
-        do 
-        {
-            if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) {
-                break
-            }
-            Start-Sleep -Seconds 20
-            $n += 1
-        } while ($n -le $MAX_RETRY_ATTEMPTS)
-        $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS
+    # It 'Creates the extension and checks that it onboards correctly with training enabled' {
+    #     Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableTraining=true" -ErrorVariable badOut
+    #     $badOut | Should -BeNullOrEmpty        
+
+    #     $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
+    #     $badOut | Should -BeNullOrEmpty
+
+    #     $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion 
+    #     $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue
+
+    #     # Loop and retry until the extension installs
+    #     $n = 0
+    #     do 
+    #     {
+    #         if (Has-ExtensionData $extensionName) {
+    #             break
+    #         }
+    #         Start-Sleep -Seconds 20
+    #         $n += 1
+    #     } while ($n -le $MAX_RETRY_ATTEMPTS)
+    #     $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS
 
-        # check if relay is populated
-        $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey
-        $relayResourceID | Should -Not -BeNullOrEmpty
-    }
-
-    It "Deletes the extension from the cluster with inference enabled" {
-        # cleanup the relay and servicebus
-        $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey
-        $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey
-        $relayNamespaceName = $relayResourceID.split("/")[8]
-        $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8]
-        az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName
-        az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName
-
-        $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
-        $badOut | Should -BeNullOrEmpty
-
-        # Extension should not be found on the cluster
-        $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
-        $badOut | Should -Not -BeNullOrEmpty
-        $output | Should -BeNullOrEmpty
-    }
-
-    It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' {
-        $sslKeyPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_key.pem"
-        $sslCertPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_cert.pem"
-        Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net inferenceLoadBalancerHA=False --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile" -ErrorVariable badOut
-        $badOut | Should -BeNullOrEmpty        
-
-        $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
-        $badOut | Should -BeNullOrEmpty
-
-        $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion 
-        $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue
-
-        # Loop and retry until the extension installs
-        $n = 0
-        do 
-        {
-            if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) {
-                break
-            }
-            Start-Sleep -Seconds 20
-            $n += 1
-        } while ($n -le $MAX_RETRY_ATTEMPTS)
-        $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS
+    #     # check if relay is populated
+    #     $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey
+    #     $relayResourceID | Should -Not -BeNullOrEmpty
+    # }
+
+    # It "Deletes the extension from the cluster" {
+    #     # cleanup the relay and servicebus
+    #     $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey
+    #     $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey
+    #     $relayNamespaceName = $relayResourceID.split("/")[8]
+    #     $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8]
+    #     az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName
+    #     az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName
+
+    #     $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
+    #     $badOut | Should -BeNullOrEmpty
+
+    #     # Extension should not be found on the cluster
+    #     $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
+    #     $badOut | Should -Not -BeNullOrEmpty
+    #     $output | Should -BeNullOrEmpty
+    # }
+
+    # It 'Creates the extension and checks that it onboards correctly with inference enabled' {
+    #     Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True inferenceLoadBalancerHA=false" -ErrorVariable badOut
+    #     $badOut | Should -BeNullOrEmpty        
+
+    #     $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
+    #     $badOut | Should -BeNullOrEmpty
+
+    #     $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion 
+    #     $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue
+
+    #     # Loop and retry until the extension installs
+    #     $n = 0
+    #     do 
+    #     {
+    #         if (Has-ExtensionData $extensionName) {
+    #             break
+    #         }
+    #         Start-Sleep -Seconds 20
+    #         $n += 1
+    #     } while ($n -le $MAX_RETRY_ATTEMPTS)
+    #     $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS
 
-        # check if relay is populated
-        $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey
-        $relayResourceID | Should -Not -BeNullOrEmpty
-    }
-
-    It "Deletes the extension from the cluster with inference enabled" {
-        # cleanup the relay and servicebus
-        $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey
-        $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey
-        $relayNamespaceName = $relayResourceID.split("/")[8]
-        $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8]
-        az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName
-        az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName
-
-        $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
-        $badOut | Should -BeNullOrEmpty
-
-        # Extension should not be found on the cluster
-        $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
-        $badOut | Should -Not -BeNullOrEmpty
-        $output | Should -BeNullOrEmpty
-    }
+    #     # check if relay is populated
+    #     $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey
+    #     $relayResourceID | Should -Not -BeNullOrEmpty
+    # }
+
+    # It "Deletes the extension from the cluster with inference enabled" {
+    #     # cleanup the relay and servicebus
+    #     $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey
+    #     $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey
+    #     $relayNamespaceName = $relayResourceID.split("/")[8]
+    #     $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8]
+    #     az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName
+    #     az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName
+
+    #     $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
+    #     $badOut | Should -BeNullOrEmpty
+
+    #     # Extension should not be found on the cluster
+    #     $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut
+    #     $badOut | Should -Not -BeNullOrEmpty
+    #     $output | Should -BeNullOrEmpty
+    # }
 }
diff --git a/testing/test/extensions/public/AzureMonitor.Tests.ps1 b/testing/test/extensions/public/AzureMonitor.Tests.ps1
@@ -23,10 +23,8 @@ Describe 'Azure Monitor Testing' {
         $n = 0
         do 
         {
-            if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) {
-                if (Get-PodStatus $extensionAgentName -Namespace $extensionAgentNamespace -eq $POD_RUNNING) {
-                    break
-                }
+            if (Has-ExtensionData $extensionName) {
+                break
             }
             Start-Sleep -Seconds 10
             $n += 1

diff --git a/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 b/testing/test/extensions/public/OpenServiceMesh.Tests.ps1
@@ -27,10 +27,8 @@ Describe 'Azure OpenServiceMesh Testing' {
         $n = 0
         do 
         {
-            if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) {
-                if (Get-PodStatus $extensionAgentName -Namespace $extensionAgentNamespace -eq $POD_RUNNING) {
-                    break
-                }
+            if (Has-ExtensionData $extensionName) {
+                break
             }
             Start-Sleep -Seconds 10
             $n += 1

diff --git a/testing/test/helper/Helper.ps1 b/testing/test/helper/Helper.ps1
@@ -7,6 +7,17 @@ function Get-ExtensionData {
     return $output.items | Where-Object { $_.metadata.name -eq $extensionName }
 }
 
+function Has-ExtensionData {
+    param(
+        [string]$extensionName
+    )
+    $extensionData = Get-ExtensionData $extensionName
+    if ($extensionData) {
+        return $true
+    }
+    return $false
+}
+
 function Get-ExtensionStatus {
     param(
         [string]$extensionName