QASE Integration (#72)

Signed-off-by: Chandan Pinjani <chandan.pinjani@suse.com>

.github/workflows/main.yaml

@@ -61,10 +61,15 @@ env:
   GKE_PROJECT_ID: ${{ secrets.GKE_PROJECT_ID }}
   PROVIDER: ${{ inputs.hosted_provider }}
   RANCHER_PASSWORD: ${{ secrets.RANCHER_PASSWORD }}
+  QASE_API_TOKEN: ${{ secrets.QASE_API_TOKEN }}
+  QASE_PROJECT_CODE: HIGHLANDER
+  QASE_REPORT: 1
+  QASE_RUN_COMPLETE: 1
   RANCHER_LOG_COLLECTOR: ${{ github.workspace }}/.github/scripts/collect-rancher-logs.sh
   GCP_RUNNER_ZONE: asia-south2-c
   DOWNSTREAM_KUBERNETES_VERSION: ${{ inputs.downstream_k8s_version }}
   DOWNSTREAM_CLUSTER_CLEANUP: ${{ inputs.downstream_cluster_cleanup }}
+  QASE_HELPER: ${{ github.workspace }}/hosted/helpers/qase/helper_qase.go
 jobs:
   create-runner:
     runs-on: ubuntu-latest
@@ -114,7 +119,7 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-            
+
       - name: Get Runner IP
         id: runner-ip
         run: echo "PUBLIC_IP=$(curl -s ifconfig.co)" >> "$GITHUB_OUTPUT"
@@ -183,12 +188,29 @@ jobs:
         with:
           go-version-file: go.mod
 
+      - name: Create/Export Qase Run
+        id: qase
+        run: |
+          # Define and export URL of GH test run in Qase run description
+          GH_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          export QASE_RUN_DESCRIPTION="${GH_RUN_URL}"
+          export QASE_RUN_NAME="${{ env.PROVIDER }}-p0-${{ inputs.rancher_version }}"
+    
+          # Create a Qase run, get its ID
+          ID=$(go run ${{ env.QASE_HELPER }} -create)
+          # Export outputs for future use
+          echo "qase_run_id=${ID}" >> ${GITHUB_OUTPUT}
+
+          # Just an info for debugging purposes
+          echo -e "Exported values:\nQASE_RUN_ID=${ID}\nQASE_RUN_DESCRIPTION=${QASE_RUN_DESCRIPTION}\nQASE_RUN_NAME=${QASE_RUN_NAME}"
+
       - name: Provisioning cluster tests
         if:  ${{ inputs.run_p0_provisioning_tests == true }}
         env:
           RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
           RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
           CATTLE_TEST_CONFIG: ${{ github.workspace }}/cattle-config-provisioning.yaml
+          QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
         run: |
           make e2e-provisioning-tests
 
@@ -198,6 +220,7 @@ jobs:
           RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
           RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
           CATTLE_TEST_CONFIG: ${{ github.workspace }}/cattle-config-import.yaml
+          QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
         run: |
           make e2e-import-tests
 
@@ -207,6 +230,7 @@ jobs:
           RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
           RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
           CATTLE_TEST_CONFIG: ${{ github.workspace }}/cattle-config-provisioning.yaml
+          QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
         run: |
           make e2e-support-matrix-provisioning-tests
 
@@ -216,25 +240,40 @@ jobs:
           RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
           RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
           CATTLE_TEST_CONFIG: ${{ github.workspace }}/cattle-config-import.yaml
+          QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
         run: |
           make e2e-support-matrix-importing-tests
 
+      - name: Finalize Qase Run and publish Results
+        env:
+          QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
+        if: ${{ !cancelled() }}
+        run: |
+          REPORT=$(go run ${{ env.QASE_HELPER }} -publish)
+          echo "${REPORT}"
+  
+      - name: Delete Qase Run if job has been cancelled
+        env:
+          QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
+        if: ${{ cancelled() }}
+        run: go run ${{ env.QASE_HELPER }} -delete
+
       - name: Collect logs
-        if: ${{ always() }}
+        if: ${{ !cancelled() }}
         run: |
           chmod +x ${{ env.RANCHER_LOG_COLLECTOR }}
           bash ${{ env.RANCHER_LOG_COLLECTOR }}
 
       - name: Upload cluster logs
-        if: ${{ always() }}
+        if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
           name: support-logs
           path: ${{ github.workspace }}/logs/*
           if-no-files-found: ignore
 
       - name: Add summary
-        if: ${{ always() }}
+        if: ${{ !cancelled() }}
         run: |
           # Add summary
           echo "## General information" >> ${GITHUB_STEP_SUMMARY}

Makefile

@@ -68,10 +68,10 @@ prepare-e2e-ci-rancher-hosted-nightly-chart: install-k3s install-helm install-ce
 prepare-e2e-ci-rancher: install-k3s install-helm install-cert-manager install-rancher ## Setup Rancher on the local machine
 
 e2e-import-tests: deps	## Run the 'P0Importing' test suite for a given ${PROVIDER}
-	ginkgo ${STANDARD_TEST_OPTIONS} -p --focus "P0Importing" ./hosted/${PROVIDER}/p0/
+	ginkgo ${STANDARD_TEST_OPTIONS} --nodes 2 --focus "P0Importing" ./hosted/${PROVIDER}/p0/
 
 e2e-provisioning-tests: deps ## Run the 'P0Provisioning' test suite for a given ${PROVIDER}
-	ginkgo ${STANDARD_TEST_OPTIONS} -p --focus "P0Provisioning" ./hosted/${PROVIDER}/p0/
+	ginkgo ${STANDARD_TEST_OPTIONS} --nodes 2 --focus "P0Provisioning" ./hosted/${PROVIDER}/p0/
 
 #TODO: Parallelize supportmatrix tests
 e2e-support-matrix-importing-tests: deps ## Run the 'SupportMatrixImporting' test suite for a given ${PROVIDER}

go.mod

@@ -8,12 +8,15 @@ require (
 	github.com/onsi/ginkgo/v2 v2.15.0
 	github.com/onsi/gomega v1.31.1
 	github.com/pkg/errors v0.9.1
+	github.com/rancher-sandbox/qase-ginkgo v1.0.1
 	github.com/rancher/shepherd v0.0.0-20240205165058-79095d1622f8
+	github.com/sirupsen/logrus v1.9.3
 	k8s.io/apimachinery v0.27.9
 	k8s.io/utils v0.0.0-20230505201702-9f6742963106
 )
 
 require (
+	github.com/antihax/optional v1.0.0 // indirect
 	github.com/aws/aws-sdk-go v1.44.322 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
@@ -36,7 +39,7 @@ require (
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
@@ -65,9 +68,9 @@ require (
 	github.com/rancher/rke v1.5.0 // indirect
 	github.com/rancher/system-upgrade-controller/pkg/apis v0.0.0-20210727200656-10b094e30007 // indirect
 	github.com/rancher/wrangler v1.1.1 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
+	go.qase.io/client v0.0.4 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
@@ -101,6 +104,7 @@ require (
 )
 
 replace (
+	go.qase.io/client => github.com/rancher/qase-go/client v0.0.0-20231114201952-65195ec001fa
 	k8s.io/api => k8s.io/api v0.27.5
 	k8s.io/client-go => github.com/rancher/client-go v1.27.4-rancher1
 )

go.sum

@@ -60,6 +60,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -341,8 +342,9 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ=
+github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -578,6 +580,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rancher-sandbox/qase-ginkgo v1.0.1 h1:LB9ITLavX3PmcOe0hp0Y7rwQCjJ3WpL8kG8v1MxPadE=
+github.com/rancher-sandbox/qase-ginkgo v1.0.1/go.mod h1:sIF43xaLHtEzmPqADKlZZV6oatc66GHz1N6gpBNn6QY=
 github.com/rancher/aks-operator v1.2.0 h1:cNB84j23Ng7GUkqIt8I1TUfkpPdA5SQ2uyosPNJM5G4=
 github.com/rancher/aks-operator v1.2.0/go.mod h1:CIU0AgI4DHYKEG3P3tHyEM/5QEud7upDOiYL6j5D/qE=
 github.com/rancher/apiserver v0.0.0-20230831052300-120e615b17ba h1:ceAHvddZkuNbUTuMgqxYAcUSQ/+YtJQO9Z1PHjmQZBY=
@@ -594,6 +598,8 @@ github.com/rancher/lasso v0.0.0-20230830164424-d684fdeb6f29 h1:+kige/h8/LnzWgPjB
 github.com/rancher/lasso v0.0.0-20230830164424-d684fdeb6f29/go.mod h1:kgk9kJVMj9FIrrXU0iyM6u/9Je4bEjPImqswkTVaKsQ=
 github.com/rancher/norman v0.0.0-20230831160711-5de27f66385d h1:Ft/iTH91TlE2oBGmpkdO4I8o8cvUmCnytdwu52a/tN4=
 github.com/rancher/norman v0.0.0-20230831160711-5de27f66385d/go.mod h1:Sm2Xqai+aecgmJ86ygyEe+TdPMLkauEpykSstBAu4Ko=
+github.com/rancher/qase-go/client v0.0.0-20231114201952-65195ec001fa h1:/qeYlQVfyvsO5yY0dZmm7mRTAsDm54jACiRDx3LAwsA=
+github.com/rancher/qase-go/client v0.0.0-20231114201952-65195ec001fa/go.mod h1:NP3xboG+t2p+XMnrcrJ/L384Ki0Cp3Pww/X+vm5Jcy0=
 github.com/rancher/rancher/pkg/apis v0.0.0-20230915232223-a9ea4ce4a5ba h1:GTSOFeKvAsgm21b6ko+XcuH3skZzuYTYyfIbFx/QNQ8=
 github.com/rancher/rancher/pkg/apis v0.0.0-20230915232223-a9ea4ce4a5ba/go.mod h1:CygL4GxLAJpxMRpqImc8+wH4zW+0kNVYW+nb0IsJ04A=
 github.com/rancher/rke v1.5.0 h1:M/YryKnBs7IwzMGA2kh1EiypVQkme6o9KSg0hlllQa4=
@@ -877,6 +883,7 @@ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
 golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=

hosted/aks/p0/p0_importing_test.go

@@ -66,6 +66,8 @@ var _ = Describe("P0Importing", func() {
 			}
 		})
 		It("should successfully import the cluster & add, delete, scale nodepool", func() {
+			// Report to Qase
+			testCaseID = 273
 
 			By("checking cluster name is same", func() {
 				Expect(cluster.Name).To(BeEquivalentTo(clusterName))
@@ -141,6 +143,9 @@ var _ = Describe("P0Importing", func() {
 			})
 
 			It("should be able to upgrade k8s version of the cluster", func() {
+				// Report to Qase
+				testCaseID = 274
+
 				By("upgrading the ControlPlane", func() {
 					var err error
 					cluster, err = helper.UpgradeClusterKubernetesVersion(cluster, upgradeToVersion, ctx.RancherClient)

hosted/aks/p0/p0_provisioning_test.go

@@ -64,6 +64,8 @@ var _ = Describe("P0Provisioning", func() {
 			}
 		})
 		It("should successfully provision the cluster & add, delete, scale nodepool", func() {
+			// Report to Qase
+			testCaseID = 216
 
 			By("checking cluster name is same", func() {
 				Expect(cluster.Name).To(BeEquivalentTo(clusterName))
@@ -139,6 +141,9 @@ var _ = Describe("P0Provisioning", func() {
 			})
 
 			It("should be able to upgrade k8s version of the cluster", func() {
+				// Report to Qase
+				testCaseID = 218
+
 				By("upgrading the ControlPlane", func() {
 					var err error
 					cluster, err = helper.UpgradeClusterKubernetesVersion(cluster, upgradeToVersion, ctx.RancherClient)

hosted/aks/p0/p0_suite_test.go

@@ -19,6 +19,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	. "github.com/rancher-sandbox/qase-ginkgo"
 	namegen "github.com/rancher/shepherd/pkg/namegenerator"
 
 	"github.com/rancher/hosted-providers-e2e/hosted/helpers"
@@ -31,6 +32,7 @@ const (
 var (
 	ctx         helpers.Context
 	clusterName string
+	testCaseID  int64
 	location    = helpers.GetAKSLocation()
 	k8sVersion  = helpers.GetK8sVersion("aks")
 )
@@ -46,3 +48,13 @@ var _ = BeforeEach(func() {
 	Expect(err).To(BeNil())
 	clusterName = namegen.AppendRandomString("akshostcluster")
 })
+
+var _ = ReportBeforeEach(func(report SpecReport) {
+	// Reset case ID
+	testCaseID = -1
+})
+
+var _ = ReportAfterEach(func(report SpecReport) {
+	// Add result in Qase if asked
+	Qase(testCaseID, report)
+})

hosted/aks/support_matrix/support_matrix_importing_test.go

@@ -70,6 +70,9 @@ var _ = Describe("SupportMatrixImporting", func() {
 			})
 
 			It("should successfully import the cluster", func() {
+				// Report to Qase
+				testCaseID = 301
+
 				By("checking cluster name is same", func() {
 					Expect(cluster.Name).To(BeEquivalentTo(clusterName))
 				})

hosted/aks/support_matrix/support_matrix_provisioning_test.go

@@ -71,6 +71,9 @@ var _ = Describe("SupportMatrixProvisioning", func() {
 			})
 
 			It("should successfully provision the cluster", func() {
+				// Report to Qase
+				testCaseID = 300
+
 				By("checking cluster name is same", func() {
 					Expect(cluster.Name).To(BeEquivalentTo(clusterName))
 				})

hosted/aks/support_matrix/support_matrix_suite_test.go

@@ -17,6 +17,7 @@ package support_matrix_test
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	. "github.com/rancher-sandbox/qase-ginkgo"
 
 	"testing"
 
@@ -26,6 +27,7 @@ import (
 
 var (
 	availableVersionList []string
+	testCaseID           int64
 	ctx                  helpers.Context
 	location             = helpers.GetAKSLocation()
 )
@@ -39,3 +41,13 @@ func TestSupportMatrix(t *testing.T) {
 	Expect(err).To(BeNil())
 	RunSpecs(t, "SupportMatrix Suite")
 }
+
+var _ = ReportBeforeEach(func(report SpecReport) {
+	// Reset case ID
+	testCaseID = -1
+})
+
+var _ = ReportAfterEach(func(report SpecReport) {
+	// Add result in Qase if asked
+	Qase(testCaseID, report)
+})

hosted/eks/p0/p0_importing_test.go

@@ -68,6 +68,8 @@ var _ = Describe("P0Importing", func() {
 		})
 
 		It("should successfully import the cluster & add, delete, scale nodepool", func() {
+			// Report to Qase
+			testCaseID = 276
 
 			By("checking cluster name is same", func() {
 				Expect(cluster.Name).To(BeEquivalentTo(clusterName))
@@ -144,6 +146,9 @@ var _ = Describe("P0Importing", func() {
 			})
 
 			It("should be able to upgrade k8s version of the cluster", func() {
+				// Report to Qase
+				testCaseID = 77
+
 				By("upgrading the ControlPlane", func() {
 					var err error
 					cluster, err = helper.UpgradeClusterKubernetesVersion(cluster, upgradeToVersion, ctx.RancherClient)

hosted/eks/p0/p0_provisioning_test.go

@@ -60,6 +60,8 @@ var _ = Describe("P0Provisioning", func() {
 		})
 
 		It("should successfully provision the cluster & add, delete, scale nodepool", func() {
+			// Report to Qase
+			testCaseID = 75
 
 			By("checking cluster name is same", func() {
 				Expect(cluster.Name).To(BeEquivalentTo(clusterName))
@@ -136,6 +138,9 @@ var _ = Describe("P0Provisioning", func() {
 			})
 
 			It("should be able to upgrade k8s version of the cluster", func() {
+				// Report to Qase
+				testCaseID = 78
+
 				By("upgrading the ControlPlane", func() {
 					var err error
 					cluster, err = helper.UpgradeClusterKubernetesVersion(cluster, upgradeToVersion, ctx.RancherClient)