Add functional test cypress workflow improvements and enable the workflow for in-house tests

.github/workflows/cypress_workflow.yml

@@ -47,7 +47,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-          group: [1, 2, 3, 4, 5, 6, 7, 8, 9]
+          group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
     container:
       image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
       options: --user 1001
@@ -101,7 +101,7 @@ jobs:
       - name: Build plugins
         run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 12
 
-      - name: Checkout
+      - name: Checkout FT repo
         uses: actions/checkout@v2
         with:
           path: ${{ env.FTR_PATH }}
@@ -113,8 +113,9 @@ jobs:
         run: |
           echo "SPEC=${{ inputs.specs }}" >> $GITHUB_ENV
 
+      # Setup spec files for existing Functional Test repo cypress tests
       - name: Setup spec files
-        if: ${{ inputs.specs == '' }}
+        if: ${{ inputs.specs == '' && matrix.group < 10 }}
         working-directory: ${{ env.FTR_PATH }}
         shell: bash
         run: |
@@ -126,11 +127,13 @@ jobs:
           echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV
 
       - name: Get Cypress version
+        if: ${{ matrix.group < 10 }}
         id: cypress_version
         run: |
           echo "name=cypress_version::$(cat ./${{ env.FTR_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT
-
+      
       - name: Cache Cypress
+        if: ${{ matrix.group < 10 }}
         id: cache-cypress
         uses: actions/cache@v1
         with:
@@ -141,36 +144,92 @@ jobs:
       - run: npx cypress cache list
       - run: npx cypress cache path
 
-      - name: Run tests
+      # Setup spec files for Dashboards in-house cypress tests
+      - name: Setup spec files for Dashboards tests
+        if: ${{ inputs.specs == '' && matrix.group > 9 }}
+        shell: bash
+        run: |
+          IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.group }})
+          DASHBOARDS_SPEC=''
+          for i in "${SPEC_ARRAY[@]}"; do
+          DASHBOARDS_SPEC+="cypress/integration/core_opensearch_dashboards/${i},"
+          done
+          echo "DASHBOARDS_SPEC=${DASHBOARDS_SPEC}" >> $GITHUB_ENV
+
+      # Run FT repo tests
+      - name: Run FT repo tests
+        if: ${{ matrix.group < 10 }}
         uses: cypress-io/github-action@v2
         with:
           working-directory: ${{ env.FTR_PATH }}
           start: ${{ env.OPENSEARCH_SNAPSHOT_CMD }}, ${{ env.START_CMD }}
           wait-on: 'http://localhost:9200, http://localhost:5601'
           command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.SPEC }}
+
+      # Clear Cypress Cache before running Dashboards tests
+      - name: Clear Cypress Cache
+        if: ${{ matrix.group > 9 }}
+        run: npx cypress cache clear
+
+      # Run Dashboards Cypress tests within the source repo
+      - name: Run Dashboards Cypress tests
+        if: ${{ matrix.group > 9 }}
+        uses: cypress-io/github-action@v6
+        with:
+          install-command: npx cypress install --force
+          start: node scripts/opensearch snapshot -E cluster.routing.allocation.disk.threshold_enabled=false, node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true
+          wait-on: 'http://localhost:9200, http://localhost:5601'
+          command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }}
 
       # Screenshots are only captured on failure, will change this once we do visual regression tests
-      - uses: actions/upload-artifact@v3
-        if: failure()
+      - name: Upload FT repo screenshots
+        uses: actions/upload-artifact@v3
+        if: failure() && (matrix.group < 10)
         with:
           name: ftr-cypress-screenshots
           path: ${{ env.FTR_PATH }}/cypress/screenshots
           retention-days: 1
-
-      - uses: actions/upload-artifact@v3
-        if: always()
+
+      - name: Upload FT repo videos
+        uses: actions/upload-artifact@v3
+        if: always() && (matrix.group < 10)
         with:
           name: ftr-cypress-videos
           path: ${{ env.FTR_PATH }}/cypress/videos
           retention-days: 1
 
-      - uses: actions/upload-artifact@v3
-        if: always()
+      - name: Upload FT repo results
+        uses: actions/upload-artifact@v3
+        if: always() && (matrix.group < 10)
         with:
           name: ftr-cypress-results
           path: ${{ env.FTR_PATH }}/cypress/results
           retention-days: 1
 
+      - name: Upload Dashboards screenshots
+        if: failure() && (matrix.group > 9)
+        uses: actions/upload-artifact@v3
+        with:
+          name: dashboards-cypress-screenshots
+          path: cypress/screenshots
+          retention-days: 1
+
+      - name: Upload Dashboards repo videos
+        uses: actions/upload-artifact@v3
+        if: always() && (matrix.group > 9)
+        with:
+          name: dashboards-cypress-videos
+          path: cypress/videos
+          retention-days: 1
+
+      - name: Upload Dashboards repo results
+        uses: actions/upload-artifact@v3
+        if: always() && (matrix.group > 9)
+        with:
+          name: dashboards-cypress-results
+          path: cypress/results
+          retention-days: 1
+
   add-comment:
     needs: [cypress-tests]
     if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.pr_number != '' }}

.github/workflows/dashboards_cypress_workflow.yml → .github/workflows/release_cypress_workflow.yml

@@ -1,5 +1,5 @@
 name: Orchestrator cypress workflow
-run-name: dashboards_cypress_workflow ${{ inputs.UNIQUE_ID != '' && inputs.UNIQUE_ID || '' }} # Unique id number appended to the workflow run-name to reference the run within the orchestrator.
+run-name: release_cypress_workflow ${{ inputs.UNIQUE_ID != '' && inputs.UNIQUE_ID || '' }} # Unique id number appended to the workflow run-name to reference the run within the orchestrator.
 # Trigger on dispatch event sent from FT repo orchestrator
 on:
   workflow_dispatch:
@@ -41,27 +41,38 @@ env:
   TEST_REPO: ${{ inputs.test_repo != '' && inputs.test_repo || github.repository }}
   TEST_BRANCH: "${{ inputs.test_branch != '' && inputs.test_branch || github.base_ref }}"
   OSD_PATH: 'osd'
-  CYPRESS_BROWSER: 'chromium'
+  CYPRESS_BROWSER: 'electron'
   JOB_ID: ${{ inputs.UNIQUE_ID}}
   OPENSEARCH:  ${{ inputs.OS_URL != '' && inputs.OS_URL  || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/linux/x64/tar/dist/opensearch/opensearch-$VERSION-linux-x64.tar.gz' }}
   DASHBOARDS: ${{ inputs.OSD_URL != '' && inputs.OSD_URL  || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/$VERSION/latest/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-$VERSION-linux-x64.tar.gz' }}
   OPENSEARCH_DIR: 'cypress/opensearch'
   DASHBOARDS_DIR: 'cypress/opensearch-dashboards'
   SECURITY_ENABLED: ${{ inputs.SECURITY_ENABLED != '' && inputs.SECURITY_ENABLED || 'false' }}
-  SPEC: 'cypress/integration/core_opensearch_dashboards/*'
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch-dashboards
+
   cypress-tests:
+    needs: Get-CI-Image-Tag
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        spec_group: [10, 11] # Add more spec files as needed
     container:
-      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
-      options: --user 1001
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
     env:
       # prevents extra Cypress installation progress messages
       CI: 1
       # avoid warnings like "tput: No value for $TERM and no -T specified"
       TERM: xterm
-    name: Run cypress tests ${{ inputs.UNIQUE_ID}}
+    name: Run cypress tests (osd:ciGroup${{ matrix.spec_group }}) ${{ inputs.UNIQUE_ID}} 
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -82,21 +93,24 @@ jobs:
           npm i -g yarn@1.22.10
           yarn config set network-timeout 1000000 -g
 
+      - name: Setup spec files
+        if: ${{ inputs.specs == '' }}
+        shell: bash
+        run: |
+          cd ${{ env.OSD_PATH }}
+          IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.spec_group }})
+          FORMATTED_SPEC=''
+          for i in "${SPEC_ARRAY[@]}"; do
+            FORMATTED_SPEC+="cypress/integration/core_opensearch_dashboards/${i},"
+          done
+          SPEC=${FORMATTED_SPEC}  # Set SPEC to FORMATTED_SPEC
+          echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV
+          echo "SPEC found: $SPEC"
+
       - name: Get Cypress version
         id: cypress_version
         run: |
           echo "name=cypress_version::$(cat ./${{ env.OSD_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT
-
-      - name: Cache Cypress
-        id: cache-cypress
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/Cypress
-          key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
-        env:
-          CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
-      - run: npx cypress cache list
-      - run: npx cypress cache path
 
       - name: Get package version (Linux)
         run: |
@@ -106,16 +120,18 @@ jobs:
       - name: Run bootstrap
         run: |
           cd ${{ env.OSD_PATH }}
-          yarn osd bootstrap
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "source $NVM_DIR/nvm.sh && nvm use && node -v && yarn -v &&
+                               yarn osd bootstrap"
 
-      - name: Download and extract Opensearch artifacts
+      - name: Download and extract OpenSearch artifacts
         run: |
           CWD=$(pwd)
           mkdir -p $CWD/${{ env.OPENSEARCH_DIR }}
           source ${{ env.OSD_PATH }}/scripts/common/utils.sh
           open_artifact $CWD/${{ env.OPENSEARCH_DIR }} ${{ env.OPENSEARCH }}
 
-      - name: Download and extract Opensearch Dashboards artifacts
+      - name: Download and extract OpenSearch Dashboards artifacts
         run: |
           CWD=$(pwd)
           mkdir -p $CWD/${{ env.DASHBOARDS_DIR }}
@@ -124,20 +140,20 @@ jobs:
 
       - name: Run Cypress tests 
         run: |
-          source ${{ env.OSD_PATH }}/scripts/cypress_tests.sh
-          run_dashboards_cypress_tests  
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "source ${{ env.OSD_PATH }}/scripts/cypress_tests.sh && run_dashboards_cypress_tests"
 
       # Screenshots are only captured on failures
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: osd-cypress-screenshots
+          name: release-osd-cypress-screenshots
           path: ${{ env.OSD_PATH }}/cypress/screenshots
           retention-days: 1
 
       - uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: osd-cypress-videos
+          name: release-osd-cypress-videos
           path: ${{ env.OSD_PATH }}/cypress/videos
           retention-days: 1

CHANGELOG.md

@@ -54,6 +54,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### 🔩 Tests
 
+- Add functional test cypress workflow improvements and enable the workflow for in-house Dashboards tests ([#6061](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6061))
+
 ## [2.12.0 - 2024-02-20](https://github.com/opensearch-project/OpenSearch-Dashboards/releases/tag/2.12.0)
 
 ### 💥 Breaking Changes

cypress/utils/commands.js

@@ -0,0 +1,15 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+// --- Typed commands --
+
+Cypress.Commands.add('getElementByTestId', (testId, options = {}) => {
+  return cy.get(`[data-test-subj="${testId}"]`, options);
+});
+
+Cypress.Commands.add('getElementsByTestIds', (testIds, options = {}) => {
+  const selectors = [testIds].flat(Infinity).map((testId) => `[data-test-subj="${testId}"]`);
+  return cy.get(selectors.join(','), options);
+});

package.json

@@ -79,8 +79,9 @@
     "spec_to_console": "scripts/use_node scripts/spec_to_console",
     "pkg-version": "scripts/use_node -e \"console.log(require('./package.json').version)\"",
     "cypress:run-without-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=false",
-    "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500"
-
+    "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500",
+    "osd:ciGroup10": "echo \"dashboard_sanity_test_spec.js\"",
+    "osd:ciGroup11": "echo \"apps/vis_builder/*.js\""
   },
   "repository": {
     "type": "git",

scripts/common/opensearch_service.sh

@@ -9,6 +9,7 @@ function setup_opensearch() {
   cd "$OPENSEARCH_DIR"
   echo "network.host: 0.0.0.0" >> config/opensearch.yml
   echo "discovery.type: single-node" >> config/opensearch.yml
+  echo "cluster.routing.allocation.disk.threshold_enabled: false" >> config/opensearch.yml
   [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/opensearch-security" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml
   # Required for IM
   [ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml

scripts/cypress_tests.sh

@@ -67,10 +67,12 @@ function check_status() {
 
 # Starts OpenSearch Dashboards and run tests in the cypress folder
 function run_dashboards_cypress_tests() {
+  run_opensearch
+  cd $CWD
   echo "[ OpenSearch Dashboards setup before it starts... ]"
   setup_dashboards >> /dev/null 2>&1 &
   sleep 100
-  cd "$DASHBOARDS_DIR"
+  cd $CWD/"$DASHBOARDS_DIR"
   if [ -x "./bin/opensearch-dashboards" ]; then
     spawn_process_and_save_PID "./bin/opensearch-dashboards &"
   else
@@ -79,11 +81,11 @@ function run_dashboards_cypress_tests() {
   fi
   check_status $DASHBOARDS_URL $DASHBOARDS_MSG
   # Run cypress tests
+  cd "$CWD"/osd
   run_cypress
 }
 
 function run_cypress() {
-  cd "$CWD"/osd
   echo "SPEC found: $SPEC"
   if [ $SECURITY_ENABLED = "true" ]; then
     echo "run security enabled tests"