test: e2e concurrency + split e2e tests (#2761)

Co-authored-by: novakzaballa <zaballa.novak@gmail.com>
Co-authored-by: Novak Zaballa <41410593+novakzaballa@users.noreply.github.com>

.github/actions/e2e-tests/action.yml

@@ -11,6 +11,9 @@ inputs:
   environment:
     description: The value of the ENV environment variable to use for npm run env.
     required: true
+  tests:
+    description: The space separated list of E2E tests to be executed.
+    required: false
 
 runs:
   using: composite
@@ -41,5 +44,5 @@ runs:
       run: |
         node -v
         npm run env
-        npm run test
+        npm run test -- ${{ inputs.tests }}
       shell: bash

.github/actions/local-e2e-tests/action.yml

@@ -0,0 +1,47 @@
+name: Run E2E tests in local environment
+description: Run the E2E tests against local environment
+
+inputs:
+  slack_token:
+    description: The slack authentication token.
+    required: true
+  tests:
+    description: The space separated list of E2E tests to be executed.
+    required: false
+    default: ''
+  concurrency:
+    description: The concurrent number of browsers to be used on testing.
+    required: false
+    default: 3
+
+runs:
+  using: composite
+
+  steps:
+    - name: Run Local API
+      id: run-local-api
+      uses: ./.github/actions/run-local-api
+      with:
+        e2e_test_token: some-token
+        # As per https://stackoverflow.com/q/65497331/421808 172.17.0.1 seems like the only way to resolve host DB
+        database_url: postgres://postgres:postgres@172.17.0.1:5432/flagsmith
+        disable_analytics_features: true
+
+    - name: Run E2E tests against local
+      uses: ./.github/actions/e2e-tests
+      env:
+        E2E_CONCURRENCY: ${{ inputs.concurrency }}
+      with:
+        e2e_test_token: some-token
+        slack_token: ${{ inputs.slack_token }}
+        environment: local
+        tests: ${{ inputs.tests }}
+
+    - name: Output API container status and logs
+      if: failure()
+      env:
+        API_CONTAINER_ID: ${{ steps.run-local-api.outputs.containerId }}
+      run: |
+        docker inspect $API_CONTAINER_ID | jq '.[0].State'
+        docker logs $API_CONTAINER_ID
+      shell: bash

.github/actions/run-local-api/action.yml

@@ -5,12 +5,6 @@ inputs:
   database_url:
     description: The database URL to connect the API to
     required: true
-  sendgrid_api_key:
-    description: The sendgrid API key to use for email sending
-    required: true
-  sentry_sdk_dsn:
-    description: The sentry SDK DSN
-    required: true
   e2e_test_token:
     description: The token to use for authenticating the E2E test process
     required: false
@@ -39,20 +33,16 @@ runs:
       env:
         E2E_TEST_AUTH_TOKEN: ${{ inputs.e2e_test_token }}
         DATABASE_URL: ${{ inputs.database_url }}
-        SENDGRID_API_KEY: ${{ inputs.sendgrid_api_key }}
-        SENTRY_SDK_DSN: ${{ inputs.sentry_sdk_dsn }}
         DISABLE_ANALYTICS_FEATURES: ${{ inputs.disable_analytics_features }}
       run: |
         CONTAINER_ID=$( docker run \
         -p 8000:8000 \
         -e DATABASE_URL=$DATABASE_URL \
         -e E2E_TEST_AUTH_TOKEN=$E2E_TEST_AUTH_TOKEN \
-        -e SENDGRID_API_KEY=$SENDGRID_API_KEY \
-        -e SENTRY_SDK_DSN=$SENTRY_SDK_DSN \
         -e DISABLE_ANALYTICS_FEATURES=$DISABLE_ANALYTICS_FEATURES \
         -e DJANGO_ALLOWED_HOSTS="*" \
         -e DJANGO_SETTINGS_MODULE=app.settings.test \
-        -e FE_E2E_TEST_USER_EMAIL=nightwatch@solidstategroup.com \
+        -e ENABLE_FE_E2E=True \
         -e ACCESS_LOG_LOCATION=- \
         -d flagsmith/flagsmith-api:e2e-${{ github.sha }} )
         echo "containerId=$CONTAINER_ID" >> "$GITHUB_OUTPUT"

.github/actions/unified-e2e-tests/action.yml

@@ -0,0 +1,40 @@
+name: Run E2E tests in docker unified environment
+description: Run the E2E tests against docker unified environment
+
+inputs:
+  github_actor:
+    description: Github actor
+    required: true
+  github_token:
+    description: Github token
+    required: true
+  e2e_test_token_staging:
+    description: The staging test token.
+    required: true
+
+runs:
+  using: composite
+
+  steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Expose GitHub Runtime
+      uses: crazy-max/ghaction-github-runtime@v2
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ inputs.github_actor }}
+        password: ${{ inputs.github_token }}
+
+    - name: Build docker-compose with unified-image
+      working-directory: frontend
+      env:
+        E2E_TEST_TOKEN_STAGING: ${{ inputs.e2e_test_token_staging }}
+        ENV: staging
+        STATIC_ASSET_CDN_URL: /
+      run: |
+        docker buildx bake -f docker-compose-e2e-tests.yml --load
+      shell: bash

.github/workflows/platform-pull-request.yml

@@ -29,9 +29,9 @@ jobs:
             style
             test
 
-  run-e2e-tests:
+  run-e2e-tests-1:
     runs-on: ubuntu-latest
-    name: Full E2E tests
+    name: E2E Local - Segments-1, Environment
 
     services:
       postgres:
@@ -47,58 +47,131 @@ jobs:
       - name: Cloning repo
         uses: actions/checkout@v3
 
-      - name: Run Local API
-        id: run-local-api
-        uses: ./.github/actions/run-local-api
+      - name: Run E2E tests against local
+        uses: ./.github/actions/local-e2e-tests
         with:
-          e2e_test_token: some-token
-          # As per https://stackoverflow.com/q/65497331/421808 172.17.0.1 seems like the only way to resolve host DB
-          database_url: postgres://postgres:postgres@172.17.0.1:5432/flagsmith
-          sentry_sdk_dsn: ${{ secrets.SENTRY_SDK_DSN }}
-          sendgrid_api_key: ${{ secrets.SENDGRID_API_KEY }}
-          disable_analytics_features: true
+          slack_token: ${{ secrets.SLACK_TOKEN }}
+          tests: segment-part-1 environment
+          concurrency: 1
+
+  run-e2e-tests-2:
+    runs-on: ubuntu-latest
+    name: E2E Local - Segments-2
+
+    services:
+      postgres:
+        image: postgres:11.12-alpine
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: flagsmith
+        ports: ['5432:5432']
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+      - name: Cloning repo
+        uses: actions/checkout@v3
 
       - name: Run E2E tests against local
-        uses: ./.github/actions/e2e-tests
+        uses: ./.github/actions/local-e2e-tests
         with:
-          e2e_test_token: some-token
           slack_token: ${{ secrets.SLACK_TOKEN }}
-          environment: local
+          tests: segment-part-2
+          concurrency: 1
+
+  run-e2e-tests-3:
+    runs-on: ubuntu-latest
+    name: E2E Local - Segments-3, Signup, Flag, Invite, Project
 
-      - name: Output API container status and logs
-        if: failure()
+    services:
+      postgres:
+        image: postgres:11.12-alpine
         env:
-          API_CONTAINER_ID: ${{ steps.run-local-api.outputs.containerId }}
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: flagsmith
+        ports: ['5432:5432']
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+      - name: Cloning repo
+        uses: actions/checkout@v3
+
+      - name: Run E2E tests against local
+        uses: ./.github/actions/local-e2e-tests
+        with:
+          slack_token: ${{ secrets.SLACK_TOKEN }}
+          tests: segment-part-3 signup flag invite project
+          concurrency: 2
+
+  run-e2e-segments-1-tests-docker-unified:
+    runs-on: ubuntu-latest
+    name: E2E Unified - Segments-1, Environment
+
+    steps:
+      - name: Cloning repo
+        uses: actions/checkout@v3
+
+      - name: Build unified docker container
+        uses: ./.github/actions/unified-e2e-tests
+        with:
+          github_actor: ${{github.actor}}
+          github_token: ${{secrets.GITHUB_TOKEN}}
+          e2e_test_token_staging: ${{ secrets.E2E_TEST_TOKEN }}
+
+      - name: Run tests on unified docker image
+        working-directory: frontend
+        env:
+          SLACK_TOKEN: ${{ inputs.slack_token }}
+          GITHUB_ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          docker compose -f docker-compose-e2e-tests.yml run frontend npx cross-env E2E_CONCURRENCY=1 npm run test -- segment-part-1 environment
+
+  run-e2e-segments-2-tests-docker-unified:
+    runs-on: ubuntu-latest
+    name: E2E Unified - Segments-2
+
+    steps:
+      - name: Cloning repo
+        uses: actions/checkout@v3
+
+      - name: Build unified docker container
+        uses: ./.github/actions/unified-e2e-tests
+        with:
+          github_actor: ${{github.actor}}
+          github_token: ${{secrets.GITHUB_TOKEN}}
+          e2e_test_token_staging: ${{ secrets.E2E_TEST_TOKEN }}
+
+      - name: Run tests on unified docker image
+        working-directory: frontend
+        env:
+          SLACK_TOKEN: ${{ inputs.slack_token }}
+          GITHUB_ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
-          docker inspect $API_CONTAINER_ID | jq '.[0].State'
-          docker logs $API_CONTAINER_ID
+          docker compose -f docker-compose-e2e-tests.yml run frontend npx cross-env E2E_CONCURRENCY=1 npm run test -- segment-part-2
 
-  run-e2e-tests-docker-unified:
+  run-e2e-other-tests-docker-unified:
     runs-on: ubuntu-latest
-    name: Full E2E tests with unified image in Docker
+    name: E2E Unified - Segments-3, Signup, Flag, Invite, Project
 
     steps:
       - name: Cloning repo
         uses: actions/checkout@v3
 
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+      - name: Build unified docker container
+        uses: ./.github/actions/unified-e2e-tests
         with:
-          registry: ghcr.io
-          username: ${{github.actor}}
-          password: ${{secrets.GITHUB_TOKEN}}
+          github_actor: ${{github.actor}}
+          github_token: ${{secrets.GITHUB_TOKEN}}
+          e2e_test_token_staging: ${{ secrets.E2E_TEST_TOKEN }}
 
-      - name: Run docker-compose with unified-image
+      - name: Run tests on unified docker image
         working-directory: frontend
         env:
-          E2E_TEST_TOKEN_STAGING: ${{ secrets.E2E_TEST_TOKEN }}
-          ENV: staging
-          SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
-          STATIC_ASSET_CDN_URL: /
+          SLACK_TOKEN: ${{ inputs.slack_token }}
           GITHUB_ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
-          docker-compose -f docker-compose-e2e-tests.yml build
-          docker-compose -f docker-compose-e2e-tests.yml run frontend npm run test
+          docker compose -f docker-compose-e2e-tests.yml run frontend npx cross-env E2E_CONCURRENCY=2 npm run test -- segment-part-3 signup flag invite project
 
   docker-build-unified:
     if: github.event.pull_request.draft == false
@@ -117,9 +190,11 @@ jobs:
 
       - name: Build
         id: docker_build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           push: false
+          cache-from: type=gha,scope=image-unified-fb
+          cache-to: type=gha,mode=max,scope=image-unified-fb
           tags: flagsmith/flagsmith:testing
 
   docker-build-api:
@@ -139,10 +214,12 @@ jobs:
 
       - name: Build
         id: docker_build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: api/Dockerfile
           push: false
+          cache-from: type=gha,scope=image-api-fb
+          cache-to: type=gha,mode=max,scope=image-api-fb
           tags: flagsmith/flagsmith-api:testing
 
   docker-build-frontend:
@@ -162,8 +239,10 @@ jobs:
 
       - name: Build
         id: docker_build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: frontend/Dockerfile
           push: false
+          cache-from: type=gha,scope=image-frontend-fb
+          cache-to: type=gha,mode=max,scope=image-frontend-fb
           tags: flagsmith/flagsmith-frontend:testing

.github/workflows/platform-test-merge-to-main.yml

@@ -31,8 +31,6 @@ jobs:
           e2e_test_token: some-token
           # As per https://stackoverflow.com/q/65497331/421808 172.17.0.1 seems like the only way to resolve host DB
           database_url: postgres://postgres:postgres@172.17.0.1:5432/flagsmith
-          sentry_sdk_dsn: ${{ secrets.SENTRY_SDK_DSN }}
-          sendgrid_api_key: ${{ secrets.SENDGRID_API_KEY }}
           disable_analytics_features: true
 
       - name: Run E2E tests against local

Dockerfile

@@ -1,11 +1,16 @@
 # Step 1 - Build Front End Application
 FROM node:16 AS build-frontend
 
-# Copy the entire project - Webpack puts compiled assets into the Django folder
+# Copy the files required to install npm packages
 WORKDIR /app
-COPY . .
+COPY frontend/package.json frontend/package-lock.json frontend/.npmrc ./frontend/.nvmrc ./frontend/
+COPY frontend/bin/ ./frontend/bin/
+COPY frontend/env/ ./frontend/env/
 
 RUN cd frontend && npm ci --quiet --production
+
+# Copy the entire project - Webpack puts compiled assets into the Django folder
+COPY . .
 ENV ENV=prod
 ENV STATIC_ASSET_CDN_URL=/static/
 RUN cd frontend && npm run bundledjango