Speed up CI pipeline (#1060)

* add build job with test job dependent on it

* spin up postgres and redis in build step

* fix indent

* fix indent again

* move env and services outside steps

* revert test yml to master minus install ff and run tests steps

* add test job dependent on build job

* run unit and feature tests in seperate jobs

* fix indent

* remove install and caching homebrew

* remove Get brew cache path

* Revert "remove Get brew cache path"

This reverts commit bf1fa73.

* Revert "remove install and caching homebrew"

This reverts commit 9d06dac.

* make reusable workflow for build convene app

* fix path invoking reusable workflow

* try to fix path again

* fix path again

* fix needs value for conditional workflow execution

* fix another ref for needs conditional

* call reusable build spp step in all test jobs

* revert yml to not use reusable step and move env to workflkow scope

Co-authored-by: Zee Spencer <zspencer@users.noreply.github.com>
Co-authored-by: Ana <anaulin@users.noreply.github.com>
Co-authored-by: Dalton <daltonrpruitt@users.noreply.github.com>

* clean up build step to just install dependencies

Co-authored-by: Zee Spencer <zspencer@users.noreply.github.com>
Co-authored-by: Ana <anaulin@users.noreply.github.com>
Co-authored-by: Dalton <daltonrpruitt@users.noreply.github.com>

* Fix name of formerly-build job.

* Remove unnecessary brew setup bits from rspec step.

* Fix bad db:test:prepare command.

Remove no longer used separate file.

* Do more setup for RSpec tests

* Add assets:precompile

* Remove unnecessary parts of setup for feature tests

* Remove duplicate "yarn install" from testing steps

* Remove inotify tweaks.

* Use same Postgres version in Docker as CI and Heroku.

* Try to reuse repeated YAML blocks.

* Bring back repeated "services" section.

---------

Co-authored-by: Zee Spencer <zspencer@users.noreply.github.com>
Co-authored-by: Ana <anaulin@users.noreply.github.com>
Co-authored-by: Dalton <daltonrpruitt@users.noreply.github.com>
Co-authored-by: Ana Ulin <ana@ulin.org>

.github/workflows/test-convene-web.yml

@@ -1,23 +1,47 @@
 name: Test
 on: push
 
+env:
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: postgres
+  # Connect to locally-running Maildev for tests
+  SMTP_PORT: 1025
+  SMTP_DOMAIN: localhost
+  SMTP_ENABLE_TLS: false
+  REDIS_HOST: redis
+  REDIS_PORT: 6379
+  HEADLESS: true
+  AWS_S3_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
+  AWS_S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}
+  AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+
 jobs:
-  test:
+  setup:
+    name: Install and Cache dependencies
     runs-on: ubuntu-latest
 
-    env:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-      # Connect to locally-running Maildev for tests
-      SMTP_PORT: 1025
-      SMTP_DOMAIN: localhost
-      SMTP_ENABLE_TLS: false
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      HEADLESS: true
-      AWS_S3_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
-      AWS_S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}
-      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Setup Ruby and install gems
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Setup Node with cache
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          cache: 'yarn'
+
+      - name: Install Node dependencies
+        run: yarn install
+
+  test-rspec:
+    name: Run Unit Tests
+    runs-on: ubuntu-latest
+    needs: [setup]
 
     services:
       postgres:
@@ -40,79 +64,108 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
-      # Temporary mitigation for https://github.com/actions/runner-images/issues/6283
-      # Perhaps we should consider not using brew in CI (though it sure is nice!)
-
-      - name: Set up Homebrew environment
-        uses: Homebrew/actions/setup-homebrew@master
-
-      - name: Install Firefox
-        uses: browser-actions/setup-firefox@latest
+        uses: actions/checkout@v3
 
       - name: Setup Ruby and install gems
         uses: ruby/setup-ruby@v1
         with:
           bundler-cache: true
 
-      - name: Increase the amount of inotify watchers
-        run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
+      - name: Setup Node with cache
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          cache: 'yarn'
 
       - name: Allow Ruby process to access port 80
         run: sudo setcap 'cap_net_bind_service=+ep' `which ruby`
 
       - name: Setup CI database.yml
         run: cp config/database.yml.github-actions config/database.yml
 
-      - name: Use Development mode env
-        run: cp .env.example .env
+      - name: Setup rails
+        run: bin/setup-rails && bin/rails assets:precompile
 
-      # Yarn caching steps borrowed from https://github.com/actions/cache/blob/main/examples.md#node---yarn
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: Run Tests
+        run: bundle exec rspec
 
-      - name: Cache dependencies
-        uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
+  test-features:
+    name: Run Browser Tests
+    runs-on: ubuntu-latest
+    needs: [setup]
+
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+        image: redis
+        ports:
+          # Maps port 6379 on service container to the host
+          - 6379:6379
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
 
-      - name: Get brew cache path
-        id: brew-cache-path
-        run: echo "::set-output name=path::$(brew --cache)"
+      - name: Setup Ruby and install gems
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
 
-      - name: Configure Homebrew cache
-        uses: actions/cache@v2
+      - name: Setup Node with cache
+        uses: actions/setup-node@v3
         with:
-          path: ${{ steps.brew-cache-path.outputs.path }}
-          key: ${{ runner.os }}-brew-${{ hashFiles('bin/setup') }}
-          restore-keys: ${{ runner.os }}-brew-
+          node-version: 16
+          cache: 'yarn'
+
+      - name: Install Firefox
+        uses: browser-actions/setup-firefox@latest
+
+      - name: Allow Ruby process to access port 80
+        run: sudo setcap 'cap_net_bind_service=+ep' `which ruby`
 
-      - name: Setup test environment
+      - name: Setup CI database.yml
+        run: cp config/database.yml.github-actions config/database.yml
+
+      - name: Use Development mode env
+        run: cp .env.example .env
+
+      - name: Install Overmind
         run: |
-            sudo chown -R $USER /usr/local/lib/node_modules
-            sudo chown -R $USER /usr/local/bin/
-            bin/setup
+          wget https://github.com/DarthSim/overmind/releases/download/v2.3.0/overmind-v2.3.0-linux-386.gz
+          gunzip -d overmind-v2.3.0-linux-386.gz
+          mv overmind-v2.3.0-linux-386 overmind
+          chmod +x overmind
+
+      - name: Install Maildev
+        run: yarn global add maildev
 
       - name: Start Rails server
         run: |
-            export PATH=$PATH:~/bin
-            bin/run &
+          export PATH=$PATH:~/bin:.
+          bin/setup-rails
+          bin/run &
 
       - name: Run Tests
         run: |
-            # To wait for asset built
-            # TODO: Start server in production mode
-            curl --connect-timeout 5 --retry 5 --retry-delay 5 --retry-max-time 40 --retry-connrefused localhost:3000 1> /dev/null
-            bin/test
-
+          # To wait for asset built
+          # TODO: Start server in production mode
+          curl --connect-timeout 5 --retry 5 --retry-delay 5 --retry-max-time 40 --retry-connrefused localhost:3000 1> /dev/null
+          yarn run test
       - name: Upload Test Results
         uses: actions/upload-artifact@v2
         if: failure()

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.1'
 
 services:
   db:
-    image: postgres
+    image: postgres:12
     restart: always
     environment:
       POSTGRES_HOST_AUTH_METHOD: trust