test: add playwright code coverage (#2014)

* test: Add playwright test coverage support

* chore: remove extraneous comment

* fix: test-e2e.yml PR comment depends on context.issue.number

* fix: .nycrc json syntax

* fix: remove coverage comment; output report in action

* chore: ensure test:build always re-builds

* chore: ensure playwright coverage saves to correct folder

* fix: ensure test:build is ran before test:e2e

* fix: remove needs: build from test:e2e

* fix: put istandbul output back into .nyc_output

* chore: add codecov action

* fix: run nyc report prior to codecov action

.github/workflows/ci.yml

@@ -149,11 +149,10 @@ jobs:
 
   test-e2e:
     name: 'test:e2e'
-    needs: build
     uses: ./.github/workflows/test-e2e.yml
 
   test-storybook:
-    name: 'test:e2e'
+    name: 'test:storybook'
     uses: ./.github/workflows/test-storybook.yml
 
   # separate check for TS

.github/workflows/test-e2e.yml

@@ -34,11 +34,20 @@ jobs:
       - name: Install dependencies
         run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
 
-      - name: Download build artifact
-        uses: actions/download-artifact@v2
-        with:
-          name: ipfs-webui_${{ github.sha }}-build
-          path: build
+      # This is required to ensure that our code is instrumented with coverage details
+      - name: Run test build
+        run: npm run test:build
 
       - name: Run E2E against ${{ matrix.backend }}-ipfs
         run: E2E_IPFSD_TYPE=${{ matrix.backend }} npm run test:e2e
+
+      - name: Generate nyc coverage report
+        id: coverage
+        run: npx nyc report --reporter=lcov
+
+      - uses: codecov/codecov-action@v3
+        with:
+          flags: e2e_tests # optional
+          name: e2e-coverage # optional
+          fail_ci_if_error: true # optional (default = false)
+          verbose: true # optional (default = false)

.gitignore

@@ -32,5 +32,7 @@ tsconfig.tsbuildinfo
 .tool-versions
 .envrc
 .cid
+.cache
+.nyc_output
 
 tx

.nycrc

@@ -0,0 +1,6 @@
+
+{
+  "include": ["src/**/*.js"],
+  "exclude": ["**/*.test.js", "**/*.stories.js"],
+  "reporter": ["html", "text", "lcov"]
+}

config-overrides.js

@@ -4,20 +4,57 @@
  * @see https://github.com/facebook/create-react-app/issues/11756#issuecomment-1184657437
  * @see https://alchemy.com/blog/how-to-polyfill-node-core-modules-in-webpack-5
  */
-const webpack = require('webpack');
+const webpack = require('webpack')
 
-module.exports = function override(config) {
+/**
+ *
+ * @param {import('webpack').RuleSetRule[]} rules
+ */
+function modifyBabelLoaderRule (rules, root = true) {
+  const foundRules = []
+  rules.forEach((rule, i) => {
+    if (rule.loader != null) {
+      if (rule.loader.includes('babel-loader')) {
+        foundRules.push(rule)
+      }
+    } else if (rule.use?.loader != null) {
+      if (typeof rule.use.loader !== 'string') {
+        if (rule.use.loader.find(loader => loader.indexOf('babel-loader') >= 0)) {
+          foundRules.push(rule)
+        }
+      } else if (rule.use.loader.indexOf('babel-loader') >= 0) {
+        foundRules.push(rule)
+      }
+    } else if (rule.oneOf) {
+      const nestedRules = modifyBabelLoaderRule(rule.oneOf, false)
+      foundRules.push(...nestedRules)
+    }
+  })
 
-  const fallback = config.resolve.fallback || {};
+  if (root) {
+    foundRules.forEach((rule, index) => {
+      if (rule.include?.indexOf('src') >= 0) {
+        console.log('Found CRA babel-loader rule for source files. Modifying it to instrument for code coverage.')
+        console.log('rule: ', rule)
+        rule.options.plugins.push('istanbul')
+      }
+    })
+  }
+
+  return foundRules
+}
+
+module.exports = function webpackOverride (config) {
+  const fallback = config.resolve.fallback || {}
 
   Object.assign(fallback, {
-    "assert": require.resolve('./src/webpack-fallbacks/assert'),
-    "stream": require.resolve('./src/webpack-fallbacks/stream'),
-    "os": require.resolve('./src/webpack-fallbacks/os'),
-    "path": require.resolve('./src/webpack-fallbacks/path'),
+    assert: require.resolve('./src/webpack-fallbacks/assert'),
+    stream: require.resolve('./src/webpack-fallbacks/stream'),
+    os: require.resolve('./src/webpack-fallbacks/os'),
+    path: require.resolve('./src/webpack-fallbacks/path')
   })
 
-  config.resolve.fallback = fallback;
+  config.resolve.fallback = fallback
 
   config.plugins = (config.plugins || []).concat([
     new webpack.ProvidePlugin({
@@ -26,5 +63,11 @@ module.exports = function override(config) {
     })
   ])
 
-  return config;
+  // Instrument for code coverage in development mode
+  const REACT_APP_ENV = process.env.REACT_APP_ENV ?? process.env.NODE_ENV ?? 'production'
+  if (REACT_APP_ENV === 'test') {
+    modifyBabelLoaderRule(config.module.rules)
+  }
+
+  return config
 }

package.json

@@ -24,8 +24,8 @@
     "test": "run-s -cl test:unit test:build test:e2e",
     "test:unit": "react-app-rewired test --env=jsdom --runInBand --watchAll=false",
     "test:unit:watch": "react-app-rewired test --env=jsdom",
-    "test:e2e": "npx playwright test -c ./test/e2e",
-    "test:build": "shx test -f build/index.html || run-s build",
+    "test:e2e": "cross-env REACT_APP_ENV=test npx playwright test -c ./test/e2e",
+    "test:build": "cross-env REACT_APP_ENV=test run-s build",
     "test:coverage": "react-app-rewired test --coverage",
     "analyze": "webpack-bundle-analyzer build/stats.json",
     "bundlesize": "bundlesize",
@@ -140,6 +140,7 @@
     "assert": "^2.0.0",
     "autoprefixer": "^10.4.7",
     "babel-eslint": "^10.1.0",
+    "babel-plugin-istanbul": "^6.1.1",
     "basic-auth": "^2.0.1",
     "big.js": "^5.2.2",
     "bundlesize": "0.18.1",
@@ -169,6 +170,7 @@
     "jest-watch-typeahead": "^2.0.0",
     "multihashing-async": "^1.0.0",
     "npm-run-all": "^4.1.5",
+    "nyc": "^15.1.0",
     "os-browserify": "^0.3.0",
     "path-browserify": "^1.0.1",
     "playwright-chromium": "^1.24.0",

test/e2e/explore.test.js

@@ -1,4 +1,4 @@
-const { test, expect } = require('@playwright/test')
+const { test, expect } = require('./setup/coverage')
 const fs = require('fs')
 const path = require('path')
 const ipfsClient = require('ipfs-http-client')

test/e2e/files.test.js

@@ -1,4 +1,4 @@
-const { test } = require('@playwright/test')
+const { test } = require('./setup/coverage')
 const { fixtureData } = require('./fixtures')
 const all = require('it-all')
 const filesize = require('filesize')

test/e2e/ipns.test.js

@@ -1,4 +1,4 @@
-const { test, expect } = require('@playwright/test')
+const { test, expect } = require('./setup/coverage')
 const { createController } = require('ipfsd-ctl')
 const ipfsClient = require('ipfs-http-client')
 

test/e2e/navigation.test.js

@@ -1,4 +1,4 @@
-const { test, expect } = require('@playwright/test')
+const { test, expect } = require('./setup/coverage')
 
 test.describe('Navigation menu', () => {
   test.beforeEach(async ({ page }) => {

test/e2e/peers.test.js

@@ -1,4 +1,4 @@
-const { test } = require('@playwright/test')
+const { test } = require('./setup/coverage')
 const { createController } = require('ipfsd-ctl')
 const ipfsClient = require('ipfs-http-client')
 

test/e2e/playwright.config.js

@@ -3,6 +3,7 @@
 const webuiPort = 3001
 const rpcPort = 55001
 
+/** @type {import('@playwright/test').Config} */
 const config = {
   testDir: './',
   timeout: process.env.CI ? 90 * 1000 : 30 * 1000,
@@ -55,9 +56,21 @@ const config = {
       command: `http-server ./build/ -c-1 -a 127.0.0.1 -p ${webuiPort}`,
       port: webuiPort,
       cwd: '../../',
-      reuseExistingServer: !process.env.CI
+      reuseExistingServer: !process.env.CI,
+      env: {
+        REACT_APP_ENV: 'test',
+        NODE_ENV: 'test',
+        PORT: webuiPort
+      }
     }
-  ]
+  ],
+  collectCoverage: true,
+  coverageConfig: {
+    include: [
+      'src/**/*.{js,jsx,ts,tsx}',
+      '!src/**/*.stories.{js,jsx,ts,tsx}'
+    ]
+  }
 }
 
 module.exports = config

test/e2e/remote-api.test.js

@@ -1,4 +1,4 @@
-const { test, expect } = require('@playwright/test')
+const { test, expect } = require('./setup/coverage')
 const { createController } = require('ipfsd-ctl')
 const getPort = require('get-port')
 const http = require('http')

test/e2e/settings.test.js

@@ -1,4 +1,4 @@
-const { test } = require('@playwright/test')
+const { test } = require('./setup/coverage')
 
 test.describe('Settings screen', () => {
   test.beforeEach(async ({ page }) => {

test/e2e/setup/coverage.js

@@ -0,0 +1,42 @@
+/**
+ * @see https://github.com/mxschmitt/playwright-test-coverage
+ * @see https://github.com/mxschmitt/playwright-test-coverage/blob/main/e2e/baseFixtures.ts
+ */
+import * as fs from 'fs'
+import * as path from 'path'
+import * as crypto from 'crypto'
+import { test as baseTest } from '@playwright/test'
+
+const istanbulCLIOutput = path.join(process.cwd(), '.nyc_output')
+
+export function generateUUID () {
+  return crypto.randomBytes(16).toString('hex')
+}
+
+export const test = baseTest.extend({
+  context: async ({ context }, use) => {
+    await context.addInitScript(() =>
+      window.addEventListener('beforeunload', () =>
+        window.collectIstanbulCoverage(JSON.stringify(window.__coverage__))
+      )
+    )
+    await fs.promises.mkdir(istanbulCLIOutput, { recursive: true })
+    await context.exposeFunction('collectIstanbulCoverage', async (coverageJSON) => {
+      if (coverageJSON) {
+        try {
+          await fs.promises.writeFile(path.join(istanbulCLIOutput, `playwright_coverage_${generateUUID()}.json`), coverageJSON)
+        } catch (err) {
+          console.error('Error writing playwright coverage file', err)
+        }
+      } else {
+        throw new Error('No coverage data')
+      }
+    })
+    await use(context)
+    for (const page of context.pages()) {
+      await page.evaluate(() => window.collectIstanbulCoverage(JSON.stringify(window.__coverage__)))
+    }
+  }
+})
+
+export const expect = test.expect

test/e2e/status.test.js

@@ -1,4 +1,4 @@
-const { test } = require('@playwright/test')
+const { test } = require('./setup/coverage')
 
 test.describe('Status page', () => {
   test.beforeEach(async ({ page }) => {