Merge pull request #3279 from jtpio/update-galata

Update to the new Galata

.github/workflows/tests.yml

@@ -151,27 +151,30 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: 3.9
     - uses: actions/cache@v1
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}-${{hashFiles('**/requirements.txt')}}
         restore-keys: |
           ${{ runner.os }}-pip-
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install jupyterlab==3.0.3
+        python -m pip install -U jupyterlab~=3.2 jupyter-packaging~=0.10
 
-    - name: Use Node.js 12.x
-      uses: actions/setup-node@v1
+    - name: Set up Node
+      uses: actions/setup-node@v2
       with:
-        node-version: 12.x
+        node-version: 14.x
+
     - name: Get yarn cache
       id: yarn-cache
       run: echo "::set-output name=dir::$(yarn cache dir)"
+
     - uses: actions/cache@v1
       with:
         path: ${{ steps.yarn-cache.outputs.dir }}
@@ -183,6 +186,7 @@ jobs:
       run: |
         yarn install --frozen-lockfile
         yarn run build
+
     - name: Build the extension
       run: |
         cd python/ipywidgets
@@ -195,42 +199,54 @@ jobs:
         pip install -e .
         jupyter labextension develop . --overwrite
         jupyter labextension list
-    - name: Install Galata
+
+    - name: Install Test Dependencies
       run: |
         cd ui-tests
         yarn install --frozen-lockfile
+        yarn playwright install chromium
+
     - name: Launch JupyterLab
       run: |
         cd ui-tests
-        yarn run start-jlab:detached
+        yarn run start:detached
+
     - name: Wait for JupyterLab
       uses: ifaxity/wait-on-action@v1
       with:
         resource: http-get://localhost:8888/api
         timeout: 20000
+
     - name: Run UI Tests
       run: |
         cd ui-tests
         yarn run test
-    - name: Upload UI Test artifacts
+
+    - name: Upload Playwright Test assets
       if: always()
       uses: actions/upload-artifact@v2
       with:
-        name: ipywidgets-ui-test-output
+        name: ipywidgets-test-assets
         path: |
-          ui-tests/test-output
-    - name: Run UI Tests
-      if: ${{ failure() }}
-      run: |
-        cd ui-tests
-        jlpm run test:create-references
-    - name: Upload UI Test new reference artifacts
-      if: ${{ failure() }}
+          ui-tests/test-results
+
+    - name: Upload Playwright Test report
+      if: always()
       uses: actions/upload-artifact@v2
       with:
-        name: ipywidgets-ui-test-new-reference
+        name: ipywidgets-test-report
         path: |
-          ui-tests/test-output/test/screenshots/*.png
+          ui-tests/playwright-report
 
-      env:
-        CI: true
+    - name: Update snapshots
+      if: failure()
+      run: |
+        cd ui-tests
+        yarn run test:update
+
+    - name: Upload updated snapshots
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: ipywidgets-updated-snapshots
+        path: ui-tests/tests

.gitignore

@@ -37,4 +37,5 @@ temp/*
 
 tsconfig.tsbuildinfo
 
-ui-tests/test-output/*
+ui-tests/test-results
+ui-tests/playwright-report

docs/source/dev_testing.md

@@ -7,20 +7,20 @@ pytest --cov=ipywidgets ./python/ipywidgets
 ```
 
 To run the Javascript tests in each package directory:
+
 ```sh
 yarn test
 ```
 
 This will run the test suite using `karma` with 'debug' level logging.
 
-
 # Visual Regression Tests
 
-ipywidgets uses [Galata](https://github.com/jupyterlab/galata) framework for visual regression testing. Galata provides a high level API to interact with JupyterLab UI programmatically and tools for capturing, comparison and report generation.
+`ipywidgets` uses the [Galata](https://github.com/jupyterlab/jupyterlab/tree/master/galata) framework for visual regression testing. Galata provides a high level API to programmatically interact with the JupyterLab UI, and tools for taking screenshots and generating test reports.
 
-Galata UI tests are written using typescript and [jest](https://github.com/facebook/jest). ipywidgets UI test suites are located in [ui-tests/tests](./ui-tests/tests) directory.
+UI tests are written in TypeScript and run with the Playwright test runner. The test suites are located in the [ui-tests/tests](../../ui-tests/tests) directory.
 
-[ui-tests/tests/widgets.test.ts](./ui-tests/tests/widgets.test.ts) test suite uploads a [notebook](./ui-tests/tests/notebooks/widgets.ipynb) into JupyterLab, runs it cell by cell and takes image captures of cell outputs. Cell outputs of this notebook are widgets of different types. Then, cell output captures are compared to [reference images](./ui-tests/reference-output/) to detect any visual regressions. Test output (diffs, result report) generated by Galata are uploaded to GitHub artifact storage and accessible from GitHub Actions page in `Artifacts` section.
+The [main test suite](../../ui-tests/tests/widgets.test.ts) uploads a [notebook](../../ui-tests/tests/notebooks/widgets.ipynb) to JupyterLab, runs it cell by cell and captures a screenshot of the cell outputs. The cell outputs correspond to widgets of different types. The cell outputs captures are then compared to the [reference snapshots](../../ui-tests/tests/widgets.test.ts-snapshots/) to detect any visual regression. The test report (diffs, result, videos) is uploaded to GitHub as an artifact and accessible from the GitHub Actions page in `Artifacts` section.
 
 ## Running Tests Locally
 
@@ -35,29 +35,35 @@ Galata needs to connect to a running instance of JupyterLab 3 to run UI tests. F
 
 ```sh
 # in ui-tests directory
-yarn run start-jlab
-```
-Alternatively, you can use the command below to launch JupyterLab which is equivalent to `start-jlab` script:
-
-```sh
-jupyter lab --expose-app-in-browser --no-browser --ServerApp.token='' --ServerApp.password='' --ServerApp.disable_check_xsrf='True'
+yarn run start
 ```
 
-Then you can run the `test` script which in turn launches Galata to run UI tests:
+Then run the `test` script:
 
 ```sh
 # in the ui-tests directory
 yarn run test
 ```
 
-You can pass additional arguments to Galata by appending parameters to the command as in `yarn run test -- --no-headless`. For a full list of Galata command-line options see [https://github.com/jupyterlab/galata#command-line-options](https://github.com/jupyterlab/galata#command-line-options).
+You can pass additional arguments to `playwright` by appending parameters to the command. For example to run the test in headed mode, `yarn run test --headed`.
+
+Checkout the [Playwright Command Line Reference](https://playwright.dev/docs/test-cli/) for more information about the available command line options.
 
 ## Adding new UI tests
 
-New test suites can be added into [ui-tests/tests](./ui-tests/tests) directory. Their names need to end with `.test.ts`. You can see some additional example test suites in [Galata repo](https://github.com/jupyterlab/galata/blob/main/packages/galata/tests). If tests in new suites are doing visual regression tests or HTML source regression tests then you also need to add their reference images / HTML files into [ui-tests/tests/reference-output](./ui-tests/tests/reference-output) directory.
+New test suites can be added to the [ui-tests/tests](../../ui-tests/tests) directory. Their names need to end with `.test.ts`. You can see some additional example test suites in the [JupyterLab repo](https://github.com/jupyterlab/jupyterlab/blob/master/galata/test). If the tests in new suites are doing visual regression tests or HTML source regression tests then you also need to add their reference images to the `-snapshots` directories.
 
 ## Reference Image Captures
 
-When doing visual regression tests, it is important to use reference images that were generated in the same environment. Otherwise, even though the same browser is used for testing, there might be minor differences in image renderings generated that could cause visual regression tests to fail.
+When doing visual regression tests, it is important to use reference images that were generated in the same environment. Otherwise, even though the same browser is used for testing, there might be minor differences that could cause visual regression tests to fail.
+
+When adding a new visual regression test, first make sure your tests pass locally on your development environment, with a reference snapshots generated in your dev environment. You can generate new reference snapshots by running `yarn run test:update`.
+
+To update the snapshots:
 
-When adding a new visual regression test, first make sure your tests pass locally on your development environment, with a reference image generated in your dev environment. You can use images captured by Galata as reference images. They will be saved as part of test output, under [ui-tests/test-output/test/screenshots](ui-tests/test-output/test/screenshots). However, you shouldn't push these references images generated in your development environment to github. Instead, have the new regression tests run and fail by GitHub Actions first, then download the artifacts from GitHub and use the captures generated in GitHub testing environment as the reference images and push those in a separate commit.
+- push the new changes to the branch
+- wait for the CI check to complete
+- go to the artifacts section and download the `ipywidgets-updated-snapshots` archive
+- extract the archive
+- copy the `-snapshots` directories to replace the existing ones
+- commmit and push the changes

ui-tests/jupyter_server_config.py

@@ -1,7 +1,10 @@
+from tempfile import mkdtemp
+
 c.ServerApp.port = 8888
+c.ServerApp.open_browser = False
+c.ServerApp.root_dir = mkdtemp(prefix='galata-test-')
 c.ServerApp.token = ""
 c.ServerApp.password = ""
 c.ServerApp.disable_check_xsrf = True
-c.ServerApp.open_browser = False
-c.LabApp.open_browser = False
+
 c.LabApp.expose_app_in_browser = True

ui-tests/package.json

@@ -1,17 +1,19 @@
 {
-  "name": "ipywidgets-ui-tests",
-  "version": "1.0.0",
-  "description": "ipywidgets UI Tests",
+  "name": "@jupyter-widgets/ui-tests",
   "private": true,
+  "version": "0.1.0",
+  "description": "ipywidgets UI Tests",
   "scripts": {
-    "start-jlab": "jupyter lab --config ./jupyter_server_config.py",
-    "start-jlab:detached": "yarn run start-jlab&",
-    "test:create-references": "galata --skip-visual-regression --skip-html-regression",
-    "test": "galata"
+    "start": "jupyter lab --config ./jupyter_server_config.py",
+    "start:detached": "yarn run start&",
+    "test": "playwright test",
+    "test:debug": "PWDEBUG=1 playwright test",
+    "test:report": "http-server ./playwright-report -a localhost -o",
+    "test:update": "playwright test --update-snapshots"
   },
   "author": "Project Jupyter",
   "license": "BSD-3-Clause",
   "dependencies": {
-    "@jupyterlab/galata": "3.0.11-2"
+    "@jupyterlab/galata": "~4.0.2"
   }
 }

ui-tests/playwright.config.js

@@ -0,0 +1,7 @@
+const baseConfig = require('@jupyterlab/galata/lib/playwright-config');
+
+module.exports = {
+  ...baseConfig,
+  timeout: 240000,
+  retries: 1,
+};

ui-tests/tests/notebooks/widgets.ipynb

@@ -6,15 +6,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import ipywidgets as widgets"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
+    "import ipywidgets as widgets\n",
+    "\n",
     "widgets.IntSlider(\n",
     "    value=7,\n",
     "    min=0,\n",

ui-tests/tests/widgets.test.ts

@@ -1,90 +1,46 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
-import { galata, describe, test } from '@jupyterlab/galata';
-import * as path from 'path';
-
-jest.setTimeout(100000);
+import { test } from '@jupyterlab/galata';
 
-describe('Widget Visual Regression', () => {
-  beforeAll(async () => {
-    await galata.resetUI();
-    galata.context.capturePrefix = 'widgets';
-  });
+import { expect } from '@playwright/test';
 
-  afterAll(async () => {
-    galata.context.capturePrefix = '';
-  });
+import * as path from 'path';
 
-  test('Upload files to JupyterLab', async () => {
-    await galata.contents.moveDirectoryToServer(
-      path.resolve(__dirname, `./notebooks`),
-      'uploaded'
+test.describe('Widget Visual Regression', () => {
+  test.beforeEach(async ({ page, tmpPath }) => {
+    await page.contents.uploadDirectory(
+      path.resolve(__dirname, './notebooks'),
+      tmpPath
     );
-    expect(
-      await galata.contents.fileExists('uploaded/widgets.ipynb')
-    ).toBeTruthy();
-    expect(
-      await galata.contents.fileExists('uploaded/WidgetArch.png')
-    ).toBeTruthy();
-  });
-
-  test('Refresh File Browser', async () => {
-    await galata.filebrowser.refresh();
-  });
-
-  test('Open directory uploaded', async () => {
-    await galata.filebrowser.openDirectory('uploaded');
-    expect(
-      await galata.filebrowser.isFileListedInBrowser('widgets.ipynb')
-    ).toBeTruthy();
+    await page.filebrowser.openDirectory(tmpPath);
   });
 
-  test('Run notebook widgets.ipynb and capture cell outputs', async () => {
+  test('Run notebook widgets.ipynb and capture cell outputs', async ({
+    page,
+    tmpPath,
+  }) => {
     const notebook = 'widgets.ipynb';
-    await galata.notebook.open(notebook);
-    expect(await galata.notebook.isOpen(notebook)).toBeTruthy();
-    await galata.notebook.activate(notebook);
-    expect(await galata.notebook.isActive(notebook)).toBeTruthy();
+    await page.notebook.openByPath(`${tmpPath}/${notebook}`);
+    await page.notebook.activate(notebook);
 
-    let numCellImages = 0;
+    const captures = new Array<Buffer>();
+    const cellCount = await page.notebook.getCellCount();
 
-    const getCaptureImageName = (id: number): string => {
-      return `cell-${id}`;
-    };
-
-    await galata.notebook.runCellByCell({
+    await page.notebook.runCellByCell({
       onAfterCellRun: async (cellIndex: number) => {
-        const cell = await galata.notebook.getCellOutput(cellIndex);
+        const cell = await page.notebook.getCellOutput(cellIndex);
         if (cell) {
-          if (
-            await galata.capture.screenshot(
-              getCaptureImageName(numCellImages),
-              cell
-            )
-          ) {
-            numCellImages++;
-          }
+          captures.push(await cell.screenshot());
         }
       },
     });
 
-    for (let c = 0; c < numCellImages; ++c) {
-      expect(
-        await galata.capture.compareScreenshot(getCaptureImageName(c))
-      ).toBe('same');
-    }
-  });
-
-  test('Close notebook widgets.ipynb', async () => {
-    await galata.notebook.close(true);
-  });
-
-  test('Open home directory', async () => {
-    await galata.filebrowser.openHomeDirectory();
-  });
+    await page.notebook.save();
 
-  test('Delete uploaded directory', async () => {
-    await galata.contents.deleteDirectory('uploaded');
+    for (let i = 0; i < cellCount; i++) {
+      const image = `widgets-cell-${i}.png`;
+      expect(captures[i]).toMatchSnapshot(image);
+    }
   });
 });