Add E2E tests

.github/workflows/e2e-tests.yml

@@ -0,0 +1,59 @@
+name: E2E Tests
+
+# suppress warning raised by https://github.com/jupyter/jupyter_core/pull/292
+env:
+  JUPYTER_PLATFORM_DIRS: '1'
+
+on:
+  push:
+    branches: main
+  pull_request:
+    branches: '*'
+
+jobs:
+  e2e-tests:
+    name: Linux
+    runs-on: ubuntu-latest
+
+    env:
+      PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Install extension dependencies and build the extension
+        run: ./scripts/install.sh
+
+      - name: Install ui-tests dependencies
+        working-directory: packages/jupyter-ai/ui-tests
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+        run: jlpm install
+
+      - name: Set up browser cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ github.workspace }}/pw-browsers
+          key: ${{ runner.os }}-${{ hashFiles('packages/jupyter-ai/ui-tests/yarn.lock') }}
+
+      - name: Install browser
+        working-directory: packages/jupyter-ai/ui-tests
+        run: jlpm install-chromium
+
+      - name: Execute e2e tests
+        working-directory: packages/jupyter-ai/ui-tests
+        run: jlpm test
+
+      - name: Upload Playwright Test report
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: jupyter-ai-playwright-tests-linux
+          path: |
+            packages/jupyter-ai/ui-tests/test-results
+            packages/jupyter-ai/ui-tests/playwright-report

.github/workflows/update-snapshots.yml

@@ -0,0 +1,52 @@
+name: Update Playwright Snapshots
+
+on:
+  issue_comment:
+    types: [created, edited]
+  workflow_dispatch:
+    inputs:
+      number:
+        description: 'PR number'
+        required: true
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update-snapshots:
+    if: ${{ github.event.inputs || (github.event.issue.pull_request && contains(github.event.comment.body, 'please update playwright snapshots')) }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Configure git to use https
+        run: git config --global hub.protocol https
+
+      - name: Checkout the branch from the PR that triggered the job
+        run: hub pr checkout ${{ github.event.inputs.number || github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Install extension dependencies and build the extension
+        run: ./scripts/install.sh
+
+      - uses: jupyterlab/maintainer-tools/.github/actions/update-snapshots@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          start_server_script: 'null'
+          test_folder: packages/jupyter-ai/ui-tests
+          npm_client: jlpm
+
+      - name: Comment back on the PR
+        run: |
+          hub api repos/${{ github.repository }}/issues/${{ github.event.inputs.number || github.event.issue.number }}/comments --raw-field 'body=Playwright snapshots updated.'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

docs/source/contributors/index.md

@@ -117,3 +117,38 @@ To uninstall your Jupyter AI development environment, deactivate and remove the
 conda deactivate
 conda env remove -n jupyter-ai
 ```
+
+## Testing
+
+### Integration / E2E tests
+
+This extension uses Playwright for the integration / E2E tests (user-level tests).
+More precisely, the JupyterLab helper
+[Galata](https://github.com/jupyterlab/jupyterlab/tree/master/galata) is used to
+test the extension in JupyterLab.
+
+Install test dependencies (needed only once):
+
+```sh
+cd ./packages/jupyter-ai/ui-tests/
+jlpm install
+jlpm playwright install
+```
+
+Tests involve snapshot comparisons against a reference snapshots generated by the Github CI. If you are using an OS other than Linux, you will need to generate local snapshots before running the tests locally for the first time. To do this, execute the command:
+
+```sh
+cd ./packages/jupyter-ai/ui-tests/
+jlpm test:update
+```
+
+To execute tests, run:
+
+```sh
+cd ./packages/jupyter-ai/ui-tests/
+jlpm test
+```
+
+You can find more information in the
+[ui-tests](https://github.com/jupyterlab/jupyter-ai/tree/main/packages/jupyter-ai/ui-tests)
+README.

packages/jupyter-ai/.gitignore

@@ -125,3 +125,7 @@ dmypy.json
 # Coverage reports
 coverage/*
 junit.xml
+
+# Ignore all non-linux snapshots
+ui-tests/tests/jupyter-ai.spec.ts-snapshots/*
+!ui-tests/tests/jupyter-ai.spec.ts-snapshots/*-linux.png

packages/jupyter-ai/ui-tests/README.md

@@ -12,35 +12,37 @@ in [jupyter_server_test_config.py](./jupyter_server_test_config.py).
 
 The default configuration will produce video for failing tests and an HTML report.
 
-## Run the tests
+> There is a new experimental UI mode that you may fall in love with; see [that video](https://www.youtube.com/watch?v=jF0yA-JLQW0).
 
 > All commands are assumed to be executed from the root directory
 
+## Run the tests
+
 To run the tests, you need to:
 
 1. Compile the extension:
 
 ```sh
-jlpm install
-jlpm build:prod
+./scripts/install.sh
 ```
 
 > Check the extension is installed in JupyterLab.
 
 2. Install test dependencies (needed only once):
 
 ```sh
-cd ./ui-tests
+cd ./packages/jupyter-ai/ui-tests/
 jlpm install
 jlpm playwright install
-cd ..
 ```
 
+> Tests involve snapshot comparisons against a reference snapshots generated by the Github CI. If you are using an OS other than Linux, you will need to generate local snapshots before running the tests locally for the first time. See [Update the tests snapshots](#update-the-tests-snapshots) section for the instructions.
+
 3. Execute the [Playwright](https://playwright.dev/docs/intro) tests:
 
 ```sh
-cd ./ui-tests
-jlpm playwright test
+cd ./packages/jupyter-ai/ui-tests/
+jlpm test
 ```
 
 Test results will be shown in the terminal. In case of any test failures, the test report
@@ -50,34 +52,12 @@ for configuring that behavior.
 
 ## Update the tests snapshots
 
-> All commands are assumed to be executed from the root directory
-
 If you are comparing snapshots to validate your tests, you may need to update
-the reference snapshots stored in the repository. To do that, you need to:
-
-1. Compile the extension:
-
-```sh
-jlpm install
-jlpm build:prod
-```
-
-> Check the extension is installed in JupyterLab.
-
-2. Install test dependencies (needed only once):
+the reference snapshots stored in the repository. To do that, you need to execute the [Playwright](https://playwright.dev/docs/intro) command:
 
 ```sh
-cd ./ui-tests
-jlpm install
-jlpm playwright install
-cd ..
-```
-
-3. Execute the [Playwright](https://playwright.dev/docs/intro) command:
-
-```sh
-cd ./ui-tests
-jlpm playwright test -u
+cd ./packages/jupyter-ai/ui-tests/
+jlpm test:update
 ```
 
 > Some discrepancy may occurs between the snapshots generated on your computer and
@@ -87,39 +67,24 @@ jlpm playwright test -u
 
 ## Create tests
 
-> All commands are assumed to be executed from the root directory
-
 To create tests, the easiest way is to use the code generator tool of playwright:
 
-1. Compile the extension:
+1. Start the server:
 
 ```sh
-jlpm install
-jlpm build:prod
+cd ./packages/jupyter-ai/ui-tests/
+jlpm start
 ```
 
-> Check the extension is installed in JupyterLab.
-
-2. Install test dependencies (needed only once):
+2. Execute the [Playwright code generator](https://playwright.dev/docs/codegen) in **another terminal**:
 
 ```sh
-cd ./ui-tests
-jlpm install
-jlpm playwright install
-cd ..
-```
-
-3. Execute the [Playwright code generator](https://playwright.dev/docs/codegen):
-
-```sh
-cd ./ui-tests
+cd ./packages/jupyter-ai/ui-tests/
 jlpm playwright codegen localhost:8888
 ```
 
 ## Debug tests
 
-> All commands are assumed to be executed from the root directory
-
 To debug tests, a good way is to use the inspector tool of playwright:
 
 1. Compile the extension:
@@ -134,15 +99,24 @@ jlpm build:prod
 2. Install test dependencies (needed only once):
 
 ```sh
-cd ./ui-tests
+cd ./packages/jupyter-ai/ui-tests/
 jlpm install
 jlpm playwright install
-cd ..
 ```
 
 3. Execute the Playwright tests in [debug mode](https://playwright.dev/docs/debug):
 
 ```sh
-cd ./ui-tests
-PWDEBUG=1 jlpm playwright test
+cd ./packages/jupyter-ai/ui-tests/
+jlpm playwright test --debug
+```
+
+## Upgrade Playwright and the browsers
+
+To update the web browser versions, you must update the package `@playwright/test`:
+
+```sh
+cd ./packages/jupyter-ai/ui-tests/
+jlpm up "@playwright/test"
+jlpm playwright install
 ```

packages/jupyter-ai/ui-tests/jupyter_server_test_config.py

@@ -4,17 +4,9 @@
 opens the server to the world and provide access to JupyterLab
 JavaScript objects through the global window variable.
 """
-from tempfile import mkdtemp
+from jupyterlab.galata import configure_jupyter_server
 
-c.ServerApp.port = 8888
-c.ServerApp.port_retries = 0
-c.ServerApp.open_browser = False
-
-c.ServerApp.root_dir = mkdtemp(prefix="galata-test-")
-c.ServerApp.token = ""
-c.ServerApp.password = ""
-c.ServerApp.disable_check_xsrf = True
-c.LabApp.expose_app_in_browser = True
+configure_jupyter_server(c)
 
 # Uncomment to set server log level to debug level
 # c.ServerApp.log_level = "DEBUG"

packages/jupyter-ai/ui-tests/package.json

@@ -1,13 +1,16 @@
 {
-  "name": "jupyter_ai-ui-tests",
+  "name": "jupyter-ai-e2e-tests",
   "version": "1.0.0",
-  "description": "JupyterLab jupyter_ai Integration Tests",
+  "description": "Jupyter AI E2E tests",
   "private": true,
   "scripts": {
     "start": "jupyter lab --config jupyter_server_test_config.py",
-    "test": "jlpm playwright test"
+    "install-chromium": "jlpm playwright install chromium",
+    "test": "jlpm playwright test",
+    "test:update": "jlpm playwright test --update-snapshots"
   },
   "devDependencies": {
-    "@jupyterlab/galata": "^4.3.0"
+    "@jupyterlab/galata": "^5.0.5",
+    "@playwright/test": "^1.37.0"
   }
 }