Merge pull request #2274 from posit-dev/kg-webview-tests

webview tests
posit-dev · Sep 19, 2024 · b5bc77f · b5bc77f
2 parents 52d6c0f + 717b39e
commit b5bc77f
Show file tree

Hide file tree

Showing 18 changed files with 330 additions and 140 deletions.
diff --git a/.github/workflows/contract.yaml b/.github/workflows/contract.yaml
@@ -1,8 +1,14 @@
 name: Nightly
 on:
   workflow_call:
+    inputs:
+      job:
+        required: true
+        type: string
+
 jobs:
   bats-tests:
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.job == 'all' || github.event.inputs.job == 'bats-tests' }}
     env:
       FUZZBUCKET_SSH_KEY: ${{ secrets.FUZZBUCKET_SSH_KEY }}
       FUZZBUCKET_URL: ${{ secrets.FUZZBUCKET_URL }}
@@ -21,7 +27,7 @@ jobs:
       - name: python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: R
         uses: r-lib/actions/setup-r@v2
         with:
@@ -63,6 +69,7 @@ jobs:
       - run: just bats test deploy
 
   vscode-ui-tests:
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.job == 'all' || github.event.inputs.job == 'vscode-ui-tests' }}
     env:
       FUZZBUCKET_SSH_KEY: ${{ secrets.FUZZBUCKET_SSH_KEY }}
       FUZZBUCKET_URL: ${{ secrets.FUZZBUCKET_URL }}
@@ -82,7 +89,8 @@ jobs:
       - name: python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
+
       - name: R
         uses: r-lib/actions/setup-r@v2
         with:
@@ -96,6 +104,7 @@ jobs:
         with:
           name: bin
           path: bin
+
       - uses: actions/checkout@v4
         with:
           repository: rstudio/connect-content
@@ -111,17 +120,25 @@ jobs:
       - run: just package
       - run: just vscode configure
       - run: just vscode-ui install
+      - run: python --version
 
-      - name: Run tests
-        run: just vscode-ui test
-        shell: bash
-        env:
-          SPEC_PATH: ./test/specs/fastapi.spec.ts
-          WORKSPACE_PATH: ../sample-content/fastapi-simple
+      - name: Run root tests
+        run: just vscode-ui test vscode-root
 
       - name: Run nested tests
-        run: just vscode-ui test
-        shell: bash
-        env:
-          SPEC_PATH: ./test/specs/nested-fastapi.spec.ts
-          WORKSPACE_PATH: ../sample-content/
+        run: just vscode-ui test vscode-nested
+
+      - name: Save screenshots
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: ${{ matrix.runs-on }}-screenshots
+          path: test/vscode-ui/errorShots
+          if-no-files-found: ignore
+          retention-days: 1
+
+      # - name: Setup tmate session
+      #   if: ${{ failure() }}
+      #   uses: mxschmitt/action-tmate@v3
+      #   with:
+      #     limit-access-to-actor: true
diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -3,7 +3,16 @@ on:
   schedule:
     - cron: "0 06 * * *"
   workflow_dispatch:
-
+    inputs:
+      job:
+        description: "Select the job to run in contract.yaml"
+        required: true
+        default: "all"
+        type: choice
+        options:
+          - all
+          - bats-tests
+          - vscode-ui-tests
 jobs:
   agent:
     uses: ./.github/workflows/agent.yaml
@@ -16,6 +25,8 @@ jobs:
     needs: [build, contract-deps]
     secrets: inherit
     uses: ./.github/workflows/contract.yaml
+    with:
+      job: ${{ github.event.inputs.job || 'all' }}
   cleanup:
     needs: nightly
     secrets: inherit

diff --git a/extensions/vscode/webviews/homeView/src/components/DeployButton.vue b/extensions/vscode/webviews/homeView/src/components/DeployButton.vue
@@ -1,5 +1,6 @@
 <template>
   <vscode-button
+    :data-automation="`deploy-button`"
     :disabled="!haveResources || isConfigInError || home.publishInProgress"
     @click="deploy"
   >

diff --git a/extensions/vscode/webviews/homeView/src/components/EvenEasierDeploy.vue b/extensions/vscode/webviews/homeView/src/components/EvenEasierDeploy.vue
@@ -36,9 +36,11 @@
         class="deployment-details-container"
       >
         <div class="deployment-details-row">
-          <span class="deployment-details-label">{{
-            home.selectedConfiguration.configuration.entrypoint
-          }}</span>
+          <span
+            class="deployment-details-label"
+            data-automation="entrypoint-label"
+            >{{ home.selectedConfiguration.configuration.entrypoint }}</span
+          >
           <span class="deployment-details-info"> (selected as entrypoint)</span>
         </div>
       </div>
@@ -100,7 +102,9 @@
           <div class="progress-container">
             <vscode-progress-ring class="progress-ring" />
             <div class="progress-desc">
-              <div>Deployment in Progress...</div>
+              <div data-automation="deployment-progress">
+                Deployment in Progress...
+              </div>
               <p class="progress-log-anchor">
                 <a
                   class="webview-link"
@@ -119,7 +123,10 @@
         </div>
       </div>
       <div v-else>
-        <div class="deployment-summary-container">
+        <div
+          class="deployment-summary-container"
+          data-automation="deploy-status"
+        >
           <h4 class="deployment-summary">
             {{ lastStatusDescription }}
           </h4>

diff --git a/extensions/vscode/webviews/homeView/src/components/tree/TreeItem.vue b/extensions/vscode/webviews/homeView/src/components/tree/TreeItem.vue
@@ -35,7 +35,7 @@
         :class="codicon"
       />
       <div class="tree-item-label-container">
-        <span class="tree-item-title">{{ title }}</span>
+        <span class="tree-item-title" data-automation="req">{{ title }}</span>
         <span v-if="description" class="tree-item-description">
           {{ description }}
         </span>

diff --git a/extensions/vscode/webviews/homeView/src/components/views/Credentials.vue b/extensions/vscode/webviews/homeView/src/components/views/Credentials.vue
@@ -1,5 +1,9 @@
 <template>
-  <TreeSection title="Credentials" :actions="sectionActions">
+  <TreeSection
+    title="Credentials"
+    data-automation="credentials"
+    :actions="sectionActions"
+  >
     <WelcomeView v-if="home.sortedCredentials.length === 0">
       <p>No credentials have been added yet.</p>
     </WelcomeView>
@@ -8,6 +12,7 @@
       v-for="credential in home.sortedCredentials"
       :title="credential.name"
       :description="credential.url"
+      :data-automation="`${credential.name}-list`"
       :codicon="
         credential.guid === CredentialGUIs.EnvironmentGUID
           ? 'codicon-bracket'

diff --git a/extensions/vscode/webviews/homeView/src/components/views/PythonPackages.vue b/extensions/vscode/webviews/homeView/src/components/views/PythonPackages.vue
@@ -1,5 +1,9 @@
 <template>
-  <TreeSection title="Python Packages" :actions="pythonPackageActions">
+  <TreeSection
+    title="Python Packages"
+    data-automation="python-packages"
+    :actions="pythonPackageActions"
+  >
     <WelcomeView v-if="showWelcomeView">
       <template v-if="showScanWelcomeView">
         <p>

diff --git a/extensions/vscode/webviews/homeView/src/components/views/RPackages.vue b/extensions/vscode/webviews/homeView/src/components/views/RPackages.vue
@@ -1,5 +1,9 @@
 <template>
-  <TreeSection title="R Packages" :actions="rPackageActions">
+  <TreeSection
+    title="R Packages"
+    data-automation="r-packages"
+    :actions="rPackageActions"
+  >
     <WelcomeView v-if="showWelcomeView">
       <template v-if="showScanWelcomeView">
         <p>
@@ -15,7 +19,7 @@
         </vscode-button>
       </template>
       <template v-if="isNotRProject">
-        <p>
+        <p data-automation="r-not-configured">
           This project is not configured to use R. To configure R, add an [r]
           section to your configuration file.
         </p>

diff --git a/extensions/vscode/webviews/homeView/src/components/views/projectFiles/ProjectFiles.vue b/extensions/vscode/webviews/homeView/src/components/views/projectFiles/ProjectFiles.vue
@@ -1,5 +1,6 @@
 <template>
   <TreeSection
+    data-automation="project-files"
     title="Project Files"
     :actions="[
       {

diff --git a/extensions/vscode/webviews/homeView/src/components/views/projectFiles/TreeProjectFiles.vue b/extensions/vscode/webviews/homeView/src/components/views/projectFiles/TreeProjectFiles.vue
@@ -31,6 +31,7 @@
           !home.flatFiles.lastDeployedFiles.has(file.id)
         "
         class="text-git-added"
+        :data-automation="`${file.id}-decorator`"
       >
         A
       </PostDecor>

diff --git a/test/scripts/test-runner.bash b/test/scripts/test-runner.bash
@@ -77,7 +77,16 @@ case "${test_case}" in
     "common")
         cli_tests
     ;;
-    "vscode-ui")
+    "vscode-root")
+        export WORKSPACE_PATH="../sample-content/fastapi-simple"
+        export PYTHON_VERSION=$(python --version | awk '{print $2}')
         setup_connect
         vscode_ui_tests
+    ;;
+    "vscode-nested")
+        export WORKSPACE_PATH="../sample-content/"
+        export PYTHON_VERSION=$(python --version | awk '{print $2}')
+        setup_connect
+        vscode_ui_tests
+    ;;
 esac
diff --git a/test/setup/connect_setup.py b/test/setup/connect_setup.py
@@ -7,7 +7,7 @@
 import os
 
 # use the perftest fuzzbucket instance since it already has all the deps
-alias = "ubuntu22-publishing-client-2024.05"
+alias = "ubuntu22-publishing-client-2024.0916"
 box_name = "connect-publishing-client"
 list_command = "fuzzbucket-client -j list"
 create_command = "fuzzbucket-client create -c -S 20 -t m5.2xlarge " + alias + " -n " + box_name
@@ -68,6 +68,8 @@ def connect_ready(box_name, max_attempts, interval):
                     update_config="fuzzbucket-client ssh " + box_name + " " + ssh_options + " sudo sed -i 's/CONNECT_IP/" + connect_box + "/g' /etc/rstudio-connect/rstudio-connect.gcfg"
                     restart_connect = "fuzzbucket-client ssh " + box_name + " " + ssh_options + " sudo systemctl restart rstudio-connect"
                     logging.info("Installing Connect on " + connect_box)
+                    # run kill script on box
+                    subprocess.check_output(check_lock, shell=True, text=True)
                     subprocess.check_output(install_connect, shell=True, text=True)
                     subprocess.check_output(update_config, shell=True, text=True)
                     subprocess.check_output(restart_connect, shell=True, text=True)
@@ -81,13 +83,8 @@ def connect_ready(box_name, max_attempts, interval):
 
 api_key=get_api_key('admin')
 connect_version=get_connect_version()
-install_connect = """
-while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do
-    echo "Waiting for package manager lock to be released...";
-    sleep 5;
-done;
-fuzzbucket-client ssh {box_name} {ssh_options} sudo -E UNATTENDED=1 bash installer-ci.sh -d {connect_version}
-""".format(box_name=box_name, ssh_options=ssh_options, connect_version=connect_version)
+check_lock = "fuzzbucket-client ssh " + box_name + " " + ssh_options + " sudo ./check_lock.sh"
+install_connect = "fuzzbucket-client ssh " + box_name + " " + ssh_options + " sudo -E UNATTENDED=1 bash installer-ci.sh -d " + connect_version
 
 response = connect_ready(box_name, 20, 5)
 

diff --git a/test/vscode-ui/justfile b/test/vscode-ui/justfile
@@ -57,14 +57,14 @@ run *args:
     {{ _with_debug }}
 
     if [ {{ os() }} == "linux" ]; then
-        xvfb-run -a npm run wdio
+        xvfb-run -a npm run {{ args }}
     else
-        npm run wdio
+        npm run {{ args }}
     fi
 
 # this can run deploy or content tests
-test:
+test *args:
     #!/usr/bin/env bash
     set -eou pipefail
     {{ _with_debug }}
-    ../scripts/test-runner.bash vscode-ui
+    ../scripts/test-runner.bash {{ args }}
diff --git a/test/vscode-ui/package.json b/test/vscode-ui/package.json
@@ -6,16 +6,18 @@
     "@testing-library/webdriverio": "^3.2.1",
     "@types/jest": "^29.5.12",
     "@wdio/cli": "^8.35.1",
+    "@wdio/dot-reporter": "^9.0.4",
     "@wdio/local-runner": "^8.35.1",
     "@wdio/mocha-framework": "^8.35.0",
-    "@wdio/spec-reporter": "^8.32.4",
+    "@wdio/spec-reporter": "^8.40.3",
     "ts-node": "^10.9.2",
     "typescript": "^5.4.3",
     "wdio-vscode-service": "^6.0.3",
     "wdio-wait-for": "^3.0.11"
   },
   "scripts": {
-    "wdio": "wdio run ./wdio.conf.ts"
+    "vscode-root": "wdio run ./wdio.conf.ts --suite root",
+    "vscode-nested": "wdio run ./wdio.conf.ts --suite nested"
   },
   "dependencies": {
     "@wdio/globals": "^8.35.1",

diff --git a/test/vscode-ui/test/specs/fastapi.spec.ts b/test/vscode-ui/test/specs/fastapi.spec.ts
@@ -1,6 +1,5 @@
 import { browser, $, expect } from "@wdio/globals";
 
-import * as os from "os";
 import * as fs from "fs";
 import * as path from "path";
 import { fileURLToPath } from "url";
@@ -30,7 +29,7 @@ describe("VS Code Extension UI Test", () => {
     await extension.click();
   });
 
-  it("can click add deployment button", async () => {
+  it("can click select button", async () => {
     await helper.switchToSubframe();
     // initialize project via button
     const selectButton = (await $('[data-automation="select-deployment"]')).$(
@@ -86,46 +85,4 @@ describe("VS Code Extension UI Test", () => {
       "type = 'python-fastapi'\nentrypoint = 'simple.py'\nvalidate = true\nfiles = [\n  '/simple.py',\n  '/requirements.txt'\n]\ntitle = 'my fastapi app'",
     );
   });
-
-  // cleanup
-  after(async () => {
-    const parentDir = path.resolve(
-      __dirname,
-      "../../../sample-content/fastapi-simple",
-    );
-    const positDir = path.join(parentDir, ".posit");
-
-    // Log the contents of the parent directory
-    console.log(fs.readdirSync(parentDir));
-
-    // Check if the directory exists before trying to delete it
-    if (fs.existsSync(positDir)) {
-      // Get the files in the directory
-      const files = fs.readdirSync(positDir);
-
-      // Delete each file in the directory
-      for (const file of files) {
-        const filePath = path.join(positDir, file);
-        if (fs.lstatSync(filePath).isDirectory()) {
-          fs.rmdirSync(filePath, { recursive: true }); // Delete directory recursively
-        } else {
-          fs.unlinkSync(filePath); // Delete file
-        }
-      }
-
-      // Delete the directory
-      fs.rmdirSync(positDir);
-    } else {
-      console.log("Directory does not exist");
-    }
-
-    // Use shell script to delete credentials
-    describe("Cleanup creds", () => {
-      it("remove credentials", async () => {
-        let scriptPath: string;
-        scriptPath = "cd ../scripts && bash cleanup.bash";
-        await helper.runShellScript(scriptPath);
-      });
-    });
-  });
 });