ci: add testrail back & test-full-suite fixes (#5729)

* Added back TestRail reporting. * Fixed an incorrect assertion in test. * Full Test Suite workflow: * Fixed trigger for full test suite not working correctly. * Added option to NOT notify on slack (i hate spamming ppl in some cases!) ### QA Notes * confirmed results upload to TR * confirmed Full Suite triggers work now
posit-dev · Dec 14, 2024 · 240e51f · 240e51f
1 parent 485eee5
commit 240e51f
Show file tree

Hide file tree

Showing 6 changed files with 69 additions and 15 deletions.
diff --git a/.github/workflows/test-e2e-linux.yml b/.github/workflows/test-e2e-linux.yml
@@ -28,6 +28,11 @@ on:
         description: "The tags to use for Currents recording."
         default: "@linux"
         type: string
+      report_testrail:
+        required: false
+        description: "Whether or not to report results to TestRail."
+        default: false
+        type: boolean
 
   workflow_dispatch:
     inputs:
@@ -125,3 +130,19 @@ jobs:
           role-to-assume: ${{ secrets.AWS_TEST_REPORTS_ROLE }}
           report-dir: ${{ env.REPORT_DIR }}
 
+      - name: Install trcli
+        if: ${{ inputs.report_testrail }}
+        shell: bash
+        run: sudo apt-get update && sudo apt-get install -y python3-pip && pip3 install trcli
+
+      - name: Upload Test Results to TestRail
+        if: ${{ inputs.report_testrail }}
+        shell: bash
+        run: |
+          TESTRAIL_TITLE="$(date +'%Y-%m-%d') ${{ env.TESTRAIL_TITLE }} - $GITHUB_REF_NAME"
+          echo "TESTRAIL_TITLE=$TESTRAIL_TITLE" >> $GITHUB_ENV
+          trcli --host "https://posit.testrail.io/" --project "${{ env.TESTRAIL_PROJECT }}" --username [email protected] --key "${{ env.TESTRAIL_API_KEY }}" parse_junit --file "./test-results/junit.xml" --case-matcher name --title "$TESTRAIL_TITLE" --close-run
+        env:
+          TESTRAIL_TITLE: ${{ inputs.project }}
+          TESTRAIL_PROJECT: "Positron"
+          TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
diff --git a/.github/workflows/test-e2e-windows.yml b/.github/workflows/test-e2e-windows.yml
@@ -22,6 +22,11 @@ on:
         description: "The tags to use for Currents recording."
         default: ""
         type: string
+      report_testrail:
+          required: false
+          description: "Whether or not to report results to TestRail."
+          default: false
+          type: boolean
 
   workflow_dispatch:
     inputs:
@@ -45,7 +50,7 @@ jobs:
     name: ${{ inputs.display_name }}
     runs-on:
       labels: [windows-latest-8x]
-    timeout-minutes: 80
+    timeout-minutes: 100
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       POSITRON_BUILD_NUMBER: 0 # CI skips building releases
@@ -162,3 +167,15 @@ jobs:
         with:
           role-to-assume: ${{ secrets.AWS_TEST_REPORTS_ROLE }}
           report-dir: ${{ env.REPORT_DIR }}
+
+      - name: Upload Test Results to TestRail
+        if: ${{ inputs.report_testrail }}
+        shell: bash
+        run: |
+          TESTRAIL_TITLE="$(date +'%Y-%m-%d') ${{ env.TESTRAIL_TITLE }} - $GITHUB_REF_NAME"
+          echo "TESTRAIL_TITLE=$TESTRAIL_TITLE" >> $GITHUB_ENV
+          trcli --host "https://posit.testrail.io/" --project "${{ env.TESTRAIL_PROJECT }}" --username [email protected] --key "${{ env.TESTRAIL_API_KEY }}" parse_junit --file "./test-results/junit.xml" --case-matcher name --title "$TESTRAIL_TITLE" --close-run
+        env:
+          TESTRAIL_TITLE: "e2e-windows"
+          TESTRAIL_PROJECT: "Positron"
+          TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
diff --git a/.github/workflows/test-full-suite.yml b/.github/workflows/test-full-suite.yml
@@ -39,56 +39,73 @@ on:
             options:
               - failure
               - always
+              - never
 
 jobs:
+  trigger:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Echo Input Details
+        run: |
+          echo "Workflow Inputs:"
+          echo "- E2E Electron (Linux): ${{ inputs.run_e2e_linux }}"
+          echo "- E2E Electron (Windows): ${{ inputs.run_e2e_windows }}"
+          echo "- E2E Chromium (Linux): ${{ inputs.run_e2e_browser }}"
+          echo "- Unit Tests: ${{ inputs.run_unit_tests }}"
+          echo "- Integration Tests: ${{ inputs.run_integration_tests }}"
+          echo "- Slack Notification On: ${{ inputs.notify_on }}"
+
   e2e-electron:
     name: e2e
-    if: github.event_name == 'schedule' || ${{ inputs.run_e2e_linux }}
+    if: ${{ github.event_name == 'schedule' || inputs.run_e2e_linux  }}
     uses: ./.github/workflows/test-e2e-linux.yml
+    secrets: inherit
     with:
       grep: ""
       project: "e2e-electron"
       display_name: "electron (linux)"
       currents_tags: ${{ github.event_name == 'schedule' && 'nightly,electron/linux' || 'electron/linux' }}
-    secrets: inherit
+      report_testrail: true
 
   e2e-windows:
     name: e2e
-    if: github.event_name == 'schedule' || ${{ inputs.run_e2e_windows }}
+    if: ${{ github.event_name == 'schedule' || inputs.run_e2e_windows }}
     uses: ./.github/workflows/test-e2e-windows.yml
+    secrets: inherit
     with:
       grep: ""
       display_name: "electron (win)"
       currents_tags: ${{ github.event_name == 'schedule' && 'nightly,electron/win' || 'electron/win' }}
-    secrets: inherit
+      report_testrail: true
 
   e2e-browser:
     name: e2e
-    if: github.event_name == 'schedule' || ${{ inputs.run_e2e_browser }}
+    if: ${{  github.event_name == 'schedule' || inputs.run_e2e_browser }}
     uses: ./.github/workflows/test-e2e-linux.yml
+    secrets: inherit
     with:
       grep: ""
       project: "e2e-browser"
       display_name: "browser (linux)"
       currents_tags: ${{ github.event_name == 'schedule' && 'nightly,browser/linux' || 'browser/linux' }}
-    secrets: inherit
+      report_testrail: true
 
   unit-tests:
     name: test
-    if: github.event_name == 'schedule' || ${{ inputs.run_unit_tests }}
+    if: ${{ github.event_name == 'schedule' || inputs.run_unit_tests }}
     uses: ./.github/workflows/test-unit.yml
     secrets: inherit
 
   integration-tests:
     name: test
-    if: github.event_name == 'schedule' || ${{ inputs.run_integration_tests }}
+    if: ${{ github.event_name == 'schedule' ||  inputs.run_integration_tests }}
     uses: ./.github/workflows/test-integration.yml
     secrets: inherit
 
   slack-notify:
     needs: [e2e-electron, e2e-windows, e2e-browser, unit-tests, integration-tests]
     runs-on: ubuntu-latest
-    if: always()
+    if: ${{ inputs.notify_on != 'never' }}
     steps:
       - name: Notify Slack
         uses: midleman/slack-workflow-status@master

diff --git a/.github/workflows/test-pull-request.yml b/.github/workflows/test-pull-request.yml
@@ -31,6 +31,7 @@ jobs:
       grep: ${{ needs.pr-tags.outputs.tags }}
       display_name: "electron (linux)"
       currents_tags: "pull-request,electron/linux,${{ needs.pr-tags.outputs.tags }}"
+      report_testrail: false
     secrets: inherit
 
   unit-tests:

diff --git a/build/secrets/.secrets.baseline b/build/secrets/.secrets.baseline
@@ -143,7 +143,7 @@
         "filename": ".github/workflows/test-full-suite.yml",
         "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0",
         "is_verified": false,
-        "line_number": 53,
+        "line_number": 62,
         "is_secret": false
       }
     ],
@@ -163,7 +163,7 @@
         "filename": ".github/workflows/test-pull-request.yml",
         "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0",
         "is_verified": false,
-        "line_number": 34,
+        "line_number": 35,
         "is_secret": false
       }
     ],
@@ -1944,5 +1944,5 @@
       }
     ]
   },
-  "generated_at": "2024-12-12T17:09:05Z"
+  "generated_at": "2024-12-14T14:27:48Z"
 }
diff --git a/test/automation/src/positron/positronVariables.ts b/test/automation/src/positron/positronVariables.ts
@@ -17,7 +17,6 @@ const VARIABLE_ITEMS = '.variable-item';
 const VARIABLE_NAMES = 'name-column';
 const VARIABLE_DETAILS = 'details-column';
 const VARIABLES_NAME_COLUMN = '.variables-instance[style*="z-index: 1"] .variable-item .name-column';
-const VARIABLES_SECTION = '[aria-label="Variables Section"]';
 const VARIABLES_INTERPRETER = '.positron-variables-container .action-bar-button-text';
 const VARIABLE_CHEVRON_ICON = '.gutter .expand-collapse-icon';
 const VARIABLE_INDENTED = '.name-column-indenter[style*="margin-left: 40px"]';
@@ -67,7 +66,6 @@ export class PositronVariables {
 		const modifier = isMac ? 'Meta' : 'Control';
 
 		await this.code.driver.getKeyboard().press(`${modifier}+Alt+B`);
-		await this.code.waitForElement(VARIABLES_SECTION);
 	}
 
 	async toggleVariable({ variableName, action }: { variableName: string; action: 'expand' | 'collapse' }) {