From 240e51fa165f6c512e586f0af105a0c8fc092607 Mon Sep 17 00:00:00 2001 From: Marie Idleman Date: Sat, 14 Dec 2024 08:29:34 -0600 Subject: [PATCH] ci: add testrail back & test-full-suite fixes (#5729) * Added back TestRail reporting. * Fixed an incorrect assertion in test. * Full Test Suite workflow: * Fixed trigger for full test suite not working correctly. * Added option to NOT notify on slack (i hate spamming ppl in some cases!) ### QA Notes * confirmed results upload to TR * confirmed Full Suite triggers work now --- .github/workflows/test-e2e-linux.yml | 21 +++++++++++ .github/workflows/test-e2e-windows.yml | 19 +++++++++- .github/workflows/test-full-suite.yml | 35 ++++++++++++++----- .github/workflows/test-pull-request.yml | 1 + build/secrets/.secrets.baseline | 6 ++-- .../src/positron/positronVariables.ts | 2 -- 6 files changed, 69 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-e2e-linux.yml b/.github/workflows/test-e2e-linux.yml index 80207eaa298..3a95face8d0 100644 --- a/.github/workflows/test-e2e-linux.yml +++ b/.github/workflows/test-e2e-linux.yml @@ -28,6 +28,11 @@ on: description: "The tags to use for Currents recording." default: "@linux" type: string + report_testrail: + required: false + description: "Whether or not to report results to TestRail." + default: false + type: boolean workflow_dispatch: inputs: @@ -125,3 +130,19 @@ jobs: role-to-assume: ${{ secrets.AWS_TEST_REPORTS_ROLE }} report-dir: ${{ env.REPORT_DIR }} + - name: Install trcli + if: ${{ inputs.report_testrail }} + shell: bash + run: sudo apt-get update && sudo apt-get install -y python3-pip && pip3 install trcli + + - name: Upload Test Results to TestRail + if: ${{ inputs.report_testrail }} + shell: bash + run: | + TESTRAIL_TITLE="$(date +'%Y-%m-%d') ${{ env.TESTRAIL_TITLE }} - $GITHUB_REF_NAME" + echo "TESTRAIL_TITLE=$TESTRAIL_TITLE" >> $GITHUB_ENV + trcli --host "https://posit.testrail.io/" --project "${{ env.TESTRAIL_PROJECT }}" --username testrailautomation@posit.co --key "${{ env.TESTRAIL_API_KEY }}" parse_junit --file "./test-results/junit.xml" --case-matcher name --title "$TESTRAIL_TITLE" --close-run + env: + TESTRAIL_TITLE: ${{ inputs.project }} + TESTRAIL_PROJECT: "Positron" + TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }} diff --git a/.github/workflows/test-e2e-windows.yml b/.github/workflows/test-e2e-windows.yml index 4269b724ca6..f23e04725a5 100644 --- a/.github/workflows/test-e2e-windows.yml +++ b/.github/workflows/test-e2e-windows.yml @@ -22,6 +22,11 @@ on: description: "The tags to use for Currents recording." default: "" type: string + report_testrail: + required: false + description: "Whether or not to report results to TestRail." + default: false + type: boolean workflow_dispatch: inputs: @@ -45,7 +50,7 @@ jobs: name: ${{ inputs.display_name }} runs-on: labels: [windows-latest-8x] - timeout-minutes: 80 + timeout-minutes: 100 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} POSITRON_BUILD_NUMBER: 0 # CI skips building releases @@ -162,3 +167,15 @@ jobs: with: role-to-assume: ${{ secrets.AWS_TEST_REPORTS_ROLE }} report-dir: ${{ env.REPORT_DIR }} + + - name: Upload Test Results to TestRail + if: ${{ inputs.report_testrail }} + shell: bash + run: | + TESTRAIL_TITLE="$(date +'%Y-%m-%d') ${{ env.TESTRAIL_TITLE }} - $GITHUB_REF_NAME" + echo "TESTRAIL_TITLE=$TESTRAIL_TITLE" >> $GITHUB_ENV + trcli --host "https://posit.testrail.io/" --project "${{ env.TESTRAIL_PROJECT }}" --username testrailautomation@posit.co --key "${{ env.TESTRAIL_API_KEY }}" parse_junit --file "./test-results/junit.xml" --case-matcher name --title "$TESTRAIL_TITLE" --close-run + env: + TESTRAIL_TITLE: "e2e-windows" + TESTRAIL_PROJECT: "Positron" + TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }} diff --git a/.github/workflows/test-full-suite.yml b/.github/workflows/test-full-suite.yml index 4a236a12fdc..f3873718175 100644 --- a/.github/workflows/test-full-suite.yml +++ b/.github/workflows/test-full-suite.yml @@ -39,56 +39,73 @@ on: options: - failure - always + - never jobs: + trigger: + runs-on: ubuntu-latest + steps: + - name: Echo Input Details + run: | + echo "Workflow Inputs:" + echo "- E2E Electron (Linux): ${{ inputs.run_e2e_linux }}" + echo "- E2E Electron (Windows): ${{ inputs.run_e2e_windows }}" + echo "- E2E Chromium (Linux): ${{ inputs.run_e2e_browser }}" + echo "- Unit Tests: ${{ inputs.run_unit_tests }}" + echo "- Integration Tests: ${{ inputs.run_integration_tests }}" + echo "- Slack Notification On: ${{ inputs.notify_on }}" + e2e-electron: name: e2e - if: github.event_name == 'schedule' || ${{ inputs.run_e2e_linux }} + if: ${{ github.event_name == 'schedule' || inputs.run_e2e_linux }} uses: ./.github/workflows/test-e2e-linux.yml + secrets: inherit with: grep: "" project: "e2e-electron" display_name: "electron (linux)" currents_tags: ${{ github.event_name == 'schedule' && 'nightly,electron/linux' || 'electron/linux' }} - secrets: inherit + report_testrail: true e2e-windows: name: e2e - if: github.event_name == 'schedule' || ${{ inputs.run_e2e_windows }} + if: ${{ github.event_name == 'schedule' || inputs.run_e2e_windows }} uses: ./.github/workflows/test-e2e-windows.yml + secrets: inherit with: grep: "" display_name: "electron (win)" currents_tags: ${{ github.event_name == 'schedule' && 'nightly,electron/win' || 'electron/win' }} - secrets: inherit + report_testrail: true e2e-browser: name: e2e - if: github.event_name == 'schedule' || ${{ inputs.run_e2e_browser }} + if: ${{ github.event_name == 'schedule' || inputs.run_e2e_browser }} uses: ./.github/workflows/test-e2e-linux.yml + secrets: inherit with: grep: "" project: "e2e-browser" display_name: "browser (linux)" currents_tags: ${{ github.event_name == 'schedule' && 'nightly,browser/linux' || 'browser/linux' }} - secrets: inherit + report_testrail: true unit-tests: name: test - if: github.event_name == 'schedule' || ${{ inputs.run_unit_tests }} + if: ${{ github.event_name == 'schedule' || inputs.run_unit_tests }} uses: ./.github/workflows/test-unit.yml secrets: inherit integration-tests: name: test - if: github.event_name == 'schedule' || ${{ inputs.run_integration_tests }} + if: ${{ github.event_name == 'schedule' || inputs.run_integration_tests }} uses: ./.github/workflows/test-integration.yml secrets: inherit slack-notify: needs: [e2e-electron, e2e-windows, e2e-browser, unit-tests, integration-tests] runs-on: ubuntu-latest - if: always() + if: ${{ inputs.notify_on != 'never' }} steps: - name: Notify Slack uses: midleman/slack-workflow-status@master diff --git a/.github/workflows/test-pull-request.yml b/.github/workflows/test-pull-request.yml index 2de5cdf9898..d043a3b1be2 100644 --- a/.github/workflows/test-pull-request.yml +++ b/.github/workflows/test-pull-request.yml @@ -31,6 +31,7 @@ jobs: grep: ${{ needs.pr-tags.outputs.tags }} display_name: "electron (linux)" currents_tags: "pull-request,electron/linux,${{ needs.pr-tags.outputs.tags }}" + report_testrail: false secrets: inherit unit-tests: diff --git a/build/secrets/.secrets.baseline b/build/secrets/.secrets.baseline index c037d870c67..df596c379e9 100644 --- a/build/secrets/.secrets.baseline +++ b/build/secrets/.secrets.baseline @@ -143,7 +143,7 @@ "filename": ".github/workflows/test-full-suite.yml", "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0", "is_verified": false, - "line_number": 53, + "line_number": 62, "is_secret": false } ], @@ -163,7 +163,7 @@ "filename": ".github/workflows/test-pull-request.yml", "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0", "is_verified": false, - "line_number": 34, + "line_number": 35, "is_secret": false } ], @@ -1944,5 +1944,5 @@ } ] }, - "generated_at": "2024-12-12T17:09:05Z" + "generated_at": "2024-12-14T14:27:48Z" } diff --git a/test/automation/src/positron/positronVariables.ts b/test/automation/src/positron/positronVariables.ts index 4be2f64134e..66f3452947c 100644 --- a/test/automation/src/positron/positronVariables.ts +++ b/test/automation/src/positron/positronVariables.ts @@ -17,7 +17,6 @@ const VARIABLE_ITEMS = '.variable-item'; const VARIABLE_NAMES = 'name-column'; const VARIABLE_DETAILS = 'details-column'; const VARIABLES_NAME_COLUMN = '.variables-instance[style*="z-index: 1"] .variable-item .name-column'; -const VARIABLES_SECTION = '[aria-label="Variables Section"]'; const VARIABLES_INTERPRETER = '.positron-variables-container .action-bar-button-text'; const VARIABLE_CHEVRON_ICON = '.gutter .expand-collapse-icon'; const VARIABLE_INDENTED = '.name-column-indenter[style*="margin-left: 40px"]'; @@ -67,7 +66,6 @@ export class PositronVariables { const modifier = isMac ? 'Meta' : 'Control'; await this.code.driver.getKeyboard().press(`${modifier}+Alt+B`); - await this.code.waitForElement(VARIABLES_SECTION); } async toggleVariable({ variableName, action }: { variableName: string; action: 'expand' | 'collapse' }) {