From 191ec234f580497796c046983e944f22b7d1f319 Mon Sep 17 00:00:00 2001 From: Citrine Date: Sun, 21 Jan 2024 12:43:25 -0800 Subject: [PATCH 1/6] feat(test_framework): Add workflow scripts to show test results in PRs The trigger for running tests is disabled until we want to enable them. In order to run, these scripts (like other workflow scripts) must be in the main branch (master). --- .github/workflows/process_test_results.yml | 41 ++++++++++++++++++++++ .github/workflows/run_tests.yml | 34 ++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 .github/workflows/process_test_results.yml create mode 100644 .github/workflows/run_tests.yml diff --git a/.github/workflows/process_test_results.yml b/.github/workflows/process_test_results.yml new file mode 100644 index 00000000000..55a7604eff5 --- /dev/null +++ b/.github/workflows/process_test_results.yml @@ -0,0 +1,41 @@ +name: Process Test Results + +on: + workflow_run: + workflows: ["Run Tests"] + types: + - completed + +permissions: {} + +jobs: + process-test-results: + name: Process Test Results + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion != 'skipped' + + permissions: + checks: write + + # needed unless run with comment_mode: off + pull-requests: write + + # required by download step to access artifacts API + actions: read + + steps: + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + time_unit: milliseconds + commit: ${{ github.event.workflow_run.head_sha }} + event_file: artifacts/Event File/event.json + event_name: ${{ github.event.workflow_run.event }} + files: "artifacts/Test Results/*.json" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml new file mode 100644 index 00000000000..19a7d0a603b --- /dev/null +++ b/.github/workflows/run_tests.yml @@ -0,0 +1,34 @@ +name: Run Tests + +on: +# pull_request + workflow_dispatch: + +jobs: + upload-event_file: + name: Upload Event File + runs-on: ubuntu-latest + steps: + - name: Upload Event File + uses: actions/upload-artifact@v4 + with: + name: Event File + path: ${{ github.event_path }} + + run-tests: + name: Run Tests + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Run Tests + run: docker-compose -f tools/headless_testing/docker-compose.yml up + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: Test Results + path: | + tools/headless_testing/testlog/results.json From 39c29caa6c3a2d21e150afcd13178eb40921e6cf Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:27:04 +0100 Subject: [PATCH 2/6] Remove globallos, set Supreme Isthmus to 1.8. --- tools/headless_testing/download-maps.sh | 2 +- tools/headless_testing/startscript.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/headless_testing/download-maps.sh b/tools/headless_testing/download-maps.sh index 8901c70e6f9..131eb233f66 100644 --- a/tools/headless_testing/download-maps.sh +++ b/tools/headless_testing/download-maps.sh @@ -1,3 +1,3 @@ #!/bin/bash -engine/*/pr-downloader --filesystem-writepath "$BAR_ROOT" --download-map "Supreme Isthmus v1.6.4" +engine/*/pr-downloader --filesystem-writepath "$BAR_ROOT" --download-map "Supreme Isthmus v1.8" diff --git a/tools/headless_testing/startscript.txt b/tools/headless_testing/startscript.txt index e41b9cf9f84..de123d076d9 100644 --- a/tools/headless_testing/startscript.txt +++ b/tools/headless_testing/startscript.txt @@ -1,6 +1,6 @@ [GAME] { - MapName=Supreme Isthmus v1.6.4; + MapName=Supreme Isthmus v1.8; GameType=Beyond All Reason $VERSION; GameStartDelay=0; StartPosType=0; @@ -10,7 +10,7 @@ FixedRNGSeed = 1; [MODOPTIONS] { - debugcommands=1:cheat|2:godmode|3:globallos|30:runtestsheadless; + debugcommands=1:cheat|2:godmode|30:runtestsheadless; deathmode=neverend; } [ALLYTEAM0] From 2b109051270c2e9f8882e1ecc179552ffcaf5b2a Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:32:20 +0100 Subject: [PATCH 3/6] Add skipped tests, cleanup error message. --- common/testing/mocha_json_reporter.lua | 33 ++++++++++++++++++++++---- luaui/Widgets/dbg_test_runner.lua | 1 + 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/testing/mocha_json_reporter.lua b/common/testing/mocha_json_reporter.lua index 56b4570fd4e..a4d9ad40a47 100644 --- a/common/testing/mocha_json_reporter.lua +++ b/common/testing/mocha_json_reporter.lua @@ -8,11 +8,13 @@ function MochaJSONReporter:new() local obj = { totalTests = 0, totalPasses = 0, + totalSkipped = 0, totalFailures = 0, startTime = nil, endTime = nil, duration = nil, - tests = {} + tests = {}, + skipped = {} } setmetatable(obj, self) self.__index = self @@ -28,21 +30,37 @@ function MochaJSONReporter:endTests(duration) self.duration = duration end -function MochaJSONReporter:testResult(label, filePath, success, duration, errorMessage) +function MochaJSONReporter:extractError(text) + local errorIndex = text:match'^%[string "[%p%a%s]*%"]:[%d]+:().*' + if errorIndex and errorIndex > 0 then + text = text:sub(errorIndex + 1) + return text + end + errorIndex = text:match'^%[t=[%d%.:]*%]%[f=[%-%d]*%] ().*' + if errorIndex and errorIndex > 0 then + text = text:sub(errorIndex) + end + return text +end + +function MochaJSONReporter:testResult(label, filePath, success, skipped, duration, errorMessage) local result = { title = label, fullTitle = label, file = filePath, duration = duration, } - if success then + if skipped then + self.totalSkipped = self.totalSkipped + 1 + result.err = {} + elseif success then self.totalPasses = self.totalPasses + 1 result.err = {} else self.totalFailures = self.totalFailures + 1 if errorMessage ~= nil then result.err = { - message = errorMessage, + message = self:extractError(errorMessage), stack = errorMessage } else @@ -54,6 +72,9 @@ function MochaJSONReporter:testResult(label, filePath, success, duration, errorM self.totalTests = self.totalTests + 1 self.tests[#(self.tests) + 1] = result + if skipped then + self.skipped[#(self.skipped) + 1] = {fullTitle = label} + end end function MochaJSONReporter:report(filePath) @@ -63,12 +84,14 @@ function MochaJSONReporter:report(filePath) ["tests"] = self.totalTests, ["passes"] = self.totalPasses, ["pending"] = 0, + ["skipped"] = self.totalSkipped, ["failures"] = self.totalFailures, ["start"] = formatTimestamp(self.startTime), ["end"] = formatTimestamp(self.endTime), ["duration"] = self.duration }, - ["tests"] = self.tests + ["tests"] = self.tests, + ["pending"] = self.skipped } local encoded = Json.encode(output) diff --git a/luaui/Widgets/dbg_test_runner.lua b/luaui/Widgets/dbg_test_runner.lua index 2c4bada6f2d..10b6fd8eddf 100644 --- a/luaui/Widgets/dbg_test_runner.lua +++ b/luaui/Widgets/dbg_test_runner.lua @@ -82,6 +82,7 @@ local function logTestResult(testResult) testResult.label, testResult.filename, (testResult.result == TestResults.TEST_RESULT.PASS), + (testResult.result == TestResults.TEST_RESULT.SKIP), testResult.milliseconds, testResult.error ) From 4b560e65c0b6d49eddb6f8d088930fb53915a553 Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:33:55 +0100 Subject: [PATCH 4/6] Run an extra test on headless checking for errors at infolog.txt. --- common/testing/infologtest.lua | 34 +++++++++++++++++++++++++++++++ luaui/Widgets/dbg_test_runner.lua | 6 ++++++ 2 files changed, 40 insertions(+) create mode 100644 common/testing/infologtest.lua diff --git a/common/testing/infologtest.lua b/common/testing/infologtest.lua new file mode 100644 index 00000000000..f9ccae903eb --- /dev/null +++ b/common/testing/infologtest.lua @@ -0,0 +1,34 @@ +-- 'hidden' test checking infolog.txt for errors, used by headless runs. + +local maxErrors = 10 + +local function skipErrors(line) + if string.find(line, 'Could not finalize projectile-texture atlas', nil, true) then + return true + end +end + +local function infologTest() + local errors = {} + local infolog = VFS.LoadFile("infolog.txt") + if infolog then + local fileLines = string.lines(infolog) + for i, line in ipairs(fileLines) do + if string.find(line, 'Error:', nil, true) and not skipErrors(line) then + errors[#errors+1] = line + if #errors > maxErrors then + return errors + end + end + end + end + return errors +end + + +function test() + local errors = infologTest() + if #errors > 0 then + error(table.concat(errors, "\n"), 0) + end +end diff --git a/luaui/Widgets/dbg_test_runner.lua b/luaui/Widgets/dbg_test_runner.lua index 10b6fd8eddf..c86317031bb 100644 --- a/luaui/Widgets/dbg_test_runner.lua +++ b/luaui/Widgets/dbg_test_runner.lua @@ -41,6 +41,7 @@ local config = { } local testReporter = nil +local headless = false -- utils -- ===== @@ -71,6 +72,7 @@ local function logEndTests(duration) testReporter:endTests(duration) testReporter:report(config.testResultsFilePath) + headless = false end local function logTestResult(testResult) @@ -142,6 +144,9 @@ local function findAllTestFiles(patterns) result[#result + 1] = testFileInfo end end + if headless then + result[#result+1] = {label="infolog", filename="common/testing/infologtest.lua"} + end return result end @@ -1258,6 +1263,7 @@ function widget:Initialize() self, "runtestsheadless", function(cmd, optLine, optWords, data, isRepeat, release, actions) + headless = true config.noColorOutput = true config.quitWhenDone = true config.gameStartTestPatterns = Util.splitPhrases(optLine) From 11676060c363922ade0f81e8824dfb105acd0a8a Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:38:37 +0100 Subject: [PATCH 5/6] Polish and merge github workflows. - Deploy now depends on successful tests. - Action processing tests now fails on inconclusive or failure. --- .github/workflows/process_test_results.yml | 41 ------------------ .github/workflows/run_tests.yml | 34 --------------- .../{quick_deploy.yml => test_and_deploy.yml} | 43 ++++++++++++++++--- 3 files changed, 38 insertions(+), 80 deletions(-) delete mode 100644 .github/workflows/process_test_results.yml delete mode 100644 .github/workflows/run_tests.yml rename .github/workflows/{quick_deploy.yml => test_and_deploy.yml} (58%) diff --git a/.github/workflows/process_test_results.yml b/.github/workflows/process_test_results.yml deleted file mode 100644 index 55a7604eff5..00000000000 --- a/.github/workflows/process_test_results.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Process Test Results - -on: - workflow_run: - workflows: ["Run Tests"] - types: - - completed - -permissions: {} - -jobs: - process-test-results: - name: Process Test Results - runs-on: ubuntu-latest - if: github.event.workflow_run.conclusion != 'skipped' - - permissions: - checks: write - - # needed unless run with comment_mode: off - pull-requests: write - - # required by download step to access artifacts API - actions: read - - steps: - - name: Download Artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - github-token: ${{ github.token }} - run-id: ${{ github.event.workflow_run.id }} - - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - time_unit: milliseconds - commit: ${{ github.event.workflow_run.head_sha }} - event_file: artifacts/Event File/event.json - event_name: ${{ github.event.workflow_run.event }} - files: "artifacts/Test Results/*.json" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml deleted file mode 100644 index 19a7d0a603b..00000000000 --- a/.github/workflows/run_tests.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Run Tests - -on: -# pull_request - workflow_dispatch: - -jobs: - upload-event_file: - name: Upload Event File - runs-on: ubuntu-latest - steps: - - name: Upload Event File - uses: actions/upload-artifact@v4 - with: - name: Event File - path: ${{ github.event_path }} - - run-tests: - name: Run Tests - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Run Tests - run: docker-compose -f tools/headless_testing/docker-compose.yml up - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: Test Results - path: | - tools/headless_testing/testlog/results.json diff --git a/.github/workflows/quick_deploy.yml b/.github/workflows/test_and_deploy.yml similarity index 58% rename from .github/workflows/quick_deploy.yml rename to .github/workflows/test_and_deploy.yml index d6de550837e..16d00fa8e28 100644 --- a/.github/workflows/quick_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -1,14 +1,44 @@ -# Workflow to push the change to deploy the change to players -# on push to master much quicker without waiting for cron jobs. -name: Deploy +name: Run Tests + on: + workflow_dispatch: + pull_request: push: branches: - - master + - 'master' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + jobs: + run-tests: + name: Run Tests + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Run Tests + run: docker compose -f tools/headless_testing/docker-compose.yml up + timeout-minutes: 30 + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + action_fail: true + action_fail_on_inconclusive: true + check_name: "Test Results" + time_unit: milliseconds + files: "tools/headless_testing/testlog/results.json" + deploy: + name: Deploy runs-on: ubuntu-latest - if: github.repository == 'beyond-all-reason/Beyond-All-Reason' + needs: run-tests + if: | + github.repository == 'beyond-all-reason/Beyond-All-Reason' && + github.event_name == 'push' && + github.ref == 'refs/heads/master' permissions: id-token: write steps: @@ -19,6 +49,7 @@ jobs: ssh -i id.key -o StrictHostKeyChecking=no debian@repos.beyondallreason.dev byar env: SSH_KEY: ${{ secrets.SSH_REPOS_DEPLOY_KEY }} + - name: Authenticate to Google Cloud id: auth uses: google-github-actions/auth@v2 @@ -28,10 +59,12 @@ jobs: token_format: id_token id_token_audience: cdnupdater id_token_include_email: true + - name: Sync files to CDN run: | curl --fail -H "Authorization: Bearer ${{ steps.auth.outputs.id_token }}" \ -X POST -d '["byar"]' https://rapidsyncer-ssd-7xiouooxaa-ey.a.run.app/sync + - name: Update CDN pointer run: | curl --fail -H "Authorization: Bearer ${{ steps.auth.outputs.id_token }}" \ From 97f6b140665efd316ea828e07d2efc5f03aa6d03 Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:54:51 +0100 Subject: [PATCH 6/6] Print infolog. --- common/testing/infologtest.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/common/testing/infologtest.lua b/common/testing/infologtest.lua index f9ccae903eb..ae71123af64 100644 --- a/common/testing/infologtest.lua +++ b/common/testing/infologtest.lua @@ -29,6 +29,7 @@ end function test() local errors = infologTest() if #errors > 0 then + Spring.Echo("INFOLOG", table.concat(errors, "\n")) error(table.concat(errors, "\n"), 0) end end