Tests #26
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Tests | |
on: | |
workflow_dispatch: | |
inputs: | |
api-url: | |
description: "API gateway URL for Golioth backend services" | |
required: false | |
default: "https://api.golioth.io" | |
api-key-id: | |
description: >- | |
Name of GitHub secret containing an API key that will be used to authenticate with the | |
Golioth backend | |
required: false | |
default: "PROD_CI_PROJECT_API_KEY" | |
coap_gateway_url: | |
description: "The CoAP gateway URL to be hardcoded into test firmware" | |
required: false | |
default: "coaps://coap.golioth.io" | |
workflow: | |
description: "Select which workflow to run" | |
required: false | |
type: choice | |
default: 'all' | |
options: | |
- all | |
- zephyr_integration | |
- zephyr_integration_native_sim | |
- zephyr_sample | |
- zephyr_sample_native_sim | |
- esp_idf_integration | |
- esp_idf_sample | |
- linux_integration | |
workflow_call: | |
inputs: | |
api-url: | |
description: "API gateway URL for Golioth backend services" | |
required: false | |
type: string | |
default: "https://api.golioth.io" | |
api-key-id: | |
description: >- | |
Name of GitHub secret containing an API key that will be used to authenticate with the | |
Golioth backend | |
required: false | |
type: string | |
default: "PROD_CI_PROJECT_API_KEY" | |
coap_gateway_url: | |
description: "The CoAP gateway URL to be hardcoded into test firmware" | |
required: false | |
type: string | |
default: "coaps://coap.golioth.io" | |
workflow: | |
description: "Select which workflow to run" | |
required: false | |
type: string | |
default: 'all' | |
jobs: | |
unit_tests: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout repository and submodules | |
uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
- name: Install gcovr and ninja | |
run: | | |
pip install gcovr ninja | |
- name: Run unit tests | |
run: | | |
ctest --build-and-test tests/unit_tests build \ | |
--build-generator Ninja \ | |
--test-command \ | |
ctest \ | |
--output-on-failure \ | |
--timeout 2 | |
- name: Coverage | |
run: | | |
gcovr \ | |
--gcov-ignore-parse-errors=negative_hits.warn_once_per_file \ | |
--merge-mode-functions=separate \ | |
--json coverage.json \ | |
build | |
- name: Upload test coverage artifacts | |
if: always() | |
uses: actions/upload-artifact@v4 | |
with: | |
name: unit-test-coverage | |
path: coverage.json | |
hil_test_zephyr: | |
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'zephyr_integration' }} | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- hil_board: esp32_devkitc_wrover | |
west_board: esp32_devkitc_wrover/esp32/procpu | |
manifest: .ci-west-zephyr.yml | |
binary_name: zephyr.bin | |
binary_blob: hal_espressif | |
- hil_board: mimxrt1024_evk | |
west_board: mimxrt1024_evk | |
manifest: .ci-west-zephyr.yml | |
binary_name: zephyr.hex | |
- hil_board: nrf52840dk | |
west_board: nrf52840dk/nrf52840 | |
manifest: .ci-west-zephyr.yml | |
binary_name: zephyr.hex | |
- hil_board: nrf9160dk | |
west_board: nrf9160dk/nrf9160/ns | |
manifest: .ci-west-ncs.yml | |
binary_name: merged.hex | |
- hil_board: rak5010 | |
west_board: rak5010/nrf52840 | |
manifest: .ci-west-zephyr.yml | |
binary_name: zephyr.hex | |
uses: ./.github/workflows/hil_test_zephyr.yml | |
with: | |
hil_board: ${{ matrix.hil_board }} | |
west_board: ${{ matrix.west_board }} | |
manifest: ${{ matrix.manifest }} | |
binary_name: ${{ matrix.binary_name}} | |
binary_blob: ${{ matrix.binary_blob }} | |
api-url: ${{ inputs.api-url }} | |
api-key-id: ${{ inputs.api-key-id }} | |
coap_gateway_url: ${{ inputs.coap_gateway_url }} | |
secrets: inherit | |
hil_test_zephyr_nsim_matrix: | |
runs-on: ubuntu-latest | |
name: zephyr-native_sim-matrix-generate | |
outputs: | |
tests: ${{ steps.output-tests.outputs.tests }} | |
steps: | |
- name: Checkout repository and submodules | |
uses: actions/checkout@v4 | |
- name: Prepare tests matrix | |
id: output-tests | |
shell: python | |
run: | | |
import json | |
import os | |
from pathlib import Path | |
tests = [p.name for p in Path('tests/hil/tests').iterdir()] | |
with open(os.environ['GITHUB_OUTPUT'], 'a') as github_output: | |
print('tests=' + json.dumps(tests), file=github_output) | |
hil_test_zephyr_nsim: | |
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'zephyr_integration_native_sim' }} | |
runs-on: ubuntu-latest | |
needs: hil_test_zephyr_nsim_matrix | |
container: | |
image: golioth/golioth-zephyr-base:0.16.8-SDK-v0 | |
env: | |
ZEPHYR_SDK_INSTALL_DIR: /opt/toolchains/zephyr-sdk-0.16.8 | |
name: zephyr-${{ matrix.platform }}-${{ matrix.test }}-test-nsim | |
strategy: | |
fail-fast: false | |
matrix: | |
test: ${{ fromJSON(needs.hil_test_zephyr_nsim_matrix.outputs.tests) }} | |
platform: | |
- native_sim_32 | |
- native_sim_64 | |
include: | |
- platform: native_sim_32 | |
west_board: native_sim | |
- platform: native_sim_64 | |
west_board: native_sim/native/64 | |
steps: | |
- name: Checkout repository and submodules | |
uses: actions/checkout@v4 | |
with: | |
path: modules/lib/golioth-firmware-sdk | |
submodules: 'recursive' | |
- name: Init and update west | |
run: | | |
mkdir -p .west | |
cat <<EOF > .west/config | |
[manifest] | |
path = modules/lib/golioth-firmware-sdk | |
file = .ci-west-zephyr.yml | |
EOF | |
west update -o=--depth=1 -n | |
git config --global user.email [email protected] | |
git config --global user.name "Git User" | |
west patch | |
pip install \ | |
-r zephyr/scripts/requirements-base.txt \ | |
-r zephyr/scripts/requirements-build-test.txt \ | |
-r zephyr/scripts/requirements-run-test.txt \ | |
pytest \ | |
pytest-timeout \ | |
modules/lib/golioth-firmware-sdk/tests/hil/scripts/pytest-hil \ | |
git+https://github.com/golioth/[email protected] | |
- name: Build test | |
env: | |
test: ${{ matrix.test }} | |
run: | | |
export EXTRA_BUILD_ARGS=-DCONFIG_GOLIOTH_COAP_HOST_URI=\"${{ inputs.coap_gateway_url }}\" | |
rm -rf hil-out | |
mkdir -p hil-out | |
west build -p -b ${{ matrix.west_board }} \ | |
modules/lib/golioth-firmware-sdk/tests/hil/platform/zephyr \ | |
-d hil-out/$test -- $EXTRA_BUILD_ARGS -DGOLIOTH_HIL_TEST=$test | |
- name: Run test | |
env: | |
test: ${{ matrix.test }} | |
run: | | |
rm -rf summary | |
mkdir summary | |
rm -rf allure-reports | |
pytest --rootdir . \ | |
modules/lib/golioth-firmware-sdk//tests/hil/tests/$test \ | |
--board native_sim \ | |
--fw-image hil-out/$test/zephyr/zephyr.exe \ | |
--api-url ${{ inputs.api-url }} \ | |
--api-key ${{ secrets[inputs.api-key-id] }} \ | |
--mask-secrets \ | |
--timeout=600 \ | |
--junitxml=summary/hil-zephyr-${{ matrix.platform }}-${{ matrix.test }}.xml \ | |
--alluredir=allure-reports \ | |
--platform=zephyr \ | |
--allure-board=${{ matrix.platform }} | |
- name: Capture coverage | |
env: | |
test: ${{ matrix.test }} | |
run: | | |
gcovr -r modules/lib/golioth-firmware-sdk \ | |
--gcov-ignore-parse-errors=negative_hits.warn_once_per_file \ | |
--merge-mode-functions=separate \ | |
--json hil-out/$test/coverage.json \ | |
hil-out/$test | |
- name: Upload test coverage artifacts | |
if: always() | |
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts | |
with: | |
secrets-json: ${{ toJson(secrets) }} | |
name: native-sim-hil-test-coverage-${{ matrix.platform }}-${{ matrix.test }} | |
path: | | |
hil-out/*/coverage.json | |
- name: Safe upload CI report summary | |
if: success() || failure() | |
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts | |
with: | |
name: ci-individual-hil-zephyr-${{ matrix.platform }}-${{ matrix.test }} | |
path: summary/* | |
- name: Safe upload Allure reports | |
if: success() || failure() | |
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts | |
with: | |
secrets-json: ${{ toJson(secrets) }} | |
name: allure-reports-hil-zephyr-${{ matrix.platform }}-${{ matrix.test }} | |
path: allure-reports | |
hil_test_zephyr_nsim_summary: | |
name: zephyr-summary-hil-nsim | |
runs-on: ubuntu-latest | |
needs: hil_test_zephyr_nsim | |
if: success() || failure() | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- platform: native_sim_32 | |
- platform: native_sim_64 | |
steps: | |
- name: Collect JUnit reports | |
uses: actions/download-artifact@v4 | |
with: | |
pattern: ci-individual-hil-zephyr-* | |
- name: Prepare CI report summary | |
if: always() | |
run: | | |
sudo apt install -y xml-twig-tools | |
rm -rf summary | |
mkdir summary | |
xml_grep --pretty_print indented \ | |
--wrap testsuites --descr '' \ | |
--cond "testsuite" \ | |
ci-individual-hil-zephyr-${{ matrix.platform }}-*/*.xml \ | |
> hil-zephyr-${{ matrix.platform }}.xml | |
- name: Upload CI report summary | |
uses: actions/upload-artifact@v4 | |
if: always() | |
with: | |
name: ci-summary-hil-zephyr-${{ matrix.platform }} | |
path: hil-zephyr-*.xml | |
hil_test_esp-idf: | |
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'esp_idf_integration' }} | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- hil_board: esp32s3_devkitc | |
idf_target: esp32s3 | |
- hil_board: esp32_devkitc_wrover | |
idf_target: esp32 | |
- hil_board: esp32c3_devkitm | |
idf_target: esp32c3 | |
uses: ./.github/workflows/hil_test_esp-idf.yml | |
with: | |
hil_board: ${{ matrix.hil_board }} | |
idf_target: ${{ matrix.idf_target }} | |
api-url: ${{ inputs.api-url }} | |
api-key-id: ${{ inputs.api-key-id }} | |
coap_gateway_url: ${{ inputs.coap_gateway_url }} | |
secrets: inherit | |
hil_test_linux: | |
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'linux_integration' }} | |
uses: ./.github/workflows/hil_test_linux.yml | |
with: | |
api-url: ${{ inputs.api-url }} | |
api-key-id: ${{ inputs.api-key-id }} | |
coap_gateway_url: ${{ inputs.coap_gateway_url }} | |
secrets: inherit | |
hil_sample_zephyr: | |
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'zephyr_sample' }} | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- hil_board: esp32_devkitc_wrover | |
west_board: esp32_devkitc_wrover/esp32/procpu | |
manifest: .ci-west-zephyr.yml | |
extra_twister_args: >- | |
--flash-before | |
--west-flash=--esp-tool=$(which esptool.py),--esp-device=${!PORT_VAR} | |
binary_blob: hal_espressif | |
run_tests: true | |
- hil_board: mimxrt1024_evk | |
west_board: mimxrt1024_evk | |
manifest: .ci-west-zephyr.yml | |
extra_twister_args: >- | |
--west-flash="--dev-id=${!SNR_VAR}" | |
--west-runner jlink | |
run_tests: true | |
- hil_board: nrf52840dk | |
west_board: nrf52840dk/nrf52840 | |
manifest: .ci-west-zephyr.yml | |
extra_twister_args: >- | |
--west-flash="--dev-id=${!SNR_VAR}" | |
run_tests: true | |
- hil_board: nrf9160dk | |
west_board: nrf9160dk/nrf9160/ns | |
manifest: .ci-west-ncs.yml | |
extra_twister_args: >- | |
--west-flash="--dev-id=${!SNR_VAR}" | |
run_tests: true | |
- hil_board: rak5010 | |
west_board: rak5010/nrf52840 | |
manifest: .ci-west-zephyr.yml | |
extra_twister_args: >- | |
--flash-before | |
--west-flash=--gdb=/usr/bin/gdb | |
--west-runner blackmagicprobe | |
run_tests: true | |
- hil_board: qemu_x86 | |
west_board: qemu_x86 | |
manifest: .ci-west-zephyr.yml | |
run_tests: false | |
uses: ./.github/workflows/hil_sample_zephyr.yml | |
with: | |
hil_board: ${{ matrix.hil_board }} | |
west_board: ${{ matrix.west_board }} | |
manifest: ${{ matrix.manifest }} | |
binary_blob: ${{ matrix.binary_blob }} | |
run_tests: ${{ matrix.run_tests }} | |
extra_twister_args: ${{ matrix.extra_twister_args }} | |
api-url: ${{ inputs.api-url }} | |
api-key-id: ${{ inputs.api-key-id }} | |
coap_gateway_url: ${{ inputs.coap_gateway_url }} | |
secrets: inherit | |
hil_sample_zephyr_nsim: | |
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'zephyr_sample_native_sim' }} | |
runs-on: ubuntu-latest | |
container: | |
image: golioth/golioth-zephyr-base:0.16.8-SDK-v0 | |
env: | |
ZEPHYR_SDK_INSTALL_DIR: /opt/toolchains/zephyr-sdk-0.16.8 | |
name: zephyr-${{ matrix.west_board }}-twister | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- west_board: native_sim | |
platform: native_sim_32 | |
- west_board: native_sim/native/64 | |
platform: native_sim_64 | |
steps: | |
- name: Checkout repository and submodules | |
uses: actions/checkout@v4 | |
with: | |
path: modules/lib/golioth-firmware-sdk | |
submodules: 'recursive' | |
- name: Init and update west | |
run: | | |
mkdir -p .west | |
cat <<EOF > .west/config | |
[manifest] | |
path = modules/lib/golioth-firmware-sdk | |
file = .ci-west-zephyr.yml | |
EOF | |
west update -o=--depth=1 -n | |
git config --global user.email [email protected] | |
git config --global user.name "Git User" | |
west patch | |
pip3 install \ | |
-r zephyr/scripts/requirements-base.txt \ | |
-r zephyr/scripts/requirements-build-test.txt \ | |
-r zephyr/scripts/requirements-run-test.txt \ | |
modules/lib/golioth-firmware-sdk/tests/hil/scripts/pytest-zephyr-samples \ | |
git+https://github.com/golioth/[email protected] | |
- name: Run tests | |
run: | | |
rm -rf allure-reports | |
zephyr/scripts/twister \ | |
--platform ${{ matrix.west_board }} \ | |
-T modules/lib/golioth-firmware-sdk/examples/zephyr \ | |
-C --coverage-basedir modules/lib/golioth-firmware-sdk \ | |
--coverage-formats txt \ | |
-x=CONFIG_GOLIOTH_COAP_HOST_URI=\"${{ inputs.coap_gateway_url }}\" \ | |
--pytest-args="--api-url=${{ inputs.api-url }}" \ | |
--pytest-args="--api-key=${{ secrets[inputs.api-key-id] }}" \ | |
--pytest-args="--mask-secrets" \ | |
--pytest-args="--alluredir=allure-reports" \ | |
--pytest-args="--runner-name=${{ runner.name }}" \ | |
--pytest-args="--hil-board=${{ matrix.platform }}" | |
- name: Safe upload twister artifacts | |
if: always() | |
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts | |
with: | |
secrets-json: ${{ toJson(secrets) }} | |
name: twister-run-artifacts-${{ matrix.platform }} | |
path: | | |
reports/* | |
twister-out/coverage.json | |
twister-out/**/*.log | |
twister-out/**/report.xml | |
twister-out/*.xml | |
twister-out/*.json | |
- name: Safe upload CI report summary | |
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts | |
if: always() | |
with: | |
name: ci-individual-samples-zephyr-${{ matrix.platform }} | |
path: twister-out/twister_suite_report.xml | |
- name: Safe upload Allure reports | |
if: always() | |
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts | |
with: | |
secrets-json: ${{ toJson(secrets) }} | |
name: allure-reports-samples-zephyr-${{ matrix.platform }} | |
path: allure-reports | |
hil_sample_zephyr_nsim_summary: | |
name: zephyr-summary-twister | |
runs-on: ubuntu-latest | |
needs: hil_sample_zephyr_nsim | |
if: success() || failure() | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- platform: native_sim_32 | |
- platform: native_sim_64 | |
steps: | |
- name: Collect JUnit reports | |
uses: actions/download-artifact@v4 | |
with: | |
pattern: ci-individual-samples-zephyr-* | |
- name: Prepare CI report summary | |
if: always() | |
run: | | |
sudo apt install -y xml-twig-tools | |
rm -rf summary | |
mkdir summary | |
xml_grep --pretty_print indented \ | |
--wrap testsuites --descr '' \ | |
--cond "testsuite" \ | |
ci-individual-samples-zephyr-${{ matrix.platform }}*/*.xml \ | |
> samples-zephyr-${{ matrix.platform }}.xml | |
- name: Upload CI report summary | |
uses: actions/upload-artifact@v4 | |
if: always() | |
with: | |
name: ci-summary-samples-zephyr-${{ matrix.platform }} | |
path: samples-zephyr-*.xml | |
coverage: | |
runs-on: ubuntu-latest | |
needs: | |
- hil_sample_zephyr_nsim | |
- hil_test_linux | |
- hil_test_zephyr_nsim | |
- unit_tests | |
name: coverage | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v4 | |
- name: Download HIL Linux coverage artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
pattern: linux-hil-test-coverage-* | |
- name: Download HIL Zephyr coverage artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
pattern: native-sim-hil-test-coverage-* | |
- name: Download Twister coverage artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
pattern: twister-run-artifacts-native* | |
- name: Download Unit Tests coverage artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
name: unit-test-coverage | |
path: unit-test-coverage | |
- name: Setup Python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: 3.x | |
- name: Install gcovr | |
shell: bash | |
run: | | |
pip install gcovr | |
- name: Merge coverage | |
shell: bash | |
run: | | |
tracefiles_hil_linux=$(for f in linux-hil-test-coverage-*/coverage.json; do | |
printf -- " $f" | |
done) | |
tracefiles_hil_zephyr=$(for t in native-sim-hil-test-coverage-*; do | |
for d in $t/*; do | |
printf -- " $d/coverage.json" | |
done | |
done) | |
tracefiles_twister=$(for d in twister-run-artifacts-*; do | |
printf -- " $d/twister-out/coverage.json" | |
done) | |
tracefiles_unit_tests=" unit-test-coverage/coverage.json" | |
shopt -s expand_aliases | |
echo -e "HIL Linux tracefiles:${tracefiles_hil_linux// /\\n - }" | |
echo -e "HIL Zephyr tracefiles:${tracefiles_hil_zephyr// /\\n - }" | |
echo -e "Twister tracefiles:${tracefiles_twister// /\\n - }" | |
echo -e "Unit Tests tracefiles:${tracefiles_unit_tests// /\\n - }" | |
tracefiles="${tracefiles_hil_linux}${tracefiles_hil_zephyr}${tracefiles_twister}${tracefiles_unit_tests}" | |
alias gcovr="gcovr \ | |
${tracefiles// / --add-tracefile } \ | |
-e examples -e external -e tests" | |
rm -rf coverage | |
mkdir -p coverage | |
gcovr --txt-summary --xml coverage.xml --html-details coverage/index.html | |
- name: Upload coverage artifacts | |
uses: actions/upload-artifact@v4 | |
if: always() | |
with: | |
name: coverage-native-sim | |
path: coverage/* | |
- name: Upload results to Codecov | |
uses: codecov/codecov-action@v4 | |
with: | |
token: ${{ secrets.CODECOV_TOKEN }} | |
files: coverage.xml | |
disable_search: true | |
- name: Coverage report | |
uses: irongut/[email protected] | |
with: | |
filename: coverage.xml | |
format: markdown | |
badge: true | |
output: file | |
hide_complexity: true | |
- name: Coverage summary | |
shell: bash | |
run: | | |
cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY | |
- name: Append Coverage PR Comment title | |
shell: bash | |
run: | | |
echo "# Code Coverage" > code-coverage-comment.md | |
cat code-coverage-results.md >> code-coverage-comment.md | |
- name: Add Coverage PR Comment | |
uses: marocchino/sticky-pull-request-comment@v2 | |
if: github.event_name == 'pull_request' | |
with: | |
path: code-coverage-comment.md | |
hil_sample_esp-idf: | |
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'esp_idf_sample' }} | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- hil_board: esp32s3_devkitc | |
idf_target: esp32s3 | |
- hil_board: esp32_devkitc_wrover | |
idf_target: esp32 | |
- hil_board: esp32c3_devkitm | |
idf_target: esp32c3 | |
uses: ./.github/workflows/hil_sample_esp-idf.yml | |
with: | |
hil_board: ${{ matrix.hil_board }} | |
idf_target: ${{ matrix.idf_target }} | |
api-url: ${{ inputs.api-url }} | |
api-key-id: ${{ inputs.api-key-id }} | |
coap_gateway_url: ${{ inputs.coap_gateway_url }} | |
secrets: inherit | |
process_allure_reports: | |
needs: | |
- hil_sample_esp-idf | |
- hil_sample_zephyr | |
- hil_sample_zephyr_nsim | |
- hil_test_esp-idf | |
- hil_test_linux | |
- hil_test_zephyr | |
- hil_test_zephyr_nsim | |
if: ${{ !cancelled() }} | |
uses: ./.github/workflows/reports-allure-publish.yml | |
secrets: inherit | |
publish_summary: | |
needs: | |
- hil_sample_esp-idf | |
- hil_sample_zephyr | |
- hil_sample_zephyr_nsim_summary | |
- hil_test_esp-idf | |
- hil_test_linux | |
- hil_test_zephyr | |
- hil_test_zephyr_nsim_summary | |
if: always() | |
uses: ./.github/workflows/reports-summary-publish.yml |