Skip to content

Tests

Tests #22

Workflow file for this run

name: Tests
on:
workflow_dispatch:
inputs:
api-url:
description: "API gateway URL for Golioth backend services"
required: false
default: "https://api.golioth.io"
api-key-id:
description: >-
Name of GitHub secret containing an API key that will be used to authenticate with the
Golioth backend
required: false
default: "PROD_CI_PROJECT_API_KEY"
coap_gateway_url:
description: "The CoAP gateway URL to be hardcoded into test firmware"
required: false
default: "coaps://coap.golioth.io"
workflow:
description: "Select which workflow to run"
required: false
type: choice
default: 'all'
options:
- all
- zephyr_integration
- zephyr_integration_native_sim
- zephyr_sample
- zephyr_sample_native_sim
- esp_idf_integration
- esp_idf_sample
- linux_integration
workflow_call:
inputs:
api-url:
description: "API gateway URL for Golioth backend services"
required: false
type: string
default: "https://api.golioth.io"
api-key-id:
description: >-
Name of GitHub secret containing an API key that will be used to authenticate with the
Golioth backend
required: false
type: string
default: "PROD_CI_PROJECT_API_KEY"
coap_gateway_url:
description: "The CoAP gateway URL to be hardcoded into test firmware"
required: false
type: string
default: "coaps://coap.golioth.io"
workflow:
description: "Select which workflow to run"
required: false
type: string
default: 'all'
jobs:
unit_tests:
runs-on: ubuntu-latest
steps:
- name: Checkout repository and submodules
uses: actions/checkout@v4
with:
submodules: 'recursive'
- name: Install gcovr and ninja
run: |
pip install gcovr ninja
- name: Run unit tests
run: |
ctest --build-and-test tests/unit_tests build \
--build-generator Ninja \
--test-command \
ctest \
--output-on-failure \
--timeout 2
- name: Coverage
run: |
gcovr \
--gcov-ignore-parse-errors=negative_hits.warn_once_per_file \
--merge-mode-functions=separate \
--json coverage.json \
build
- name: Upload test coverage artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: unit-test-coverage
path: coverage.json
hil_test_zephyr:
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'zephyr_integration' }}
strategy:
fail-fast: false
matrix:
include:
- hil_board: esp32_devkitc_wrover
west_board: esp32_devkitc_wrover/esp32/procpu
manifest: .ci-west-zephyr.yml
binary_name: zephyr.bin
binary_blob: hal_espressif
- hil_board: mimxrt1024_evk
west_board: mimxrt1024_evk
manifest: .ci-west-zephyr.yml
binary_name: zephyr.hex
- hil_board: nrf52840dk
west_board: nrf52840dk/nrf52840
manifest: .ci-west-zephyr.yml
binary_name: zephyr.hex
- hil_board: nrf9160dk
west_board: nrf9160dk/nrf9160/ns
manifest: .ci-west-ncs.yml
binary_name: merged.hex
- hil_board: rak5010
west_board: rak5010/nrf52840
manifest: .ci-west-zephyr.yml
binary_name: zephyr.hex
uses: ./.github/workflows/hil_test_zephyr.yml
with:
hil_board: ${{ matrix.hil_board }}
west_board: ${{ matrix.west_board }}
manifest: ${{ matrix.manifest }}
binary_name: ${{ matrix.binary_name}}
binary_blob: ${{ matrix.binary_blob }}
api-url: ${{ inputs.api-url }}
api-key-id: ${{ inputs.api-key-id }}
coap_gateway_url: ${{ inputs.coap_gateway_url }}
secrets: inherit
hil_test_zephyr_nsim_matrix:
runs-on: ubuntu-latest
name: zephyr-native_sim-matrix-generate
outputs:
tests: ${{ steps.output-tests.outputs.tests }}
steps:
- name: Checkout repository and submodules
uses: actions/checkout@v4
- name: Prepare tests matrix
id: output-tests
shell: python
run: |
import json
import os
from pathlib import Path
tests = [p.name for p in Path('tests/hil/tests').iterdir()]
with open(os.environ['GITHUB_OUTPUT'], 'a') as github_output:
print('tests=' + json.dumps(tests), file=github_output)
hil_test_zephyr_nsim:
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'zephyr_integration_native_sim' }}
runs-on: ubuntu-latest
needs: hil_test_zephyr_nsim_matrix
container:
image: golioth/golioth-zephyr-base:0.16.8-SDK-v0
env:
ZEPHYR_SDK_INSTALL_DIR: /opt/toolchains/zephyr-sdk-0.16.8
name: zephyr-${{ matrix.platform }}-${{ matrix.test }}-test-nsim
strategy:
fail-fast: false
matrix:
test: ${{ fromJSON(needs.hil_test_zephyr_nsim_matrix.outputs.tests) }}
platform:
- native_sim_32
- native_sim_64
include:
- platform: native_sim_32
west_board: native_sim
- platform: native_sim_64
west_board: native_sim/native/64
steps:
- name: Checkout repository and submodules
uses: actions/checkout@v4
with:
path: modules/lib/golioth-firmware-sdk
submodules: 'recursive'
- name: Init and update west
run: |
mkdir -p .west
cat <<EOF > .west/config
[manifest]
path = modules/lib/golioth-firmware-sdk
file = .ci-west-zephyr.yml
EOF
west update -o=--depth=1 -n
git config --global user.email [email protected]
git config --global user.name "Git User"
west patch
pip install \
-r zephyr/scripts/requirements-base.txt \
-r zephyr/scripts/requirements-build-test.txt \
-r zephyr/scripts/requirements-run-test.txt \
pytest \
pytest-timeout \
modules/lib/golioth-firmware-sdk/tests/hil/scripts/pytest-hil \
git+https://github.com/golioth/[email protected]
- name: Build test
env:
test: ${{ matrix.test }}
run: |
export EXTRA_BUILD_ARGS=-DCONFIG_GOLIOTH_COAP_HOST_URI=\"${{ inputs.coap_gateway_url }}\"
rm -rf hil-out
mkdir -p hil-out
west build -p -b ${{ matrix.west_board }} \
modules/lib/golioth-firmware-sdk/tests/hil/platform/zephyr \
-d hil-out/$test -- $EXTRA_BUILD_ARGS -DGOLIOTH_HIL_TEST=$test
- name: Run test
env:
test: ${{ matrix.test }}
run: |
rm -rf summary
mkdir summary
rm -rf allure-reports
pytest --rootdir . \
modules/lib/golioth-firmware-sdk//tests/hil/tests/$test \
--board native_sim \
--fw-image hil-out/$test/zephyr/zephyr.exe \
--api-url ${{ inputs.api-url }} \
--api-key ${{ secrets[inputs.api-key-id] }} \
--mask-secrets \
--timeout=600 \
--junitxml=summary/hil-zephyr-${{ matrix.platform }}-${{ matrix.test }}.xml \
--alluredir=allure-reports \
--platform=zephyr \
--allure-board=${{ matrix.platform }}
- name: Capture coverage
env:
test: ${{ matrix.test }}
run: |
gcovr -r modules/lib/golioth-firmware-sdk \
--gcov-ignore-parse-errors=negative_hits.warn_once_per_file \
--merge-mode-functions=separate \
--json hil-out/$test/coverage.json \
hil-out/$test
- name: Upload test coverage artifacts
if: always()
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts
with:
secrets-json: ${{ toJson(secrets) }}
name: native-sim-hil-test-coverage-${{ matrix.platform }}-${{ matrix.test }}
path: |
hil-out/*/coverage.json
- name: Safe upload CI report summary
if: success() || failure()
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts
with:
name: ci-individual-hil-zephyr-${{ matrix.platform }}-${{ matrix.test }}
path: summary/*
- name: Safe upload Allure reports
if: success() || failure()
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts
with:
secrets-json: ${{ toJson(secrets) }}
name: allure-reports-hil-zephyr-${{ matrix.platform }}-${{ matrix.test }}
path: allure-reports
hil_test_zephyr_nsim_summary:
name: zephyr-summary-hil-nsim
runs-on: ubuntu-latest
needs: hil_test_zephyr_nsim
if: success() || failure()
strategy:
fail-fast: false
matrix:
include:
- platform: native_sim_32
- platform: native_sim_64
steps:
- name: Collect JUnit reports
uses: actions/download-artifact@v4
with:
pattern: ci-individual-hil-zephyr-*
- name: Prepare CI report summary
if: always()
run: |
sudo apt install -y xml-twig-tools
rm -rf summary
mkdir summary
xml_grep --pretty_print indented \
--wrap testsuites --descr '' \
--cond "testsuite" \
ci-individual-hil-zephyr-${{ matrix.platform }}-*/*.xml \
> hil-zephyr-${{ matrix.platform }}.xml
- name: Upload CI report summary
uses: actions/upload-artifact@v4
if: always()
with:
name: ci-summary-hil-zephyr-${{ matrix.platform }}
path: hil-zephyr-*.xml
hil_test_esp-idf:
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'esp_idf_integration' }}
strategy:
fail-fast: false
matrix:
include:
- hil_board: esp32s3_devkitc
idf_target: esp32s3
- hil_board: esp32_devkitc_wrover
idf_target: esp32
- hil_board: esp32c3_devkitm
idf_target: esp32c3
uses: ./.github/workflows/hil_test_esp-idf.yml
with:
hil_board: ${{ matrix.hil_board }}
idf_target: ${{ matrix.idf_target }}
api-url: ${{ inputs.api-url }}
api-key-id: ${{ inputs.api-key-id }}
coap_gateway_url: ${{ inputs.coap_gateway_url }}
secrets: inherit
hil_test_linux:
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'linux_integration' }}
uses: ./.github/workflows/hil_test_linux.yml
with:
api-url: ${{ inputs.api-url }}
api-key-id: ${{ inputs.api-key-id }}
coap_gateway_url: ${{ inputs.coap_gateway_url }}
secrets: inherit
hil_sample_zephyr:
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'zephyr_sample' }}
strategy:
fail-fast: false
matrix:
include:
- hil_board: esp32_devkitc_wrover
west_board: esp32_devkitc_wrover/esp32/procpu
manifest: .ci-west-zephyr.yml
extra_twister_args: >-
--flash-before
--west-flash=--esp-tool=$(which esptool.py),--esp-device=${!PORT_VAR}
binary_blob: hal_espressif
run_tests: true
- hil_board: mimxrt1024_evk
west_board: mimxrt1024_evk
manifest: .ci-west-zephyr.yml
extra_twister_args: >-
--west-flash="--dev-id=${!SNR_VAR}"
--west-runner jlink
run_tests: true
- hil_board: nrf52840dk
west_board: nrf52840dk/nrf52840
manifest: .ci-west-zephyr.yml
extra_twister_args: >-
--west-flash="--dev-id=${!SNR_VAR}"
run_tests: true
- hil_board: nrf9160dk
west_board: nrf9160dk/nrf9160/ns
manifest: .ci-west-ncs.yml
extra_twister_args: >-
--west-flash="--dev-id=${!SNR_VAR}"
run_tests: true
- hil_board: rak5010
west_board: rak5010/nrf52840
manifest: .ci-west-zephyr.yml
extra_twister_args: >-
--flash-before
--west-flash=--gdb=/usr/bin/gdb
--west-runner blackmagicprobe
run_tests: true
- hil_board: qemu_x86
west_board: qemu_x86
manifest: .ci-west-zephyr.yml
run_tests: false
uses: ./.github/workflows/hil_sample_zephyr.yml
with:
hil_board: ${{ matrix.hil_board }}
west_board: ${{ matrix.west_board }}
manifest: ${{ matrix.manifest }}
binary_blob: ${{ matrix.binary_blob }}
run_tests: ${{ matrix.run_tests }}
extra_twister_args: ${{ matrix.extra_twister_args }}
api-url: ${{ inputs.api-url }}
api-key-id: ${{ inputs.api-key-id }}
coap_gateway_url: ${{ inputs.coap_gateway_url }}
secrets: inherit
hil_sample_zephyr_nsim:
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'zephyr_sample_native_sim' }}
runs-on: ubuntu-latest
container:
image: golioth/golioth-zephyr-base:0.16.8-SDK-v0
env:
ZEPHYR_SDK_INSTALL_DIR: /opt/toolchains/zephyr-sdk-0.16.8
name: zephyr-${{ matrix.west_board }}-twister
strategy:
fail-fast: false
matrix:
include:
- west_board: native_sim
platform: native_sim_32
- west_board: native_sim/native/64
platform: native_sim_64
steps:
- name: Checkout repository and submodules
uses: actions/checkout@v4
with:
path: modules/lib/golioth-firmware-sdk
submodules: 'recursive'
- name: Init and update west
run: |
mkdir -p .west
cat <<EOF > .west/config
[manifest]
path = modules/lib/golioth-firmware-sdk
file = .ci-west-zephyr.yml
EOF
west update -o=--depth=1 -n
git config --global user.email [email protected]
git config --global user.name "Git User"
west patch
pip3 install \
-r zephyr/scripts/requirements-base.txt \
-r zephyr/scripts/requirements-build-test.txt \
-r zephyr/scripts/requirements-run-test.txt \
modules/lib/golioth-firmware-sdk/tests/hil/scripts/pytest-zephyr-samples \
git+https://github.com/golioth/[email protected]
- name: Run tests
run: |
rm -rf allure-reports
zephyr/scripts/twister \
--platform ${{ matrix.west_board }} \
-T modules/lib/golioth-firmware-sdk/examples/zephyr \
-C --coverage-basedir modules/lib/golioth-firmware-sdk \
--coverage-formats txt \
-x=CONFIG_GOLIOTH_COAP_HOST_URI=\"${{ inputs.coap_gateway_url }}\" \
--pytest-args="--api-url=${{ inputs.api-url }}" \
--pytest-args="--api-key=${{ secrets[inputs.api-key-id] }}" \
--pytest-args="--mask-secrets" \
--pytest-args="--alluredir=allure-reports" \
--pytest-args="--runner-name=${{ runner.name }}" \
--pytest-args="--hil-board=${{ matrix.platform }}"
- name: Safe upload twister artifacts
if: always()
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts
with:
secrets-json: ${{ toJson(secrets) }}
name: twister-run-artifacts-${{ matrix.platform }}
path: |
reports/*
twister-out/coverage.json
twister-out/**/*.log
twister-out/**/report.xml
twister-out/*.xml
twister-out/*.json
- name: Safe upload CI report summary
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts
if: always()
with:
name: ci-individual-samples-zephyr-${{ matrix.platform }}
path: twister-out/twister_suite_report.xml
- name: Safe upload Allure reports
if: always()
uses: ./modules/lib/golioth-firmware-sdk/.github/actions/safe-upload-artifacts
with:
secrets-json: ${{ toJson(secrets) }}
name: allure-reports-samples-zephyr-${{ matrix.platform }}
path: allure-reports
hil_sample_zephyr_nsim_summary:
name: zephyr-summary-twister
runs-on: ubuntu-latest
needs: hil_sample_zephyr_nsim
if: success() || failure()
strategy:
fail-fast: false
matrix:
include:
- platform: native_sim_32
- platform: native_sim_64
steps:
- name: Collect JUnit reports
uses: actions/download-artifact@v4
with:
pattern: ci-individual-samples-zephyr-*
- name: Prepare CI report summary
if: always()
run: |
sudo apt install -y xml-twig-tools
rm -rf summary
mkdir summary
xml_grep --pretty_print indented \
--wrap testsuites --descr '' \
--cond "testsuite" \
ci-individual-samples-zephyr-${{ matrix.platform }}*/*.xml \
> samples-zephyr-${{ matrix.platform }}.xml
- name: Upload CI report summary
uses: actions/upload-artifact@v4
if: always()
with:
name: ci-summary-samples-zephyr-${{ matrix.platform }}
path: samples-zephyr-*.xml
coverage:
runs-on: ubuntu-latest
needs:
- hil_sample_zephyr_nsim
- hil_test_linux
- hil_test_zephyr_nsim
- unit_tests
name: coverage
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Download HIL Linux coverage artifacts
uses: actions/download-artifact@v4
with:
pattern: linux-hil-test-coverage-*
- name: Download HIL Zephyr coverage artifacts
uses: actions/download-artifact@v4
with:
pattern: native-sim-hil-test-coverage-*
- name: Download Twister coverage artifacts
uses: actions/download-artifact@v4
with:
pattern: twister-run-artifacts-native*
- name: Download Unit Tests coverage artifacts
uses: actions/download-artifact@v4
with:
name: unit-test-coverage
path: unit-test-coverage
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: 3.x
- name: Install gcovr
shell: bash
run: |
pip install gcovr
- name: Merge coverage
shell: bash
run: |
tracefiles_hil_linux=$(for f in linux-hil-test-coverage-*/coverage.json; do
printf -- " $f"
done)
tracefiles_hil_zephyr=$(for t in native-sim-hil-test-coverage-*; do
for d in $t/*; do
printf -- " $d/coverage.json"
done
done)
tracefiles_twister=$(for d in twister-run-artifacts-*; do
printf -- " $d/twister-out/coverage.json"
done)
tracefiles_unit_tests=" unit-test-coverage/coverage.json"
shopt -s expand_aliases
echo -e "HIL Linux tracefiles:${tracefiles_hil_linux// /\\n - }"
echo -e "HIL Zephyr tracefiles:${tracefiles_hil_zephyr// /\\n - }"
echo -e "Twister tracefiles:${tracefiles_twister// /\\n - }"
echo -e "Unit Tests tracefiles:${tracefiles_unit_tests// /\\n - }"
tracefiles="${tracefiles_hil_linux}${tracefiles_hil_zephyr}${tracefiles_twister}${tracefiles_unit_tests}"
alias gcovr="gcovr \
${tracefiles// / --add-tracefile } \
-e examples -e external -e tests"
rm -rf coverage
mkdir -p coverage
gcovr --txt-summary --xml coverage.xml --html-details coverage/index.html
- name: Upload coverage artifacts
uses: actions/upload-artifact@v4
if: always()
with:
name: coverage-native-sim
path: coverage/*
- name: Upload results to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: coverage.xml
disable_search: true
- name: Coverage report
uses: irongut/[email protected]
with:
filename: coverage.xml
format: markdown
badge: true
output: file
hide_complexity: true
- name: Coverage summary
shell: bash
run: |
cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY
- name: Append Coverage PR Comment title
shell: bash
run: |
echo "# Code Coverage" > code-coverage-comment.md
cat code-coverage-results.md >> code-coverage-comment.md
- name: Add Coverage PR Comment
uses: marocchino/sticky-pull-request-comment@v2
if: github.event_name == 'pull_request'
with:
path: code-coverage-comment.md
hil_sample_esp-idf:
if: ${{ inputs.workflow == 'all' || inputs.workflow == 'esp_idf_sample' }}
strategy:
fail-fast: false
matrix:
include:
- hil_board: esp32s3_devkitc
idf_target: esp32s3
- hil_board: esp32_devkitc_wrover
idf_target: esp32
- hil_board: esp32c3_devkitm
idf_target: esp32c3
uses: ./.github/workflows/hil_sample_esp-idf.yml
with:
hil_board: ${{ matrix.hil_board }}
idf_target: ${{ matrix.idf_target }}
api-url: ${{ inputs.api-url }}
api-key-id: ${{ inputs.api-key-id }}
coap_gateway_url: ${{ inputs.coap_gateway_url }}
secrets: inherit
process_allure_reports:
needs:
- hil_sample_esp-idf
- hil_sample_zephyr
- hil_sample_zephyr_nsim
- hil_test_esp-idf
- hil_test_linux
- hil_test_zephyr
- hil_test_zephyr_nsim
if: ${{ !cancelled() }}
uses: ./.github/workflows/reports-allure-publish.yml
secrets: inherit
publish_summary:
needs:
- hil_sample_esp-idf
- hil_sample_zephyr
- hil_sample_zephyr_nsim_summary
- hil_test_esp-idf
- hil_test_linux
- hil_test_zephyr
- hil_test_zephyr_nsim_summary
if: always()
uses: ./.github/workflows/reports-summary-publish.yml