From 7ac18e2858953c491246c664343711b1a9f82040 Mon Sep 17 00:00:00 2001 From: Leon Schuermann Date: Tue, 24 Dec 2024 10:45:11 +0000 Subject: [PATCH 1/2] hwci: move system setup from Actions workflow to script --- .github/workflows/treadmill-ci.yml | 32 ++---- hwci/.gitignore | 178 +++++++++++++++++++++++++++++ hwci/boards/nrf52dk.py | 12 +- hwci/boards/tockloader_board.py | 6 +- hwci/core/main.py | 6 +- hwci/hwci.py | 7 ++ hwci/setup.sh | 92 +++++++++++++++ 7 files changed, 302 insertions(+), 31 deletions(-) create mode 100644 hwci/.gitignore create mode 100644 hwci/hwci.py create mode 100755 hwci/setup.sh diff --git a/.github/workflows/treadmill-ci.yml b/.github/workflows/treadmill-ci.yml index c854cf4..be5f1e4 100644 --- a/.github/workflows/treadmill-ci.yml +++ b/.github/workflows/treadmill-ci.yml @@ -216,7 +216,9 @@ jobs: # Avoid overwriting the RUSTFLAGS environment variable rustflags: '' - - name: Install required system packages + # This is required for the actions/checkout steps to perform a + # proper git clone that also supports checking out submodules: + - name: Install git run: | # TODO: currently, the Netboot NBD targets have no access to their # boot parition (e.g., mounted on /boot/firmware) on a Raspberry Pi OS @@ -224,16 +226,7 @@ jobs: # to fail. Thus we ignore errors in these steps until we figure this # part out. sudo DEBIAN_FRONTEND=noninteractive apt update || true - sudo DEBIAN_FRONTEND=noninteractive apt install -y \ - git cargo openocd python3 python3-pip python3-serial \ - python3-pexpect gcc-arm-none-eabi libnewlib-arm-none-eabi \ - pkg-config libudev-dev cmake libusb-1.0-0-dev udev make \ - gdb-multiarch gcc-arm-none-eabi build-essential jq || true - - # Install probe-rs: - curl --proto '=https' --tlsv1.2 -LsSf \ - https://github.com/probe-rs/probe-rs/releases/latest/download/probe-rs-tools-installer.sh \ - | sh + sudo DEBIAN_FRONTEND=noninteractive apt install -y git - name: Checkout the Tock Hardware CI scripts uses: actions/checkout@v4 @@ -244,33 +237,32 @@ jobs: - name: Checkout the Tock kernel repository uses: actions/checkout@v4 with: - path: tock + path: hwci/repos/tock repository: tock/tock ref: ${{ inputs.tock-kernel-ref }} - name: Checkout the libtock-c repository uses: actions/checkout@v4 with: - path: libtock-c + path: hwci/repos/libtock-c repository: tock/libtock-c ref: ${{ inputs.libtock-c-ref }} - fetch-depth: 0 + fetch-depth: 0 submodules: false persist-credentials: true - - name: Create Python virtual environment and install required dependencies + - name: Run setup script run: | - python3 -m venv ./hwcienv - source ./hwcienv/bin/activate - pip install -r hwci/requirements.txt -c hwci/requirements-frozen.txt + cd ./hwci/ + ./setup.sh - name: Run tests env: JSON_TEST_ARRAY: ${{ toJSON(fromJSON(needs.test-prepare.outputs.tml-jobs)[matrix.tml-job-id].tests) }} run: | - source ./hwcienv/bin/activate cd ./hwci - export PYTHONPATH="$PWD:$PYTHONPATH" + source ./.venv/bin/activate + STEP_FAIL=0 # Generate a summary of all the tests executed: diff --git a/hwci/.gitignore b/hwci/.gitignore new file mode 100644 index 0000000..d75335b --- /dev/null +++ b/hwci/.gitignore @@ -0,0 +1,178 @@ +# Ignore local repository checkouts used for the Hardware CI +repos/ + +# ----------------------------------------------------------------------------- +# https://raw.githubusercontent.com/github/gitignore/refs/heads/main/Python.gitignore +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# PyPI configuration file +.pypirc + + diff --git a/hwci/boards/nrf52dk.py b/hwci/boards/nrf52dk.py index 88a58d0..9384b85 100644 --- a/hwci/boards/nrf52dk.py +++ b/hwci/boards/nrf52dk.py @@ -18,9 +18,10 @@ class Nrf52dk(TockloaderBoard): def __init__(self): super().__init__() self.arch = "cortex-m4" + self.kernel_path = os.path.join( + self.base_dir, "repos/tock") self.kernel_board_path = os.path.join( - self.base_dir, "tock/boards/nordic/nrf52840dk" - ) + self.kernel_path, "boards/nordic/nrf52840dk") self.uart_port = self.get_uart_port() self.uart_baudrate = self.get_uart_baudrate() self.openocd_board = "nrf52dk" @@ -67,10 +68,9 @@ def cleanup(self): def flash_kernel(self): logging.info("Flashing the Tock OS kernel") - tock_dir = os.path.join(self.base_dir, "tock") - if not os.path.exists(tock_dir): - logging.error(f"Tock directory {tock_dir} not found") - raise FileNotFoundError(f"Tock directory {tock_dir} not found") + if not os.path.exists(self.kernel_path): + logging.error(f"Tock directory {self.kernel_path} not found") + raise FileNotFoundError(f"Tock directory {self.kernel_path} not found") # Run make flash-openocd from the board directory subprocess.run( diff --git a/hwci/boards/tockloader_board.py b/hwci/boards/tockloader_board.py index 89f3e7e..6578f49 100644 --- a/hwci/boards/tockloader_board.py +++ b/hwci/boards/tockloader_board.py @@ -15,14 +15,12 @@ def __init__(self): super().__init__() self.board = None # Should be set in subclass self.arch = None # Should be set in subclass - self.base_dir = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) + self.base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) def flash_app(self, app_path): app_name = os.path.basename(app_path) logging.info(f"Flashing app: {app_name}") - libtock_c_dir = os.path.join(self.base_dir, "libtock-c") + libtock_c_dir = os.path.join(self.base_dir, "repos", "libtock-c") if not os.path.exists(libtock_c_dir): logging.error(f"libtock-c directory {libtock_c_dir} not found") raise FileNotFoundError(f"libtock-c directory {libtock_c_dir} not found") diff --git a/hwci/core/main.py b/hwci/core/main.py index 73a0367..c704b63 100644 --- a/hwci/core/main.py +++ b/hwci/core/main.py @@ -6,7 +6,7 @@ import logging import importlib.util import sys - +from pathlib import Path def main(): parser = argparse.ArgumentParser(description="Run tests on Tock OS") @@ -20,6 +20,10 @@ def main(): format="%(asctime)s - %(levelname)s - %(message)s", ) + # Ensure that imported modules can find the top-level hwci modules + # (appends the hwci root to the PYTHONPATH): + sys.path.append(str(Path(__file__).parent.parent)) + # 1. Load board module board_spec = importlib.util.spec_from_file_location("board_module", args.board) board_module = importlib.util.module_from_spec(board_spec) diff --git a/hwci/hwci.py b/hwci/hwci.py new file mode 100644 index 0000000..f154119 --- /dev/null +++ b/hwci/hwci.py @@ -0,0 +1,7 @@ +# Licensed under the Apache License, Version 2.0 or the MIT License. +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Copyright Tock Contributors 2024. + +if __name__ == "__main__": + import core + core.main() diff --git a/hwci/setup.sh b/hwci/setup.sh new file mode 100755 index 0000000..49819e6 --- /dev/null +++ b/hwci/setup.sh @@ -0,0 +1,92 @@ +#! /usr/bin/env bash + +function promptContinue() { + while true; do + read -p "$1 (y/n)? " CHOICE + case "$CHOICE" in + y|Y ) return 0;; + n|N ) return 1;; + * ) continue;; + esac + done +} + +# If not running in CI, prompt the user before continuing. This script will +# attempt to modify the global system environment. +if [ "$CI" != "true" ]; then + echo "This script will attempt to install all system dependencies" >&2 + echo "necessary to run the Tock hardware CI Python scripts." >&2 + echo "It will attempt to use sudo, install packages using your" >&2 + echo "system's package manager, and create a Python virtual env." >&2 + echo "You should only run it in an environment that's ephemeral" >&2 + echo "or specifically used for the Tock hardware CI." >&2 + echo "" >&2 + promptContinue "THIS SCRIPT MAY MESS WITH YOUR SYSTEM! Continue" + if [ $? -ne 0 ]; then + echo "Aborting on user request." >&2 + exit 1 + fi +fi + +# From this point onward, echo all commands and exit on the first error: +set -e -x + +# Require rustup to be installed. We don't want to mess with the user's +# installation and GitHub actions will install this for us. Tock will then +# use rustup to ensure the correct target toolchain is installed as part +# of its own build process: +type rustup || (echo "rustup is not installed, aborting."; exit 1) + +# Ensure that `elf2tab` is installed: +if ! type elf2tab; then + # We may not have a rustup default toolchain selected. In this case, + # select the stable toolchain for elf2tab. + if ! rustup default; then + rustup toolchain add stable + cargo '+stable' install elf2tab + else + cargo install elf2tab + fi + elf2tab --version +fi + +# Install all required system dependencies. For now, we only support Debian +# hosts (such as Raspberry Pi OS). +# +# TODO: currently, the Treadmill Netboot NBD targets have no access to their +# boot parition (e.g., mounted on /boot/firmware) on a Raspberry Pi OS host. +# This causes certain hooks in response to dpkg / apt commands to fail. Thus +# we ignore errors in these steps until we figure this part out. +sudo DEBIAN_FRONTEND=noninteractive apt update || true +sudo DEBIAN_FRONTEND=noninteractive apt install -y \ + git cargo openocd python3 python3-pip python3-serial \ + python3-pexpect gcc-arm-none-eabi libnewlib-arm-none-eabi \ + pkg-config libudev-dev cmake libusb-1.0-0-dev udev make \ + gdb-multiarch gcc-arm-none-eabi build-essential jq || true + +# If we don't have any of the tock or libtock-c repos checked out, clone them +# here. We never want to do this for CI, as that'll want to check out specific +# revisions of those repositories (and we don't accidentally want to always +# test the current HEAD). +if [ "$CI" != "true" ]; then + test ! -d "./repos/tock" \ + && git clone "https://github.com/tock/tock.git" "./repos/tock" + test ! -d "./repos/libtock-c" \ + && git clone "https://github.com/tock/libtock-c.git" "./repos/libtock-c" +fi + +# Create a Python virtual environment and install the required dependencies: +python3 -m venv ./.venv +source ./.venv/bin/activate +pip install -r ./requirements.txt -c ./requirements-frozen.txt + +set +x + +# Fin! +echo "" >&2 +echo "All packages installed successfully! To continue, activate the" >&2 +echo "Python virtual environment:" >&2 +echo "" >&2 +echo " source .venv/bin/activate" >&2 +echo "" >&2 + From 0c17ffa8fef7373043045155b41a5b6e13b72f53 Mon Sep 17 00:00:00 2001 From: Leon Schuermann Date: Tue, 24 Dec 2024 07:59:43 -0500 Subject: [PATCH 2/2] .github/workflows/treadmill-ci-test: deploy on treadmill-ci env for workflow_dispatch --- .github/workflows/treadmill-ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/treadmill-ci-test.yml b/.github/workflows/treadmill-ci-test.yml index 4b9b964..79b332f 100644 --- a/.github/workflows/treadmill-ci-test.yml +++ b/.github/workflows/treadmill-ci-test.yml @@ -96,7 +96,7 @@ jobs: # Provide access to the required Treadmill secrets by running in the # appropriate environment (depending on the on: triggers above) - job-environment: ${{ github.event_name == 'pull_request' && 'treadmill-ci' || 'treadmill-ci-merged' }} + job-environment: ${{ (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') && 'treadmill-ci' || 'treadmill-ci-merged' }} # This workflow tests the tock-hardware-ci scripts itself, so take the