From f9193f95794ceabf903bf216e29271c62c115bcf Mon Sep 17 00:00:00 2001 From: h2zero Date: Fri, 15 Jul 2022 17:43:12 -0600 Subject: [PATCH] Update build script and add workflows for PR/release. --- .github/workflows/build.yml | 23 ++++++ .github/workflows/release.yml | 43 +++++++++++ .gitignore | 3 + README.md | 23 +++--- scripts/build-firmware | 133 ++++++++++++++++------------------ scripts/flash | 7 +- 6 files changed, 143 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..6696c95 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,23 @@ +name: Build + +on: [push, pull_request] + +jobs: + build-upload: + runs-on: ubuntu-latest + name: Build + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install build tools + run: | + sudo apt-get update + sudo apt-get -y install libusb-1.0-0 cmake srecord + pip install -U pyserial + - name: Build + run: ./scripts/build-firmware diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..283eb44 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Release + +on: + release: + types: [created] + +jobs: + build-upload: + runs-on: ubuntu-latest + name: Build + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install build tools + run: | + sudo apt-get update + sudo apt-get -y install libusb-1.0-0 cmake srecord + pip install -U pyserial + + - name: Get release + id: get_release + uses: bruceadams/get-release@v1.2.3 + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: Prepare Release Assets + run: | + ./scripts/build-firmware + mkdir assets + cp build/konnected-esp8266-latest.bin assets/konnected-esp8266-${{ steps.get_release.outputs.tag_name }}.bin + + - name: Upload asset to release + uses: bgpat/release-asset-action@03b0c30db1c4031ce3474740b0e4275cd7e126a3 + with: + pattern: "assets/*" + release-url: ${{ steps.get_release.outputs.upload_url }} + allow-overwrite: true + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 6c56811..8c03d32 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,6 @@ firmware/builds # Private files private + +# NodeMCU +nodemcu-firmware diff --git a/README.md b/README.md index 87a0db8..3a35d88 100644 --- a/README.md +++ b/README.md @@ -51,34 +51,31 @@ Go to the [releases section](https://github.com/konnected-io/konnected-security/ that you can flash on your Konnected Alarm Panel or ESP8266 device. ### Building the Firmware Yourself -Konnected leverages the [NodeMCU](https://github.com/nodemcu/nodemcu-firmware) codebase and [Docker builder](https://hub.docker.com/r/marcelstoer/nodemcu-build/) to create a base nodeMCU firmware image and a filesystem containing the Konnected application. Building only requires a few steps. +Konnected leverages the [NodeMCU](https://github.com/nodemcu/nodemcu-firmware) codebase to create a base nodeMCU firmware image and a filesystem containing the Konnected application. Building only requires a few steps. -1. Download and install [Docker](https://www.docker.com/products/docker-desktop) 1. Clone the Konnected repo git clone https://github.com/konnected-io/konnected-security.git -1. Use the build-firmware script to kick off the build - providing a semantic version command line argument as shown below. The build script will automatically pull down the correct nodeMCU image, and use this nodeMCU docker builder to create base firmware, an LFS image, and a SPIFFS file system containing the entire Konnected application. +1. Use the build-firmware script to kick off the build. The build script will automatically pull down the correct nodeMCU image, and use this to create base firmware, an LFS image, and a SPIFFS file system containing the entire Konnected application. cd konnected-security - ./scripts/build-firmware 2-2-99 + ./scripts/build-firmware -1. Once the build completes a folder will be created in `firmware/builds` named after the version specified in the previous step. This folder will contain three files also reflecting the version. - 1. konnected-filesystem-0x100000-2-2-99.img - 1. konnected-firmware-2-2-99.bin - 1. konnected-esp8266-2-2-99.bin +1. Once the build completes a folder will be created in `build`. This folder will contain three files reflecting the version or branch. + 1. konnected-filesystem-0x100000-{BRANCH/VERSION}.img + 1. konnected-firmware-{BRANCH/VERSION}.bin + 1. konnected-esp8266-*.bin The `konnected-firmware-*` contains the firmware partition and should be flashed at location `0x0`. The `konnected-filesystem-*` image contains the Konnected application and should be flashed at the memory location in the filename. -For convenience, the `konnected-esp8266-*` image is an all-in-one image to be flashed at memory location `0x0` containing the two images above. +For convenience, the `konnected-esp8266-latest` image is an all-in-one image to be flashed at memory location `0x0` containing the two images above. -*Note: Each time you build it will remove any prior build outputs corresponding to the same version.* -*Note: Versions in this project should always be formatted `--`.* ### Flashing a Build Flashing a build is simple with the [NodeMCU PyFlasher](https://github.com/marcelstoer/nodemcu-pyflasher/releases). Simply flash -the `konnected-esp8266-*.bin` file to your device. Typically use baud rate 115200 and flash mode `dio`. +the `konnected-esp8266-latest.bin` file to your device. Typically use baud rate 115200 and flash mode `dio`. Mac and Linux users can also easily flash from the command line using `esptool` @@ -91,7 +88,7 @@ Mac and Linux users can also easily flash from the command line using `esptool` 1. Flash the downloaded image using `esptool`: - esptool.py --port=/dev/cu.SLAB_USBtoUART write_flash --flash_mode dio --flash_size detect 0x0 konnected-esp8266-3-0-0.bin + esptool.py --port=/dev/cu.SLAB_USBtoUART write_flash --flash_mode dio --flash_size detect 0x0 konnected-esp8266-latest.bin *Note: The USB port may vary depending on your computer platform and board.* diff --git a/scripts/build-firmware b/scripts/build-firmware index b39e363..3e4ad96 100755 --- a/scripts/build-firmware +++ b/scripts/build-firmware @@ -2,118 +2,111 @@ function usage() { echo - echo "usage: $0 " + echo "usage: $0 [-h][-c]" echo - echo ' -v version should be formatted --' - echo ' -c optional abs path to custom nodemcu-firmware repo' + echo ' -h displays usage information' + echo ' -c perform a clean build' echo - echo 'Execute from the root dir of the repo. Upon successful completion the release will be created in firmware/releases/' - echo " ex: '$0 2-3-0'" + echo 'Execute from the root dir of the repo. Upon successful completion the firmware image will be build/konnected-pro-fw_*' + echo " ex: '$0'" exit 1 } -if [ "$#" == 0 ]; then - usage -fi set -e +if ! git describe --exact-match --tags &> /dev/null; then +BRANCH="$(git branch | sed -n -e 's/^\* \(.*\)/\1/p')" +else +BRANCH="$(git describe --exact-match --tags $(git log -n1 --pretty='%h'))" +fi + FW_TAG="3.0.0-release_20210201" -VERSION="$1" FIRMWARE_OVERLAY_PATH="firmware/nodemcu-firmware-overlay" - LFS_BASE="${PWD}/src/lfs" OUTPUT_BASE="${PWD}/build" OUTPUT_APP="${OUTPUT_BASE}/app" -FW_BASE="${OUTPUT_BASE}/nodemcu-firmware" +FW_BASE="${PWD}/nodemcu-firmware" +IMAGE_NAME="konnected-firmware-${BRANCH}" # process cmd args -while getopts "v:c:h" opt; do - case ${opt} in - h ) - usage - ;; - c ) - FW_BASE="$OPTARG" - FW_TAG='' - echo "using custom nodemcu-repo ${FW_BASE}" - ;; - v ) - VERSION="$OPTARG" - if ! [[ "${VERSION}" =~ ^[0-9]+-[0-9]+-[0-9]+ ]] ; then - echo "Version not formatted correctly..." - usage - fi - echo "tagging as version ${VERSION}" - ;; - \? ) - echo "Invalid Option: -$OPTARG" 1>&2 - exit 1 - ;; - esac +while getopts "ch" opt; do + case ${opt} in + h ) + usage + ;; + c ) + cd "${FW_BASE}" + make clean + cd - + # copy in our config overlay + cp -rf "${FIRMWARE_OVERLAY_PATH}"/* "${FW_BASE}/" + rm -f "${FW_BASE}/local/fs/*" + echo "performing clean build" + ;; + \? ) + echo "Invalid Option: -$OPTARG" 1>&2 + exit 1 + ;; + esac done -BUILD_PATH="firmware/builds/${VERSION}" -IMAGE_NAME="konnected-firmware-${VERSION}" - -rm -rf ${OUTPUT_BASE} +rm -rf "${OUTPUT_BASE}" mkdir -p "${OUTPUT_BASE}"; chown -R $USER "${OUTPUT_BASE}" -# pull and update the FW repo if none is supplied -if [ ! -z "$FW_TAG" ] ; then +# pull and update the FW repo if not already existing +if [ ! -d "$FW_BASE" ] ; then echo "Checking out nodemcu-firmware:${FW_TAG}" git clone --recurse-submodules https://github.com/nodemcu/nodemcu-firmware.git \ "${FW_BASE}" + # Copy firmware configuration from this repository to the nodemcu-firmware repo + cp -rf "${FIRMWARE_OVERLAY_PATH}"/* "${FW_BASE}/" + rm -f "${FW_BASE}/local/fs/*" cd "${FW_BASE}" git checkout "${FW_TAG}" git submodule update --recursive cd - fi -# Copy firmware configuration from this repository to the nodemcu-firmware repo -cp -rf "${FIRMWARE_OVERLAY_PATH}"/* "${FW_BASE}/" - -rm -f "${FW_BASE}/local/fs/*" +# Build it +cd "${FW_BASE}" +make +srec_cat -output "${OUTPUT_BASE}/${IMAGE_NAME}.bin" -binary bin/0x00000.bin -binary -fill 0xff 0x00000 0x10000 bin/0x10000.bin -binary -offset 0x10000 +cd - -# Build NodeMCU firmware image -docker run -e "IMAGE_NAME=${IMAGE_NAME}" \ - --rm -ti -v "${FW_BASE}:/opt/nodemcu-firmware" marcelstoer/nodemcu-build build +# Build spiffsimg tool (manual for now because of https://github.com/nodemcu/nodemcu-firmware/issues/2943) +make -C "${FW_BASE}"/tools/spiffsimg # Build LFS image with application files -LUA_FILES=$(find ${LFS_BASE} -iname "*.lua" | sed "s|${LFS_BASE}|/lfs|g") +mkdir -p "${OUTPUT_APP}" +LUA_FILES=$(find ${LFS_BASE} -iname "*.lua") echo Adding files to LFS: echo ${LUA_FILES} -docker run --rm -ti \ - -v ${FW_BASE}:/nodemcu-firmware \ - -v ${OUTPUT_APP}:/build \ - -v ${LFS_BASE}:/lfs \ - marcelstoer/nodemcu-build /nodemcu-firmware/luac.cross.int -m 0x40000 -f -o "/build/lfs.img" ${LUA_FILES} echo +"${FW_BASE}/luac.cross.int" -m 0x40000 -f -o "${OUTPUT_APP}/lfs.img" ${LUA_FILES} # transfer source files not in the lfs to the build output (ignore the status) cp -f ${LFS_BASE}/../* "${OUTPUT_APP}" 2>/dev/null || true; echo "Building SPIFFS..." -# Build spiffsimg tool (manual for now because of https://github.com/nodemcu/nodemcu-firmware/issues/2943) -docker run --rm -ti \ - -v ${FW_BASE}:/nodemcu-firmware \ - marcelstoer/nodemcu-build make -C /nodemcu-firmware/tools/spiffsimg +# create the fs list from /build/app +rm -rf "${OUTPUT_BASE}/spiffs.lst" +find "${OUTPUT_APP}" -type f | sed "s|${OUTPUT_APP}/||" | awk '{printf "import build/app/%s %s\n", $1, $1}' > "${OUTPUT_BASE}/spiffs.lst" -# Create SPIFFS image -docker run \ - --rm -ti -v "${FW_BASE}:/opt/nodemcu-firmware" \ - -v "${PWD}/scripts:/scripts" \ - -v "${OUTPUT_APP}:/opt/lua" marcelstoer/nodemcu-build bash /scripts/build-spiffs +echo Adding files to spiffs: +cat "${OUTPUT_BASE}/spiffs.lst" +echo -# Create or clear the output directory and transfer files -mkdir -p "${BUILD_PATH}" -rm -rf "${BUILD_PATH}/*" -cp "${FW_BASE}/bin/nodemcu_integer_${IMAGE_NAME}.bin" "${BUILD_PATH}/${IMAGE_NAME}.bin" -cp "${FW_BASE}/bin/konnected-filesystem-0x100000.img" "${BUILD_PATH}/konnected-filesystem-0x100000-${VERSION}.img" +"${FW_BASE}/tools/spiffsimg/spiffsimg" -f "${OUTPUT_BASE}/konnected-filesystem-0x100000-${BRANCH}.img" -S 10m -U 0x100000 -r "${OUTPUT_BASE}/spiffs.lst" -d -srec_cat -output "${BUILD_PATH}/konnected-esp8266-${VERSION}.bin" -binary "${BUILD_PATH}/${IMAGE_NAME}.bin" -binary -fill 0xff 0x0000 0x100000 "${BUILD_PATH}/konnected-filesystem-0x100000-${VERSION}.img" -binary -offset 0x100000 +# Transfer firmware file +#cp "${FW_BASE}/bin/nodemcu_integer_${IMAGE_NAME}.bin" "${OUTPUT_BASE}/${IMAGE_NAME}.bin" + +# Build full image +srec_cat -output "${OUTPUT_BASE}/konnected-esp8266-${BRANCH}.bin" -binary "${OUTPUT_BASE}/${IMAGE_NAME}.bin" -binary -fill 0xff 0x0000 0x100000 "${OUTPUT_BASE}/konnected-filesystem-0x100000-${BRANCH}.img" -binary -offset 0x100000 # Copy to common name for ease of flashing -cp "${BUILD_PATH}/konnected-esp8266-${VERSION}.bin" "${BUILD_PATH}/../konnected-esp8266-latest.bin" +cp "${OUTPUT_BASE}/konnected-esp8266-${BRANCH}.bin" "${OUTPUT_BASE}/konnected-esp8266-latest.bin" + +echo "Build Complete: Flash this build with './scripts/flash ${BRANCH} '" -echo "Build Complete: Flash this build with './scripts/flash ${VERSION} '" \ No newline at end of file diff --git a/scripts/flash b/scripts/flash index fac3f6e..5a3f797 100755 --- a/scripts/flash +++ b/scripts/flash @@ -15,13 +15,8 @@ if [ "$#" == 0 ]; then fi VERSION="$1" -if ! [[ "${VERSION}" =~ ^[0-9]+-[0-9]+-[0-9]+ ]] ; then - echo "Version not formatted correctly..." - usage -fi - PORT=${2:-/dev/cu.wchusbserial1410} -BUILD_PATH="firmware/builds/${VERSION}" +BUILD_PATH=build if [ ! -d "${BUILD_PATH}" ] ; then echo "No matching build version - trying releases"