Merge pull request #166 from h2zero/update-build-script

Update build scripts and add workflows for PR/release.

.github/workflows/build.yml

@@ -0,0 +1,23 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build-upload:
+    runs-on: ubuntu-latest
+    name: Build
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - name: Install build tools
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install libusb-1.0-0 cmake srecord
+          pip install -U pyserial
+      - name: Build
+        run: ./scripts/build-firmware

.github/workflows/release.yml

@@ -0,0 +1,43 @@
+name: Release
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build-upload:
+    runs-on: ubuntu-latest
+    name: Build
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - name: Install build tools
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install libusb-1.0-0 cmake srecord
+          pip install -U pyserial
+
+      - name: Get release
+        id: get_release
+        uses: bruceadams/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Prepare Release Assets
+        run: |
+          ./scripts/build-firmware
+          mkdir assets
+          cp build/konnected-esp8266-latest.bin assets/konnected-esp8266-${{ steps.get_release.outputs.tag_name }}.bin
+
+      - name: Upload asset to release
+        uses: bgpat/release-asset-action@03b0c30db1c4031ce3474740b0e4275cd7e126a3
+        with:
+          pattern: "assets/*"
+          release-url: ${{ steps.get_release.outputs.upload_url }}
+          allow-overwrite: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -48,3 +48,6 @@ firmware/builds
 
 # Private files
 private
+
+# NodeMCU
+nodemcu-firmware

README.md

@@ -51,34 +51,31 @@ Go to the [releases section](https://github.com/konnected-io/konnected-security/
 that you can flash on your Konnected Alarm Panel or ESP8266 device.
 
 ### Building the Firmware Yourself
-Konnected leverages the [NodeMCU](https://github.com/nodemcu/nodemcu-firmware) codebase and [Docker builder](https://hub.docker.com/r/marcelstoer/nodemcu-build/) to create a base nodeMCU firmware image and a filesystem containing the Konnected application. Building only requires a few steps.
+Konnected leverages the [NodeMCU](https://github.com/nodemcu/nodemcu-firmware) codebase to create a base nodeMCU firmware image and a filesystem containing the Konnected application. Building only requires a few steps.
 
-1. Download and install [Docker](https://www.docker.com/products/docker-desktop)
 1. Clone the Konnected repo
 
         git clone https://github.com/konnected-io/konnected-security.git
 
-1. Use the build-firmware script to kick off the build - providing a semantic version command line argument as shown below. The build script will automatically pull down the correct nodeMCU image, and use this nodeMCU docker builder to create base firmware, an LFS image, and a SPIFFS file system containing the entire Konnected application.
+1. Use the build-firmware script to kick off the build. The build script will automatically pull down the correct nodeMCU image, and use this to create base firmware, an LFS image, and a SPIFFS file system containing the entire Konnected application.
 
         cd konnected-security
-        ./scripts/build-firmware 2-2-99
+        ./scripts/build-firmware
 
-1. Once the build completes a folder will be created in `firmware/builds` named after the version specified in the previous step. This folder will contain three files also reflecting the version.
-   1. konnected-filesystem-0x100000-2-2-99.img
-   1. konnected-firmware-2-2-99.bin
-   1. konnected-esp8266-2-2-99.bin
+1. Once the build completes a folder will be created in `build`. This folder will contain three files reflecting the version or branch.
+   1. konnected-filesystem-0x100000-{BRANCH/VERSION}.img
+   1. konnected-firmware-{BRANCH/VERSION}.bin
+   1. konnected-esp8266-*.bin
 
 The `konnected-firmware-*` contains the firmware partition and should be flashed at location `0x0`.
 The `konnected-filesystem-*` image contains the Konnected application and should be flashed at the memory location in
 the filename.
-For convenience, the `konnected-esp8266-*` image is an all-in-one image to be flashed at memory location `0x0` containing the two images above.
+For convenience, the `konnected-esp8266-latest` image is an all-in-one image to be flashed at memory location `0x0` containing the two images above.
 
-*Note: Each time you build it will remove any prior build outputs corresponding to the same version.*
-*Note: Versions in this project should always be formatted `<major>-<minor>-<patch>`.*
 
 ### Flashing a Build
 Flashing a build is simple with the [NodeMCU PyFlasher](https://github.com/marcelstoer/nodemcu-pyflasher/releases). Simply flash
-the `konnected-esp8266-*.bin` file to your device. Typically use baud rate 115200 and flash mode `dio`.
+the `konnected-esp8266-latest.bin` file to your device. Typically use baud rate 115200 and flash mode `dio`.
 
 Mac and Linux users can also easily flash from the command line using `esptool`
 
@@ -91,7 +88,7 @@ Mac and Linux users can also easily flash from the command line using `esptool`
 
  1. Flash the downloaded image using `esptool`:
 
-         esptool.py --port=/dev/cu.SLAB_USBtoUART write_flash --flash_mode dio --flash_size detect 0x0 konnected-esp8266-3-0-0.bin
+         esptool.py --port=/dev/cu.SLAB_USBtoUART write_flash --flash_mode dio --flash_size detect 0x0 konnected-esp8266-latest.bin
 
  *Note: The USB port may vary depending on your computer platform and board.*
 

scripts/build-firmware

@@ -2,118 +2,111 @@
 
 function usage() {
     echo
-    echo "usage: $0 <version>"
+    echo "usage: $0 [-h][-c]"
     echo
-    echo '    -v version should be formatted <major>-<minor>-<patch>'
-    echo '    -c <nodemcu-repo> optional abs path to custom nodemcu-firmware repo'
+    echo '    -h displays usage information'
+    echo '    -c perform a clean build'
     echo
-    echo 'Execute from the root dir of the repo. Upon successful completion the release will be created in firmware/releases/<version>'
-    echo "    ex: '$0 2-3-0'"
+    echo 'Execute from the root dir of the repo. Upon successful completion the firmware image will be build/konnected-pro-fw_*'
+    echo "    ex: '$0'"
     exit 1
 }
-if [ "$#" == 0 ]; then
-    usage
-fi
 
 set -e
 
+if ! git describe --exact-match --tags &> /dev/null; then
+BRANCH="$(git branch | sed -n -e 's/^\* \(.*\)/\1/p')"
+else
+BRANCH="$(git describe --exact-match --tags $(git log -n1 --pretty='%h'))"
+fi
+
 FW_TAG="3.0.0-release_20210201"
-VERSION="$1"
 FIRMWARE_OVERLAY_PATH="firmware/nodemcu-firmware-overlay"
-
 LFS_BASE="${PWD}/src/lfs"
 OUTPUT_BASE="${PWD}/build"
 OUTPUT_APP="${OUTPUT_BASE}/app"
-FW_BASE="${OUTPUT_BASE}/nodemcu-firmware"
+FW_BASE="${PWD}/nodemcu-firmware"
+IMAGE_NAME="konnected-firmware-${BRANCH}"
 
 # process cmd args
-while getopts "v:c:h" opt; do
-    case ${opt} in
-        h )
-            usage
-            ;;
-        c )
-            FW_BASE="$OPTARG"
-            FW_TAG=''
-            echo "using custom nodemcu-repo ${FW_BASE}"
-            ;;
-        v )
-            VERSION="$OPTARG"
-            if ! [[ "${VERSION}" =~ ^[0-9]+-[0-9]+-[0-9]+ ]] ; then
-                echo "Version not formatted correctly..."
-                usage
-            fi
-            echo "tagging as version ${VERSION}"
-            ;;
-        \? )
-            echo "Invalid Option: -$OPTARG" 1>&2
-            exit 1
-            ;;
-    esac
+while getopts "ch" opt; do
+  case ${opt} in
+    h )
+      usage
+      ;;
+    c )
+      cd "${FW_BASE}"
+      make clean
+      cd -
+      # copy in our config overlay
+      cp -rf "${FIRMWARE_OVERLAY_PATH}"/* "${FW_BASE}/"
+      rm -f "${FW_BASE}/local/fs/*"
+      echo "performing clean build"
+      ;;
+   \? )
+     echo "Invalid Option: -$OPTARG" 1>&2
+     exit 1
+     ;;
+  esac
 done
 
-BUILD_PATH="firmware/builds/${VERSION}"
-IMAGE_NAME="konnected-firmware-${VERSION}"
-
-rm -rf ${OUTPUT_BASE}
+rm -rf "${OUTPUT_BASE}"
 mkdir -p "${OUTPUT_BASE}";
 chown -R $USER "${OUTPUT_BASE}"
 
-# pull and update the FW repo if none is supplied
-if [ ! -z "$FW_TAG" ] ; then
+# pull and update the FW repo if not already existing
+if [ ! -d "$FW_BASE" ] ; then
     echo "Checking out nodemcu-firmware:${FW_TAG}"
     git clone --recurse-submodules https://github.com/nodemcu/nodemcu-firmware.git \
         "${FW_BASE}"
+    # Copy firmware configuration from this repository to the nodemcu-firmware repo
+    cp -rf "${FIRMWARE_OVERLAY_PATH}"/* "${FW_BASE}/"
+    rm -f "${FW_BASE}/local/fs/*"
     cd "${FW_BASE}"
     git checkout "${FW_TAG}"
     git submodule update --recursive
     cd -
 fi
 
-# Copy firmware configuration from this repository to the nodemcu-firmware repo
-cp -rf "${FIRMWARE_OVERLAY_PATH}"/* "${FW_BASE}/"
-
-rm -f "${FW_BASE}/local/fs/*"
+# Build it
+cd "${FW_BASE}"
+make
+srec_cat -output "${OUTPUT_BASE}/${IMAGE_NAME}.bin" -binary bin/0x00000.bin -binary -fill 0xff 0x00000 0x10000 bin/0x10000.bin -binary -offset 0x10000
+cd -
 
-# Build NodeMCU firmware image
-docker run -e "IMAGE_NAME=${IMAGE_NAME}" \
-    --rm -ti -v "${FW_BASE}:/opt/nodemcu-firmware" marcelstoer/nodemcu-build build
+# Build spiffsimg tool (manual for now because of https://github.com/nodemcu/nodemcu-firmware/issues/2943)
+make -C "${FW_BASE}"/tools/spiffsimg
 
 # Build LFS image with application files
-LUA_FILES=$(find ${LFS_BASE} -iname "*.lua" | sed "s|${LFS_BASE}|/lfs|g")
+mkdir -p "${OUTPUT_APP}"
+LUA_FILES=$(find ${LFS_BASE} -iname "*.lua")
 
 echo Adding files to LFS:
 echo ${LUA_FILES}
-docker run --rm -ti \
-  -v ${FW_BASE}:/nodemcu-firmware \
-  -v ${OUTPUT_APP}:/build \
-  -v ${LFS_BASE}:/lfs \
-  marcelstoer/nodemcu-build /nodemcu-firmware/luac.cross.int -m 0x40000 -f -o "/build/lfs.img" ${LUA_FILES}
 echo
+"${FW_BASE}/luac.cross.int" -m 0x40000 -f -o "${OUTPUT_APP}/lfs.img" ${LUA_FILES}
 
 # transfer source files not in the lfs to the build output (ignore the status)
 cp -f ${LFS_BASE}/../* "${OUTPUT_APP}" 2>/dev/null || true; echo "Building SPIFFS..."
 
-# Build spiffsimg tool (manual for now because of https://github.com/nodemcu/nodemcu-firmware/issues/2943)
-docker run --rm -ti \
-  -v ${FW_BASE}:/nodemcu-firmware \
-  marcelstoer/nodemcu-build make -C /nodemcu-firmware/tools/spiffsimg
+# create the fs list from /build/app
+rm -rf "${OUTPUT_BASE}/spiffs.lst"
+find "${OUTPUT_APP}" -type f | sed "s|${OUTPUT_APP}/||" | awk '{printf "import build/app/%s %s\n", $1, $1}' > "${OUTPUT_BASE}/spiffs.lst"
 
-# Create SPIFFS image
-docker run \
-    --rm -ti -v "${FW_BASE}:/opt/nodemcu-firmware" \
-    -v "${PWD}/scripts:/scripts" \
-    -v "${OUTPUT_APP}:/opt/lua" marcelstoer/nodemcu-build bash /scripts/build-spiffs
+echo Adding files to spiffs:
+cat "${OUTPUT_BASE}/spiffs.lst"
+echo
 
-# Create or clear the output directory and transfer files
-mkdir -p "${BUILD_PATH}"
-rm -rf "${BUILD_PATH}/*"
-cp "${FW_BASE}/bin/nodemcu_integer_${IMAGE_NAME}.bin" "${BUILD_PATH}/${IMAGE_NAME}.bin"
-cp "${FW_BASE}/bin/konnected-filesystem-0x100000.img" "${BUILD_PATH}/konnected-filesystem-0x100000-${VERSION}.img"
+"${FW_BASE}/tools/spiffsimg/spiffsimg" -f "${OUTPUT_BASE}/konnected-filesystem-0x100000-${BRANCH}.img" -S 10m -U 0x100000 -r "${OUTPUT_BASE}/spiffs.lst" -d
 
-srec_cat -output "${BUILD_PATH}/konnected-esp8266-${VERSION}.bin" -binary "${BUILD_PATH}/${IMAGE_NAME}.bin" -binary -fill 0xff 0x0000 0x100000 "${BUILD_PATH}/konnected-filesystem-0x100000-${VERSION}.img" -binary -offset 0x100000
+# Transfer firmware file
+#cp "${FW_BASE}/bin/nodemcu_integer_${IMAGE_NAME}.bin" "${OUTPUT_BASE}/${IMAGE_NAME}.bin"
+
+# Build full image
+srec_cat -output "${OUTPUT_BASE}/konnected-esp8266-${BRANCH}.bin" -binary "${OUTPUT_BASE}/${IMAGE_NAME}.bin" -binary -fill 0xff 0x0000 0x100000 "${OUTPUT_BASE}/konnected-filesystem-0x100000-${BRANCH}.img" -binary -offset 0x100000
 
 # Copy to common name for ease of flashing
-cp "${BUILD_PATH}/konnected-esp8266-${VERSION}.bin" "${BUILD_PATH}/../konnected-esp8266-latest.bin"
+cp "${OUTPUT_BASE}/konnected-esp8266-${BRANCH}.bin" "${OUTPUT_BASE}/konnected-esp8266-latest.bin"
+
+echo "Build Complete: Flash this build with './scripts/flash ${BRANCH} <port>'"
 
-echo "Build Complete: Flash this build with './scripts/flash ${VERSION} <port>'"

scripts/flash

@@ -15,13 +15,8 @@ if [ "$#" == 0 ]; then
 fi
 
 VERSION="$1"
-if ! [[ "${VERSION}" =~ ^[0-9]+-[0-9]+-[0-9]+ ]] ; then
-    echo "Version not formatted correctly..."
-    usage
-fi
-
 PORT=${2:-/dev/cu.wchusbserial1410}
-BUILD_PATH="firmware/builds/${VERSION}"
+BUILD_PATH=build
 
 if [ ! -d "${BUILD_PATH}" ] ; then
     echo "No matching build version - trying releases"