Add card.binary HIL tests.

.github/workflows/md5srv-tests.yml

@@ -0,0 +1,42 @@
+name: MD5 Server Tests
+
+on:
+  workflow_dispatch:
+  workflow_call:  # reusable workflow
+
+jobs:
+  test-md5srv: # job id
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    env:
+        TERM: xterm-256color
+        MD5SRV_TIMEOUT: 5
+        MD5SRV_DIR: ./test/hitl/scripts
+        BATS_VERSION: 1.10.0
+        BATS_LIB_PATH: /usr/lib
+        # /usr/local/lib on OSX
+    steps:
+      - name: Setup Bats and bats libs
+        uses: brokenpip3/[email protected]
+        with:
+            bats-install: true
+            file-install: false
+            detik-install: false
+      - name: Setup BATS_LIB_PATH
+        run: |
+          if [ -e /usr/local/lib/bats-support ]; then
+            echo "BATS_LIB_PATH=/usr/local/lib" >> $GITHUB_ENV
+          fi
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Run Tests
+        run: |
+            cd ${{env.MD5SRV_DIR}}
+            $HOME/.local/bin/bats -p --print-output-on-failure .
+      - name: Rerun Tests
+        if: failure()
+        run: |
+          cd ${{env.MD5SRV_DIR}}
+          $HOME/.local/bin/bats -p --print-output-on-failure -x .

.github/workflows/notecard-binary-tests.yml

@@ -0,0 +1,245 @@
+name: Note Binary CI
+
+on:
+  pull_request:
+    branches: [ master ]
+  workflow_dispatch:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron:  '45 4 * * 1'  # 4.45am every Monday
+
+permissions:
+  checks: write
+
+jobs:
+  md5srv-test:
+     uses: ./.github/workflows/md5srv-tests.yml
+
+  notecard-binary-test:
+    # needs: md5srv-test
+    runs-on: [self-hosted, swan, notecard, stlink, notecard-serial, md5srv, notehub-client]
+    defaults:
+      run:
+        shell: bash
+    env:
+        NOTEHUB: "notehub.io"
+        NOTEHUB_API: "api.notefile.net"
+        NOTEHUB_ROUTE_TIMEOUT: 180
+        PIO_PROJECT_DIR: ./test/hitl/card.binary
+        NOTEHUB_PROXY_ROUTE_ALIAS: card.binary.${{github.run_id}}
+        NOTEHUB_PROXY_ROUTE_LABEL: card.binary.proxy.${{github.run_id}}
+        NOTEHUB_HTTP_ROUTE_LABEL: card.binary.http.${{github.run_id}}
+
+        # Troubleshooting helpers
+        # DELETE_NOTEHUB_ROUTES set to false to see the created routes on notehub
+        DELETE_NOTEHUB_ROUTES: true
+        # CREATE_NOTEHUB_ROUTES set to false to use the already created routes on notehub
+        CREATE_NOTEHUB_ROUTES: true
+        # FLASH_TEST_FIRMWARE set to false to skip flashing firmware to the Host (Swan).
+        # Be sure to press reset on Swan before running the workflow unless you deliberately want to skip running the tests.
+        FLASH_TEST_FIRMWARE: true
+        # START_MD5SRV set to false to skip starting the MD5 server. There should be one
+        # already running locally with MD5SRV_PORT/ADDRESS/TOKEN set correspondingly.
+        START_MD5SRV: true
+        # START_LOCALTUNNEL, set to false to skip starting the localtunnel.
+        START_LOCALTUNNEL: false
+        # START_TUNNELMOLE: set to false to skip starting tunnel mole.
+        START_TUNNELMOLE: true
+        # When neither tunneling solution is used (because they're already instantiated outside of the workflow)
+        # be sure to set MD5SRV_URL in the environment
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Generate MD5 Server Token
+        run: |
+          [ -n "$MD5SRV_TOKEN" ] || echo "MD5SRV_TOKEN=`uuidgen`" >> $GITHUB_ENV
+          # propagate the environment variable so that it's available in the `env` context
+          echo "MD5SRV_PORT=$MD5SRV_PORT" >> $GITHUB_ENV
+      - name: Check Env Vars
+        run: |
+          . scripts/check_runner_config.sh
+          echo NOTEHUB_PROXY_ROUTE_ALIAS=$NOTEHUB_PROXY_ROUTE_ALIAS
+      - name: Prep MD5 Server
+        uses: pyTooling/Actions/[email protected]
+        with:
+          main: |
+            [ -e md5srv-files ] || mkdir md5srv-files
+            rm -rf md5srv-files/*
+            md5url=http://${MD5SRV_ADDRESS}:${MD5SRV_PORT}/
+          post: |
+            # the JarvusInnovations/background-action@v1 that launches the background
+            # process doesn't clean them up. We do that here. MD5SRV_PID is set in the
+            # ./scripts/run_md5srv.sh script
+            echo Stop MD5 Server
+            [ -n "$MD5SRV_PID" ] || (echo "MD5SRV_PID not set" && exit 1)
+            [ -z "$MD5SRV_PID" ] || kill $MD5SRV_PID
+            rm -rf md5srv-files
+
+      - name: Install PlatformIO dependencies
+        if: env.FLASH_TEST_FIRMWARE!='false'
+        run: |
+          python3 -m venv venv    # python venv is also used by the md5server, so this comes first.
+          source venv/bin/activate
+          pip install platformio
+          cd $PIO_PROJECT_DIR
+          pio pkg install -l "Blues Wireless Notecard" -e debug
+          # Remove the bundled note-c and put the local working copy there
+          NOTE_C_DEP="$GITHUB_WORKSPACE/$PIO_PROJECT_DIR/.pio/libdeps/debug/Blues Wireless Notecard/src/note-c"
+          rm -rf "$NOTE_C_DEP"
+          mkdir "$NOTE_C_DEP"
+          # copy only files in note-c
+          find "$GITHUB_WORKSPACE" -maxdepth 1 -type f -exec cp "{}" "${NOTE_C_DEP}" \;
+
+      - name: Start MD5 Server
+        uses: JarvusInnovations/background-action@v1
+        with:
+          run: |
+            bash ./scripts/run_md5srv.sh
+          wait-on:
+            # just a dummy wait-on since this is required.
+            file:${{github.workspace}}/scripts/run_md5srv.sh
+
+      # When done this way, the background process is terminated at the end of the step,
+      # At least when running with `act`. The same may be true of github runners also.
+      # - name: Start MD5 Server
+      #   uses: pyTooling/Actions/[email protected]
+      #   with:
+      #     main: |
+      #       ./run_md5srv.sh
+      #       echo "MD5SRV_PID=$MD5SRV_PID"
+      #       echo "MD5SRV_PID=$MD5SRV_PID" >> $GITHUB_ENV
+      #       echo "writing server log to `realpath md5srv.log`"
+      #       md5url=http://${MD5SRV_ADDRESS}:${MD5SRV_PORT}/
+      #     post: |
+      #       echo Stop MD5 Server
+      #       [ -n "$MD5SRV_PID" ] || (echo "MD5SRV_PID not set" && exit 1)
+      #       # [ -z "$MD5SRV_PID" ] || kill $MD5SRV_PID
+      #       rm -rf md5srv-files
+
+      - name: Build and Upload Test Firmware
+        if: env.FLASH_TEST_FIRMWARE!='false'
+        run: |
+          source venv/bin/activate
+          export PLATFORMIO_BUILD_FLAGS="'-D NOTEHUB_PROXY_ROUTE_ALIAS=\"$NOTEHUB_PROXY_ROUTE_ALIAS\"' '-D PRODUCT_UID=\"$NOTEHUB_PRODUCT_UID\"'"
+          echo "build flags $PLATFORMIO_BUILD_FLAGS"
+          timeout 10 ./scripts/wait_for_file.sh "$STLINK_PROGRAMMER_PORT"
+          platformio test -e debug --without-testing --upload-port "$STLINK_PROGRAMMER_PORT" --project-dir "$PIO_PROJECT_DIR"
+          timeout 10 ./scripts/wait_for_file.sh "$SWAN_SERIAL"
+
+      - name: Start localtunnel
+        if: env.START_LOCALTUNNEL!='false'
+        id: localtunnel
+        uses: Rei-x/expose-localtunnel-action@main
+        with:
+          ports: ${{ env.MD5SRV_PORT }}
+      - name: Fetch localtunnel URL
+        if: env.START_LOCALTUNNEL!='false'
+        run: |
+          MD5SRV_URL="${{steps.localtunnel.outputs.url-1}}"
+          echo "MD5SRV_URL=$MD5SRV_URL" >> $GITHUB_ENV
+
+      - name: Prep tunnelmole
+        if: env.START_TUNNELMOLE!='false'
+        uses: pyTooling/Actions/[email protected]
+        with:
+          main: |
+            [ ! -e tmole.log ] || rm tmole.log
+            sleep 2 # otherwise it thinks we exited early
+          post: |
+            # the JarvusInnovations/background-action@v1 that launches the background
+            # process doesn't clean them up. We do that here. TMOLE_PID is set in the
+            # ./scripts/run_tunnelmole.sh script
+            echo Stop Tunnelmole
+            [ -n "$TMOLE_PID" ] || (echo "TMOLE_PID not set" && exit 1)
+            [ -z "$TMOLE_PID" ] || kill $TMOLE_PID
+
+      - name: Start tunnelmole
+        uses: JarvusInnovations/background-action@v1
+        if: env.START_TUNNELMOLE!='false'
+        with:
+          run: |
+            bash ./scripts/run_tunnelmole.sh
+          wait-on:
+            # just a dummy wait-on since this is required.
+            file:${{github.workspace}}/tmole.log
+
+      - name: Check server is available
+        run: |
+          # the request will return a 401 from md5srv, but that's expected without the access token
+          # Curl still returns success because it could contact the server
+          code=`curl -s -o /dev/null -w "%{http_code}" $MD5SRV_URL`
+          [ "$code" -lt "500" ] || ( echo "5xx error ($code) from tunnel." && exit 1 )
+
+      - name: Create Notehub accesss token
+        if: env.CREATE_NOTEHUB_ROUTES!='false'
+        run: |
+            curl -f -X POST \
+                -L 'https://${{env.NOTEHUB}}/oauth2/token' \
+                -H 'content-type: application/x-www-form-urlencoded' \
+                -d grant_type=client_credentials \
+                -d client_id=$NOTEHUB_CLIENT_ID \
+                -d client_secret=$NOTEHUB_CLIENT_SECRET | \
+                { token=$(jq -r .access_token); echo "NOTEHUB_ACCESS_TOKEN=$token" >> $GITHUB_ENV; }
+
+      - name: Create Notehub HTTP Route
+        if: env.CREATE_NOTEHUB_ROUTES!='false'
+        uses: pyTooling/Actions/[email protected]
+        with:
+          main: |
+            # ?note=1 instructs the MD5 server to process the content as an event, extracting the path
+            # from the event body.
+            route_req=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_HTTP_ROUTE_LABEL" --arg URL "$MD5SRV_URL/?note=1" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
+              '{ "label":$LABEL, "type":"http", "http":{ "timeout":$TIMEOUT, "filter": { "type":"include", "files": ["cardbinary.qo"] }, "url":$URL, "http_headers": { "X-Access-Token":$TOKEN } } }'`
+            echo $route_req
+            route=`echo "$route_req" | curl -s -f -X POST -L "https://$NOTEHUB_API/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
+                    -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
+            echo $route
+            route_uid=`echo $route | jq -r .uid`
+            [ -n "$route_uid" ]
+            echo "NOTEHUB_HTTP_ROUTE_UID=$route_uid" >> $GITHUB_ENV
+          post: |
+            echo Delete Notehub HTTP Route
+            [ "$DELETE_NOTEHUB_ROUTES" == "false" ] || ([ -n "$NOTEHUB_HTTP_ROUTE_UID" ] && curl -f -s -X DELETE \
+                -L "https://$NOTEHUB_API/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_HTTP_ROUTE_UID" \
+                -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN")
+
+      - name: Create Notehub Proxy Route
+        if: env.CREATE_NOTEHUB_ROUTES!='false'
+        uses: pyTooling/Actions/[email protected]
+        with:
+          main: |
+            ALIAS=$NOTEHUB_PROXY_ROUTE_ALIAS
+            route=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_PROXY_ROUTE_LABEL" --arg URL "$MD5SRV_URL" --arg ALIAS "$ALIAS" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
+              '{ "label":$LABEL, "type":"proxy", "proxy":{ "timeout":$TIMEOUT, "url":$URL, "alias":$ALIAS, "http_headers": { "X-Access-Token":$TOKEN } } }' \
+              | curl -s -f -X POST -L "https://api.notefile.net/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
+                    -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
+            echo $route
+            route_uid=`echo $route | jq -r .uid`
+            [ -n $route_uid ]
+            echo "NOTEHUB_PROXY_ROUTE_UID=$route_uid" >> $GITHUB_ENV
+            echo "NOTEHUB_PROXY_ROUTE_ALIAS=$ALIAS" >> $GITHUB_ENV
+          post: |
+            echo Delete Notehub Proxy Route
+            [ "$DELETE_NOTEHUB_ROUTES" == "false" ] || ([ -n "$NOTEHUB_PROXY_ROUTE_UID" ] && curl -f -s -X DELETE \
+                -L "https://api.notefile.net/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_PROXY_ROUTE_UID" \
+                -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN")
+
+      - name: Run Tests
+        run: |
+          source venv/bin/activate
+          cd $PIO_PROJECT_DIR
+          platformio test -v -e debug \
+             --without-building --without-uploading \
+             --test-port "$SWAN_SERIAL" \
+             --json-output-path test.json \
+             --junit-output-path test.xml \
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        if: env.GITHUB_TOKEN && (success() || failure()) # always run even if the previous step fails
+        with:
+          report_paths: '**/test/hitl/card.binary/test*.xml'
+          check_name: Notecard Binary HIL Tests
+          require_tests: true

scripts/check_runner_config.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+function env_var_defined() {
+    [ -v $1 ] || echo "Environment variable '$1' not set."
+}
+
+function check_all() {
+    env_var_defined "NOTEHUB_CLIENT_ID"
+    env_var_defined "NOTEHUB_CLIENT_SECRET"
+    env_var_defined "NOTEHUB_PROJECT_UID"
+    env_var_defined "NOTEHUB_PRODUCT_UID"
+    env_var_defined "MD5SRV_PORT"
+    env_var_defined "MD5SRV_ADDRESS"
+    env_var_defined "MD5SRV_TOKEN"
+    env_var_defined "SWAN_SERIAL"
+    env_var_defined "STLINK_PROGRAMMER_PORT"
+
+    # these are defined in the workflow itself, but good to check in case of errors causing them to be unset
+    env_var_defined "NOTEHUB_API"
+    env_var_defined "NOTEHUB"
+    env_var_defined "PIO_PROJECT_DIR"
+    env_var_defined "NOTEHUB_PROXY_ROUTE_LABEL"
+    env_var_defined "NOTEHUB_PROXY_ROUTE_ALIAS"
+    env_var_defined "NOTEHUB_HTTP_ROUTE_LABEL"
+
+
+}
+
+errors=$(check_all)
+if [ -n "$errors" ]; then
+    echo "$errors"   # quoted to preserve newlines
+    echo "There are configuration errors. See the log above for details."
+    exit 1
+fi
+
+exit 0

scripts/run_md5srv.sh

@@ -0,0 +1,7 @@
+. venv/bin/activate
+python3 ./test/hitl/scripts/md5srv.py --dir md5srv-files --save > md5srv.log 2>&1 &
+MD5SRV_PID=$!
+echo "MD5SRV_PID=$MD5SRV_PID" >> $GITHUB_ENV
+
+# Prevent the action from thinking we exited early
+sleep 2

scripts/run_tunnelmole.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+[ -n "$MD5SRV_PORT" ] || (echo "MD5SRV_PORT not set" && exit 1)
+which tmole
+tmole $MD5SRV_PORT > tmole.log 2>&1 &
+TMOLE_PID=$!
+echo "TMOLE_PID=$TMOLE_PID"
+echo "TMOLE_PID=$TMOLE_PID" >> $GITHUB_ENV
+timeout 10 bash -c "until test -e tmole.log; do sleep 0.1; done"
+sleep 1
+# find the url
+exec 3< tmole.log
+# ignore the first line
+read <&3 output
+read <&3 output
+MD5SRV_URL=`echo "$output" | grep https | cut -d " " -f1`
+echo "MD5SRV_URL=$MD5SRV_URL"
+[ -n "$MD5SRV_URL" ] || (echo "Could not fetch tunnelmole URL" && exit 1)
+echo "MD5SRV_URL=$MD5SRV_URL" >> $GITHUB_ENV
+sleep 2
+echo "Tunnel mole started. Writing logs to `realpath tmole.log`"

scripts/wait_for_file.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# $1 filename to wait for
+set -e
+
+if [ -z "$1" ]; then
+  echo "Expected 1 argument: <filename>"
+  exit 1
+else
+  echo "Waiting for file $1..."
+fi
+
+while ! test -e "$1"; do
+  sleep 0.5
+done

test/hitl/card.binary/lib/notecard_binary/NotecardBinary.cpp

@@ -0,0 +1,13 @@
+#include "NotecardBinary.h"
+
+Notecard notecard;
+
+
+BinaryImage small_binary = { .name = "small", .length=small_img_len, .data=small_img_map };
+
+
+BufferBinaryGenerator small_image(small_binary);
+
+
+// ensure the virtual destructor is defined.
+BinaryGenerator::~BinaryGenerator() {}