Feat/systemd readme

.gitignore

@@ -1,9 +1,3 @@
-deploy/*
-work/*
-config
-postrun.sh
-SKIP
-SKIP_IMAGES
-.pc
-*-pc
-apt-cacher-ng/
+*.secret
+work/
+deploy/

README.md

@@ -46,6 +46,28 @@ Upon execution, `build.sh` will source the file `config` in the current
 working directory.  This bash shell fragment is intended to set needed
 environment variables.
 
+You can use your own configs with the command line switch 
+`sudo ./build.sh -c config.path.secret`. If the suffix is `.secret` 
+it'll be ignored by git, via .gitignore.
+
+A sample config is as follows:
+
+```bash
+. config
+
+IMG_NAME="${IMG_NAME}-sample"
+
+TARGET_HOSTNAME=sth-pi
+FIRST_USER_NAME=pi
+ENABLE_SSH=1
+# PUBKEY_SSH_FIRST_USER="<your ssh key>"
+PUBKEY_ONLY_SSH=1
+
+WPA_ESSID="<ESSID>"
+WPA_PASSWORD="<password>"
+WPA_COUNTRY=PL
+```
+
 The following environment variables are supported:
 
  * `IMG_NAME` **required** (Default: unset)

build.sh

@@ -242,6 +242,10 @@ export TIMEZONE_DEFAULT="${TIMEZONE_DEFAULT:-Europe/London}"
 
 export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"}
 
+export WORK_PARTITION_SIZE=${WORK_PARTITION_SIZE:-8192}
+export DEPLOY_PARTITION_SIZE=${DEPLOY_PARTITION_SIZE:-4096}
+export ROOT_PART_EXTRA_SPACE=${ROOT_PART_EXTRA_SPACE:-8192}
+
 export PUBKEY_SSH_FIRST_USER
 
 export CLEAN

config

@@ -0,0 +1,7 @@
+IMG_NAME="rasppi-os"
+STAGE_LIST=${STAGE_LIST:-"stage0 stage1 stage2 stage99-sth"}
+
+WORK_PARTITION_SIZE=8192
+DEPLOY_PARTITION_SIZE=1024
+
+DEPLOY_COMPRESSION=none

export-image/04-set-partuuid/00-run.sh

@@ -7,10 +7,14 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 	IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
 
 	BOOT_PARTUUID="${IMGID}-01"
-	ROOT_PARTUUID="${IMGID}-02"
+	ROOT_PARTUUID="${IMGID}-04"
+	DEPLOY_PARTUUID="${IMGID}-02"
+	WORK_PARTUUID="${IMGID}-03"
 
 	sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+	sed -i "s/DEPLOYDEV/PARTUUID=${DEPLOY_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+	sed -i "s/WORKDEV/PARTUUID=${WORK_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 
 	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"
 

export-image/prerun.sh

@@ -10,29 +10,48 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 	rm -rf "${ROOTFS_DIR}"
 	mkdir -p "${ROOTFS_DIR}"
 
-	BOOT_SIZE="$((256 * 1024 * 1024))"
+	MEGABYTE="$((1024 * 1024))"
+
+	BOOT_SIZE="$((256 * $MEGABYTE))"
 	ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1)
+	WORK_SIZE="$(($WORK_PARTITION_SIZE * $MEGABYTE))"
+	DEPLOY_SIZE="$(($DEPLOY_PARTITION_SIZE * $MEGABYTE))"
 
 	# All partition sizes and starts will be aligned to this size
 	ALIGN="$((4 * 1024 * 1024))"
 	# Add this much space to the calculated file size. This allows for
 	# some overhead (since actual space usage is usually rounded up to the
 	# filesystem block size) and gives some free space on the resulting
 	# image.
-	ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)"
+	ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + $ROOT_PART_EXTRA_SPACE * ${MEGABYTE}) / 1" | bc)"
 
 	BOOT_PART_START=$((ALIGN))
 	BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN))
-	ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE))
+	DEPLOY_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE))
+	DEPLOY_PART_SIZE=$(((DEPLOY_SIZE + ALIGN  - 1) / ALIGN * ALIGN))
+	WORK_PART_START=$((DEPLOY_PART_START + DEPLOY_PART_SIZE))
+	WORK_PART_SIZE=$(((WORK_SIZE + ALIGN  - 1) / ALIGN * ALIGN))
+	ROOT_PART_START=$((WORK_PART_START + WORK_PART_SIZE))
 	ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN  - 1) / ALIGN * ALIGN))
-	IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE))
 
+	IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + DEPLOY_PART_SIZE + WORK_PART_SIZE + ROOT_PART_SIZE))
+
+	echo Creating paritions:
+
 	truncate -s "${IMG_SIZE}" "${IMG_FILE}"
 
 	parted --script "${IMG_FILE}" mklabel msdos
+	echo BOOT: start="$(($BOOT_PART_START / $MEGABYTE))"MB size="$(( (BOOT_PART_SIZE - 1 ) / $MEGABYTE))"MB
 	parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))"
+	echo DEPLOY: start="$(($DEPLOY_PART_START / $MEGABYTE))"MB size="$(( (DEPLOY_PART_SIZE - 1 ) / $MEGABYTE))"MB
+	parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${DEPLOY_PART_START}" "$((DEPLOY_PART_START + DEPLOY_PART_SIZE - 1))"
+	echo WORK: start="$(($WORK_PART_START / $MEGABYTE))"MB size="$(( (WORK_PART_SIZE - 1 ) / $MEGABYTE))"MB
+	parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${WORK_PART_START}" "$((WORK_PART_START + WORK_PART_SIZE - 1))"
+	echo ROOT: start="$(($ROOT_PART_START / $MEGABYTE))"MB size="$(( (ROOT_PART_SIZE - 1 ) / $MEGABYTE))"MB
 	parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))"
 
+	echo TOTAL IMAGE SIZE: $(($IMG_SIZE / $MEGABYTE))MB
+
 	echo "Creating loop device..."
 	cnt=0
 	until LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do
@@ -47,21 +66,37 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 	done
 
 	BOOT_DEV="${LOOP_DEV}p1"
-	ROOT_DEV="${LOOP_DEV}p2"
+	ROOT_DEV="${LOOP_DEV}p4"
+	DEPLOY_DEV="${LOOP_DEV}p2"
+	WORK_DEV="${LOOP_DEV}p3"
 
 	ROOT_FEATURES="^huge_file"
 	for FEATURE in 64bit; do
 	if grep -q "$FEATURE" /etc/mke2fs.conf; then
 		ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES"
 	fi
 	done
+
+	echo "Formatting filesystems..."
 	mkdosfs -n bootfs -F 32 -s 4 -v "$BOOT_DEV" > /dev/null
+	mkdosfs -n deploy -F 32 -s 4 -v "$DEPLOY_DEV" > /dev/null
+	mkfs.ext4 -L work -O "$ROOT_FEATURES" "$WORK_DEV" > /dev/null
 	mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null
 
 	mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4
 	mkdir -p "${ROOTFS_DIR}/boot"
 	mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat
-
-	rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
+	
+	rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot --exclude /srv --exclude /opt/sth/deploy "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
 	rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/"
+
+	if [[ -d "${EXPORT_ROOTFS_DIR}/srv/sth" ]]; then
+		mkdir "${ROOTFS_DIR}/srv" 
+		mkdir "${ROOTFS_DIR}/opt/sth/deploy"
+
+		mount -v "$WORK_DEV" "${ROOTFS_DIR}/srv" -t ext4
+		mount -v "$DEPLOY_DEV" "${ROOTFS_DIR}/opt/sth/deploy" -t vfat
+		rsync -artx "${EXPORT_ROOTFS_DIR}/srv/" "${ROOTFS_DIR}/srv"
+		rsync -rtx "${EXPORT_ROOTFS_DIR}/opt/sth/deploy/" "${ROOTFS_DIR}/opt/sth/deploy"
+	fi
 fi

scripts/qcow2_handling

@@ -225,13 +225,21 @@ function make_bootable_image() {
 	mkdir -p $MOUNTROOT
 
 	MOUNTPT=$MOUNTROOT
-	PARTITION=2
+	PARTITION=4
 	mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
 
 	MOUNTPT=$MOUNTROOT/boot
 	PARTITION=1
 	mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
 
+	MOUNTPT=$MOUNTROOT/opt/sth/deploy
+	PARTITION=2
+	mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
+
+	MOUNTPT=$MOUNTROOT/srv/sth
+	PARTITION=3
+	mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
+
 	if [ ! -d "${MOUNTROOT}/root" ]; then
 		echo "Image damaged or not mounted. Exit."
 		exit 1
@@ -242,6 +250,8 @@ function make_bootable_image() {
 		echo "Set UUIDs to make it bootable"
 		sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab"
 		sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab"
+		sed -i "s/DEPLOYDEV/PARTUUID=${DEPLOY_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+		sed -i "s/WORKDEV/PARTUUID=${WORK_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 		sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt"
 	fi
 

stage1/01-sys-tweaks/files/fstab

@@ -1,3 +1,5 @@
-proc            /proc           proc    defaults          0       0
-BOOTDEV  /boot           vfat    defaults          0       2
-ROOTDEV  /               ext4    defaults,noatime  0       1
+proc       /proc           proc    defaults          0       0
+BOOTDEV    /boot           vfat    defaults          0       2
+ROOTDEV    /               ext4    defaults,noatime  0       1
+WORKDEV    /srv            ext4    defaults,noatime  0       1
+DEPLOYDEV  /opt/sth/deploy vfat    defaults,noatime  0       1

stage99-sth/01-setup-user/00-run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -e
+
+echo "## Creating STH user"
+
+on_chroot << EOF
+adduser sth --system --home /usr/lib/sth
+addgroup sth --system
+
+# Add them to groups
+usermod -aG sudo sth
+usermod -aG gpio sth
+
+# Create a sth-owned folder to put the app in
+# You call this whatever you like, probably the name of your application
+mkdir -p /usr/lib/sth
+EOF

stage99-sth/02-install-node/00-run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -e
+
+NODE_VERSION=18
+
+echo "Installing node.js $NODE_VERSION"
+
+on_chroot << EOF
+    curl -sL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash -
+    sudo apt-get install -y nodejs jq python3-pip python-is-python3
+EOF

stage99-sth/03-install-sth/00-run.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -e
+
+on_chroot << EOF
+    npm i -g @scramjet/sth
+    pip install pyee==9.0.4 scramjet-framework-py
+EOF

stage99-sth/04-setup-systemd/00-run.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+install -d "${ROOTFS_DIR}/srv/sth"
+install -d "${ROOTFS_DIR}/var/log/sth"
+install -d "${ROOTFS_DIR}/opt/sth/deploy"
+install -d "${ROOTFS_DIR}/opt/sth/deploy/conf"
+install -d "${ROOTFS_DIR}/opt/sth/deploy/sequences"
+
+install -m 755 files/start-sth               "${ROOTFS_DIR}/usr/bin/start-sth"
+install -d                                   "${ROOTFS_DIR}/etc/sth"
+install -d                                   "${ROOTFS_DIR}/etc/sth/sequences"
+install -m 644 files/sth-config              "${ROOTFS_DIR}/etc/sth/"
+install -m 644 files/sth.service             "${ROOTFS_DIR}/etc/systemd/system/"
+install -m 644 files/sth-config-deploy.json  "${ROOTFS_DIR}/opt/sth/deploy/conf/sth-config.json"
+
+on_chroot << EOF
+chown -R sth:sth /usr/lib/sth
+chown -R sth:sth /srv/sth
+chown -R sth:sth /var/log/sth
+systemctl enable sth
+EOF

stage99-sth/04-setup-systemd/files/README.md

@@ -0,0 +1,18 @@
+STH in Systemd on an image
+============================
+
+Here's a list of files to start STH as a service in Linux:
+
+* start-sth - start script
+* sth-config - the basic config for STH
+* sth.service - the systemctl config file
+
+Here's where these are installed.
+
+| File                   | Destination
++------------------------+---------------------------------------
+| start-sth              | /usr/bin/start-sth
+| sth-config             | /etc/sth/sth-config
+| sth.service            | /etc/systemd/system/sth.service
+| sth-config-deploy.json | /opt/sth/deploy/conf/sth-config.json
+

stage99-sth/04-setup-systemd/files/start-sth

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+. /etc/sth/sth-config
+
+shopt -s extglob
+
+findfirstconf () {
+    TARGET=$1
+    for FILE in "${TARGET%%/}".{json,yaml,yml}; do
+        [[ -e "$FILE" ]] && echo $FILE && break;
+    done
+}
+
+findsequences() {
+    TARGET=$1
+    for FILE in "${TARGET%%/}"/*.{tgz,tar.gz}; do
+        [[ -e "$FILE" ]] && echo $FILE
+    done
+}
+
+settargetname () {
+    BASE=$(basename "$1")
+    TARGET=$2
+
+    echo ${TARGET%%/}/$BASE
+}
+
+DEPLOY_CONFIG_LOCATION=$(findfirstconf "${DEPLOY_PATH}/conf/sth-config")
+HAS_SEQUENCES=0
+EXTRA_OPTS=("--no-colors" "-D" "$WORK_PATH/sequences")
+
+if [[ -n "${DEPLOY_CONFIG_LOCATION}" ]]; then
+    RUNTIME_CONFIG_LOCATION=$(settargetname "$DEPLOY_CONFIG_LOCATION" "$WORK_PATH")
+    cp "${DEPLOY_CONFIG_LOCATION}" "${RUNTIME_CONFIG_LOCATION}"
+    EXTRA_OPTS+=("--config=${RUNTIME_CONFIG_LOCATION}")
+
+    echo "Using STH config from ${DEPLOY_CONFIG_LOCATION}" | logger -e
+fi
+
+if [[ -e "${DEPLOY_PATH}/conf/purge.txt" ]]; then
+    echo "Purging all sequences" | logger -e
+    rm -rf "$WORK_PATH/sequences"
+    mkdir "$WORK_PATH/sequences"
+fi
+
+SEQUENCES="$(findsequences ${DEPLOY_PATH}/sequences/)"
+if [[ -n "$SEQUENCES" ]]; then
+    function unpack() {
+        PACKAGE=$1
+        TARGET=$2
+
+        mkdir -p "$TARGET"
+        tar -C "$TARGET" -zxf "$PACKAGE"
+    }
+
+    STARTUP_CONFIG=$(findfirstconf "${DEPLOY_PATH}/conf/startup-config")
+    if [[ -n "${STARTUP_CONFIG}" ]]; then
+        RUNTIME_CONFIG_LOCATION=$(settargetname "$STARTUP_CONFIG" "$WORK_PATH")
+        cp "${STARTUP_CONFIG}" "${RUNTIME_CONFIG_LOCATION}"
+        EXTRA_OPTS+=("--startup-config=${RUNTIME_CONFIG_LOCATION}")
+
+        echo "Using startup config from ${DEPLOY_CONFIG_LOCATION}" | logger -e
+    fi
+
+
+    for SEQ_FILE in "$(findsequences ${DEPLOY_PATH}/sequences/)"; do
+        BASENAME=`basename "${SEQ_FILE}"`
+        TARGET="${BASENAME%.@(tar.gz|tgz)}"
+
+        echo "Unpacking sequence '${BASENAME}' to '$WORK_PATH/sequences/$TARGET'" | logger -e
+        unpack "$SEQ_FILE" "$WORK_PATH/sequences/$TARGET"
+
+        EXTRA_OPTS+=("-E")
+    done
+fi
+
+echo Running STH with opts: "${EXTRA_OPTS[@]}"
+sth "${EXTRA_OPTS[@]}" 2>&1 | logger -e

stage99-sth/04-setup-systemd/files/sth-config

@@ -0,0 +1,4 @@
+LOGCOLORS="--no-colors"
+SEQUENCES_ROOT="/srv/sth/sequences"
+DEPLOY_PATH=/opt/sth/deploy
+WORK_PATH=/srv/sth

stage99-sth/04-setup-systemd/files/sth-config-deploy.json

@@ -0,0 +1,13 @@
+{
+    "host": {
+        "id": "mini-pi-1"
+    },
+    "safeOperationLimit": 64,
+    "instanceRequirements": {
+        "freeMem": 128
+    },
+    "telemetry": {
+        "status": true,
+        "environment": "raspberry-pi-sth"
+    }
+}