balena-deploy: Use docker cache when creating compose file

automation/entry_scripts/balena-build-block.sh

@@ -3,15 +3,22 @@
 set -e
 
 script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${script_dir}/balena-api.inc"
-source "${script_dir}/balena-lib.inc"
+if [ -f "/balena-lib.inc" ] && [ -f "/balena-api.inc" ] && [ -f "/balena-docker.inc" ]; then
+	source "/balena-lib.inc"
+	source "/balena-api.inc"
+else
+	automation_dir=$( cd "${script_dir}/.." && pwd )
+	source "${automation_dir}/include/balena-lib.inc"
+	source "${automation_dir}/include/balena-api.inc"
+fi
 
 # Input checks
 [ -z "${APPNAME}" ] && echo "The block's app name needs to be defined" && exit 1
 [ -z "${MACHINE}" ] && echo "Machine needs to be defined" && exit 1
 [ -z "${PACKAGES}" ] && echo "list of packages to install without dependencies" && exit 1
 [ -z "${RELEASE_VERSION}" ] && echo "A release version needs to be defined" && exit 1
 [ -z "${WORKSPACE}" ] && echo "Workspace needs to be defined" && exit 1
+DEPLOY_DIR="${DEPLOY_DIR:-"${WORKSPACE}/deploy-jenkins"}"
 
 [ -z "${PACKAGE_TYPE}" ] && PACKAGE_TYPE="ipk"
 
@@ -63,10 +70,10 @@ echo "LABEL ${BALENA_HOSTOS_BLOCK_REQUIRES_REBOOT}=1"  >> "${TMPDIR}/Dockerfile"
 echo "LABEL ${BALENA_HOSTOS_BLOCK_STORE}=data" >> "${TMPDIR}/Dockerfile"
 
 # Copy local package feed to context if available from previous build step
-if [ -d "${WORKSPACE}/deploy-jenkins/${PACKAGE_TYPE}" ]; then
+if [ -d "${DEPLOY_DIR}/${PACKAGE_TYPE}" ]; then
 	ARCH_LIST=""
 	mkdir -p "${TMPDIR}/feed"
-	cp -r "${WORKSPACE}/deploy-jenkins/${PACKAGE_TYPE}" "${TMPDIR}/feed/"
+	cp -r "${DEPLOY_DIR}/${PACKAGE_TYPE}" "${TMPDIR}/feed/"
 	# Extract package architecture list from feed
 	# Each architecture is one directory
 	while IFS=$'\n' read -r dir; do
@@ -75,7 +82,7 @@ if [ -d "${WORKSPACE}/deploy-jenkins/${PACKAGE_TYPE}" ]; then
 		else
 			ARCH_LIST="${ARCH_LIST} ${dir}"
 		fi
-	done< <(find "${WORKSPACE}/deploy-jenkins/${PACKAGE_TYPE}" -mindepth 1 -maxdepth 1 -type d | xargs -I{} basename {})
+	done< <(find "${DEPLOY_DIR}/${PACKAGE_TYPE}" -mindepth 1 -maxdepth 1 -type d | xargs -I{} basename {})
 else
 	proto=${FEED_URL%:*}
 	if [ -z "${FEED_URL}" ] || [ "${proto}" = "file" ]; then
@@ -91,8 +98,8 @@ docker rmi -f $(docker images --filter "label=${BALENA_HOSTOS_BLOCK_CLASS}" --fo
 
 if balena build --logs --nocache --deviceType "${MACHINE}" --arch "${ARCH}" --buildArg PACKAGES="${PACKAGES}" --buildArg ARCH_LIST="${ARCH_LIST}" --buildArg NAMESPACE="${NAMESPACE:-resin}"; then
 	image_id=$(docker images --filter "label=${BALENA_HOSTOS_BLOCK_CLASS}" --format "{{.ID}}")
-	mkdir -p "${WORKSPACE}/deploy-jenkins"
-	docker save "${image_id}" > "${WORKSPACE}/deploy-jenkins/${APPNAME}-${RELEASE_VERSION}.docker"
+	mkdir -p "${DEPLOY_DIR}"
+	docker save "${image_id}" > "${DEPLOY_DIR}/${APPNAME}-${RELEASE_VERSION}.docker"
 else
 	echo "[ERROR] Fail to build"
 	exit 1

automation/entry_scripts/balena-deploy-block.sh

@@ -1,9 +1,17 @@
 #!/bin/bash
 set -e
 
-source /balena-docker.inc
-source /balena-lib.inc
-source /balena-api.inc
+script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+if [ -f "/balena-lib.inc" ] && [ -f "/balena-api.inc" ] && [ -f "/balena-docker.inc" ]; then
+	source "/balena-lib.inc"
+	source "/balena-api.inc"
+	source "/balena-docker.inc"
+else
+	automation_dir=$( cd "${script_dir}/.." && pwd )
+	source "${automation_dir}/include/balena-lib.inc"
+	source "${automation_dir}/include/balena-api.inc"
+	source "${automation_dir}/include/balena-docker.inc"
+fi
 
 trap 'balena_docker_stop fail' SIGINT SIGTERM
 
@@ -12,9 +20,6 @@ balena_docker_start "/scratch/docker" "/var/run" "/var/log/docker.log"
 balena_docker_wait
 
 BALENAOS_ACCOUNT="${BALENAOS_ACCOUNT:-"balena_os"}"
-if [ -f "/host/appimage.docker" ]; then
-	_local_image=$(docker load -i /host/appimage.docker | cut -d: -f1 --complement | tr -d " " )
-fi
 
 echo "[INFO] Logging into $API_ENV as ${BALENAOS_ACCOUNT}"
 export BALENARC_BALENA_URL=${API_ENV}
@@ -25,22 +30,32 @@ if [ "$ESR" = "true" ]; then
 	APPNAME="${APPNAME}-esr"
 fi
 
+# Use /deploy folder to generate compose file to use local images that live there
+# Use a release dir to limit context
+RELEASE_DIR=$(balena_docker_create_compose_file "${MACHINE}" "${API_ENV}" "${RELEASE_VERSION}" "${BALENAOS_TOKEN}" "${BLOCKS}" "/deploy")
+if [ ! -f "${RELEASE_DIR}/docker-compose.yml" ]; then
+	echo "[ERROR] Failed to generate compose file"
+	exit 1
+fi
+
 if [ -f "/deploy/balena.yml" ]; then
-	echo -e "\nversion: $(balena_lib_get_os_version)" >> "/deploy/balena.yml"
+	cp "/deploy/balena.yml" "${RELEASE_DIR}"
+	echo -e "\nversion: $(balena_lib_get_os_version)" >> "${RELEASE_DIR}/balena.yml"
 fi
 
 echo "[INFO] Deploying  to ${BALENAOS_ACCOUNT}/$APPNAME"
 balena_api_create_public_app "${APPNAME}" "${BALENARC_BALENA_URL}" "${MACHINE}" "${balenaCloudEmail}" "${balenaCloudPassword}" "${ESR}" "${BOOTABLE}"
-_releaseID=$(balena_lib_release "${BALENAOS_ACCOUNT}/$APPNAME" "${FINAL}" "/deploy" "${API_ENV}" "$_local_image")
+_releaseID=$(balena_lib_release "${BALENAOS_ACCOUNT}/$APPNAME" "${FINAL}" "${RELEASE_DIR}" "${API_ENV}")
 if [ -z "${_releaseID}" ]; then
 	echo "[INFO] Failed to deploy to ${BALENAOS_ACCOUNT}/$APPNAME"
 	exit 1
 fi
 
 # Legacy hostapp tagging
-if [ "${DEPLOY}" = "yes" ] && [ "${FINAL}" = "yes" ]; then
+if [ "${FINAL}" = "yes" ]; then
 	balena_lib_release_finalize "${_releaseID}" "${BALENAOS_ACCOUNT}/${APPNAME}" "${API_ENV}" "${BALENAOS_TOKEN}" "${ESR}"
 fi
 
 balena_docker_stop
+rm -rf "${RELEASE_DIR:?}"
 exit 0

automation/include/balena-deploy.inc

@@ -315,47 +315,44 @@ EOSU
 		'
 }
 
-# Builds and deploys the specified block to BalenaCloud
+# Builds and deploys to BalenaCloud
 # Input arguments;
 #  $1: App name to deploy into
 #  $2: Device type for the app
-#  $3: Bootable flag
-#  $4: Path to the image to deploy
-#  $5: Deploy release to S3 and balenaCloud fleet
-#  $6: Deploy to balenaCloud fleet as draft (default) or final release
-#  $7: Path to the working directory (defaults to device repository root)
-#  $8: BalenaCloud admin account (defaults to balena_os)
-#  $9: BalenaCloud email address (defaults to the balenaCloudEmail enviromental variable)
-#  $10: BalenaCloud password (defaults to the balenaCloudPassword enviromental variable)
-#  $11: ESR release flag (default to "ESR" environment variable)
+#  $3: Deploy to balenaCloud fleet as draft (default) or final release
+#  $4: Block names to deploy (defaults to hostapp)
+#  $5: Path to the working directory (defaults to device repository root)
+#  $6: Bootable flag - defaults to 1
+#  $7: BalenaCloud admin account (defaults to balena_os)
+#  $8: BalenaCloud email address (defaults to the balenaCloudEmail enviromental variable)
+#  $9: BalenaCloud password (defaults to the balenaCloudPassword enviromental variable)
+#  $10: ESR release flag (default to "ESR" environment variable)
 #
-balena_deploy_block() {
+balena_deploy() {
 	local _appName="$1"
 	local _device_type="${2:-${MACHINE}}"
-	local _bootable=${3:-"0"}
-	local _image_path="${4:-""}"
-	local _deploy="${5:-no}"
-	local _final="${6:-no}"
-	local _work_dir="${7:-"${device_dir}"}"
-	local _balenaos_account="${8:-"balena_os"}"
-	local _balenaCloudEmail="${9:-"${balenaCloudEmail}"}"
-	local _balenaCloudPassword="${10:-"${balenaCloudPassword}"}"
-	local _esr="${11}"
-	local _discontinued
+	local _final="${3:-no}"
+	local _blocks="${4:-"${MACHINE}"}"
+	local _work_dir="${5:-"${device_dir}"}"
+	local _bootable=${6:-"1"}
+	local _balenaos_account="${7:-"balena_os"}"
+	local _balenaCloudEmail="${8:-"${balenaCloudEmail}"}"
+	local _balenaCloudPassword="${9:-"${balenaCloudPassword}"}"
+	local _esr="${10}"
 	local _api_env=$(balena_lib_environment)
+	local _discontinued
 
 	[ -z "${_device_type}" ] && echo "Device type is required" && return
 	_discontinued=$(balena_lib_get_dt_state "${_device_type}")
 	if [ "${_discontinued}" = "DISCONTINUED" ]; then
-		echo "[INFO]: balena_deploy_block: Not deploying discontinued device type."
+		echo "[INFO]: balena_deploy: Not deploying discontinued device type."
 		return
 	fi
 
 	[ -z "${_appName}" ] && echo "App name is required" && return
-	if [ -f "${_image_path}" ]; then
-		_image_path="$(readlink --canonicalize "${_image_path}")"
-	fi
+	[ -z "${_blocks}" ] && echo "Block names are required" && return
 	_esr=${_esr:-"${ESR}"}
+	[ ! -d "${_work_dir}" ] && mkdir "${_work_dir}"
 
 	if [ ! -f "${_work_dir}/balena.yml" ]; then
 		if [ -f "${device_dir}/balena.yml" ]; then
@@ -373,20 +370,19 @@ balena_deploy_block() {
 	docker run --rm -t \
 		-e APPNAME="${_appName}" \
 		-e API_ENV="${_api_env}" \
+		-e BLOCKS="${_blocks}" \
 		-e BALENAOS_TOKEN="$(balena_lib_token "${_api_env}")" \
 		-e BALENAOS_ACCOUNT="${_balenaos_account}" \
 		-e META_BALENA_VERSION="$(balena_lib_get_meta_balena_base_version)" \
 		-e RELEASE_VERSION="$(balena_lib_get_os_version)" \
 		-e MACHINE="${_device_type}" \
 		-e VERBOSE="${VERBOSE}" \
 		-e BOOTABLE="${_bootable}" \
-		-e DEPLOY="${_deploy}" \
 		-e FINAL="${_final}" \
 		-e ESR="${_esr:-"false"}" \
 		-e balenaCloudEmail="${_balenaCloudEmail}" \
 		-e balenaCloudPassword="${_balenaCloudPassword}" \
 		-e ESR="${_esr}" \
-		-v "${_image_path}":/host/appimage.docker \
 		-v "${device_dir}":/work \
 		-v "${_work_dir}":/deploy \
 		--privileged \
@@ -395,7 +391,7 @@ balena_deploy_block() {
 	balena_lib_docker_remove_helper_images "balena-push-env"
 }
 
-# Builds and deploys the specified block to BalenaCloud
+# Builds the specified block
 # Input arguments;
 #  $1: App name to deploy into
 #  $2: Device type for the app
@@ -436,19 +432,6 @@ balena_build_block() {
 	balena_lib_docker_remove_helper_images "yocto-block-build-env"
 }
 
-# Initialize a compose file in the specified path
-#
-# Input:
-# $1: Path to create the compose file into
-__init_compose() {
-	local _path="${1}"
-	[ -z "${_path}" ] && return
-	cat << EOF > "${_path}/docker-compose.yml"
-version: '2'
-services:
-EOF
-}
-
 # Deploy a package feed locally
 #
 # Inputs:
@@ -473,147 +456,3 @@ balena_deploy_feed() {
 		cp -r "${device_dir}/build/tmp/deploy/${_package_type}" "$_deploy_dir/"
 	fi
 }
-
-# Add a compose service
-#
-# Inputs:
-# $1: Path to the directory holding the compose file - will be created if needed
-# $2: Name of the service to be added
-# $3: Image digest for the service
-# $4: Image class: fileset, overlay or service (default)
-# $5: Image reboot required: 0 (default) or 1
-# $6: Image engine type: boot, root or data (default)
-# $6: Image is bootable, false (default) or true
-#
-# Outputs:
-#    Compose file in the specified path
-#
-__add_compose_service() {
-	local _path=$1
-	local _service_name=$2
-	local _image=$3
-	local _image_class=$4
-	local _image_reboot=$5
-	local _image_engine=$6
-	local _bootable=$7
-
-	[ -z "${_path}" ] || [ -z "${_service_name}" ] || [ -z "${_image}" ] && return
-	_image_class=${_image_class:-"service"}
-	_image_reboot=${_image_reboot:-0}
-	_image_engine=${_image_engine:-"data"}
-	_bootable=${_bootable:-"false"}
-
-	if [ ! -f "${_path}/docker-compose.yml" ]; then
-		__init_compose "${_path}"
-	fi
-	printf "  %s:\n" "${_service_name}" >> "${_path}/docker-compose.yml"
-	printf "    image: %s\n" "${_image}" >> "${_path}/docker-compose.yml"
-	printf "    labels:\n" >> "${_path}/docker-compose.yml"
-	if [ -n "${_image_class}" ]; then
-		printf "      %s: %s\n" \""${BALENA_HOSTOS_BLOCK_CLASS}"\" \""${_image_class}"\" >> "${_path}/docker-compose.yml"
-	fi
-	if [ "${_image_reboot}" = "1" ]; then
-		printf "      %s: '1'\n" \""${BALENA_HOSTOS_BLOCK_REQUIRES_REBOOT}"\" >> "${_path}/docker-compose.yml"
-	fi
-	if [ -n "${_image_engine}" ]; then
-		printf "      %s: %s\n" \""${BALENA_HOSTOS_BLOCK_STORE}"\" \""${_image_engine}"\" >> "${_path}/docker-compose.yml"
-	fi
-	if [ "${_bootable}" = "true" ]; then
-		printf "      %s: %s\n" \""${BALENA_HOSTOS_BLOCK_BOOTABLE}"\" \""${_bootable}"\" >> "${_path}/docker-compose.yml"
-	fi
-}
-
-# Creates a compose file
-#
-# Inputs:
-# $1: Device type to build for
-# $2: Balena API environment (default to balena-cloud.com)
-# $3: BalenaOS version - defaults to current device repository tag
-# $4: BalenaOS token
-# $5: HostOS blocks - default to none
-#
-# Outputs:
-#    Path where the compose file is created
-#
-__create_compose_file() {
-	local _device_type="$1"
-	local _apiEnv="$2"
-	local _version="$3"
-	local _token="$4"
-	local _blocks="$5"
-	local _path
-	local _block_image
-	local _class
-	local _store
-	local _reboot_required
-	local _block
-	local _image_id
-	local _bootable
-
-	[ -z "${_device_type}" ] && return
-	_version=${_version:-$(balena_lib_get_os_version)}
-	_apiEnv=${_apiEnv:-"balena-cloud.com"}
-	[ -z "${_path}" ] && _path=$(mktemp -d)
-
-	[ -z "${_blocks}" ] && >&2 echo "Blocks are required" && return 1
-	for _block in ${_blocks}; do
-		_block_image=$(balena_api_fetch_image_from_app "${_block}" "${_version}" "${_apiEnv}")
-		_image_id=$(balena_docker_image_retrieve "${_block_image}")
-		if [ -z "${_block_image}" ] || [ "${_block_image}" = "" ]; then
-			>&2 echo "[${_block}] No such image for ${_version} in ${_apiEnv}"
-			continue
-		fi
-		# Container images created by importing a tarball and saving (like the Yocto docker image class does) do not contain labels
-		# Hence, if no labels are found default to hostapp values, class service, root store and bootable.
-		_class=$(balena_lib_get_label_from_image "${_image_id}" "${BALENA_HOSTOS_BLOCK_CLASS}")
-		[ -z "${_class}" ] && class="service"
-		_store=$(balena_lib_get_label_from_image "${_image_id}" "${BALENA_HOSTOS_BLOCK_STORE}")
-		[ -z "${_store}" ] && _store="root"
-		_reboot_required=$(balena_lib_get_label_from_image "${_image_id}" "${BALENA_HOSTOS_BLOCK_REQUIRES_REBOOT}")
-		[ -z "${_reboot_required}" ] && _reboot_required="1"
-		_bootable=$(balena_api_is_bootable "${_block}" "${_apiEnv}" "${_token}")
-		__add_compose_service "${_path}" "${_block}" "${_block_image}" "${_class}" "${_reboot_required}" "${_store}" "${_bootable}"
-	done
-	echo "${_path}"
-}
-
-# Deploys a multi-container hostOS
-#
-# Inputs:
-# $1: Application name
-# $2: HostOS blocks - required
-# $3: Device type for the application
-# $4: Release final version
-# $5: Balena API environment (default to balena-cloud.com)
-# $6: Balena API token (defaults to ~/.balena/token)
-# $7: Balena cloud account (defaults to balena_os)
-# $8: Bootable 0 or 1 (default)
-#
-# Outputs:
-#    None
-#
-balena_deploy_hostos() {
-	local _appName="$1"
-	local _blocks="$2"
-	local _device_type="$3"
-	local _final="${4:-no}"
-	local _apiEnv="$5"
-	local _token="$6"
-	local _account="$7"
-	local _bootable="${8:-1}"
-	local _path
-	local _version
-
-	_apiEnv=${_apiEnv:-"$(balena_lib_environment)"}
-	_account=${_account:-"balena_os"}
-	_token=${_token:-"$(balena_lib_token "${_apiEnv}")"}
-	_version=$(balena_lib_get_os_version)
-	[ -z "${_version}" ] && >&2 echo "Invalid version" && return
-	[ -z "${_device_type}" ] && >&2 echo "Required device type" && return
-	_path=$(__create_compose_file "${_device_type}" "${_apiEnv}" "${_version}" "${_token}" "${_blocks}")
-	if [ ! -f "${_path}/docker-compose.yml" ]; then
-		>&2 echo "No compose file in ${_path}"
-		return
-	fi
-	balena_deploy_block "${_appName}" "${_device_type}" "${_bootable}" "${_path}"
-}