From 6f5ebb35f5736434f15123f2382f4d4eaa156713 Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Tue, 14 Jun 2022 19:43:44 +0200 Subject: [PATCH 01/11] balena-lib: get_label_from_image: Return nothing if not found It's easier to check its return value if it returns nothing without an actual match. Change-type: patch Signed-off-by: Alex Gonzalez --- automation/include/balena-lib.inc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/automation/include/balena-lib.inc b/automation/include/balena-lib.inc index 6915ff487..c5d75a663 100644 --- a/automation/include/balena-lib.inc +++ b/automation/include/balena-lib.inc @@ -430,7 +430,7 @@ balena_lib_get_dt_arch() { # # # Output: -# Label value in stdout or null +# Label value in stdout or nothing # balena_lib_get_label_from_image() { local _image_id="${1}" @@ -438,7 +438,9 @@ balena_lib_get_label_from_image() { local _value _value=$(docker inspect "${_image_id}" | jq -r '.[].Config.Labels["'"${_label}"'"]') - echo "${_value}" + if [ "${_value}" != "null" ]; then + echo "${_value}" + fi } From cac3f1edb93e1d3eb5e6ba2a1d77d9c28e5b9574 Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Thu, 16 Jun 2022 16:49:26 +0200 Subject: [PATCH 02/11] balena-lib: balena_lib_release: Use local cache if available Use a `--cache-from :` to allow deployment to use images cached locally, for example when a hostapp or an OS block has been built and loaded into the local docker cache. Change-type: patch Signed-off-by: Alex Gonzalez --- automation/include/balena-lib.inc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/automation/include/balena-lib.inc b/automation/include/balena-lib.inc index c5d75a663..2d2531bef 100644 --- a/automation/include/balena-lib.inc +++ b/automation/include/balena-lib.inc @@ -465,13 +465,14 @@ balena_lib_release() { local _releaseID="" local _status="" local _debug="" + local _version [ -z "${_appName}" ] && >&2 echo "balena_lib_release: Application name is required" && exit 1 _apiEnv=${_apiEnv:-"$(balena_lib_environment)"} _path=${_path:-"$(pwd)"} + _version=$(balena_lib_get_os_version) if [ -f "${_path}/balena.yml" ]; then _contract_version=$(awk '/version:/ {print $2}' "${_path}/balena.yml") - _version=$(balena_lib_get_os_version) if [ "${_contract_version}" != "${_version}" ]; then >&2 echo "balena_lib_release: Version mismatch, contract ${_contract_version} os ${_version}" return 1 @@ -489,7 +490,7 @@ balena_lib_release() { if [ -n "${_image_id}" ]; then _releaseID=$(BALENARC_BALENA_URL="${_apiEnv}" balena deploy "${_appName}" "${_image_id}" --source "${_path}" ${_status} ${_debug} | sed -n 's/.*Release: //p') else - _releaseID=$(BALENARC_BALENA_URL="${_apiEnv}" balena deploy "${_appName}" --build --source "${_path}" ${_status} ${_debug} | sed -n 's/.*Release: //p') + _releaseID=$(BALENARC_BALENA_URL="${_apiEnv}" balena deploy "${_appName}" --build --source "${_path}" --cache-from ${_appName#*/}:${_version} ${_status} ${_debug} | sed -n 's/.*Release: //p') fi [ -n "${_releaseID}" ] && >&2 echo "Deployed ${_image_id} to ${_appName} as ${_status##--} at ${_releaseID}" echo "${_releaseID}" From 73dcf9e7f210fff43c9a984fb561b7acb6743b3a Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Wed, 15 Jun 2022 20:58:05 +0200 Subject: [PATCH 03/11] balena-docker: Move compose file creation function from balena-deploy.inc The function is only called from within the balena-push-env container that already contains the balena-docker.inc script include. Change-type: patch Signed-off-by: Alex Gonzalez --- automation/include/balena-deploy.inc | 157 --------------------------- automation/include/balena-docker.inc | 142 ++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 157 deletions(-) diff --git a/automation/include/balena-deploy.inc b/automation/include/balena-deploy.inc index 9ec37d981..ce9ef3d3d 100644 --- a/automation/include/balena-deploy.inc +++ b/automation/include/balena-deploy.inc @@ -436,19 +436,6 @@ balena_build_block() { balena_lib_docker_remove_helper_images "yocto-block-build-env" } -# Initialize a compose file in the specified path -# -# Input: -# $1: Path to create the compose file into -__init_compose() { - local _path="${1}" - [ -z "${_path}" ] && return - cat << EOF > "${_path}/docker-compose.yml" -version: '2' -services: -EOF -} - # Deploy a package feed locally # # Inputs: @@ -473,147 +460,3 @@ balena_deploy_feed() { cp -r "${device_dir}/build/tmp/deploy/${_package_type}" "$_deploy_dir/" fi } - -# Add a compose service -# -# Inputs: -# $1: Path to the directory holding the compose file - will be created if needed -# $2: Name of the service to be added -# $3: Image digest for the service -# $4: Image class: fileset, overlay or service (default) -# $5: Image reboot required: 0 (default) or 1 -# $6: Image engine type: boot, root or data (default) -# $6: Image is bootable, false (default) or true -# -# Outputs: -# Compose file in the specified path -# -__add_compose_service() { - local _path=$1 - local _service_name=$2 - local _image=$3 - local _image_class=$4 - local _image_reboot=$5 - local _image_engine=$6 - local _bootable=$7 - - [ -z "${_path}" ] || [ -z "${_service_name}" ] || [ -z "${_image}" ] && return - _image_class=${_image_class:-"service"} - _image_reboot=${_image_reboot:-0} - _image_engine=${_image_engine:-"data"} - _bootable=${_bootable:-"false"} - - if [ ! -f "${_path}/docker-compose.yml" ]; then - __init_compose "${_path}" - fi - printf " %s:\n" "${_service_name}" >> "${_path}/docker-compose.yml" - printf " image: %s\n" "${_image}" >> "${_path}/docker-compose.yml" - printf " labels:\n" >> "${_path}/docker-compose.yml" - if [ -n "${_image_class}" ]; then - printf " %s: %s\n" \""${BALENA_HOSTOS_BLOCK_CLASS}"\" \""${_image_class}"\" >> "${_path}/docker-compose.yml" - fi - if [ "${_image_reboot}" = "1" ]; then - printf " %s: '1'\n" \""${BALENA_HOSTOS_BLOCK_REQUIRES_REBOOT}"\" >> "${_path}/docker-compose.yml" - fi - if [ -n "${_image_engine}" ]; then - printf " %s: %s\n" \""${BALENA_HOSTOS_BLOCK_STORE}"\" \""${_image_engine}"\" >> "${_path}/docker-compose.yml" - fi - if [ "${_bootable}" = "true" ]; then - printf " %s: %s\n" \""${BALENA_HOSTOS_BLOCK_BOOTABLE}"\" \""${_bootable}"\" >> "${_path}/docker-compose.yml" - fi -} - -# Creates a compose file -# -# Inputs: -# $1: Device type to build for -# $2: Balena API environment (default to balena-cloud.com) -# $3: BalenaOS version - defaults to current device repository tag -# $4: BalenaOS token -# $5: HostOS blocks - default to none -# -# Outputs: -# Path where the compose file is created -# -__create_compose_file() { - local _device_type="$1" - local _apiEnv="$2" - local _version="$3" - local _token="$4" - local _blocks="$5" - local _path - local _block_image - local _class - local _store - local _reboot_required - local _block - local _image_id - local _bootable - - [ -z "${_device_type}" ] && return - _version=${_version:-$(balena_lib_get_os_version)} - _apiEnv=${_apiEnv:-"balena-cloud.com"} - [ -z "${_path}" ] && _path=$(mktemp -d) - - [ -z "${_blocks}" ] && >&2 echo "Blocks are required" && return 1 - for _block in ${_blocks}; do - _block_image=$(balena_api_fetch_image_from_app "${_block}" "${_version}" "${_apiEnv}") - _image_id=$(balena_docker_image_retrieve "${_block_image}") - if [ -z "${_block_image}" ] || [ "${_block_image}" = "" ]; then - >&2 echo "[${_block}] No such image for ${_version} in ${_apiEnv}" - continue - fi - # Container images created by importing a tarball and saving (like the Yocto docker image class does) do not contain labels - # Hence, if no labels are found default to hostapp values, class service, root store and bootable. - _class=$(balena_lib_get_label_from_image "${_image_id}" "${BALENA_HOSTOS_BLOCK_CLASS}") - [ -z "${_class}" ] && class="service" - _store=$(balena_lib_get_label_from_image "${_image_id}" "${BALENA_HOSTOS_BLOCK_STORE}") - [ -z "${_store}" ] && _store="root" - _reboot_required=$(balena_lib_get_label_from_image "${_image_id}" "${BALENA_HOSTOS_BLOCK_REQUIRES_REBOOT}") - [ -z "${_reboot_required}" ] && _reboot_required="1" - _bootable=$(balena_api_is_bootable "${_block}" "${_apiEnv}" "${_token}") - __add_compose_service "${_path}" "${_block}" "${_block_image}" "${_class}" "${_reboot_required}" "${_store}" "${_bootable}" - done - echo "${_path}" -} - -# Deploys a multi-container hostOS -# -# Inputs: -# $1: Application name -# $2: HostOS blocks - required -# $3: Device type for the application -# $4: Release final version -# $5: Balena API environment (default to balena-cloud.com) -# $6: Balena API token (defaults to ~/.balena/token) -# $7: Balena cloud account (defaults to balena_os) -# $8: Bootable 0 or 1 (default) -# -# Outputs: -# None -# -balena_deploy_hostos() { - local _appName="$1" - local _blocks="$2" - local _device_type="$3" - local _final="${4:-no}" - local _apiEnv="$5" - local _token="$6" - local _account="$7" - local _bootable="${8:-1}" - local _path - local _version - - _apiEnv=${_apiEnv:-"$(balena_lib_environment)"} - _account=${_account:-"balena_os"} - _token=${_token:-"$(balena_lib_token "${_apiEnv}")"} - _version=$(balena_lib_get_os_version) - [ -z "${_version}" ] && >&2 echo "Invalid version" && return - [ -z "${_device_type}" ] && >&2 echo "Required device type" && return - _path=$(__create_compose_file "${_device_type}" "${_apiEnv}" "${_version}" "${_token}" "${_blocks}") - if [ ! -f "${_path}/docker-compose.yml" ]; then - >&2 echo "No compose file in ${_path}" - return - fi - balena_deploy_block "${_appName}" "${_device_type}" "${_bootable}" "${_path}" -} diff --git a/automation/include/balena-docker.inc b/automation/include/balena-docker.inc index cb05fe085..8132c78a1 100644 --- a/automation/include/balena-docker.inc +++ b/automation/include/balena-docker.inc @@ -165,3 +165,145 @@ balena_docker_image_retrieve() { export IMAGEID=${_image_id} echo "${_image_id}" } + +# Initialize a compose file in the specified path +# +# Input: +# $1: Path to create the compose file into +__init_compose() { + local _path="${1}" + [ -z "${_path}" ] && return + cat << EOF > "${_path}/docker-compose.yml" +version: '2' +services: +EOF +} + +# Add a compose service +# +# Inputs: +# $1: Path to the directory holding the compose file - will be created if needed +# $2: Name of the service to be added +# $3: Image digest for the service +# $4: Image class: fileset, overlay or service (default) +# $5: Image reboot required: 0 (default) or 1 +# $6: Image engine type: boot, root or data (default) +# $6: Image is bootable, false (default) or true +# +# Outputs: +# Compose file in the specified path +# +__add_compose_service() { + local _path=$1 + local _service_name=$2 + local _image=$3 + local _image_class=$4 + local _image_reboot=$5 + local _image_engine=$6 + local _bootable=$7 + + [ -z "${_path}" ] || [ -z "${_service_name}" ] || [ -z "${_image}" ] && return + _image_class=${_image_class:-"service"} + _image_reboot=${_image_reboot:-0} + _image_engine=${_image_engine:-"data"} + _bootable=${_bootable:-"false"} + + if [ ! -f "${_path}/docker-compose.yml" ]; then + __init_compose "${_path}" + fi + printf " %s:\n" "${_service_name}" >> "${_path}/docker-compose.yml" + # Check if image is local (service_name:version) + if [ "${_image}" != "${_service_name}:*" ]; then + printf " image: %s\n" "${_image}" >> "${_path}/docker-compose.yml" + else + printf " build: %s\n" "${_service_name}" >> "${_path}/docker-compose.yml" + mkdir "${_path}/${_service_name}" + echo "FROM ${_image}" > "${_path}/${_service_name}/Dockerfile" + fi + printf " labels:\n" >> "${_path}/docker-compose.yml" + if [ -n "${_image_class}" ]; then + printf " %s: %s\n" \""${BALENA_HOSTOS_BLOCK_CLASS}"\" \""${_image_class}"\" >> "${_path}/docker-compose.yml" + fi + if [ "${_image_reboot}" = "1" ]; then + printf " %s: '1'\n" \""${BALENA_HOSTOS_BLOCK_REQUIRES_REBOOT}"\" >> "${_path}/docker-compose.yml" + fi + if [ -n "${_image_engine}" ]; then + printf " %s: %s\n" \""${BALENA_HOSTOS_BLOCK_STORE}"\" \""${_image_engine}"\" >> "${_path}/docker-compose.yml" + fi + if [ "${_bootable}" = "true" ]; then + printf " %s: %s\n" \""${BALENA_HOSTOS_BLOCK_BOOTABLE}"\" \""${_bootable}"\" >> "${_path}/docker-compose.yml" + fi +} + +# Creates a compose file +# +# Inputs: +# $1: Device type to build for +# $2: Balena API environment (default to balena-cloud.com) +# $3: BalenaOS version - defaults to current device repository tag +# $4: BalenaOS token +# $5: HostOS blocks - defaults to hostapp block +# $6: Working dir - if not provided uses a temporary directory +# +# Outputs: +# Path where the compose file is created +# +balena_docker_create_compose_file() { + local _device_type="$1" + local _apiEnv="$2" + local _version="$3" + local _token="$4" + local _blocks="${5:-${_device_type}}" + local _work_dir="${6:-$(mktemp -d)}" + local _block_image + local _class + local _store + local _reboot_required + local _block + local _image_id + local _local_image + local _bootable + local _release_path="${_work_dir}/release" + + [ -z "${_device_type}" ] && return + _version=${_version:-$(balena_lib_get_os_version)} + _apiEnv=${_apiEnv:-"balena-cloud.com"} + + mkdir -p "${_release_path}" + [ -z "${_blocks}" ] && >&2 echo "Blocks are required" && return 1 + for _block in ${_blocks}; do + # Try to find a hostapp container image first + _local_image=$(readlink --canonicalize $(find "${_work_dir}" -name "balena-image-${_block}.docker") || true) + if [ -z "${_local_image}" ]; then + # If not, try to find a block container image + _local_image=$(readlink --canonicalize $(find "${_work_dir}" -name "${_block}-${_version}.docker") || true) + fi + if [ -n "${_local_image}" ]; then + _image_id=$(balena_docker_image_retrieve "${_local_image}") + docker tag "${_image_id}" "${_block}:${_version}" + _block_image="${_block}:${_version}" + else + _block_image=$(balena_api_fetch_image_from_app "${_block}" "${_version}" "${_apiEnv}") + if [ -z "${_block_image}" ] || [ "${_block_image}" = "" ]; then + >&2 echo "[${_block}] No such image for ${_version} in ${_apiEnv}" + continue + fi + _image_id=$(balena_docker_image_retrieve "${_block_image}") + fi + if [ -z "${_image_id}" ] || [ "${_image_id}" = "" ]; then + >&2 echo "[${_block}] Could not find image for ${_version} in ${_apiEnv}" + return 1 + fi + # Container images created by importing a tarball and saving (like the Yocto docker image class does) do not contain labels + # Hence, if no labels are found default to hostapp values, class service, root store and bootable. + _class=$(balena_lib_get_label_from_image "${_image_id}" "${BALENA_HOSTOS_BLOCK_CLASS}") + [ -z "${_class}" ] && class="service" + _store=$(balena_lib_get_label_from_image "${_image_id}" "${BALENA_HOSTOS_BLOCK_STORE}") + [ -z "${_store}" ] && _store="root" + _reboot_required=$(balena_lib_get_label_from_image "${_image_id}" "${BALENA_HOSTOS_BLOCK_REQUIRES_REBOOT}") + [ -z "${_reboot_required}" ] && _reboot_required="1" + _bootable=$(balena_api_is_bootable "${_block}" "${_apiEnv}" "${_token}") + __add_compose_service "${_release_path}" "${_block}" "${_block_image}" "${_class}" "${_reboot_required}" "${_store}" "${_bootable}" + done + echo "${_release_path}" +} From 430872d3363cd5e53daa4b6daa4faef9541bdaf5 Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Mon, 13 Jun 2022 20:30:53 +0200 Subject: [PATCH 04/11] balena-deploy: Deploy using compose files Change-type: patch Signed-off-by: Alex Gonzalez --- automation/include/balena-deploy.inc | 52 +++++++++++++--------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/automation/include/balena-deploy.inc b/automation/include/balena-deploy.inc index ce9ef3d3d..60749adc1 100644 --- a/automation/include/balena-deploy.inc +++ b/automation/include/balena-deploy.inc @@ -315,47 +315,44 @@ EOSU ' } -# Builds and deploys the specified block to BalenaCloud +# Builds and deploys to BalenaCloud # Input arguments; # $1: App name to deploy into # $2: Device type for the app -# $3: Bootable flag -# $4: Path to the image to deploy -# $5: Deploy release to S3 and balenaCloud fleet -# $6: Deploy to balenaCloud fleet as draft (default) or final release -# $7: Path to the working directory (defaults to device repository root) -# $8: BalenaCloud admin account (defaults to balena_os) -# $9: BalenaCloud email address (defaults to the balenaCloudEmail enviromental variable) -# $10: BalenaCloud password (defaults to the balenaCloudPassword enviromental variable) -# $11: ESR release flag (default to "ESR" environment variable) +# $3: Deploy to balenaCloud fleet as draft (default) or final release +# $4: Block names to deploy (defaults to hostapp) +# $5: Path to the working directory (defaults to device repository root) +# $6: Bootable flag - defaults to 1 +# $7: BalenaCloud admin account (defaults to balena_os) +# $8: BalenaCloud email address (defaults to the balenaCloudEmail enviromental variable) +# $9: BalenaCloud password (defaults to the balenaCloudPassword enviromental variable) +# $10: ESR release flag (default to "ESR" environment variable) # -balena_deploy_block() { +balena_deploy() { local _appName="$1" local _device_type="${2:-${MACHINE}}" - local _bootable=${3:-"0"} - local _image_path="${4:-""}" - local _deploy="${5:-no}" - local _final="${6:-no}" - local _work_dir="${7:-"${device_dir}"}" - local _balenaos_account="${8:-"balena_os"}" - local _balenaCloudEmail="${9:-"${balenaCloudEmail}"}" - local _balenaCloudPassword="${10:-"${balenaCloudPassword}"}" - local _esr="${11}" - local _discontinued + local _final="${3:-no}" + local _blocks="${4:-"${MACHINE}"}" + local _work_dir="${5:-"${device_dir}"}" + local _bootable=${6:-"1"} + local _balenaos_account="${7:-"balena_os"}" + local _balenaCloudEmail="${8:-"${balenaCloudEmail}"}" + local _balenaCloudPassword="${9:-"${balenaCloudPassword}"}" + local _esr="${10}" local _api_env=$(balena_lib_environment) + local _discontinued [ -z "${_device_type}" ] && echo "Device type is required" && return _discontinued=$(balena_lib_get_dt_state "${_device_type}") if [ "${_discontinued}" = "DISCONTINUED" ]; then - echo "[INFO]: balena_deploy_block: Not deploying discontinued device type." + echo "[INFO]: balena_deploy: Not deploying discontinued device type." return fi [ -z "${_appName}" ] && echo "App name is required" && return - if [ -f "${_image_path}" ]; then - _image_path="$(readlink --canonicalize "${_image_path}")" - fi + [ -z "${_blocks}" ] && echo "Block names are required" && return _esr=${_esr:-"${ESR}"} + [ ! -d "${_work_dir}" ] && mkdir "${_work_dir}" if [ ! -f "${_work_dir}/balena.yml" ]; then if [ -f "${device_dir}/balena.yml" ]; then @@ -373,6 +370,7 @@ balena_deploy_block() { docker run --rm -t \ -e APPNAME="${_appName}" \ -e API_ENV="${_api_env}" \ + -e BLOCKS="${_blocks}" \ -e BALENAOS_TOKEN="$(balena_lib_token "${_api_env}")" \ -e BALENAOS_ACCOUNT="${_balenaos_account}" \ -e META_BALENA_VERSION="$(balena_lib_get_meta_balena_base_version)" \ @@ -380,13 +378,11 @@ balena_deploy_block() { -e MACHINE="${_device_type}" \ -e VERBOSE="${VERBOSE}" \ -e BOOTABLE="${_bootable}" \ - -e DEPLOY="${_deploy}" \ -e FINAL="${_final}" \ -e ESR="${_esr:-"false"}" \ -e balenaCloudEmail="${_balenaCloudEmail}" \ -e balenaCloudPassword="${_balenaCloudPassword}" \ -e ESR="${_esr}" \ - -v "${_image_path}":/host/appimage.docker \ -v "${device_dir}":/work \ -v "${_work_dir}":/deploy \ --privileged \ @@ -395,7 +391,7 @@ balena_deploy_block() { balena_lib_docker_remove_helper_images "balena-push-env" } -# Builds and deploys the specified block to BalenaCloud +# Builds the specified block # Input arguments; # $1: App name to deploy into # $2: Device type for the app From 1ecd77cf659489eddbba323cc542b9ef9e1e1980 Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Mon, 13 Jun 2022 20:36:49 +0200 Subject: [PATCH 05/11] jenkins_build.sh: Deploy using compose file Using compose files allows to deploy both single and multi-container apps. Change-type: patch Signed-off-by: Alex Gonzalez SQ jenkind_build.sh --- automation/jenkins_build.sh | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/automation/jenkins_build.sh b/automation/jenkins_build.sh index 0b4800035..1cf023471 100755 --- a/automation/jenkins_build.sh +++ b/automation/jenkins_build.sh @@ -166,16 +166,8 @@ if [ "$deploy" = "yes" ]; then balena_deploy_to_s3 "$MACHINE" "${ESR}" "${deployTo}" balena_deploy_to_dockerhub "${MACHINE}" - _image_path=$(find "${WORKSPACE}/build/tmp/deploy/" -name "balena-image-${MACHINE}.docker" -type l || true) - if [ -n "${_image_path}" ] && [ -f "${_image_path}" ]; then - balena_deploy_block "$(balena_lib_get_slug "${MACHINE}")" "${MACHINE}" "${_bootable:-1}" "${_image_path}" "${deploy}" "${final}" - else - _state=$(balena_lib_get_dt_state "${MACHINE}") - if [ "${_state}" != "DISCONTINUED" ]; then - echo "[ERROR]:balena_deploy_hostapp: No hostapp to release" - exit 1 - fi - fi + blocks=${BLOCKS:-${MACHINE}} + balena_deploy "$(balena_lib_get_slug "${MACHINE}")" "${blocks}" "${final}" "${MACHINE}" "${WORKSPACE}/build/tmp/deploy/images/${MACHINE}/" if [ "$AMI" = "true" ]; then echo "[INFO] Generating AMI" From c912a1400e64b34ee0ff92e50d0bcd029fdc13a5 Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Wed, 15 Jun 2022 19:10:40 +0200 Subject: [PATCH 06/11] balena-deploy-block.sh: Deploy using compose files Change-type: patch Signed-off-by: Alex Gonzalez --- .../entry_scripts/balena-deploy-block.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/automation/entry_scripts/balena-deploy-block.sh b/automation/entry_scripts/balena-deploy-block.sh index 914bfdff1..bdb6a0ae5 100755 --- a/automation/entry_scripts/balena-deploy-block.sh +++ b/automation/entry_scripts/balena-deploy-block.sh @@ -12,9 +12,6 @@ balena_docker_start "/scratch/docker" "/var/run" "/var/log/docker.log" balena_docker_wait BALENAOS_ACCOUNT="${BALENAOS_ACCOUNT:-"balena_os"}" -if [ -f "/host/appimage.docker" ]; then - _local_image=$(docker load -i /host/appimage.docker | cut -d: -f1 --complement | tr -d " " ) -fi echo "[INFO] Logging into $API_ENV as ${BALENAOS_ACCOUNT}" export BALENARC_BALENA_URL=${API_ENV} @@ -25,22 +22,32 @@ if [ "$ESR" = "true" ]; then APPNAME="${APPNAME}-esr" fi +# Use /deploy folder to generate compose file to use local images that live there +# Use a release dir to limit context +RELEASE_DIR=$(balena_docker_create_compose_file "${MACHINE}" "${API_ENV}" "${RELEASE_VERSION}" "${BALENAOS_TOKEN}" "${BLOCKS}" "/deploy") +if [ ! -f "${RELEASE_DIR}/docker-compose.yml" ]; then + echo "[ERROR] Failed to generate compose file" + exit 1 +fi + if [ -f "/deploy/balena.yml" ]; then - echo -e "\nversion: $(balena_lib_get_os_version)" >> "/deploy/balena.yml" + cp "/deploy/balena.yml" "${RELEASE_DIR}" + echo -e "\nversion: $(balena_lib_get_os_version)" >> "${RELEASE_DIR}/balena.yml" fi echo "[INFO] Deploying to ${BALENAOS_ACCOUNT}/$APPNAME" balena_api_create_public_app "${APPNAME}" "${BALENARC_BALENA_URL}" "${MACHINE}" "${balenaCloudEmail}" "${balenaCloudPassword}" "${ESR}" "${BOOTABLE}" -_releaseID=$(balena_lib_release "${BALENAOS_ACCOUNT}/$APPNAME" "${FINAL}" "/deploy" "${API_ENV}" "$_local_image") +_releaseID=$(balena_lib_release "${BALENAOS_ACCOUNT}/$APPNAME" "${FINAL}" "${RELEASE_DIR}" "${API_ENV}") if [ -z "${_releaseID}" ]; then echo "[INFO] Failed to deploy to ${BALENAOS_ACCOUNT}/$APPNAME" exit 1 fi # Legacy hostapp tagging -if [ "${DEPLOY}" = "yes" ] && [ "${FINAL}" = "yes" ]; then +if [ "${FINAL}" = "yes" ]; then balena_lib_release_finalize "${_releaseID}" "${BALENAOS_ACCOUNT}/${APPNAME}" "${API_ENV}" "${BALENAOS_TOKEN}" "${ESR}" fi balena_docker_stop +rm -rf "${RELEASE_DIR:?}" exit 0 From 80d175ae9da544c2224bbd8649c4f244ea74c6c1 Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Thu, 16 Jun 2022 12:25:09 +0200 Subject: [PATCH 07/11] SQ jenkins_buil-blocks --- automation/jenkins_build-blocks.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/automation/jenkins_build-blocks.sh b/automation/jenkins_build-blocks.sh index b3a66bc05..e129cf918 100755 --- a/automation/jenkins_build-blocks.sh +++ b/automation/jenkins_build-blocks.sh @@ -14,12 +14,13 @@ usage() { Usage: ${script_name} [OPTIONS] -d Device type name -a Balena API environment - -b HostOS block names - -p Deploy as final version + -b OS block names + -p Deploy (defaults to no) + -f Deploy as final version (defaults to draft) -t Balena API token -n Registry namespace -s Shared build directory - -c Balena account (defaults to balena_os) + -c Balena organization (defaults to balena_os) -h Display usage EOF exit 0 @@ -35,7 +36,8 @@ __build_hostos_blocks() { local _blocks="${3}" local _api_env="${4}" local _balenaos_account="${5}" - local _final="${6:-"no"}" + local _final="${6:-no}" + local _deploy="${7:-no}" local _hostos_blocks="" local _appname local _appnames @@ -79,15 +81,15 @@ __build_hostos_blocks() { _packages=$(balena_lib_contract_fetch_composedOf_list "${_block}" "${_device_type}" "${_version}" "sw.package.yocto.${_package_type}") for _block in ${_blocks}; do - local _release_version _appname="${_device_type}-${_block}" balena_build_block "${_appname}" "${_device_type}" "${_packages}" "${_balenaos_account}" "${_api_env}" - _release_version=$(balena_lib_get_os_version) - balena_deploy_block "${_appname}" "${_device_type}" "${_bootable:-0}" "${_image_path:-"${WORKSPACE}/deploy-jenkins/${_appName}-${_release_version}.docker"}" + if [ "${_deploy}" = "yes" ]; then + balena_deploy "${_appname}" "${_device_type}" "${_final:-no}" "${_block}" "${_bootable:-0}" "${_image_path:-"${WORKSPACE}/deploy-jenkins/"}" + fi done # Remove packages folder from deploy directory - rm -rf "${_deploy_dir}/${_package_type}" + rm -rf "${_deploy_dir:?}/${_package_type}" fi echo "${_hostos_blocks}" fi @@ -102,7 +104,8 @@ main() { local _shared_dir local _hostos_blocks local _balenaos_account - local _final + local _final="no" + local _deploy="no" local _esr=false ## Sanity checks if [ ${#} -lt 1 ] ; then @@ -119,7 +122,8 @@ main() { s) _shared_dir="${OPTARG}" ;; c) _balenaos_account="${OPTARG}" ;; e) _esr=true ;; - p) _final="yes";; + p) _deploy="${OPTARG}" ;; + f) _final="${OPTARG}";; h) usage;; *) usage;exit 1;; esac @@ -137,7 +141,7 @@ main() { [ -n "${_namespace}" ] && echo "Setting dockerhub account to ${_namespace}" && export NAMESPACE=${_namespace} _balenaos_account=${_balenaos_account:-balena_os} - _hostos_blocks=$(__build_hostos_blocks "${_device_type}" "${_shared_dir}" "${_blocks}" "${_api_env}" "${_balenaos_account}" "${_final}") + _hostos_blocks=$(__build_hostos_blocks "${_device_type}" "${_shared_dir}" "${_blocks}" "${_api_env}" "${_balenaos_account}" "${_final}" "${_deploy}") fi } From 893e75f648b343ad9afc4c16bf13b64e608f9a26 Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Thu, 16 Jun 2022 19:55:14 +0200 Subject: [PATCH 08/11] balena-lib: Use balena.yml if present Change-type: patch Signed-off-by: Alex Gonzalez --- automation/include/balena-lib.inc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/automation/include/balena-lib.inc b/automation/include/balena-lib.inc index 2d2531bef..d5a44f47f 100644 --- a/automation/include/balena-lib.inc +++ b/automation/include/balena-lib.inc @@ -98,7 +98,10 @@ __find_contract_by_slug() { local _contracts local _slug - _contracts=$(find "${device_dir}/contracts/" -name "contract.json" -print 2>/dev/null) + _contracts=$(find "${device_dir}" -maxdepth 1 -name "balena.yml" -print 2>/dev/null) + if [ -z "${_contracts}" ]; then + _contracts=$(find "${device_dir}/contracts/" -name "contract.json" -print 2>/dev/null) + fi for contract in ${_contracts}; do _slug=$(jq --raw-output .slug < "${contract}") if [ "${_slug}" = "${_tslug}" ]; then From 88a6ce75f8c9dc4a114dcae498bc67dc346ca0e8 Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Sat, 18 Jun 2022 20:01:59 +0200 Subject: [PATCH 09/11] balena-build-block: Allow to specify the deploy dir This allows to use the script outside of the Jenkins context. Change-type: patch Signed-off-by: Alex Gonzalez --- automation/entry_scripts/balena-build-block.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/automation/entry_scripts/balena-build-block.sh b/automation/entry_scripts/balena-build-block.sh index 2d8e81ccc..22f4ba746 100755 --- a/automation/entry_scripts/balena-build-block.sh +++ b/automation/entry_scripts/balena-build-block.sh @@ -12,6 +12,7 @@ source "${script_dir}/balena-lib.inc" [ -z "${PACKAGES}" ] && echo "list of packages to install without dependencies" && exit 1 [ -z "${RELEASE_VERSION}" ] && echo "A release version needs to be defined" && exit 1 [ -z "${WORKSPACE}" ] && echo "Workspace needs to be defined" && exit 1 +DEPLOY_DIR="${DEPLOY_DIR:-"${WORKSPACE}/deploy-jenkins"}" [ -z "${PACKAGE_TYPE}" ] && PACKAGE_TYPE="ipk" @@ -63,10 +64,10 @@ echo "LABEL ${BALENA_HOSTOS_BLOCK_REQUIRES_REBOOT}=1" >> "${TMPDIR}/Dockerfile" echo "LABEL ${BALENA_HOSTOS_BLOCK_STORE}=data" >> "${TMPDIR}/Dockerfile" # Copy local package feed to context if available from previous build step -if [ -d "${WORKSPACE}/deploy-jenkins/${PACKAGE_TYPE}" ]; then +if [ -d "${DEPLOY_DIR}/${PACKAGE_TYPE}" ]; then ARCH_LIST="" mkdir -p "${TMPDIR}/feed" - cp -r "${WORKSPACE}/deploy-jenkins/${PACKAGE_TYPE}" "${TMPDIR}/feed/" + cp -r "${DEPLOY_DIR}/${PACKAGE_TYPE}" "${TMPDIR}/feed/" # Extract package architecture list from feed # Each architecture is one directory while IFS=$'\n' read -r dir; do @@ -75,7 +76,7 @@ if [ -d "${WORKSPACE}/deploy-jenkins/${PACKAGE_TYPE}" ]; then else ARCH_LIST="${ARCH_LIST} ${dir}" fi - done< <(find "${WORKSPACE}/deploy-jenkins/${PACKAGE_TYPE}" -mindepth 1 -maxdepth 1 -type d | xargs -I{} basename {}) + done< <(find "${DEPLOY_DIR}/${PACKAGE_TYPE}" -mindepth 1 -maxdepth 1 -type d | xargs -I{} basename {}) else proto=${FEED_URL%:*} if [ -z "${FEED_URL}" ] || [ "${proto}" = "file" ]; then @@ -91,8 +92,8 @@ docker rmi -f $(docker images --filter "label=${BALENA_HOSTOS_BLOCK_CLASS}" --fo if balena build --logs --nocache --deviceType "${MACHINE}" --arch "${ARCH}" --buildArg PACKAGES="${PACKAGES}" --buildArg ARCH_LIST="${ARCH_LIST}" --buildArg NAMESPACE="${NAMESPACE:-resin}"; then image_id=$(docker images --filter "label=${BALENA_HOSTOS_BLOCK_CLASS}" --format "{{.ID}}") - mkdir -p "${WORKSPACE}/deploy-jenkins" - docker save "${image_id}" > "${WORKSPACE}/deploy-jenkins/${APPNAME}-${RELEASE_VERSION}.docker" + mkdir -p "${DEPLOY_DIR}" + docker save "${image_id}" > "${DEPLOY_DIR}/${APPNAME}-${RELEASE_VERSION}.docker" else echo "[ERROR] Fail to build" exit 1 From 0ab922b434a2db50132446962c98df16a5586b26 Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Tue, 21 Jun 2022 15:44:48 +0200 Subject: [PATCH 10/11] SQ balena-build-block --- automation/entry_scripts/balena-build-block.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/entry_scripts/balena-build-block.sh b/automation/entry_scripts/balena-build-block.sh index 22f4ba746..f6ed9d833 100755 --- a/automation/entry_scripts/balena-build-block.sh +++ b/automation/entry_scripts/balena-build-block.sh @@ -3,8 +3,8 @@ set -e script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -source "${script_dir}/balena-api.inc" -source "${script_dir}/balena-lib.inc" +source "/balena-api.inc" +source "/balena-lib.inc" # Input checks [ -z "${APPNAME}" ] && echo "The block's app name needs to be defined" && exit 1 From 7f9e9787e2489a9214eca59c0bf91e0d622118a2 Mon Sep 17 00:00:00 2001 From: Alex Gonzalez Date: Tue, 21 Jun 2022 20:53:38 +0200 Subject: [PATCH 11/11] balena-*-block: Source includes from / as well as inside device repositories This allows to use these scripts from container images that contain the include files in the container root folder. It simplifies the Dockerfiles for block builder containers. Change-type: patch Signed-off-by: Alex Gonzalez --- automation/entry_scripts/balena-build-block.sh | 10 ++++++++-- automation/entry_scripts/balena-deploy-block.sh | 14 +++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/automation/entry_scripts/balena-build-block.sh b/automation/entry_scripts/balena-build-block.sh index f6ed9d833..b7a66e82b 100755 --- a/automation/entry_scripts/balena-build-block.sh +++ b/automation/entry_scripts/balena-build-block.sh @@ -3,8 +3,14 @@ set -e script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -source "/balena-api.inc" -source "/balena-lib.inc" +if [ -f "/balena-lib.inc" ] && [ -f "/balena-api.inc" ] && [ -f "/balena-docker.inc" ]; then + source "/balena-lib.inc" + source "/balena-api.inc" +else + automation_dir=$( cd "${script_dir}/.." && pwd ) + source "${automation_dir}/include/balena-lib.inc" + source "${automation_dir}/include/balena-api.inc" +fi # Input checks [ -z "${APPNAME}" ] && echo "The block's app name needs to be defined" && exit 1 diff --git a/automation/entry_scripts/balena-deploy-block.sh b/automation/entry_scripts/balena-deploy-block.sh index bdb6a0ae5..7a89c8867 100755 --- a/automation/entry_scripts/balena-deploy-block.sh +++ b/automation/entry_scripts/balena-deploy-block.sh @@ -1,9 +1,17 @@ #!/bin/bash set -e -source /balena-docker.inc -source /balena-lib.inc -source /balena-api.inc +script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +if [ -f "/balena-lib.inc" ] && [ -f "/balena-api.inc" ] && [ -f "/balena-docker.inc" ]; then + source "/balena-lib.inc" + source "/balena-api.inc" + source "/balena-docker.inc" +else + automation_dir=$( cd "${script_dir}/.." && pwd ) + source "${automation_dir}/include/balena-lib.inc" + source "${automation_dir}/include/balena-api.inc" + source "${automation_dir}/include/balena-docker.inc" +fi trap 'balena_docker_stop fail' SIGINT SIGTERM