Change ubuntu version in tests workflow, fix delivery.yaml (#774)

- Change jobs.tests.runs-on ubuntu version
- Use vm_config.type: linux
- Adjust scripts for a more strict shellcheck version

.github/workflows/tests.yaml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
 
     steps:
     - uses: actions/checkout@v1

delivery.yaml

@@ -8,6 +8,8 @@ pipeline:
     IMAGE: registry-write.opensource.zalan.do/acid/spilo
     PGVERSION: 14
   type: script
+  vm_config:
+    type: linux
   commands:
   - desc: Build spilo docker image
     cmd: |

postgres-appliance/launch.sh

@@ -11,7 +11,7 @@ if [ -f /a.tar.xz ]; then
     fi
 fi
 
-if [ "x$1" = "xinit" ]; then
+if [ "$1" = "init" ]; then
     exec /usr/bin/dumb-init -c --rewrite 1:0 -- /bin/sh /launch.sh
 fi
 

postgres-appliance/scripts/basebackup.sh

@@ -21,13 +21,14 @@ done
 
 if which pg_receivewal &> /dev/null; then
     PG_RECEIVEWAL=pg_receivewal
-    PG_BASEBACKUP_OPTS="-X none"
+    PG_BASEBACKUP_OPTS=(-X none)
 else
     PG_RECEIVEWAL=pg_receivexlog
-    PG_BASEBACKUP_OPTS=""
+    PG_BASEBACKUP_OPTS=()
 fi
 
-readonly WAL_FAST=$(dirname "$DATA_DIR")/wal_fast
+WAL_FAST=$(dirname "$DATA_DIR")/wal_fast
+readonly WAL_FAST
 mkdir -p "$WAL_FAST"
 
 rm -fr "$DATA_DIR" "${WAL_FAST:?}"/*
@@ -96,7 +97,7 @@ fi
 
 ATTEMPT=0
 while [[ $((ATTEMPT++)) -le $RETRIES ]]; do
-    pg_basebackup --pgdata="${DATA_DIR}" ${PG_BASEBACKUP_OPTS} --dbname="${CONNSTR}" &
+    pg_basebackup --pgdata="${DATA_DIR}" "${PG_BASEBACKUP_OPTS[@]}" --dbname="${CONNSTR}" &
     basebackup_pid=$!
     wait $basebackup_pid
     EXITCODE=$?
@@ -108,5 +109,5 @@ while [[ $((ATTEMPT++)) -le $RETRIES ]]; do
     fi
 done
 
-[[ $EXITCODE != 0 && ! -z $receivewal_pid ]] && kill "$receivewal_pid"
+[[ $EXITCODE != 0 && -n $receivewal_pid ]] && kill "$receivewal_pid"
 exit $EXITCODE

postgres-appliance/scripts/patroni_wait.sh

@@ -60,10 +60,10 @@ do
 done
 
 if [ $# -gt 0 ]; then
-    [ ! -z "$TIMEOUT" ] && CUTOFF=$(($(date +%s)+TIMEOUT))
+    [ -n "$TIMEOUT" ] && CUTOFF=$(($(date +%s)+TIMEOUT))
 
     while [ "$(curl -so /dev/null -w '%{http_code}' "http://localhost:8008/$ROLE")" != "200" ]; do
-        [ ! -z "$TIMEOUT" ] && [ $CUTOFF -le "$(date +%s)" ] && exit 2
+        [ -n "$TIMEOUT" ] && [ $CUTOFF -le "$(date +%s)" ] && exit 2
         sleep "$INTERVAL"
     done
 

postgres-appliance/scripts/post_init.sh

@@ -176,14 +176,14 @@ while IFS= read -r db_name; do
     # In case if timescaledb binary is missing the first query fails with the error
     # ERROR:  could not access file "$libdir/timescaledb-$OLD_VERSION": No such file or directory
     UPGRADE_TIMESCALEDB=$(echo -e "SELECT NULL;\nSELECT default_version != installed_version FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1)
-    if [ "x$UPGRADE_TIMESCALEDB" = "xt" ]; then
+    if [ "$UPGRADE_TIMESCALEDB" = "t" ]; then
         echo "ALTER EXTENSION timescaledb UPDATE;"
     fi
     UPGRADE_POSTGIS=$(echo -e "SELECT COUNT(*) FROM pg_catalog.pg_extension WHERE extname = 'postgis'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1)
-    if [ "x$UPGRADE_POSTGIS" = "x1" ]; then
+    if [ "$UPGRADE_POSTGIS" = "1" ]; then
         # public.postgis_lib_version() is available only if postgis extension is created
         UPGRADE_POSTGIS=$(echo -e "SELECT extversion != public.postgis_lib_version() FROM pg_catalog.pg_extension WHERE extname = 'postgis'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1)
-        if [ "x$UPGRADE_POSTGIS" = "xt" ]; then
+        if [ "$UPGRADE_POSTGIS" = "t" ]; then
             echo "ALTER EXTENSION postgis UPDATE;"
             echo "SELECT public.postgis_extensions_upgrade();"
         fi
@@ -200,7 +200,7 @@ GRANT EXECUTE ON FUNCTION public.pg_stat_statements_reset($RESET_ARGS) TO admin;
     else
         echo "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO admin;"
     fi
-    if [ "x$ENABLE_PG_MON" = "xtrue" ] && [ "$PGVER" -ge 11 ]; then echo "CREATE EXTENSION IF NOT EXISTS pg_mon SCHEMA public;"; fi
+    if [ "$ENABLE_PG_MON" = "true" ] && [ "$PGVER" -ge 11 ]; then echo "CREATE EXTENSION IF NOT EXISTS pg_mon SCHEMA public;"; fi
     cat metric_helpers.sql
 done < <(psql -d "$2" -tAc 'select pg_catalog.quote_ident(datname) from pg_catalog.pg_database where datallowconn')
 ) | PGOPTIONS="-c synchronous_commit=local" psql -Xd "$2"

postgres-appliance/scripts/postgres_backup.sh

@@ -13,7 +13,8 @@ log "I was called as: $0 $*"
 readonly PGDATA=$1
 DAYS_TO_RETAIN=$BACKUP_NUM_TO_RETAIN
 
-readonly IN_RECOVERY=$(psql -tXqAc "select pg_is_in_recovery()")
+IN_RECOVERY=$(psql -tXqAc "select pg_is_in_recovery()")
+readonly IN_RECOVERY
 if [[ $IN_RECOVERY == "f" ]]; then
     [[ "$WALG_BACKUP_FROM_REPLICA" == "true" ]] && log "Cluster is not in recovery, not running backup" && exit 0
 elif [[ $IN_RECOVERY == "t" ]]; then
@@ -41,7 +42,8 @@ fi
 BEFORE=""
 LEFT=0
 
-readonly NOW=$(date +%s -u)
+NOW=$(date +%s -u)
+readonly NOW
 while read -r name last_modified rest; do
     last_modified=$(date +%s -ud "$last_modified")
     if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then
@@ -56,7 +58,7 @@ while read -r name last_modified rest; do
 done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d')
 
 # we want keep at least N backups even if the number of days exceeded
-if [ ! -z "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then
+if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then
     if [[ "$USE_WALG_BACKUP" == "true" ]]; then
         $WAL_E delete before FIND_FULL "$BEFORE" --confirm
     else

postgres-appliance/scripts/restore_command.sh

@@ -25,8 +25,10 @@ readonly wal_destination=$2
 
 [[ -z $wal_filename || -z $wal_destination ]] && exit 1
 
-readonly wal_dir=$(dirname "$wal_destination")
-readonly wal_fast_source=$(dirname "$(dirname "$(realpath "$wal_dir")")")/wal_fast/$wal_filename
+wal_dir=$(dirname "$wal_destination")
+readonly wal_dir
+wal_fast_source=$(dirname "$(dirname "$(realpath "$wal_dir")")")/wal_fast/$wal_filename
+readonly wal_fast_source
 
 [[ -f $wal_fast_source ]] && exec mv "${wal_fast_source}" "${wal_destination}"
 

postgres-appliance/scripts/wal-e-wal-fetch.sh

@@ -71,12 +71,12 @@ else
     exit 1
 fi
 
-if [[ ! -z $WALE_S3_ENDPOINT && $WALE_S3_ENDPOINT =~ ^([a-z\+]{2,10}://)?([^:\/?]+) ]]; then
+if [[ -n $WALE_S3_ENDPOINT && $WALE_S3_ENDPOINT =~ ^([a-z\+]{2,10}://)?([^:\/?]+) ]]; then
     S3_HOST=${BASH_REMATCH[2]}
 fi
 
 if [[ -z $AWS_REGION ]]; then
-    if [[ ! -z $WALE_S3_ENDPOINT && $WALE_S3_ENDPOINT =~ ^([a-z\+]{2,10}://)?s3-([^\.]+) ]]; then
+    if [[ -n $WALE_S3_ENDPOINT && $WALE_S3_ENDPOINT =~ ^([a-z\+]{2,10}://)?s3-([^\.]+) ]]; then
         AWS_REGION=${BASH_REMATCH[2]}
     elif [[ "$AWS_INSTANCE_PROFILE" == "true" ]]; then
         load_region_from_aws_instance_profile
@@ -95,7 +95,8 @@ fi
 readonly SERVICE=s3
 readonly REQUEST=aws4_request
 readonly HOST=$BUCKET.$S3_HOST
-readonly TIME=$(date +%Y%m%dT%H%M%SZ)
+TIME=$(date +%Y%m%dT%H%M%SZ)
+readonly TIME
 readonly DATE=${TIME%T*}
 readonly DRSR="$DATE/$AWS_REGION/$SERVICE/$REQUEST"
 readonly EMPTYHASH=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
@@ -105,10 +106,14 @@ function hmac_sha256() {
 }
 
 # Four-step signing key calculation
-readonly DATE_KEY=$(hmac_sha256 key:"AWS4$AWS_SECRET_ACCESS_KEY" "$DATE")
-readonly DATE_REGION_KEY=$(hmac_sha256 "hexkey:$DATE_KEY" "$AWS_REGION")
-readonly DATE_REGION_SERVICE_KEY=$(hmac_sha256 "hexkey:$DATE_REGION_KEY" "$SERVICE")
-readonly SIGNING_KEY=$(hmac_sha256 "hexkey:$DATE_REGION_SERVICE_KEY" "$REQUEST")
+DATE_KEY=$(hmac_sha256 key:"AWS4$AWS_SECRET_ACCESS_KEY" "$DATE")
+readonly DATE_KEY
+DATE_REGION_KEY=$(hmac_sha256 "hexkey:$DATE_KEY" "$AWS_REGION")
+readonly DATE_REGION_KEY
+DATE_REGION_SERVICE_KEY=$(hmac_sha256 "hexkey:$DATE_REGION_KEY" "$SERVICE")
+readonly DATE_REGION_SERVICE_KEY
+SIGNING_KEY=$(hmac_sha256 "hexkey:$DATE_REGION_SERVICE_KEY" "$REQUEST")
+readonly SIGNING_KEY
 
 if [[ -z $AWS_INSTANCE_PROFILE ]]; then
     readonly SIGNED_HEADERS="host;x-amz-content-sha256;x-amz-date"
@@ -156,14 +161,14 @@ function generate_next_segments() {
 
 function clear_except() {
     set +e
-    for dir in $PREFETCHDIR/running/0*; do
+    for dir in "$PREFETCHDIR"/running/0*; do
         item=$(basename "$dir")
         if [[ $item =~ ^[0-9A-F]{24}$ ]]; then
             [[ " ${PREFETCHES[*]} " =~ \ $item\  ]] || rm -fr "$dir"
         fi
     done
 
-    for file in $PREFETCHDIR/0*; do
+    for file in "$PREFETCHDIR"/0*; do
         item=$(basename "$file")
         if [[ $item =~ ^[0-9A-F]{24}$ ]]; then
             [[ " ${PREFETCHES[*]} " =~ \ $item\  ]] || rm -f "$file"
@@ -182,9 +187,11 @@ function try_to_promote_prefetched() {
 
 echo "$$ $SEGMENT"
 
-readonly PREFETCHDIR=$(dirname "$DESTINATION")/.wal-e/prefetch
+PREFETCHDIR=$(dirname "$DESTINATION")/.wal-e/prefetch
+readonly PREFETCHDIR
 if [[ $prefetch -gt 0 && $SEGMENT =~ ^[0-9A-F]{24}$ ]]; then
-    readonly PREFETCHES=($(generate_next_segments "$prefetch"))
+    mapfile -t PREFETCHES < <(generate_next_segments "$prefetch")
+    readonly PREFETCHES
     for segment in "${PREFETCHES[@]}"; do
         running="$PREFETCHDIR/running/$segment"
         [[ -d $running || -f $PREFETCHDIR/$segment ]] && continue

postgres-appliance/scripts/wale_restore.sh

@@ -43,9 +43,9 @@ while true; do
     [[ -z $wal_segment_backup_start ]] && wal_segment_backup_start=$($WAL_E backup-list 2> /dev/null \
         | sed '0,/^name\s*\(last_\)\?modified\s*/d' | sort -bk2 | tail -n1 | awk '{print $3;}' | sed 's/_.*$//')
 
-    [[ ! -z "$CONNSTR" && $server_version == "-1" ]] && server_version=$(psql -d "$CONNSTR" -tAc 'show server_version_num' 2> /dev/null || echo "-1")
+    [[ -n "$CONNSTR" && $server_version == "-1" ]] && server_version=$(psql -d "$CONNSTR" -tAc 'show server_version_num' 2> /dev/null || echo "-1")
 
-    [[ ! -z $wal_segment_backup_start && ( -z "$CONNSTR" || $server_version != "-1") ]] && break
+    [[ -n $wal_segment_backup_start && ( -z "$CONNSTR" || $server_version != "-1") ]] && break
     [[ $((ATTEMPT++)) -ge $RETRIES ]] && break
     sleep 1
 done
@@ -69,7 +69,7 @@ if [[ $server_version != "-1" ]]; then
     while true; do
         [[ -z $diff_in_bytes ]] && diff_in_bytes=$(psql -d "$CONNSTR" -tAc "$query")
         [[ -z $cluster_size ]] && cluster_size=$(psql -d "$CONNSTR" -tAc "SELECT SUM(pg_catalog.pg_database_size(datname)) FROM pg_catalog.pg_database")
-        [[ ! -z $diff_in_bytes && ! -z $cluster_size ]] && break
+        [[ -n $diff_in_bytes && -n $cluster_size ]] && break
         [[ $((ATTEMPT++)) -ge $RETRIES ]] && break
         sleep 1
     done