From f1bc015ca186263861a1a19eebd408a03961627a Mon Sep 17 00:00:00 2001 From: dappnodedev Date: Wed, 22 Nov 2023 11:35:37 +0100 Subject: [PATCH 1/5] Reduce flyway size --- flyway/Dockerfile | 20 +++++++++++--------- flyway/entrypoint.sh | 4 ++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/flyway/Dockerfile b/flyway/Dockerfile index 89373f4..ea19d12 100644 --- a/flyway/Dockerfile +++ b/flyway/Dockerfile @@ -1,24 +1,26 @@ -ARG UPSTREAM_VERSION - ############## # MIGRATIONS # ############## -FROM debian:bullseye-slim as postgres-migrations +FROM alpine:3.15.10 as postgres-migrations ARG UPSTREAM_VERSION WORKDIR /usr/src/app -RUN apt update && apt install -y wget -# Get migrations from consensys web3signer repo -# path is /usr/src/app/web3signer-21.10.0/slashing-protection/src/main/resources/migrations/postgresql + +# Install wget and other dependencies, if necessary +RUN apk --no-cache add wget + +# Get migrations from ConsenSys web3signer repository RUN wget -q https://github.com/ConsenSys/web3signer/archive/refs/tags/${UPSTREAM_VERSION}.tar.gz && \ - tar -xvf ${UPSTREAM_VERSION}.tar.gz + tar -xvf ${UPSTREAM_VERSION}.tar.gz && \ + rm ${UPSTREAM_VERSION}.tar.gz ########## # FLYWAY # ########## FROM flyway/flyway:9.16.1-alpine ARG UPSTREAM_VERSION -RUN apk update && apk add postgresql-client + +RUN apk update && apk --no-cache add postgresql-client COPY entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/entrypoint.sh && rm /flyway/sql/put-your-sql-migrations-here.txt +RUN chmod +x /usr/local/bin/entrypoint.sh && rm -rf /flyway/sql/* COPY --from=postgres-migrations /usr/src/app/web3signer-${UPSTREAM_VERSION}/slashing-protection/src/main/resources/migrations/postgresql/* /flyway/sql/ ENTRYPOINT ["entrypoint.sh"] \ No newline at end of file diff --git a/flyway/entrypoint.sh b/flyway/entrypoint.sh index e54f166..691ec7b 100644 --- a/flyway/entrypoint.sh +++ b/flyway/entrypoint.sh @@ -3,7 +3,7 @@ set -e # Get postgresql database version and trim whitespaces -DATABASE_VERSION=$(PGPASSWORD=gnosis psql --tuples-only -U postgres -h postgres.web3signer-gnosis.dappnode -p 5432 -d web3signer-gnosis -c "SELECT version FROM database_version WHERE id=1;" | awk '{print $1}' | tr -d '[:space:]') +DATABASE_VERSION=$(PGPASSWORD=password psql --tuples-only -U postgres -h postgres.web3signer-gnosis.dappnode -p 5432 -d web3signer -c "SELECT version FROM database_version WHERE id=1;" | awk '{print $1}' | tr -d '[:space:]') # Get the latest migration file version (ending in .sql) and trim whitespaces LATEST_MIGRATION_VERSION=$(ls -1 /flyway/sql/ | grep -E "V[0-9]+__.*.sql$" | tail -n 1 | cut -d'_' -f1 | cut -d'V' -f2 | sed 's/^0*//' | tr -d '[:space:]') @@ -22,7 +22,7 @@ if [ "$DATABASE_VERSION" -ge "$LATEST_MIGRATION_VERSION" ]; then exit 0 else echo "Database version is less than the latest migration file version. Migrating..." - flyway -baselineOnMigrate="true" -baselineVersion="${DATABASE_VERSION}" -url=jdbc:postgresql://postgres.web3signer-gnosis.dappnode:5432/web3signer-gnosis -user=postgres -password=gnosis -connectRetries=60 migrate + flyway -baselineOnMigrate="true" -baselineVersion="${DATABASE_VERSION}" -url=jdbc:postgresql://postgres.web3signer-gnosis.dappnode:5432/web3signer -user=postgres -password=password -connectRetries=60 migrate echo "Migration completed" exit 0 fi From 656e4c7029da5e90ac990b7eef41493613ad5fff Mon Sep 17 00:00:00 2001 From: dappnodedev Date: Wed, 22 Nov 2023 11:35:48 +0100 Subject: [PATCH 2/5] Reduce postgres size --- postgres/Dockerfile | 46 ++++++++++++++++++++-------------------- postgres/rename_files.sh | 15 +++++++++++++ 2 files changed, 38 insertions(+), 23 deletions(-) create mode 100644 postgres/rename_files.sh diff --git a/postgres/Dockerfile b/postgres/Dockerfile index fbe5683..e17134e 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -1,37 +1,37 @@ -ARG UPSTREAM_VERSION - -########## -# FLYWAY # -########## -#FROM flyway/flyway:8.0.5-alpine as binary-flyway -#COPY --from=binary-flyway /flyway/flyway /usr/local/bin/flyway - ############## # MIGRATIONS # ############## -FROM debian:bullseye-slim as postgres-migrations +FROM alpine:3.15.10 as postgres-migrations ARG UPSTREAM_VERSION WORKDIR /usr/src/app -RUN apt update && apt install -y wget -# Get migrations from consensys web3signer repo -# path is /usr/src/app/web3signer-21.10.0/slashing-protection/src/main/resources/migrations/postgresql +# Install wget and other dependencies, if necessary +RUN apk --no-cache add wget + +# Get migrations from ConsenSys web3signer repository RUN wget -q https://github.com/ConsenSys/web3signer/archive/refs/tags/${UPSTREAM_VERSION}.tar.gz && \ - tar -xvf ${UPSTREAM_VERSION}.tar.gz + tar -xvf ${UPSTREAM_VERSION}.tar.gz && \ + rm ${UPSTREAM_VERSION}.tar.gz ############ # POSTGRES # ############ -FROM postgres:14.1-bullseye +FROM postgres:14.1-alpine3.15 + ARG UPSTREAM_VERSION -ENV POSTGRES_PASSWORD=gnosis -ENV PGPASSWORD=gnosis -ENV POSTGRES_USER=postgres -ENV POSTGRES_DB=web3signer-gnosis -RUN apt update && apt install -y rename -COPY --from=postgres-migrations /usr/src/app/web3signer-${UPSTREAM_VERSION}/slashing-protection/src/main/resources/migrations/postgresql/* /docker-entrypoint-initdb.d/ -# Rename scripts to be executed in alfabetical order -RUN rename 's/(\d+)(?=.*\.)/sprintf("%03d",$1)/eg' /docker-entrypoint-initdb.d/* +ENV POSTGRES_PASSWORD=password \ + PGPASSWORD=password \ + POSTGRES_USER=postgres \ + POSTGRES_DB=web3signer \ + INITDB_DIR=/docker-entrypoint-initdb.d/ + +COPY rename_files.sh /usr/local/bin/rename_files.sh + +COPY --from=postgres-migrations /usr/src/app/web3signer-${UPSTREAM_VERSION}/slashing-protection/src/main/resources/migrations/postgresql/* ${INITDB_DIR} + +RUN /usr/local/bin/rename_files.sh + +USER postgres -CMD ["postgres"] +CMD ["postgres"] \ No newline at end of file diff --git a/postgres/rename_files.sh b/postgres/rename_files.sh new file mode 100644 index 0000000..419eac1 --- /dev/null +++ b/postgres/rename_files.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# Loop over each file in the /docker-entrypoint-initdb.d directory +for file in ${INITDB_DIR}*; do + # Extract the base name and directory of the file + dir=$(dirname "$file") + base=$(basename "$file") + + # Use sed to modify the file name + # This will pad the first number in the file name with zeros to make it three digits long + new_base=$(echo "$base" | sed -r 's/([0-9]+)/000\1/g; s/0*([0-9]{3})/\1/g') + + # Move (rename) the file to its new name + mv "$dir/$base" "$dir/$new_base" +done From 178197f94bd210a671efcf3d973f34c71357c46e Mon Sep 17 00:00:00 2001 From: dappnodedev Date: Wed, 22 Nov 2023 11:35:59 +0100 Subject: [PATCH 3/5] Reduce w3s size --- web3signer/Dockerfile | 2 +- web3signer/entrypoint.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/web3signer/Dockerfile b/web3signer/Dockerfile index a65067d..3bffaea 100644 --- a/web3signer/Dockerfile +++ b/web3signer/Dockerfile @@ -6,7 +6,7 @@ ARG UPSTREAM_VERSION FROM consensys/web3signer:$UPSTREAM_VERSION USER root -RUN apt update && apt install ca-certificates unzip --yes +RUN apt update && apt install ca-certificates --yes COPY /security /security COPY entrypoint.sh /usr/bin/entrypoint.sh diff --git a/web3signer/entrypoint.sh b/web3signer/entrypoint.sh index d05aeaa..2f05a41 100755 --- a/web3signer/entrypoint.sh +++ b/web3signer/entrypoint.sh @@ -44,7 +44,7 @@ esac mkdir -p "$KEYFILES_DIR" mkdir -p "/opt/web3signer/manual_migration" -if grep -Fq "/opt/web3signer/keyfiles" ${KEYFILES_DIR}/*.yaml ;then +if grep -Fq "/opt/web3signer/keyfiles" ${KEYFILES_DIR}/*.yaml; then sed -i "s|/opt/web3signer/keyfiles|$KEYFILES_DIR|g" ${KEYFILES_DIR}/*.yaml fi @@ -64,9 +64,9 @@ exec /opt/web3signer/bin/web3signer \ eth2 \ --network=gnosis \ --Xnetwork-capella-fork-epoch=648704 \ - --slashing-protection-db-url=jdbc:postgresql://postgres.web3signer-gnosis.dappnode:5432/web3signer-gnosis \ + --slashing-protection-db-url=jdbc:postgresql://postgres.web3signer-gnosis.dappnode:5432/web3signer \ --slashing-protection-db-username=postgres \ - --slashing-protection-db-password=gnosis \ + --slashing-protection-db-password=password \ --slashing-protection-pruning-enabled=true \ --slashing-protection-pruning-epochs-to-keep=500 \ --key-manager-api-enabled=true \ From 0fb51c409b3dc2ebd11675e4e7011dfcfa5068f8 Mon Sep 17 00:00:00 2001 From: dappnodedev Date: Wed, 22 Nov 2023 11:56:30 +0100 Subject: [PATCH 4/5] Give exec permission to script --- postgres/rename_files.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 postgres/rename_files.sh diff --git a/postgres/rename_files.sh b/postgres/rename_files.sh old mode 100644 new mode 100755 From 030370a357a291ca522cf6072050b348d070f894 Mon Sep 17 00:00:00 2001 From: dappnodedev Date: Wed, 22 Nov 2023 12:05:43 +0100 Subject: [PATCH 5/5] Recover previous pass --- flyway/entrypoint.sh | 4 ++-- postgres/Dockerfile | 6 +++--- web3signer/entrypoint.sh | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flyway/entrypoint.sh b/flyway/entrypoint.sh index 691ec7b..e54f166 100644 --- a/flyway/entrypoint.sh +++ b/flyway/entrypoint.sh @@ -3,7 +3,7 @@ set -e # Get postgresql database version and trim whitespaces -DATABASE_VERSION=$(PGPASSWORD=password psql --tuples-only -U postgres -h postgres.web3signer-gnosis.dappnode -p 5432 -d web3signer -c "SELECT version FROM database_version WHERE id=1;" | awk '{print $1}' | tr -d '[:space:]') +DATABASE_VERSION=$(PGPASSWORD=gnosis psql --tuples-only -U postgres -h postgres.web3signer-gnosis.dappnode -p 5432 -d web3signer-gnosis -c "SELECT version FROM database_version WHERE id=1;" | awk '{print $1}' | tr -d '[:space:]') # Get the latest migration file version (ending in .sql) and trim whitespaces LATEST_MIGRATION_VERSION=$(ls -1 /flyway/sql/ | grep -E "V[0-9]+__.*.sql$" | tail -n 1 | cut -d'_' -f1 | cut -d'V' -f2 | sed 's/^0*//' | tr -d '[:space:]') @@ -22,7 +22,7 @@ if [ "$DATABASE_VERSION" -ge "$LATEST_MIGRATION_VERSION" ]; then exit 0 else echo "Database version is less than the latest migration file version. Migrating..." - flyway -baselineOnMigrate="true" -baselineVersion="${DATABASE_VERSION}" -url=jdbc:postgresql://postgres.web3signer-gnosis.dappnode:5432/web3signer -user=postgres -password=password -connectRetries=60 migrate + flyway -baselineOnMigrate="true" -baselineVersion="${DATABASE_VERSION}" -url=jdbc:postgresql://postgres.web3signer-gnosis.dappnode:5432/web3signer-gnosis -user=postgres -password=gnosis -connectRetries=60 migrate echo "Migration completed" exit 0 fi diff --git a/postgres/Dockerfile b/postgres/Dockerfile index e17134e..f5e7012 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -20,10 +20,10 @@ FROM postgres:14.1-alpine3.15 ARG UPSTREAM_VERSION -ENV POSTGRES_PASSWORD=password \ - PGPASSWORD=password \ +ENV POSTGRES_PASSWORD=gnosis \ + PGPASSWORD=gnosis \ POSTGRES_USER=postgres \ - POSTGRES_DB=web3signer \ + POSTGRES_DB=web3signer-gnosis \ INITDB_DIR=/docker-entrypoint-initdb.d/ COPY rename_files.sh /usr/local/bin/rename_files.sh diff --git a/web3signer/entrypoint.sh b/web3signer/entrypoint.sh index 2f05a41..7f34c09 100755 --- a/web3signer/entrypoint.sh +++ b/web3signer/entrypoint.sh @@ -64,9 +64,9 @@ exec /opt/web3signer/bin/web3signer \ eth2 \ --network=gnosis \ --Xnetwork-capella-fork-epoch=648704 \ - --slashing-protection-db-url=jdbc:postgresql://postgres.web3signer-gnosis.dappnode:5432/web3signer \ + --slashing-protection-db-url=jdbc:postgresql://postgres.web3signer-gnosis.dappnode:5432/web3signer-gnosis \ --slashing-protection-db-username=postgres \ - --slashing-protection-db-password=password \ + --slashing-protection-db-password=gnosis \ --slashing-protection-pruning-enabled=true \ --slashing-protection-pruning-epochs-to-keep=500 \ --key-manager-api-enabled=true \