From 969c8b4c8919c3a98a57242881dbdc9ebd0d6976 Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Fri, 12 Apr 2024 16:07:12 +0200 Subject: [PATCH 1/5] ci: new prod deploy --- .github/workflows/container-deploy.yml | 28 ++++++++++++++------------ Makefile | 4 ++-- README.md | 2 +- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 77dc927..0c64a77 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -16,17 +16,17 @@ jobs: env: # FIXME name you envs, see reuse below # it's also the name of the directory for the application - - ${{ startsWith(github.ref, 'refs/tags/v') && 'off-query-org' || 'off-query-net' }} + # FIXME - REMOVE BEFORE MERGING !!! + - 'off-query-new-org' + #- ${{ startsWith(github.ref, 'refs/tags/v') && 'off-query-org' || 'off-query-net' }} environment: ${{ matrix.env }} concurrency: ${{ matrix.env }} steps: - - name: Set common variables - run: | - echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV - echo "SSH_USERNAME=off" >> $GITHUB_ENV - name: Set various variable for staging deployment if: matrix.env == 'off-query-net' run: | + echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV + echo "SSH_USERNAME=off" >> $GITHUB_ENV # deploy target echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV # configurations @@ -34,16 +34,18 @@ jobs: echo "MONGO_URI=mongodb://10.1.0.200:27017" >> $GITHUB_ENV echo "REDIS_URL=redis://redis:6379" >> $GITHUB_ENV - name: Set various variable for production deployment - if: matrix.env == 'off-query-org' + # FIXME - CHANGE BEFORE MERGE + if: matrix.env == 'off-query-new-org' run: | + echo "SSH_PROXY_HOST=off1.openfoodfacts.org" >> $GITHUB_ENV + echo "SSH_USERNAME=off" >> $GITHUB_ENV # deploy target - echo "SSH_HOST=10.1.0.201" >> $GITHUB_ENV + echo "SSH_HOST=10.1.0.115" >> $GITHUB_ENV # configurations echo "COMMON_NET_NAME=" >> $GITHUB_ENV # mongodb and redis (through stunnel) - echo "MONGO_URI=mongodb://10.1.0.113:27017" >> $GITHUB_ENV - echo "REDIS_URL=redis://10.1.0.113:6379" >> $GITHUB_ENV - + echo "MONGO_URI=mongodb://10.1.0.102:27017" >> $GITHUB_ENV + echo "REDIS_URL=redis://10.1.0.122:6379" >> $GITHUB_ENV - name: Wait for container build workflow uses: tomchv/wait-my-workflow@v1.1.0 id: wait-build @@ -152,7 +154,7 @@ jobs: script_stop: false script: | cd ${{ matrix.env }} - docker-compose up -d 2>&1 + docker compose up -d 2>&1 - name: Check services are up uses: appleboy/ssh-action@master @@ -169,8 +171,8 @@ jobs: script: | cd ${{ matrix.env }} exit_code=0 - for service in `docker-compose config --service | tr '\n' ' '`; do - if [ -z `docker-compose ps -q $service` ] || [ -z `docker ps -q --no-trunc | grep $(docker-compose ${{ env.compose_args }} ps -q $service)` ]; then + for service in `docker compose config --service | tr '\n' ' '`; do + if [ -z `docker compose ps -q $service` ] || [ -z `docker ps -q --no-trunc | grep $(docker compose ${{ env.compose_args }} ps -q $service)` ]; then echo "$service: DOWN" exit_code=1 else diff --git a/Makefile b/Makefile index 4f1e98d..9a01849 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ # Use this to start both the query service and associated database in Docker up: - docker-compose up -d --build + docker compose up -d --build # This task starts a Postgres database in Docker and then prepares the local environment for development dev: - docker-compose up -d query_postgres + docker compose up -d query_postgres npm install npm run migration:up diff --git a/README.md b/README.md index 0525597..ab990df 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ To get started... Run the following: ``` -docker-compose up -d query_postgres +docker compose up -d query_postgres ``` ### Use an existing Postgres database From 36f44c54be7806ea2b0d77190e8d4bdfb15c16cb Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Fri, 12 Apr 2024 16:24:46 +0200 Subject: [PATCH 2/5] ci: add create_external_networks --- .github/workflows/container-deploy.yml | 31 +++++++++++++------------- Makefile | 24 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 0c64a77..c9b1b31 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -126,21 +126,22 @@ jobs: echo "REDIS_URL=${{ env.REDIS_URL }}" >> .env echo "LOG_LEVEL=log" >> .env - # Not worried about this at the moment as data is a cache - - # - name: Create external Docker volumes - # uses: appleboy/ssh-action@master - # with: - # host: ${{ env.SSH_HOST }} - # username: ${{ env.SSH_USERNAME }} - # key: ${{ secrets.SSH_PRIVATE_KEY }} - # proxy_host: ${{ env.SSH_PROXY_HOST }} - # proxy_username: ${{ env.SSH_USERNAME }} - # proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} - # script_stop: false - # script: | - # cd ${{ matrix.env }} - # docker volume create + + - name: Create external Docker networks and volumes + uses: appleboy/ssh-action@master + with: + host: ${{ env.SSH_HOST }} + username: ${{ env.SSH_USERNAME }} + key: ${{ secrets.SSH_PRIVATE_KEY }} + proxy_host: ${{ env.SSH_PROXY_HOST }} + proxy_username: ${{ env.SSH_USERNAME }} + proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} + script_stop: false + script: | + cd ${{ matrix.env }} + make create_external_volumes && \ + make create_external_networks + - name: Start services uses: appleboy/ssh-action@master diff --git a/Makefile b/Makefile index 9a01849..e9fc7be 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,18 @@ +#!/usr/bin/make + +# use bash everywhere ! +SHELL := /bin/bash +# some vars +ENV_FILE ?= .env + +# load env variables to be able to use them in this file +# also takes into account envrc (direnv file) +ifneq (,$(wildcard ./${ENV_FILE})) + -include ${ENV_FILE} + -include .envrc + export +endif + # Use this to start both the query service and associated database in Docker up: docker compose up -d --build @@ -13,3 +28,12 @@ tests: lint: npm run lint + +# PRODUCTION +create_external_volumes: + @echo "🎣 No external volumes (it's all cache !)" + +create_external_networks: + @echo "🎣 Creating external networks (production only) …" + docker network create --driver=bridge --subnet="172.30.0.0/16" ${COMMON_NET_NAME} \ + || echo "network already exists" From 1652de0c961b44ca664bd0121363fec03d15bfa0 Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Fri, 12 Apr 2024 16:41:53 +0200 Subject: [PATCH 3/5] ci: add create_external_networks --- .github/workflows/container-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index c9b1b31..60d99c0 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -42,7 +42,7 @@ jobs: # deploy target echo "SSH_HOST=10.1.0.115" >> $GITHUB_ENV # configurations - echo "COMMON_NET_NAME=" >> $GITHUB_ENV + echo "COMMON_NET_NAME=common_net" >> $GITHUB_ENV # mongodb and redis (through stunnel) echo "MONGO_URI=mongodb://10.1.0.102:27017" >> $GITHUB_ENV echo "REDIS_URL=redis://10.1.0.122:6379" >> $GITHUB_ENV From c6f606026a1bc180ada6ff72db3b92067cd4b859 Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Fri, 12 Apr 2024 16:47:33 +0200 Subject: [PATCH 4/5] ci: deploy to right directory --- .github/workflows/container-deploy.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 60d99c0..a52a529 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -22,6 +22,11 @@ jobs: environment: ${{ matrix.env }} concurrency: ${{ matrix.env }} steps: + - name: Set various common variable for deployment + run: | + echo "PROJECT_DIR=${{ matrix.env }}" >> $GITHUB_ENV + + - name: Set various variable for staging deployment if: matrix.env == 'off-query-net' run: | @@ -37,6 +42,8 @@ jobs: # FIXME - CHANGE BEFORE MERGE if: matrix.env == 'off-query-new-org' run: | + # FIXME - CHANGE BEFORE MERGE + echo "PROJECT_DIR=off-query-org" >> $GITHUB_ENV echo "SSH_PROXY_HOST=off1.openfoodfacts.org" >> $GITHUB_ENV echo "SSH_USERNAME=off" >> $GITHUB_ENV # deploy target @@ -80,10 +87,10 @@ jobs: script_stop: false script: | # Clone Git repository if not already there - [ ! -d '${{ matrix.env }}' ] && git clone --depth 1 https://github.com/${{ github.repository }} ${{ matrix.env }} --no-single-branch 2>&1 + [ ! -d '${{ env.PROJECT_DIR }}' ] && git clone --depth 1 https://github.com/${{ github.repository }} ${{ env.PROJECT_DIR }} --no-single-branch 2>&1 # Go to repository directory - cd ${{ matrix.env }} + cd ${{ env.PROJECT_DIR }} # Fetch newest commits (in case it wasn't freshly cloned) git fetch --depth 1 @@ -103,7 +110,7 @@ jobs: script_stop: false script: | # Go to repository directory - cd ${{ matrix.env }} + cd ${{ env.PROJECT_DIR }} # new env rm .env @@ -138,7 +145,7 @@ jobs: proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} script_stop: false script: | - cd ${{ matrix.env }} + cd ${{ env.PROJECT_DIR }} make create_external_volumes && \ make create_external_networks @@ -154,7 +161,7 @@ jobs: proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} script_stop: false script: | - cd ${{ matrix.env }} + cd ${{ env.PROJECT_DIR }} docker compose up -d 2>&1 - name: Check services are up @@ -170,7 +177,7 @@ jobs: proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} script_stop: false script: | - cd ${{ matrix.env }} + cd ${{ env.PROJECT_DIR }} exit_code=0 for service in `docker compose config --service | tr '\n' ' '`; do if [ -z `docker compose ps -q $service` ] || [ -z `docker ps -q --no-trunc | grep $(docker compose ${{ env.compose_args }} ps -q $service)` ]; then @@ -194,5 +201,5 @@ jobs: proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} script_stop: false script: | - cd ${{ matrix.env }} + cd ${{ env.PROJECT_DIR }} docker system prune -af From b68124bbbf1ecfa8e668b229235f2264ea2a9fc1 Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Tue, 16 Apr 2024 18:28:17 +0200 Subject: [PATCH 5/5] =?UTF-8?q?ci:=C2=A0finalize=20new=20prod=20deploy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/container-deploy.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index a52a529..bf173ac 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -16,17 +16,13 @@ jobs: env: # FIXME name you envs, see reuse below # it's also the name of the directory for the application - # FIXME - REMOVE BEFORE MERGING !!! - - 'off-query-new-org' - #- ${{ startsWith(github.ref, 'refs/tags/v') && 'off-query-org' || 'off-query-net' }} + - ${{ startsWith(github.ref, 'refs/tags/v') && 'off-query-org' || 'off-query-net' }} environment: ${{ matrix.env }} concurrency: ${{ matrix.env }} steps: - name: Set various common variable for deployment run: | echo "PROJECT_DIR=${{ matrix.env }}" >> $GITHUB_ENV - - - name: Set various variable for staging deployment if: matrix.env == 'off-query-net' run: | @@ -39,11 +35,8 @@ jobs: echo "MONGO_URI=mongodb://10.1.0.200:27017" >> $GITHUB_ENV echo "REDIS_URL=redis://redis:6379" >> $GITHUB_ENV - name: Set various variable for production deployment - # FIXME - CHANGE BEFORE MERGE - if: matrix.env == 'off-query-new-org' + if: matrix.env == 'off-query-org' run: | - # FIXME - CHANGE BEFORE MERGE - echo "PROJECT_DIR=off-query-org" >> $GITHUB_ENV echo "SSH_PROXY_HOST=off1.openfoodfacts.org" >> $GITHUB_ENV echo "SSH_USERNAME=off" >> $GITHUB_ENV # deploy target