From 7e934167ad4e2636e78487bccb1d952f874bedc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:47:33 +0000 Subject: [PATCH 01/13] build(deps): bump GoogleCloudPlatform/release-please-action Bumps [GoogleCloudPlatform/release-please-action](https://github.com/googlecloudplatform/release-please-action) from 3.7.13 to 4.0.2. - [Release notes](https://github.com/googlecloudplatform/release-please-action/releases) - [Changelog](https://github.com/google-github-actions/release-please-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/googlecloudplatform/release-please-action/compare/v3.7.13...v4.0.2) --- updated-dependencies: - dependency-name: GoogleCloudPlatform/release-please-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/release-please.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 328a982..feaefa9 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -7,7 +7,7 @@ jobs: release-please: runs-on: ubuntu-latest steps: - - uses: GoogleCloudPlatform/release-please-action@v3.7.13 + - uses: GoogleCloudPlatform/release-please-action@v4.0.2 with: token: ${{ secrets.GITHUB_TOKEN }} release-type: simple From 302d32d590741bc3414369a0da31ae46f5aaa4ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jan 2024 21:53:15 +0000 Subject: [PATCH 02/13] build(deps): bump toshimaru/auto-author-assign from 2.0.1 to 2.1.0 Bumps [toshimaru/auto-author-assign](https://github.com/toshimaru/auto-author-assign) from 2.0.1 to 2.1.0. - [Release notes](https://github.com/toshimaru/auto-author-assign/releases) - [Changelog](https://github.com/toshimaru/auto-author-assign/blob/main/CHANGELOG.md) - [Commits](https://github.com/toshimaru/auto-author-assign/compare/v2.0.1...v2.1.0) --- updated-dependencies: - dependency-name: toshimaru/auto-author-assign dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-assign-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-assign-pr.yml b/.github/workflows/auto-assign-pr.yml index 1f2a66b..7488ae7 100644 --- a/.github/workflows/auto-assign-pr.yml +++ b/.github/workflows/auto-assign-pr.yml @@ -12,4 +12,4 @@ jobs: assign-author: runs-on: ubuntu-latest steps: - - uses: toshimaru/auto-author-assign@v2.0.1 + - uses: toshimaru/auto-author-assign@v2.1.0 From 77f62a5f091d3c9d8b919cdc67d102049585b8a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 21:54:18 +0000 Subject: [PATCH 03/13] build(deps): bump pre-commit/action from 3.0.0 to 3.0.1 Bumps [pre-commit/action](https://github.com/pre-commit/action) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/pre-commit/action/releases) - [Commits](https://github.com/pre-commit/action/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: pre-commit/action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/pre-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 12518d4..3df92c4 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -13,4 +13,4 @@ jobs: - uses: actions/setup-python@v5 with: python-version: 3.11 - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 From e15f6cf2103bec088906439dc4c684e185242808 Mon Sep 17 00:00:00 2001 From: Pierre Slamich Date: Thu, 15 Feb 2024 10:40:53 +0100 Subject: [PATCH 04/13] docs: mention Wikipage (#27) * docs: mention Wikipage * Update README.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index c77ab98..5e97097 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,14 @@ Open Food Facts moderation tool (WIP). +### Wikipage +- https://wiki.openfoodfacts.org/Moderation +- [Meeting minutes](https://docs.google.com/document/d/1B9Ci42kl_jrFt2hi3PiWW9tM9l6B1sI5kQMI9Zd6QS4/edit) + +### Meeting +Valentin and Raphael are working on this tool. They meet weekly. Please ping them on Slack if you'd like to contribute. +We have more general quality meeting every month. + ### Pre-Commit This repo uses [pre-commit](https://pre-commit.com/) to enforce code styling, etc. To use it: ```console From a0b29d21df2250e43c7675be6c9791b78102ed88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 10:45:12 +0100 Subject: [PATCH 05/13] build(deps): bump jinja2 from 3.1.2 to 3.1.3 (#21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.2...3.1.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Raphaël Bournhonesque --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1aff57f..ac11390 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ openfoodfacts==0.1.10 requests==2.31.0 pydantic-settings==2.0.3 sentry-sdk[fastapi]==1.31.0 -jinja2==3.1.2 +jinja2==3.1.3 peewee==3.17.0 psycopg2-binary==2.9.9 \ No newline at end of file From 3dfe6aea84e4df69f658fac61458cd9d4705ec81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bournhonesque?= Date: Thu, 15 Feb 2024 10:45:45 +0100 Subject: [PATCH 06/13] fix: fix flake8 error --- app/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/api.py b/app/api.py index dc293cb..cc5f900 100644 --- a/app/api.py +++ b/app/api.py @@ -82,7 +82,9 @@ class TicketCreate(BaseModel): ) type: IssueType = Field(..., description="Type of the issue") url: str = Field(..., description="URL of the product or of the flagged image") - status: TicketStatus = Field(TicketStatus.open, description="Status of the ticket") + status: TicketStatus = Field( + default=TicketStatus.open, description="Status of the ticket" + ) image_id: str | None = Field( None, description="ID of the flagged image, if the ticket type is `image`", From 464a387e71ec37af7c0446fc4134e70102f1dd51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bournhonesque?= Date: Fri, 16 Feb 2024 13:01:59 +0100 Subject: [PATCH 07/13] feat: make nutripatrol production-ready (#29) - fix container-deploy.yml for staging deployment - add environment info to sentry (dev, staging, prod) - add peewee_migrate to apply create/apply DB migrations. - remove remaining search-a-licious specific config - expose API on port 9010 --- .env | 7 ++- .github/workflows/container-deploy.yml | 43 ++++++++--------- Dockerfile | 1 + Makefile | 41 ++++++++++++++++ app/__main__.py | 4 ++ app/cli.py | 35 ++++++++++++++ app/config.py | 5 ++ app/models.py | 47 ++++++++++++------ app/utils.py | 3 ++ docker-compose.yml | 3 +- docker/dev.yml | 2 + migrations/001_initial.py | 67 ++++++++++++++++++++++++++ requirements.txt | 4 +- 13 files changed, 218 insertions(+), 44 deletions(-) create mode 100644 app/__main__.py create mode 100644 app/cli.py create mode 100644 migrations/001_initial.py diff --git a/.env b/.env index 6a9a3cc..db3cd67 100644 --- a/.env +++ b/.env @@ -5,7 +5,7 @@ COMPOSE_PATH_SEPARATOR=; # dev is default target COMPOSE_FILE=docker-compose.yml;docker/dev.yml -API_PORT=127.0.0.1:8000 +API_EXPOSE=127.0.0.1:8000 # by default on dev desktop, no restart RESTART_POLICY=no @@ -25,4 +25,7 @@ POSTGRES_PASSWORD=postgres # The top-level domain used for Open Food Facts, # it's either `net` (staging) or `org` (production) -OFF_TLD=net \ No newline at end of file +OFF_TLD=net + +# Environment name (mostly used for Sentry): dev, staging, prod +ENVIRONMENT=dev \ No newline at end of file diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index c335525..260d8a9 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -29,15 +29,19 @@ jobs: environment: ${{ matrix.env }} concurrency: ${{ matrix.env }} steps: + - name: Set common variables + run: | + echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV + echo "SSH_USERNAME=off" >> $GITHUB_ENV - name: Set various variable for staging (net) deployment if: matrix.env == 'nutripatrol-net' run: | - # direct container access - echo "OPENFOODFACTS_API_URL=https://off:off@world.openfoodfacts.net" >> $GITHUB_ENV - # deploy target echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV - echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV - echo "SSH_USERNAME=off" >> $GITHUB_ENV + echo "ENVIRONMENT=staging" >> $GITHUB_ENV + if: matrix.env == 'nutripatrol-org' + run: | + echo "SSH_HOST=10.1.0.201" >> $GITHUB_ENV + echo "ENVIRONMENT=prod" >> $GITHUB_ENV - name: Wait for docker image container build workflow uses: tomchv/wait-my-workflow@v1.1.0 id: wait-build @@ -99,32 +103,20 @@ jobs: mv .env .env-dev # init .env - echo "# Env file generated by container-deploy action"> .env + echo "# Env file generated by container-deploy action" > .env # Set Docker Compose variables echo "DOCKER_CLIENT_TIMEOUT=180" >> .env echo "COMPOSE_HTTP_TIMEOUT=180" >> .env echo "COMPOSE_PROJECT_NAME=nutripatrol" >> .env echo "COMPOSE_PATH_SEPARATOR=;" >> .env echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml" >> .env - # Copy variables that are same as dev - grep '\(STACK_VERSION\|ES_PORT\)' .env-dev >> .env # Set docker variables echo "TAG=sha-${{ github.sha }}" >> .env echo "RESTART_POLICY=always" >> .env - # Set App variables - echo "CLUSTER_NAME=${{ matrix.env }}-es-cluster" >> .env - echo "SEARCH_PORT=8180" >> .env - echo "ES_VUE_PORT=8181" >> .env - echo "REDIS_PORT=8182" >> .env - echo "MEM_LIMIT=4294967296" >> .env - # this is the network shared with productopener - echo "COMMON_NET_NAME=po_webnet">> .env - echo "OPENFOODFACTS_API_URL=${{ env.OPENFOODFACTS_API_URL }}" >> .env - # This secret is to be generated using htpasswd, see .env file - # use simple quotes to avoid interpolation of $apr1$ ! - echo 'NGINX_BASIC_AUTH_USER_PASSWD=${{ secrets.NGINX_BASIC_AUTH_USER_PASSWD }}' >> .env echo "SENTRY_DNS=${{ secrets.SENTRY_DSN }}" >> .env - echo "CONFIG_PATH=data/config/openfoodfacts.yml" >> .env + echo "ENVIRONMENT=${{ env.ENVIRONMENT }}" >> .env + # Expose API on port 9010 + echo "API_EXPOSE=0.0.0.0:9010" >> .env - name: Create Docker volumes uses: appleboy/ssh-action@master @@ -152,8 +144,11 @@ jobs: script_stop: false script: | cd ${{ matrix.env }} - docker-compose down - docker-compose up -d --remove-orphans 2>&1 + make pull + # Apply migrations + make migrate-db + # Launch new version + make up - name: Check services are up uses: appleboy/ssh-action@master @@ -183,7 +178,7 @@ jobs: script_stop: false script: | cd ${{ matrix.env }} - docker system prune -af + make prune - uses: frankie567/grafana-annotation-action@v1.0.3 if: ${{ always() }} diff --git a/Dockerfile b/Dockerfile index 0fe6597..97dba93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN groupadd -g $USER_GID off && \ chown off:off -R /opt/nutripatrol /home/off WORKDIR /opt/nutripatrol COPY --chown=off:off requirements.txt requirements.txt +COPY --chown=off:off migrations /opt/nutripatrol/migrations RUN pip install --no-cache-dir --upgrade -r requirements.txt USER off:off diff --git a/Makefile b/Makefile index bfc8c8c..b70a9d5 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,10 @@ build: docker-compose build +# pull images from image repository +pull: + ${DOCKER_COMPOSE} pull + up: ifdef service ${DOCKER_COMPOSE} up -d ${service} 2>&1 @@ -52,3 +56,40 @@ endif down: @echo "🥫 Bringing down containers …" ${DOCKER_COMPOSE} down + + +#-----------# +# Utilities # +#-----------# + +guard-%: # guard clause for targets that require an environment variable (usually used as an argument) + @ if [ "${${*}}" = "" ]; then \ + echo "Environment variable '$*' is mandatory"; \ + echo use "make ${MAKECMDGOALS} $*=you-args"; \ + exit 1; \ + fi; + + +#------------# +# Database # +#------------# + +# apply DB migrations +migrate-db: + ${DOCKER_COMPOSE} run --rm --no-deps api python -m app migrate-db + +# add a new DB revision +add-revision: guard-name + ${DOCKER_COMPOSE} run --rm --no-deps api python -m app add-revision ${name} + + +#---------# +# Cleanup # +#---------# +prune: + @echo "🥫 Pruning unused Docker artifacts (save space) …" + docker system prune -af + +prune_cache: + @echo "🥫 Pruning Docker builder cache …" + docker builder prune -f diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 0000000..c0bcdad --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,4 @@ +from app.cli import main + +if __name__ == "__main__": + main() diff --git a/app/cli.py b/app/cli.py new file mode 100644 index 0000000..5958d30 --- /dev/null +++ b/app/cli.py @@ -0,0 +1,35 @@ +import typer + +app = typer.Typer() + + +@app.command() +def migrate_db(): + """Run unapplied DB migrations.""" + from openfoodfacts.utils import get_logger + + from app.models import db, run_migration + + get_logger() + + with db.connection_context(): + run_migration() + + +@app.command() +def add_revision( + name: str = typer.Argument(..., help="name of the revision"), +): + """Create a new migration file using peewee_migrate.""" + from openfoodfacts.utils import get_logger + + from app.models import add_revision, db + + get_logger() + + with db.connection_context(): + add_revision(name) + + +def main() -> None: + app() diff --git a/app/config.py b/app/config.py index 40a22ce..16b90c9 100644 --- a/app/config.py +++ b/app/config.py @@ -1,8 +1,11 @@ from enum import StrEnum +from pathlib import Path from openfoodfacts import Environment from pydantic_settings import BaseSettings +PROJECT_DIR = Path(__file__).parent.parent + class LoggingLevel(StrEnum): NOTSET = "NOTSET" @@ -36,6 +39,8 @@ class Settings(BaseSettings): postgres_password: str = "postgres" postgres_port: int = 5432 off_tld: Environment = Environment.net + environment: str = "dev" + migration_dir: Path = PROJECT_DIR / "migrations" settings = Settings() diff --git a/app/models.py b/app/models.py index e4426db..81c5f7f 100644 --- a/app/models.py +++ b/app/models.py @@ -5,7 +5,9 @@ ForeignKeyField, Model, PostgresqlDatabase, + TextField, ) +from peewee_migrate import Router from .config import settings @@ -19,12 +21,13 @@ class TicketModel(Model): - barcode = CharField(null=True) - type = CharField() - url = CharField() - status = CharField() + # barcode of the product, if any + barcode = TextField(null=True) + type = CharField(max_length=50) + url = TextField() + status = CharField(max_length=50) image_id = CharField(null=True) - flavor = CharField() + flavor = CharField(max_length=20) created_at = DateTimeField() class Meta: @@ -33,8 +36,8 @@ class Meta: class ModeratorActionModel(Model): - action_type = CharField() - user_id = CharField() + action_type = CharField(max_length=20) + user_id = TextField() ticket = ForeignKeyField(TicketModel, backref="moderator_actions") created_at = DateTimeField() @@ -45,19 +48,33 @@ class Meta: class FlagModel(Model): ticket = ForeignKeyField(TicketModel, backref="flags") - barcode = CharField(null=True) - type = CharField() - url = CharField() - user_id = CharField() - device_id = CharField() + barcode = TextField(null=True) + type = CharField(max_length=50) + url = TextField() + user_id = TextField() + device_id = TextField() source = CharField() confidence = FloatField(null=True) image_id = CharField(null=True) - flavor = CharField() - reason = CharField(null=True) - comment = CharField(max_length=500, null=True) + flavor = CharField(max_length=20) + reason = TextField(null=True) + comment = TextField(null=True) created_at = DateTimeField() class Meta: database = db table_name = "flags" + + +def run_migration(): + """Run all unapplied migrations.""" + # embedding schema does not exist at DB initialization + router = Router(db, migrate_dir=settings.migration_dir) + # Run all unapplied migrations + router.run() + + +def add_revision(name: str): + """Create a migration revision.""" + router = Router(db, migrate_dir=settings.migration_dir) + router.create(name, auto=True) diff --git a/app/utils.py b/app/utils.py index 3062a7a..f6870ca 100644 --- a/app/utils.py +++ b/app/utils.py @@ -4,6 +4,8 @@ from sentry_sdk.integrations import Integration from sentry_sdk.integrations.logging import LoggingIntegration +from app.config import settings + def init_sentry(sentry_dsn: str | None, integrations: list[Integration] | None = None): if sentry_dsn: @@ -17,4 +19,5 @@ def init_sentry(sentry_dsn: str | None, integrations: list[Integration] | None = sentry_sdk.init( # type:ignore # mypy say it's abstract sentry_dsn, integrations=integrations, + environment=settings.environment, ) diff --git a/docker-compose.yml b/docker-compose.yml index 8846ce9..5af2c7e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,6 @@ x-api-common: &api-common - POSTGRES_PASSWORD - POSTGRES_DB - POSTGRES_HOST - - OFF_TLD networks: - default @@ -18,7 +17,7 @@ services: api: <<: *api-common ports: - - "${API_PORT}:8000" + - "${API_EXPOSE:-127.0.0.1:8000}:8000" postgres: restart: $RESTART_POLICY diff --git a/docker/dev.yml b/docker/dev.yml index 2caec6f..95c266a 100644 --- a/docker/dev.yml +++ b/docker/dev.yml @@ -10,6 +10,8 @@ x-api-base: &api-base volumes: # mount code dynamically - "./app:/opt/nutripatrol/app" + # mount migrations dynamically + - "./migrations:/opt/nutripatrol/migrations" services: api: diff --git a/migrations/001_initial.py b/migrations/001_initial.py new file mode 100644 index 0000000..df0ea1e --- /dev/null +++ b/migrations/001_initial.py @@ -0,0 +1,67 @@ +"""Peewee migrations -- 001_initial.py.""" + +import peewee as pw +from peewee_migrate import Migrator + + +def migrate(migrator: Migrator, database: pw.Database, *, fake=False): + """Write your migrations here.""" + + @migrator.create_model + class TicketModel(pw.Model): + id = pw.AutoField() + barcode = pw.TextField(null=True) + type = pw.CharField(max_length=50) + url = pw.TextField() + status = pw.CharField(max_length=50) + image_id = pw.CharField(max_length=255, null=True) + flavor = pw.CharField(max_length=20) + created_at = pw.DateTimeField() + + class Meta: + table_name = "tickets" + + @migrator.create_model + class FlagModel(pw.Model): + id = pw.AutoField() + ticket = pw.ForeignKeyField( + column_name="ticket_id", field="id", model=migrator.orm["tickets"] + ) + barcode = pw.TextField(null=True) + type = pw.CharField(max_length=50) + url = pw.TextField() + user_id = pw.TextField() + device_id = pw.TextField() + source = pw.CharField(max_length=255) + confidence = pw.FloatField(null=True) + image_id = pw.CharField(max_length=255, null=True) + flavor = pw.CharField(max_length=20) + reason = pw.TextField(null=True) + comment = pw.TextField(null=True) + created_at = pw.DateTimeField() + + class Meta: + table_name = "flags" + + @migrator.create_model + class ModeratorActionModel(pw.Model): + id = pw.AutoField() + action_type = pw.CharField(max_length=20) + user_id = pw.TextField() + ticket = pw.ForeignKeyField( + column_name="ticket_id", field="id", model=migrator.orm["tickets"] + ) + created_at = pw.DateTimeField() + + class Meta: + table_name = "moderator_actions" + + +def rollback(migrator: Migrator, database: pw.Database, *, fake=False): + """Write your rollback migrations here.""" + + migrator.remove_model("moderator_actions") + + migrator.remove_model("flags") + + migrator.remove_model("tickets") diff --git a/requirements.txt b/requirements.txt index ac11390..e19123c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,6 @@ pydantic-settings==2.0.3 sentry-sdk[fastapi]==1.31.0 jinja2==3.1.3 peewee==3.17.0 -psycopg2-binary==2.9.9 \ No newline at end of file +peewee-migrate==1.12.2 +psycopg2-binary==2.9.9 +typer==0.9.0 \ No newline at end of file From 89d465d12d5a0dd0b687a1fb3b1504b2c2eb9446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bournhonesque?= Date: Fri, 16 Feb 2024 13:08:02 +0100 Subject: [PATCH 08/13] fix: fix container-deploy.yml config --- .github/workflows/container-deploy.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 260d8a9..80f8ee2 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -38,10 +38,11 @@ jobs: run: | echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV echo "ENVIRONMENT=staging" >> $GITHUB_ENV - if: matrix.env == 'nutripatrol-org' - run: | - echo "SSH_HOST=10.1.0.201" >> $GITHUB_ENV - echo "ENVIRONMENT=prod" >> $GITHUB_ENV + - name: Set various variable for staging (org) deployment + if: matrix.env == 'nutripatrol-org' + run: | + echo "SSH_HOST=10.1.0.201" >> $GITHUB_ENV + echo "ENVIRONMENT=prod" >> $GITHUB_ENV - name: Wait for docker image container build workflow uses: tomchv/wait-my-workflow@v1.1.0 id: wait-build From 64ca6f894db7cec499399ce9f5b9e0c26d8b0853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bournhonesque?= Date: Fri, 16 Feb 2024 13:12:18 +0100 Subject: [PATCH 09/13] chore: setup config for external volumes (Postgres) --- Makefile | 5 +++++ docker-compose.yml | 3 ++- docker/prod.yml | 6 ++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b70a9d5..5af0635 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,11 @@ down: ${DOCKER_COMPOSE} down +# Create all external volumes needed for production. Using external volumes is useful to prevent data loss (as they are not deleted when performing docker down -v) +create_external_volumes: + @echo "🥫 Creating external volumes (production only) …" + docker volume create nutripatrol-postgres-data + #-----------# # Utilities # #-----------# diff --git a/docker-compose.yml b/docker-compose.yml index 5af2c7e..8469ab3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,4 +35,5 @@ services: - "${POSTGRES_EXPOSE:-127.0.0.1:5432}:5432" volumes: - postgres-data: \ No newline at end of file + postgres-data: + name: ${COMPOSE_PROJECT_NAME:-nutripatrol}-postgres-data \ No newline at end of file diff --git a/docker/prod.yml b/docker/prod.yml index c404b8b..316c287 100644 --- a/docker/prod.yml +++ b/docker/prod.yml @@ -1,3 +1,5 @@ -version: "3.7" +version: "3.9" -# modifications to docker-compose for production settings \ No newline at end of file +volumes: + postgres-data: + external: true From 18110b89c135e2b4bff0a00c0d09c0030435e326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bournhonesque?= Date: Fri, 16 Feb 2024 13:17:27 +0100 Subject: [PATCH 10/13] chore: fix postgres config --- .github/workflows/container-deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 80f8ee2..626743e 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -118,6 +118,9 @@ jobs: echo "ENVIRONMENT=${{ env.ENVIRONMENT }}" >> .env # Expose API on port 9010 echo "API_EXPOSE=0.0.0.0:9010" >> .env + echo "POSTGRES_HOST=postgres" >> .env + # Expose PostgreSQL locally on port 5434 + echo "POSTGRES_EXPOSE=127.0.0.1:5434" >> .env - name: Create Docker volumes uses: appleboy/ssh-action@master From c924464b72bbfbdeed939bb7ee7030993cc3b98e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bournhonesque?= Date: Fri, 16 Feb 2024 13:26:13 +0100 Subject: [PATCH 11/13] chore: fix volume name for prod postgres-data --- docker/prod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/prod.yml b/docker/prod.yml index 316c287..87523d7 100644 --- a/docker/prod.yml +++ b/docker/prod.yml @@ -3,3 +3,4 @@ version: "3.9" volumes: postgres-data: external: true + name: ${COMPOSE_PROJECT_NAME:-nutripatrol}-postgres-data From 28aed8d721ad6c9e7a994b6ab591c71c4538c01d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bournhonesque?= Date: Fri, 16 Feb 2024 14:17:52 +0100 Subject: [PATCH 12/13] chore: add postgres password during deployment --- .github/workflows/container-deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 626743e..ee92f01 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -119,6 +119,9 @@ jobs: # Expose API on port 9010 echo "API_EXPOSE=0.0.0.0:9010" >> .env echo "POSTGRES_HOST=postgres" >> .env + echo "POSTGRES_DB=postgres" >> .env + echo "POSTGRES_USER=postgres" >> .env + echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .env # Expose PostgreSQL locally on port 5434 echo "POSTGRES_EXPOSE=127.0.0.1:5434" >> .env From 0a85f3ace2edb66062ef439055fadd4e81cf29c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bournhonesque?= Date: Fri, 16 Feb 2024 15:55:06 +0100 Subject: [PATCH 13/13] chore: enable deployment in prod & fix release-please configuration --- .github/workflows/container-deploy.yml | 11 +++------ .github/workflows/release-please.yml | 34 ++++++++++++++++++-------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index ee92f01..5d75275 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -5,9 +5,8 @@ on: branches: - main - deploy-* -# only staging for now, not prod -# tags: -# - v*.*.* + tags: + - v*.*.* # Note on secrets used for connection @@ -23,9 +22,7 @@ jobs: strategy: matrix: env: - # only stagging for now - # Note: env is also the name of the directory on the server - - nutripatrol-net + - ${{ startsWith(github.ref, 'refs/tags/v') && 'robotoff-org' || 'robotoff-net' }} environment: ${{ matrix.env }} concurrency: ${{ matrix.env }} steps: @@ -193,4 +190,4 @@ jobs: apiHost: https://grafana.openfoodfacts.org apiToken: ${{ secrets.GRAFANA_API_TOKEN }} text: Deployment ${{ steps.livecheck.outcome }} on ${{ matrix.env }} - tags: type:deployment,origin:github,status:${{ steps.livecheck.outcome }},repo:${{ github.repository }},sha:${{ github.sha }},app:robotoff,env:${{ matrix.env }} + tags: type:deployment,origin:github,status:${{ steps.livecheck.outcome }},repo:${{ github.repository }},sha:${{ github.sha }},app:nutripatrol,env:${{ matrix.env }} diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index feaefa9..101b419 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -1,13 +1,27 @@ name: Run release-please on: - push: - branches: - - main + push: + branches: + - main + jobs: - release-please: - runs-on: ubuntu-latest - steps: - - uses: GoogleCloudPlatform/release-please-action@v4.0.2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - release-type: simple + release-please: + runs-on: ubuntu-latest + steps: + - uses: GoogleCloudPlatform/release-please-action@v3.7.13 + with: + # We can't use GITHUB_TOKEN here because, github actions can't provocate actions + # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow + # So this is a personnal access token + token: ${{ secrets.RELEASE_PLEASE_TOKEN }} + release-type: simple + changelog-types: | + [ + {"type":"feat","section":"Features","hidden":false}, + {"type":"fix","section":"Bug Fixes","hidden":false}, + {"type":"style","section":"Technical","hidden":false}, + {"type":"docs","section":"Technical","hidden":false}, + {"type":"test","section":"Technical","hidden":false}, + {"type":"chore","section":"Technical","hidden":false}, + {"type":"refactor","section":"Technical","hidden":false} + ]