Merge branch 'main' into dependabot/pip/fastapi-0.109.1

.env

@@ -5,7 +5,7 @@ COMPOSE_PATH_SEPARATOR=;
 # dev is default target
 COMPOSE_FILE=docker-compose.yml;docker/dev.yml
 
-API_PORT=127.0.0.1:8000
+API_EXPOSE=127.0.0.1:8000
 
 # by default on dev desktop, no restart
 RESTART_POLICY=no
@@ -25,4 +25,7 @@ POSTGRES_PASSWORD=postgres
 
 # The top-level domain used for Open Food Facts,
 # it's either `net` (staging) or `org` (production) 
-OFF_TLD=net
+OFF_TLD=net
+
+# Environment name (mostly used for Sentry): dev, staging, prod
+ENVIRONMENT=dev

.github/workflows/container-deploy.yml

@@ -5,9 +5,8 @@ on:
     branches:
       - main
       - deploy-*
-# only staging for now, not prod
-#     tags:
-#       - v*.*.*
+    tags:
+      - v*.*.*
 
 
 # Note on secrets used for connection
@@ -23,21 +22,24 @@ jobs:
     strategy:
       matrix:
         env:
-          # only stagging for now
-          # Note: env is also the name of the directory on the server
-          - nutripatrol-net
+          - ${{ startsWith(github.ref, 'refs/tags/v') && 'robotoff-org' || 'robotoff-net' }}
     environment: ${{ matrix.env }}
     concurrency: ${{ matrix.env }}
     steps:
+    - name: Set common variables
+      run: |
+        echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
+        echo "SSH_USERNAME=off" >> $GITHUB_ENV
     - name: Set various variable for staging (net) deployment
       if: matrix.env == 'nutripatrol-net'
       run: |
-        # direct container access
-        echo "OPENFOODFACTS_API_URL=https://off:[email protected]" >> $GITHUB_ENV
-        # deploy target
         echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV
-        echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
-        echo "SSH_USERNAME=off" >> $GITHUB_ENV
+        echo "ENVIRONMENT=staging" >> $GITHUB_ENV
+    - name: Set various variable for staging (org) deployment
+      if: matrix.env == 'nutripatrol-org'
+      run: |
+        echo "SSH_HOST=10.1.0.201" >> $GITHUB_ENV
+        echo "ENVIRONMENT=prod" >> $GITHUB_ENV
     - name: Wait for docker image container build workflow
       uses: tomchv/[email protected]
       id: wait-build
@@ -99,32 +101,26 @@ jobs:
 
           mv .env .env-dev
           # init .env
-          echo "# Env file generated by container-deploy action"> .env
+          echo "# Env file generated by container-deploy action" > .env
           # Set Docker Compose variables
           echo "DOCKER_CLIENT_TIMEOUT=180" >> .env
           echo "COMPOSE_HTTP_TIMEOUT=180" >> .env
           echo "COMPOSE_PROJECT_NAME=nutripatrol" >> .env
           echo "COMPOSE_PATH_SEPARATOR=;" >> .env
           echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml" >> .env
-          # Copy variables that are same as dev
-          grep '\(STACK_VERSION\|ES_PORT\)' .env-dev >> .env
           # Set docker variables
           echo "TAG=sha-${{ github.sha }}" >> .env
           echo "RESTART_POLICY=always" >> .env
-          # Set App variables
-          echo "CLUSTER_NAME=${{ matrix.env }}-es-cluster" >> .env
-          echo "SEARCH_PORT=8180" >> .env
-          echo "ES_VUE_PORT=8181" >> .env
-          echo "REDIS_PORT=8182" >> .env
-          echo "MEM_LIMIT=4294967296" >> .env
-          # this is the network shared with productopener
-          echo "COMMON_NET_NAME=po_webnet">> .env
-          echo "OPENFOODFACTS_API_URL=${{ env.OPENFOODFACTS_API_URL }}" >> .env
-          # This secret is to be generated using htpasswd, see .env file
-          # use simple quotes to avoid interpolation of $apr1$ !
-          echo 'NGINX_BASIC_AUTH_USER_PASSWD=${{ secrets.NGINX_BASIC_AUTH_USER_PASSWD }}' >> .env
           echo "SENTRY_DNS=${{ secrets.SENTRY_DSN }}" >> .env
-          echo "CONFIG_PATH=data/config/openfoodfacts.yml" >> .env
+          echo "ENVIRONMENT=${{ env.ENVIRONMENT }}" >> .env
+          # Expose API on port 9010
+          echo "API_EXPOSE=0.0.0.0:9010" >> .env
+          echo "POSTGRES_HOST=postgres" >> .env
+          echo "POSTGRES_DB=postgres" >> .env
+          echo "POSTGRES_USER=postgres" >> .env
+          echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .env
+          # Expose PostgreSQL locally on port 5434
+          echo "POSTGRES_EXPOSE=127.0.0.1:5434" >> .env
 
     - name: Create Docker volumes
       uses: appleboy/ssh-action@master
@@ -152,8 +148,11 @@ jobs:
         script_stop: false
         script: |
           cd ${{ matrix.env }}
-          docker-compose down
-          docker-compose up -d --remove-orphans 2>&1
+          make pull
+          # Apply migrations
+          make migrate-db
+          # Launch new version
+          make up
 
     - name: Check services are up
       uses: appleboy/ssh-action@master
@@ -183,12 +182,12 @@ jobs:
         script_stop: false
         script: |
           cd ${{ matrix.env }}
-          docker system prune -af
+          make prune
 
     - uses: frankie567/[email protected]
       if: ${{ always() }}
       with:
         apiHost: https://grafana.openfoodfacts.org
         apiToken: ${{ secrets.GRAFANA_API_TOKEN }}
         text: <a href="https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}">Deployment ${{ steps.livecheck.outcome }} on ${{ matrix.env }}</a>
-        tags: type:deployment,origin:github,status:${{ steps.livecheck.outcome }},repo:${{ github.repository }},sha:${{ github.sha }},app:robotoff,env:${{ matrix.env }}
+        tags: type:deployment,origin:github,status:${{ steps.livecheck.outcome }},repo:${{ github.repository }},sha:${{ github.sha }},app:nutripatrol,env:${{ matrix.env }}

.github/workflows/release-please.yml

@@ -1,13 +1,27 @@
 name: Run release-please
 on:
-   push:
-     branches:
-       - main
+  push:
+    branches:
+      - main
+
 jobs:
-   release-please:
-     runs-on: ubuntu-latest
-     steps:
-       - uses: GoogleCloudPlatform/[email protected]
-         with:
-           token: ${{ secrets.GITHUB_TOKEN }}
-           release-type: simple
+  release-please:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: GoogleCloudPlatform/[email protected]
+        with:
+          # We can't use GITHUB_TOKEN here because, github actions can't provocate actions
+          # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow
+          # So this is a personnal access token
+          token: ${{ secrets.RELEASE_PLEASE_TOKEN }}
+          release-type: simple
+          changelog-types: |
+            [
+              {"type":"feat","section":"Features","hidden":false},
+              {"type":"fix","section":"Bug Fixes","hidden":false},
+              {"type":"style","section":"Technical","hidden":false},
+              {"type":"docs","section":"Technical","hidden":false},
+              {"type":"test","section":"Technical","hidden":false},
+              {"type":"chore","section":"Technical","hidden":false},
+              {"type":"refactor","section":"Technical","hidden":false}
+            ]

Dockerfile

@@ -15,6 +15,7 @@ RUN groupadd -g $USER_GID off && \
     chown off:off -R /opt/nutripatrol /home/off
 WORKDIR /opt/nutripatrol
 COPY --chown=off:off requirements.txt requirements.txt
+COPY --chown=off:off migrations /opt/nutripatrol/migrations
 RUN  pip install --no-cache-dir --upgrade -r requirements.txt
 
 USER off:off

Makefile

@@ -41,6 +41,10 @@ build:
 	docker-compose build
 
 
+# pull images from image repository
+pull:
+	${DOCKER_COMPOSE} pull
+
 up:
 ifdef service
 	${DOCKER_COMPOSE} up -d ${service} 2>&1
@@ -52,3 +56,45 @@ endif
 down:
 	@echo "🥫 Bringing down containers …"
 	${DOCKER_COMPOSE} down
+
+
+# Create all external volumes needed for production. Using external volumes is useful to prevent data loss (as they are not deleted when performing docker down -v)
+create_external_volumes:
+	@echo "🥫 Creating external volumes (production only) …"
+	docker volume create nutripatrol-postgres-data
+
+#-----------#
+# Utilities #
+#-----------#
+
+guard-%: # guard clause for targets that require an environment variable (usually used as an argument)
+	@ if [ "${${*}}" = "" ]; then \
+   		echo "Environment variable '$*' is mandatory"; \
+   		echo use "make ${MAKECMDGOALS} $*=you-args"; \
+   		exit 1; \
+	fi;
+
+
+#------------#
+#  Database  #
+#------------#
+
+# apply DB migrations
+migrate-db:
+	${DOCKER_COMPOSE} run --rm --no-deps api python -m app migrate-db
+
+# add a new DB revision
+add-revision: guard-name
+	${DOCKER_COMPOSE} run --rm --no-deps api python -m app add-revision ${name}
+
+
+#---------#
+# Cleanup #
+#---------#
+prune:
+	@echo "🥫 Pruning unused Docker artifacts (save space) …"
+	docker system prune -af
+
+prune_cache:
+	@echo "🥫 Pruning Docker builder cache …"
+	docker builder prune -f

app/__main__.py

@@ -0,0 +1,4 @@
+from app.cli import main
+
+if __name__ == "__main__":
+    main()

app/api.py

@@ -82,7 +82,9 @@ class TicketCreate(BaseModel):
     )
     type: IssueType = Field(..., description="Type of the issue")
     url: str = Field(..., description="URL of the product or of the flagged image")
-    status: TicketStatus = Field(TicketStatus.open, description="Status of the ticket")
+    status: TicketStatus = Field(
+        default=TicketStatus.open, description="Status of the ticket"
+    )
     image_id: str | None = Field(
         None,
         description="ID of the flagged image, if the ticket type is `image`",

app/cli.py

@@ -0,0 +1,35 @@
+import typer
+
+app = typer.Typer()
+
+
+@app.command()
+def migrate_db():
+    """Run unapplied DB migrations."""
+    from openfoodfacts.utils import get_logger
+
+    from app.models import db, run_migration
+
+    get_logger()
+
+    with db.connection_context():
+        run_migration()
+
+
+@app.command()
+def add_revision(
+    name: str = typer.Argument(..., help="name of the revision"),
+):
+    """Create a new migration file using peewee_migrate."""
+    from openfoodfacts.utils import get_logger
+
+    from app.models import add_revision, db
+
+    get_logger()
+
+    with db.connection_context():
+        add_revision(name)
+
+
+def main() -> None:
+    app()

app/config.py

@@ -1,8 +1,11 @@
 from enum import StrEnum
+from pathlib import Path
 
 from openfoodfacts import Environment
 from pydantic_settings import BaseSettings
 
+PROJECT_DIR = Path(__file__).parent.parent
+
 
 class LoggingLevel(StrEnum):
     NOTSET = "NOTSET"
@@ -36,6 +39,8 @@ class Settings(BaseSettings):
     postgres_password: str = "postgres"
     postgres_port: int = 5432
     off_tld: Environment = Environment.net
+    environment: str = "dev"
+    migration_dir: Path = PROJECT_DIR / "migrations"
 
 
 settings = Settings()

app/models.py

@@ -5,7 +5,9 @@
     ForeignKeyField,
     Model,
     PostgresqlDatabase,
+    TextField,
 )
+from peewee_migrate import Router
 
 from .config import settings
 
@@ -19,12 +21,13 @@
 
 
 class TicketModel(Model):
-    barcode = CharField(null=True)
-    type = CharField()
-    url = CharField()
-    status = CharField()
+    # barcode of the product, if any
+    barcode = TextField(null=True)
+    type = CharField(max_length=50)
+    url = TextField()
+    status = CharField(max_length=50)
     image_id = CharField(null=True)
-    flavor = CharField()
+    flavor = CharField(max_length=20)
     created_at = DateTimeField()
 
     class Meta:
@@ -33,8 +36,8 @@ class Meta:
 
 
 class ModeratorActionModel(Model):
-    action_type = CharField()
-    user_id = CharField()
+    action_type = CharField(max_length=20)
+    user_id = TextField()
     ticket = ForeignKeyField(TicketModel, backref="moderator_actions")
     created_at = DateTimeField()
 
@@ -45,19 +48,33 @@ class Meta:
 
 class FlagModel(Model):
     ticket = ForeignKeyField(TicketModel, backref="flags")
-    barcode = CharField(null=True)
-    type = CharField()
-    url = CharField()
-    user_id = CharField()
-    device_id = CharField()
+    barcode = TextField(null=True)
+    type = CharField(max_length=50)
+    url = TextField()
+    user_id = TextField()
+    device_id = TextField()
     source = CharField()
     confidence = FloatField(null=True)
     image_id = CharField(null=True)
-    flavor = CharField()
-    reason = CharField(null=True)
-    comment = CharField(max_length=500, null=True)
+    flavor = CharField(max_length=20)
+    reason = TextField(null=True)
+    comment = TextField(null=True)
     created_at = DateTimeField()
 
     class Meta:
         database = db
         table_name = "flags"
+
+
+def run_migration():
+    """Run all unapplied migrations."""
+    # embedding schema does not exist at DB initialization
+    router = Router(db, migrate_dir=settings.migration_dir)
+    # Run all unapplied migrations
+    router.run()
+
+
+def add_revision(name: str):
+    """Create a migration revision."""
+    router = Router(db, migrate_dir=settings.migration_dir)
+    router.create(name, auto=True)