Merge branch 'main' into feature/ui

.env

@@ -5,7 +5,7 @@ COMPOSE_PATH_SEPARATOR=;
 # dev is default target
 COMPOSE_FILE=docker-compose.yml;docker/dev.yml
 
-API_PORT=127.0.0.1:8000
+API_EXPOSE=127.0.0.1:8000
 
 # by default on dev desktop, no restart
 RESTART_POLICY=no
@@ -25,4 +25,7 @@ POSTGRES_EXPOSE=127.0.0.1:5432
 
 # The top-level domain used for Open Food Facts,
 # it's either `net` (staging) or `org` (production) 
-OFF_TLD=net
+OFF_TLD=net
+
+# Environment name (mostly used for Sentry): dev, staging, prod
+ENVIRONMENT=dev

.github/workflows/auto-assign-pr.yml

@@ -12,4 +12,4 @@ jobs:
   assign-author:
     runs-on: ubuntu-latest
     steps:
-      - uses: toshimaru/auto-author-assign@v2.0.1
+      - uses: toshimaru/auto-author-assign@v2.1.0

.github/workflows/container-deploy.yml

@@ -5,9 +5,8 @@ on:
     branches:
       - main
       - deploy-*
-# only staging for now, not prod
-#     tags:
-#       - v*.*.*
+    tags:
+      - v*.*.*
 
 
 # Note on secrets used for connection
@@ -23,21 +22,24 @@ jobs:
     strategy:
       matrix:
         env:
-          # only stagging for now
-          # Note: env is also the name of the directory on the server
-          - nutripatrol-net
+          - ${{ startsWith(github.ref, 'refs/tags/v') && 'robotoff-org' || 'robotoff-net' }}
     environment: ${{ matrix.env }}
     concurrency: ${{ matrix.env }}
     steps:
+    - name: Set common variables
+      run: |
+        echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
+        echo "SSH_USERNAME=off" >> $GITHUB_ENV
     - name: Set various variable for staging (net) deployment
       if: matrix.env == 'nutripatrol-net'
       run: |
-        # direct container access
-        echo "OPENFOODFACTS_API_URL=https://off:[email protected]" >> $GITHUB_ENV
-        # deploy target
         echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV
-        echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
-        echo "SSH_USERNAME=off" >> $GITHUB_ENV
+        echo "ENVIRONMENT=staging" >> $GITHUB_ENV
+    - name: Set various variable for staging (org) deployment
+      if: matrix.env == 'nutripatrol-org'
+      run: |
+        echo "SSH_HOST=10.1.0.201" >> $GITHUB_ENV
+        echo "ENVIRONMENT=prod" >> $GITHUB_ENV
     - name: Wait for docker image container build workflow
       uses: tomchv/[email protected]
       id: wait-build
@@ -99,32 +101,26 @@ jobs:
 
           mv .env .env-dev
           # init .env
-          echo "# Env file generated by container-deploy action"> .env
+          echo "# Env file generated by container-deploy action" > .env
           # Set Docker Compose variables
           echo "DOCKER_CLIENT_TIMEOUT=180" >> .env
           echo "COMPOSE_HTTP_TIMEOUT=180" >> .env
           echo "COMPOSE_PROJECT_NAME=nutripatrol" >> .env
           echo "COMPOSE_PATH_SEPARATOR=;" >> .env
           echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml" >> .env
-          # Copy variables that are same as dev
-          grep '\(STACK_VERSION\|ES_PORT\)' .env-dev >> .env
           # Set docker variables
           echo "TAG=sha-${{ github.sha }}" >> .env
           echo "RESTART_POLICY=always" >> .env
-          # Set App variables
-          echo "CLUSTER_NAME=${{ matrix.env }}-es-cluster" >> .env
-          echo "SEARCH_PORT=8180" >> .env
-          echo "ES_VUE_PORT=8181" >> .env
-          echo "REDIS_PORT=8182" >> .env
-          echo "MEM_LIMIT=4294967296" >> .env
-          # this is the network shared with productopener
-          echo "COMMON_NET_NAME=po_webnet">> .env
-          echo "OPENFOODFACTS_API_URL=${{ env.OPENFOODFACTS_API_URL }}" >> .env
-          # This secret is to be generated using htpasswd, see .env file
-          # use simple quotes to avoid interpolation of $apr1$ !
-          echo 'NGINX_BASIC_AUTH_USER_PASSWD=${{ secrets.NGINX_BASIC_AUTH_USER_PASSWD }}' >> .env
           echo "SENTRY_DNS=${{ secrets.SENTRY_DSN }}" >> .env
-          echo "CONFIG_PATH=data/config/openfoodfacts.yml" >> .env
+          echo "ENVIRONMENT=${{ env.ENVIRONMENT }}" >> .env
+          # Expose API on port 9010
+          echo "API_EXPOSE=0.0.0.0:9010" >> .env
+          echo "POSTGRES_HOST=postgres" >> .env
+          echo "POSTGRES_DB=postgres" >> .env
+          echo "POSTGRES_USER=postgres" >> .env
+          echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .env
+          # Expose PostgreSQL locally on port 5434
+          echo "POSTGRES_EXPOSE=127.0.0.1:5434" >> .env
 
     - name: Create Docker volumes
       uses: appleboy/ssh-action@master
@@ -152,8 +148,11 @@ jobs:
         script_stop: false
         script: |
           cd ${{ matrix.env }}
-          docker-compose down
-          docker-compose up -d --remove-orphans 2>&1
+          make pull
+          # Apply migrations
+          make migrate-db
+          # Launch new version
+          make up
 
     - name: Check services are up
       uses: appleboy/ssh-action@master
@@ -183,12 +182,12 @@ jobs:
         script_stop: false
         script: |
           cd ${{ matrix.env }}
-          docker system prune -af
+          make prune
 
     - uses: frankie567/[email protected]
       if: ${{ always() }}
       with:
         apiHost: https://grafana.openfoodfacts.org
         apiToken: ${{ secrets.GRAFANA_API_TOKEN }}
         text: <a href="https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}">Deployment ${{ steps.livecheck.outcome }} on ${{ matrix.env }}</a>
-        tags: type:deployment,origin:github,status:${{ steps.livecheck.outcome }},repo:${{ github.repository }},sha:${{ github.sha }},app:robotoff,env:${{ matrix.env }}
+        tags: type:deployment,origin:github,status:${{ steps.livecheck.outcome }},repo:${{ github.repository }},sha:${{ github.sha }},app:nutripatrol,env:${{ matrix.env }}

.github/workflows/pre-commit.yml

@@ -13,4 +13,4 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: 3.11
-    - uses: pre-commit/[email protected].0
+    - uses: pre-commit/[email protected].1

.github/workflows/release-please.yml

@@ -1,13 +1,27 @@
 name: Run release-please
 on:
-   push:
-     branches:
-       - main
+  push:
+    branches:
+      - main
+
 jobs:
-   release-please:
-     runs-on: ubuntu-latest
-     steps:
-       - uses: GoogleCloudPlatform/[email protected]
-         with:
-           token: ${{ secrets.GITHUB_TOKEN }}
-           release-type: simple
+  release-please:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: GoogleCloudPlatform/[email protected]
+        with:
+          # We can't use GITHUB_TOKEN here because, github actions can't provocate actions
+          # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow
+          # So this is a personnal access token
+          token: ${{ secrets.RELEASE_PLEASE_TOKEN }}
+          release-type: simple
+          changelog-types: |
+            [
+              {"type":"feat","section":"Features","hidden":false},
+              {"type":"fix","section":"Bug Fixes","hidden":false},
+              {"type":"style","section":"Technical","hidden":false},
+              {"type":"docs","section":"Technical","hidden":false},
+              {"type":"test","section":"Technical","hidden":false},
+              {"type":"chore","section":"Technical","hidden":false},
+              {"type":"refactor","section":"Technical","hidden":false}
+            ]

Dockerfile

@@ -15,6 +15,7 @@ RUN groupadd -g $USER_GID off && \
     chown off:off -R /opt/nutripatrol /home/off
 WORKDIR /opt/nutripatrol
 COPY --chown=off:off requirements.txt requirements.txt
+COPY --chown=off:off migrations /opt/nutripatrol/migrations
 RUN  pip install --no-cache-dir --upgrade -r requirements.txt
 
 USER off:off

Makefile

@@ -41,6 +41,10 @@ build:
 	docker-compose build
 
 
+# pull images from image repository
+pull:
+	${DOCKER_COMPOSE} pull
+
 up:
 ifdef service
 	${DOCKER_COMPOSE} up -d ${service} 2>&1
@@ -52,3 +56,45 @@ endif
 down:
 	@echo "🥫 Bringing down containers …"
 	${DOCKER_COMPOSE} down
+
+
+# Create all external volumes needed for production. Using external volumes is useful to prevent data loss (as they are not deleted when performing docker down -v)
+create_external_volumes:
+	@echo "🥫 Creating external volumes (production only) …"
+	docker volume create nutripatrol-postgres-data
+
+#-----------#
+# Utilities #
+#-----------#
+
+guard-%: # guard clause for targets that require an environment variable (usually used as an argument)
+	@ if [ "${${*}}" = "" ]; then \
+   		echo "Environment variable '$*' is mandatory"; \
+   		echo use "make ${MAKECMDGOALS} $*=you-args"; \
+   		exit 1; \
+	fi;
+
+
+#------------#
+#  Database  #
+#------------#
+
+# apply DB migrations
+migrate-db:
+	${DOCKER_COMPOSE} run --rm --no-deps api python -m app migrate-db
+
+# add a new DB revision
+add-revision: guard-name
+	${DOCKER_COMPOSE} run --rm --no-deps api python -m app add-revision ${name}
+
+
+#---------#
+# Cleanup #
+#---------#
+prune:
+	@echo "🥫 Pruning unused Docker artifacts (save space) …"
+	docker system prune -af
+
+prune_cache:
+	@echo "🥫 Pruning Docker builder cache …"
+	docker builder prune -f

README.md

@@ -2,6 +2,14 @@
 
 Open Food Facts moderation tool (WIP).
 
+### Wikipage 
+- https://wiki.openfoodfacts.org/Moderation
+- [Meeting minutes](https://docs.google.com/document/d/1B9Ci42kl_jrFt2hi3PiWW9tM9l6B1sI5kQMI9Zd6QS4/edit)
+
+### Meeting
+Valentin and Raphael are working on this tool. They meet weekly. Please ping them on Slack if you'd like to contribute. 
+We have more general quality meeting every month.
+
 ### Pre-Commit
 This repo uses [pre-commit](https://pre-commit.com/) to enforce code styling, etc. To use it:
 ```console

app/__main__.py

@@ -0,0 +1,4 @@
+from app.cli import main
+
+if __name__ == "__main__":
+    main()

app/api.py

@@ -99,7 +99,9 @@ class TicketCreate(BaseModel):
     )
     type: IssueType = Field(..., description="Type of the issue")
     url: str = Field(..., description="URL of the product or of the flagged image")
-    status: TicketStatus = Field(TicketStatus.open, description="Status of the ticket")
+    status: TicketStatus = Field(
+        default=TicketStatus.open, description="Status of the ticket"
+    )
     image_id: str | None = Field(
         None,
         description="ID of the flagged image, if the ticket type is `image`",

app/cli.py

@@ -0,0 +1,35 @@
+import typer
+
+app = typer.Typer()
+
+
+@app.command()
+def migrate_db():
+    """Run unapplied DB migrations."""
+    from openfoodfacts.utils import get_logger
+
+    from app.models import db, run_migration
+
+    get_logger()
+
+    with db.connection_context():
+        run_migration()
+
+
+@app.command()
+def add_revision(
+    name: str = typer.Argument(..., help="name of the revision"),
+):
+    """Create a new migration file using peewee_migrate."""
+    from openfoodfacts.utils import get_logger
+
+    from app.models import add_revision, db
+
+    get_logger()
+
+    with db.connection_context():
+        add_revision(name)
+
+
+def main() -> None:
+    app()

app/config.py

@@ -1,8 +1,11 @@
 from enum import StrEnum
+from pathlib import Path
 
 from openfoodfacts import Environment
 from pydantic_settings import BaseSettings
 
+PROJECT_DIR = Path(__file__).parent.parent
+
 
 class LoggingLevel(StrEnum):
     NOTSET = "NOTSET"
@@ -36,6 +39,8 @@ class Settings(BaseSettings):
     postgres_password: str = "postgres"
     postgres_port: int = 5432
     off_tld: Environment = Environment.net
+    environment: str = "dev"
+    migration_dir: Path = PROJECT_DIR / "migrations"
 
 
 settings = Settings()