diff --git a/.env b/.env index c2c8d5d..067e3d6 100644 --- a/.env +++ b/.env @@ -5,7 +5,7 @@ COMPOSE_PATH_SEPARATOR=; # dev is default target COMPOSE_FILE=docker-compose.yml;docker/dev.yml -API_PORT=127.0.0.1:8000 +API_EXPOSE=127.0.0.1:8000 # by default on dev desktop, no restart RESTART_POLICY=no @@ -25,4 +25,7 @@ POSTGRES_EXPOSE=127.0.0.1:5432 # The top-level domain used for Open Food Facts, # it's either `net` (staging) or `org` (production) -OFF_TLD=net \ No newline at end of file +OFF_TLD=net + +# Environment name (mostly used for Sentry): dev, staging, prod +ENVIRONMENT=dev \ No newline at end of file diff --git a/.github/workflows/auto-assign-pr.yml b/.github/workflows/auto-assign-pr.yml index 1f2a66b..7488ae7 100644 --- a/.github/workflows/auto-assign-pr.yml +++ b/.github/workflows/auto-assign-pr.yml @@ -12,4 +12,4 @@ jobs: assign-author: runs-on: ubuntu-latest steps: - - uses: toshimaru/auto-author-assign@v2.0.1 + - uses: toshimaru/auto-author-assign@v2.1.0 diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index c335525..5d75275 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -5,9 +5,8 @@ on: branches: - main - deploy-* -# only staging for now, not prod -# tags: -# - v*.*.* + tags: + - v*.*.* # Note on secrets used for connection @@ -23,21 +22,24 @@ jobs: strategy: matrix: env: - # only stagging for now - # Note: env is also the name of the directory on the server - - nutripatrol-net + - ${{ startsWith(github.ref, 'refs/tags/v') && 'robotoff-org' || 'robotoff-net' }} environment: ${{ matrix.env }} concurrency: ${{ matrix.env }} steps: + - name: Set common variables + run: | + echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV + echo "SSH_USERNAME=off" >> $GITHUB_ENV - name: Set various variable for staging (net) deployment if: matrix.env == 'nutripatrol-net' run: | - # direct container access - echo "OPENFOODFACTS_API_URL=https://off:off@world.openfoodfacts.net" >> $GITHUB_ENV - # deploy target echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV - echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV - echo "SSH_USERNAME=off" >> $GITHUB_ENV + echo "ENVIRONMENT=staging" >> $GITHUB_ENV + - name: Set various variable for staging (org) deployment + if: matrix.env == 'nutripatrol-org' + run: | + echo "SSH_HOST=10.1.0.201" >> $GITHUB_ENV + echo "ENVIRONMENT=prod" >> $GITHUB_ENV - name: Wait for docker image container build workflow uses: tomchv/wait-my-workflow@v1.1.0 id: wait-build @@ -99,32 +101,26 @@ jobs: mv .env .env-dev # init .env - echo "# Env file generated by container-deploy action"> .env + echo "# Env file generated by container-deploy action" > .env # Set Docker Compose variables echo "DOCKER_CLIENT_TIMEOUT=180" >> .env echo "COMPOSE_HTTP_TIMEOUT=180" >> .env echo "COMPOSE_PROJECT_NAME=nutripatrol" >> .env echo "COMPOSE_PATH_SEPARATOR=;" >> .env echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml" >> .env - # Copy variables that are same as dev - grep '\(STACK_VERSION\|ES_PORT\)' .env-dev >> .env # Set docker variables echo "TAG=sha-${{ github.sha }}" >> .env echo "RESTART_POLICY=always" >> .env - # Set App variables - echo "CLUSTER_NAME=${{ matrix.env }}-es-cluster" >> .env - echo "SEARCH_PORT=8180" >> .env - echo "ES_VUE_PORT=8181" >> .env - echo "REDIS_PORT=8182" >> .env - echo "MEM_LIMIT=4294967296" >> .env - # this is the network shared with productopener - echo "COMMON_NET_NAME=po_webnet">> .env - echo "OPENFOODFACTS_API_URL=${{ env.OPENFOODFACTS_API_URL }}" >> .env - # This secret is to be generated using htpasswd, see .env file - # use simple quotes to avoid interpolation of $apr1$ ! - echo 'NGINX_BASIC_AUTH_USER_PASSWD=${{ secrets.NGINX_BASIC_AUTH_USER_PASSWD }}' >> .env echo "SENTRY_DNS=${{ secrets.SENTRY_DSN }}" >> .env - echo "CONFIG_PATH=data/config/openfoodfacts.yml" >> .env + echo "ENVIRONMENT=${{ env.ENVIRONMENT }}" >> .env + # Expose API on port 9010 + echo "API_EXPOSE=0.0.0.0:9010" >> .env + echo "POSTGRES_HOST=postgres" >> .env + echo "POSTGRES_DB=postgres" >> .env + echo "POSTGRES_USER=postgres" >> .env + echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .env + # Expose PostgreSQL locally on port 5434 + echo "POSTGRES_EXPOSE=127.0.0.1:5434" >> .env - name: Create Docker volumes uses: appleboy/ssh-action@master @@ -152,8 +148,11 @@ jobs: script_stop: false script: | cd ${{ matrix.env }} - docker-compose down - docker-compose up -d --remove-orphans 2>&1 + make pull + # Apply migrations + make migrate-db + # Launch new version + make up - name: Check services are up uses: appleboy/ssh-action@master @@ -183,7 +182,7 @@ jobs: script_stop: false script: | cd ${{ matrix.env }} - docker system prune -af + make prune - uses: frankie567/grafana-annotation-action@v1.0.3 if: ${{ always() }} @@ -191,4 +190,4 @@ jobs: apiHost: https://grafana.openfoodfacts.org apiToken: ${{ secrets.GRAFANA_API_TOKEN }} text: Deployment ${{ steps.livecheck.outcome }} on ${{ matrix.env }} - tags: type:deployment,origin:github,status:${{ steps.livecheck.outcome }},repo:${{ github.repository }},sha:${{ github.sha }},app:robotoff,env:${{ matrix.env }} + tags: type:deployment,origin:github,status:${{ steps.livecheck.outcome }},repo:${{ github.repository }},sha:${{ github.sha }},app:nutripatrol,env:${{ matrix.env }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 12518d4..3df92c4 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -13,4 +13,4 @@ jobs: - uses: actions/setup-python@v5 with: python-version: 3.11 - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 328a982..101b419 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -1,13 +1,27 @@ name: Run release-please on: - push: - branches: - - main + push: + branches: + - main + jobs: - release-please: - runs-on: ubuntu-latest - steps: - - uses: GoogleCloudPlatform/release-please-action@v3.7.13 - with: - token: ${{ secrets.GITHUB_TOKEN }} - release-type: simple + release-please: + runs-on: ubuntu-latest + steps: + - uses: GoogleCloudPlatform/release-please-action@v3.7.13 + with: + # We can't use GITHUB_TOKEN here because, github actions can't provocate actions + # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow + # So this is a personnal access token + token: ${{ secrets.RELEASE_PLEASE_TOKEN }} + release-type: simple + changelog-types: | + [ + {"type":"feat","section":"Features","hidden":false}, + {"type":"fix","section":"Bug Fixes","hidden":false}, + {"type":"style","section":"Technical","hidden":false}, + {"type":"docs","section":"Technical","hidden":false}, + {"type":"test","section":"Technical","hidden":false}, + {"type":"chore","section":"Technical","hidden":false}, + {"type":"refactor","section":"Technical","hidden":false} + ] diff --git a/Dockerfile b/Dockerfile index 0fe6597..97dba93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN groupadd -g $USER_GID off && \ chown off:off -R /opt/nutripatrol /home/off WORKDIR /opt/nutripatrol COPY --chown=off:off requirements.txt requirements.txt +COPY --chown=off:off migrations /opt/nutripatrol/migrations RUN pip install --no-cache-dir --upgrade -r requirements.txt USER off:off diff --git a/Makefile b/Makefile index bfc8c8c..5af0635 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,10 @@ build: docker-compose build +# pull images from image repository +pull: + ${DOCKER_COMPOSE} pull + up: ifdef service ${DOCKER_COMPOSE} up -d ${service} 2>&1 @@ -52,3 +56,45 @@ endif down: @echo "🥫 Bringing down containers …" ${DOCKER_COMPOSE} down + + +# Create all external volumes needed for production. Using external volumes is useful to prevent data loss (as they are not deleted when performing docker down -v) +create_external_volumes: + @echo "🥫 Creating external volumes (production only) …" + docker volume create nutripatrol-postgres-data + +#-----------# +# Utilities # +#-----------# + +guard-%: # guard clause for targets that require an environment variable (usually used as an argument) + @ if [ "${${*}}" = "" ]; then \ + echo "Environment variable '$*' is mandatory"; \ + echo use "make ${MAKECMDGOALS} $*=you-args"; \ + exit 1; \ + fi; + + +#------------# +# Database # +#------------# + +# apply DB migrations +migrate-db: + ${DOCKER_COMPOSE} run --rm --no-deps api python -m app migrate-db + +# add a new DB revision +add-revision: guard-name + ${DOCKER_COMPOSE} run --rm --no-deps api python -m app add-revision ${name} + + +#---------# +# Cleanup # +#---------# +prune: + @echo "🥫 Pruning unused Docker artifacts (save space) …" + docker system prune -af + +prune_cache: + @echo "🥫 Pruning Docker builder cache …" + docker builder prune -f diff --git a/README.md b/README.md index a7e2f05..c6830a2 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,14 @@ Open Food Facts moderation tool (WIP). +### Wikipage +- https://wiki.openfoodfacts.org/Moderation +- [Meeting minutes](https://docs.google.com/document/d/1B9Ci42kl_jrFt2hi3PiWW9tM9l6B1sI5kQMI9Zd6QS4/edit) + +### Meeting +Valentin and Raphael are working on this tool. They meet weekly. Please ping them on Slack if you'd like to contribute. +We have more general quality meeting every month. + ### Pre-Commit This repo uses [pre-commit](https://pre-commit.com/) to enforce code styling, etc. To use it: ```console diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 0000000..c0bcdad --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,4 @@ +from app.cli import main + +if __name__ == "__main__": + main() diff --git a/app/api.py b/app/api.py index 513ee19..f97d390 100644 --- a/app/api.py +++ b/app/api.py @@ -99,7 +99,9 @@ class TicketCreate(BaseModel): ) type: IssueType = Field(..., description="Type of the issue") url: str = Field(..., description="URL of the product or of the flagged image") - status: TicketStatus = Field(TicketStatus.open, description="Status of the ticket") + status: TicketStatus = Field( + default=TicketStatus.open, description="Status of the ticket" + ) image_id: str | None = Field( None, description="ID of the flagged image, if the ticket type is `image`", diff --git a/app/cli.py b/app/cli.py new file mode 100644 index 0000000..5958d30 --- /dev/null +++ b/app/cli.py @@ -0,0 +1,35 @@ +import typer + +app = typer.Typer() + + +@app.command() +def migrate_db(): + """Run unapplied DB migrations.""" + from openfoodfacts.utils import get_logger + + from app.models import db, run_migration + + get_logger() + + with db.connection_context(): + run_migration() + + +@app.command() +def add_revision( + name: str = typer.Argument(..., help="name of the revision"), +): + """Create a new migration file using peewee_migrate.""" + from openfoodfacts.utils import get_logger + + from app.models import add_revision, db + + get_logger() + + with db.connection_context(): + add_revision(name) + + +def main() -> None: + app() diff --git a/app/config.py b/app/config.py index 40a22ce..16b90c9 100644 --- a/app/config.py +++ b/app/config.py @@ -1,8 +1,11 @@ from enum import StrEnum +from pathlib import Path from openfoodfacts import Environment from pydantic_settings import BaseSettings +PROJECT_DIR = Path(__file__).parent.parent + class LoggingLevel(StrEnum): NOTSET = "NOTSET" @@ -36,6 +39,8 @@ class Settings(BaseSettings): postgres_password: str = "postgres" postgres_port: int = 5432 off_tld: Environment = Environment.net + environment: str = "dev" + migration_dir: Path = PROJECT_DIR / "migrations" settings = Settings() diff --git a/app/models.py b/app/models.py index e4426db..81c5f7f 100644 --- a/app/models.py +++ b/app/models.py @@ -5,7 +5,9 @@ ForeignKeyField, Model, PostgresqlDatabase, + TextField, ) +from peewee_migrate import Router from .config import settings @@ -19,12 +21,13 @@ class TicketModel(Model): - barcode = CharField(null=True) - type = CharField() - url = CharField() - status = CharField() + # barcode of the product, if any + barcode = TextField(null=True) + type = CharField(max_length=50) + url = TextField() + status = CharField(max_length=50) image_id = CharField(null=True) - flavor = CharField() + flavor = CharField(max_length=20) created_at = DateTimeField() class Meta: @@ -33,8 +36,8 @@ class Meta: class ModeratorActionModel(Model): - action_type = CharField() - user_id = CharField() + action_type = CharField(max_length=20) + user_id = TextField() ticket = ForeignKeyField(TicketModel, backref="moderator_actions") created_at = DateTimeField() @@ -45,19 +48,33 @@ class Meta: class FlagModel(Model): ticket = ForeignKeyField(TicketModel, backref="flags") - barcode = CharField(null=True) - type = CharField() - url = CharField() - user_id = CharField() - device_id = CharField() + barcode = TextField(null=True) + type = CharField(max_length=50) + url = TextField() + user_id = TextField() + device_id = TextField() source = CharField() confidence = FloatField(null=True) image_id = CharField(null=True) - flavor = CharField() - reason = CharField(null=True) - comment = CharField(max_length=500, null=True) + flavor = CharField(max_length=20) + reason = TextField(null=True) + comment = TextField(null=True) created_at = DateTimeField() class Meta: database = db table_name = "flags" + + +def run_migration(): + """Run all unapplied migrations.""" + # embedding schema does not exist at DB initialization + router = Router(db, migrate_dir=settings.migration_dir) + # Run all unapplied migrations + router.run() + + +def add_revision(name: str): + """Create a migration revision.""" + router = Router(db, migrate_dir=settings.migration_dir) + router.create(name, auto=True) diff --git a/app/utils.py b/app/utils.py index 3062a7a..f6870ca 100644 --- a/app/utils.py +++ b/app/utils.py @@ -4,6 +4,8 @@ from sentry_sdk.integrations import Integration from sentry_sdk.integrations.logging import LoggingIntegration +from app.config import settings + def init_sentry(sentry_dsn: str | None, integrations: list[Integration] | None = None): if sentry_dsn: @@ -17,4 +19,5 @@ def init_sentry(sentry_dsn: str | None, integrations: list[Integration] | None = sentry_sdk.init( # type:ignore # mypy say it's abstract sentry_dsn, integrations=integrations, + environment=settings.environment, ) diff --git a/docker-compose.yml b/docker-compose.yml index 8846ce9..8469ab3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,6 @@ x-api-common: &api-common - POSTGRES_PASSWORD - POSTGRES_DB - POSTGRES_HOST - - OFF_TLD networks: - default @@ -18,7 +17,7 @@ services: api: <<: *api-common ports: - - "${API_PORT}:8000" + - "${API_EXPOSE:-127.0.0.1:8000}:8000" postgres: restart: $RESTART_POLICY @@ -36,4 +35,5 @@ services: - "${POSTGRES_EXPOSE:-127.0.0.1:5432}:5432" volumes: - postgres-data: \ No newline at end of file + postgres-data: + name: ${COMPOSE_PROJECT_NAME:-nutripatrol}-postgres-data \ No newline at end of file diff --git a/docker/dev.yml b/docker/dev.yml index 2caec6f..95c266a 100644 --- a/docker/dev.yml +++ b/docker/dev.yml @@ -10,6 +10,8 @@ x-api-base: &api-base volumes: # mount code dynamically - "./app:/opt/nutripatrol/app" + # mount migrations dynamically + - "./migrations:/opt/nutripatrol/migrations" services: api: diff --git a/docker/prod.yml b/docker/prod.yml index c404b8b..87523d7 100644 --- a/docker/prod.yml +++ b/docker/prod.yml @@ -1,3 +1,6 @@ -version: "3.7" +version: "3.9" -# modifications to docker-compose for production settings \ No newline at end of file +volumes: + postgres-data: + external: true + name: ${COMPOSE_PROJECT_NAME:-nutripatrol}-postgres-data diff --git a/migrations/001_initial.py b/migrations/001_initial.py new file mode 100644 index 0000000..df0ea1e --- /dev/null +++ b/migrations/001_initial.py @@ -0,0 +1,67 @@ +"""Peewee migrations -- 001_initial.py.""" + +import peewee as pw +from peewee_migrate import Migrator + + +def migrate(migrator: Migrator, database: pw.Database, *, fake=False): + """Write your migrations here.""" + + @migrator.create_model + class TicketModel(pw.Model): + id = pw.AutoField() + barcode = pw.TextField(null=True) + type = pw.CharField(max_length=50) + url = pw.TextField() + status = pw.CharField(max_length=50) + image_id = pw.CharField(max_length=255, null=True) + flavor = pw.CharField(max_length=20) + created_at = pw.DateTimeField() + + class Meta: + table_name = "tickets" + + @migrator.create_model + class FlagModel(pw.Model): + id = pw.AutoField() + ticket = pw.ForeignKeyField( + column_name="ticket_id", field="id", model=migrator.orm["tickets"] + ) + barcode = pw.TextField(null=True) + type = pw.CharField(max_length=50) + url = pw.TextField() + user_id = pw.TextField() + device_id = pw.TextField() + source = pw.CharField(max_length=255) + confidence = pw.FloatField(null=True) + image_id = pw.CharField(max_length=255, null=True) + flavor = pw.CharField(max_length=20) + reason = pw.TextField(null=True) + comment = pw.TextField(null=True) + created_at = pw.DateTimeField() + + class Meta: + table_name = "flags" + + @migrator.create_model + class ModeratorActionModel(pw.Model): + id = pw.AutoField() + action_type = pw.CharField(max_length=20) + user_id = pw.TextField() + ticket = pw.ForeignKeyField( + column_name="ticket_id", field="id", model=migrator.orm["tickets"] + ) + created_at = pw.DateTimeField() + + class Meta: + table_name = "moderator_actions" + + +def rollback(migrator: Migrator, database: pw.Database, *, fake=False): + """Write your rollback migrations here.""" + + migrator.remove_model("moderator_actions") + + migrator.remove_model("flags") + + migrator.remove_model("tickets") diff --git a/requirements.txt b/requirements.txt index 1aff57f..e19123c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ openfoodfacts==0.1.10 requests==2.31.0 pydantic-settings==2.0.3 sentry-sdk[fastapi]==1.31.0 -jinja2==3.1.2 +jinja2==3.1.3 peewee==3.17.0 -psycopg2-binary==2.9.9 \ No newline at end of file +peewee-migrate==1.12.2 +psycopg2-binary==2.9.9 +typer==0.9.0 \ No newline at end of file