Merge branch 'develop' into M2-3148

# Conflicts: # src/apps/answers/crud/answers.py # src/apps/applets/crud/applets.py
ChildMindInstitute · Jan 3, 2024 · 3eab5bc · 3eab5bc
2 parents fd5d84c + 24757dd
commit 3eab5bc
Show file tree

Hide file tree

Showing 232 changed files with 12,652 additions and 3,935 deletions.
diff --git a/.env.default b/.env.default
@@ -25,8 +25,8 @@ CORS__ALLOW_HEADERS=*
 
 
 # Authentication
-AUTHENTICATION__ACCESS_TOKEN__SECRET_KEY="e51bcf5f4cb8550ff3f6a8bb4dfe112a3da2cf5142929e1b281cd974c88fa66c"
-AUTHENTICATION__REFRESH_TOKEN__SECRET_KEY="5da342d54ed5659f123cdd1cefe439c5aaf7e317a0aba1405375c07d32e097cc"
+AUTHENTICATION__ACCESS_TOKEN__SECRET_KEY="secret1"
+AUTHENTICATION__REFRESH_TOKEN__SECRET_KEY="secret2"
 AUTHENTICATION__ACCESS_TOKEN__EXPIRATION=30
 AUTHENTICATION__REFRESH_TOKEN__EXPIRATION=540
 AUTHENTICATION__ALGORITHM="HS256"
@@ -53,12 +53,14 @@ MAILING__VALIDATE_CERTS=False
 # FCM Notification api key
 NOTIFICATION__API_KEY=
 
-# CDN configs
-CDN__SECRET_KEY=
-CDN__ACCESS_KEY=
+# CDN configs (container by default)
+CDN__ENDPOINT_URL=http://localhost:9000
+CDN__SECRET_KEY=miniosecret
+CDN__ACCESS_KEY=minioaccess
 CDN__REGION=
-CDN__BUCKET_ANSWERS=
-CDN__BUCKET=
+CDN__BUCKET_ANSWER=media
+CDN__BUCKET=media
+CDN__STORAGE_ADDRESS=http://localhost:9000
 CDN__LEGACY_REGION=
 CDN__LEGACY_BUCKET=
 CDN__LEGACY_SECRET_KEY=

diff --git a/.github/workflows/code_quality.yaml b/.github/workflows/code_quality.yaml
@@ -6,10 +6,22 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+      with:
+        repository: 'awslabs/git-secrets'
+        ref: 'master'
+    - name: Install git-secrets
+      run: sudo make install
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v3
       with:
         python-version: '3.10'
+    - name: Install git-secrets in the repository
+      run: git secrets --install
+    - name: Install git-secrets aws register in the repository 
+      run: git secrets --register-aws
+    - name: Scan aws secrets
+      run: git secrets --scan
     - name: Install pipenv
       run: python -m pip install --upgrade pip && pip install pipenv
     - name: Install deps

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -29,8 +29,8 @@ repos:
         entry: mypy
         language: system
         types: [python]
-        exclude: "apps/girderformindlogger"
-      
+        exclude: "apps/girderformindlogger|apps/migrate"
+
       - id: git-secrets
         name: git-secrets
         entry: git secrets --scan

diff --git a/Makefile b/Makefile
@@ -37,6 +37,10 @@ cq:
 migrate:
 	python src/apps/migrate/run.py
 
+.PHONY: migrate_answer
+migrate_answer:
+	python src/apps/migrate/answers/run.py
+
 # ###############
 # Docker
 # ###############

diff --git a/Pipfile b/Pipfile
@@ -7,7 +7,7 @@ name = "pypi"
 aioredis = "~=2.0.1"
 alembic = "~=1.8.1"
 asyncpg = "~=0.27.0"
-asynctest = "*"
+asynctest = "==0.13.0"
 boto3 = "==1.26.10"
 fastapi = "==0.87.0"
 # The latest version of the fastapi is not taken because of the issue
@@ -16,6 +16,7 @@ fastapi = "==0.87.0"
 fastapi-mail = "~=1.2.2"
 httpx = "~=0.23"
 jinja2 = "~=3.1.2"
+bcrypt = "==4.0.1"
 passlib = {version = "~=1.7.4", extras = ["bcrypt"]}
 pillow = "~=9.3.0"
 psutil = "~=5.9.4"
@@ -24,17 +25,20 @@ pydantic = {extras = ["email"], version = "~=1.10.2"}
 python-jose = {version = "~=3.3.0", extras = ["cryptography"]}
 python-multipart = "~=0.0.5"
 sentry-sdk = "~=1.6"
-sqlalchemy = {extras = ["asyncio"], version = "~=1.4.46"}
+sqlalchemy = {extras = ["asyncio"], version = "==1.4.49"}
 uvicorn = {extras = ["standard"], version = "==0.19"}
-aiohttp = "*"
-firebase-admin = "*"
-aio-pika = "*"
-pyld = "*"
-azure-storage-blob = "*"
-taskiq-fastapi = "*"
-taskiq-redis = "*"
-taskiq-aio-pika = "*"
+taskiq = {extras = ["reload"], version = "==0.9.1"}
+aiohttp = "==3.8.5"
+firebase-admin = "==6.2.0"
+aio-pika = "==9.3.0"
+pyld = "==2.0.3"
+azure-storage-blob = "==12.18.2"
+taskiq-fastapi = "==0.3.0"
+taskiq-redis = "==0.5.0"
+taskiq-aio-pika = "==0.4.0"
 sqlalchemy-utils = "==0.41.1"
+typer = {extras = ["all"], version = "==0.9.0"}
+aiofiles = "==23.2.1"
 
 [dev-packages]
 black = "~=22.6"
@@ -78,9 +82,10 @@ pycryptodomex = "==3.9.7"
 psycopg2-binary = "==2.9.6"
 cachetools = "==5.3.0"
 pyld = "==2.0.3"
-types-requests = "*"
-taskiq = {extras = ["reload"], version = "*"}
-types-pytz = "*"
-
+types-requests = "==2.31.0.10"
+types-pytz = "==2023.3.1.1"
+gevent = "~=23.9"
+types-aiofiles = "==23.2.0.0"
+types-cachetools = "==5.3.0.7"
 [requires]
 python_version = "3.10"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/README.md b/README.md
@@ -12,15 +12,13 @@
 - ✅ [Redis](https://redis.io)
 - ✅ [Docker](https://docs.docker.com/get-docker/)
 - ✅ [Pydantic](https://pydantic-docs.helpmanual.io)
-- ✅ [FastAPI](https://fastapi.tiangolo.com/)
 - ✅ [SQLAlchemy](https://www.sqlalchemy.org/)
 
 And
 
 - ✅ [The 12-Factor App](https://12factor.net)
-- ✅ [Domain driven design](https://www.amazon.com/Domain-Driven-Design-Tackling-Complexity-Software-ebook/dp/B00794TAUG)
 
-</br>
+<br/>
 
 🔌 **Code quality tools:**
 - ✅ [flake8](https://github.com/pycqa/flake8)
@@ -29,7 +27,7 @@ And
 - ✅ [mypy](https://github.com/python/mypy)
 - ✅ [pytest](https://github.com/pytest-dev/pytest)
 
-</br>
+<br/>
 
 ## ✋ <span style="color:#9DB7FF">Mandatory steps</span>
 
@@ -49,23 +47,22 @@ git clone [email protected]:ChildMindInstitute/mindlogger-backend-refactor.git
 
 
 #### 2.1 Description 📜
-| Key | Default value                                                    | Description                                                                                                                                                                   |
-| --- |------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| PYTHONPATH | src/                                                             | This variable makes all folders inside `src/` reachable in a runtime. </br> ***NOTE:*** You don't need to do this if you use Docker as far as it is hardcoded in `Dockerfile` |
-| DATABASE__HOST | postgres                                                         | Database Host                                                                                                                                                                 |
-| DATABASE__USER | postgres                                                         | User name for Postgresql Database user                                                                                                                                        |
-| DATABASE__PASSWORD | postgres                                                         | Password for Postgresql Database user                                                                                                                                         |
-| DATABASE__DB | mindlogger_backend                                               | Database name                                                                                                                                                                 |
-| CORS__ALLOW_ORIGINS | `*`                                                              | Represents the list of allowed origins. Set the `Access-Control-Allow-Origin` header. Example: `https://dev.com,http://localohst:8000`                                        |
-| CORS__ALLOW_CREDENTIALS | true                                                             | Set the `Access-Control-Allow-Credentials` header                                                                                                                             |
-| CORS__ALLOW_METHODS | `*`                                                              | Set the `Access-Control-Allow-Methods` header                                                                                                                                 |
-| CORS__ALLOW_HEADERS | `*`                                                              | Set the `Access-Control-Allow-Headers` header                                                                                                                                 |
-| AUTHENTICATION__SECRET_KEY | e51bcf5f4cb8550ff3f6a8bb4dfe112a                                 | Access token's salt                                                                                                                                                           |
-| AUTHENTICATION__REFRESH_SECRET_KEY | 5da342d54ed5659f123cdd1cefe439c5aaf7e317a0aba1405375c07d32e097cc | Refresh token salt                                                                                                                                                            |
-| AUTHENTICATION__ALGORITHM | HS256                                                            | The JWT's algorithm                                                                                                                                                           |
-| AUTHENTICATION__ACCESS_TOKEN_EXPIRATION_TIME | 30                                                               | Time in minutes after which the access token will stop working                                                                                                                |
-| AUTHENTICATION__REFRESH_TOKEN_EXPIRATION_TIME | 30                                                               | Time in minutes after which the refresh token will stop working                                                                                                               |
-| ADMIN_DOMAIN | -                                                                | Admin panel domain                                                                                                                                                            |
+| Key | Default value      | Description                                                                                                                                                                   |
+| --- |--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| DATABASE__HOST | postgres           | Database Host                                                                                                                                                                 |
+| DATABASE__USER | postgres           | User name for Postgresql Database user                                                                                                                                        |
+| DATABASE__PASSWORD | postgres           | Password for Postgresql Database user                                                                                                                                         |
+| DATABASE__DB | mindlogger_backend | Database name                                                                                                                                                                 |
+| CORS__ALLOW_ORIGINS | `*`                | Represents the list of allowed origins. Set the `Access-Control-Allow-Origin` header. Example: `https://dev.com,http://localohst:8000`                                        |
+| CORS__ALLOW_CREDENTIALS | true               | Set the `Access-Control-Allow-Credentials` header                                                                                                                             |
+| CORS__ALLOW_METHODS | `*`                | Set the `Access-Control-Allow-Methods` header                                                                                                                                 |
+| CORS__ALLOW_HEADERS | `*`                | Set the `Access-Control-Allow-Headers` header                                                                                                                                 |
+| AUTHENTICATION__ACCESS_TOKEN__SECRET_KEY | secret1            | Access token's salt                                                                                                                                                           |
+| AUTHENTICATION__REFRESH_TOKEN__SECRET_KEY | secret2            | Refresh token salt                                                                                                                                                            |
+| AUTHENTICATION__ALGORITHM | HS256              | The JWT's algorithm                                                                                                                                                           |
+| AUTHENTICATION__ACCESS_TOKEN__EXPIRATION | 30                 | Time in minutes after which the access token will stop working                                                                                                                |
+| AUTHENTICATION__REFRESH_TOKEN__EXPIRATION | 30                 | Time in minutes after which the refresh token will stop working                                                                                                               |
+| ADMIN_DOMAIN | -                  | Admin panel domain                                                                                                                                                            |
 
 ##### ✋ Mandatory:
 
@@ -82,7 +79,7 @@ cp .env.default .env
 ```
 
 
-</br>
+<br/>
 
 
 ## 👨‍🦯 <span style="color:#9DB7FF">Local development</span>
@@ -115,7 +112,7 @@ pipenv shell
 pipenv sync --dev
 ```
 
-</br>
+<br/>
 
 > 🛑 **NOTE:** if you don't use `pipenv` for some reason remember that you will not have automatically exported variables from your `.env` file.
 >
@@ -136,7 +133,7 @@ set -o allexport; source .env; set +o allexport
 
 > 🛑 **NOTE:** Please do not forget about environment variables! Now all environment variables for the Postgres Database which runs in docker are already passed to docker-compose.yaml from the .env file.
 
-</br>
+<br/>
 
 
 ### 3. Provide code quality ✨
@@ -177,7 +174,7 @@ P.S. You don't need to do this additional step if you run application via Docker
 uvicorn src.main:app --proxy-headers --port {PORT} --reload
 ```
 
-</br>
+<br/>
 
 ### 5. Running Tests ▶️
 
@@ -216,8 +213,18 @@ psql# create user test;
 # Set password for the user
 psql# alter user test with password 'test';
 ```
-</br>
-</br>
+
+#### Test coverage
+
+To correctly calculate test coverage, you need to run the coverage with the `--concurrency=thread,gevent` parameter:
+
+```bash
+coverage run --concurrency=thread,gevent -m pytest
+coverage report -m
+```
+
+<br/>
+<br/>
 
 ## 🐳 <span style="color:#9DB7FF">Docker development</span>
 
@@ -631,4 +638,4 @@ postgresql+asyncpg://<username>:<password>@<hostname>:port/database
 For AWS S3 bucket next fields are required: 
 `storage_region`,`storage_bucket`, `storage_access_key`,`storage_secret_key`.
 ### 3. Azure Blob
-In case of Azure blob, specify your connection string into field `storage_secret_key`
+In case of Azure blob, specify your connection string into field `storage_secret_key`
diff --git a/compose/fastapi/Dockerfile b/compose/fastapi/Dockerfile
@@ -1,4 +1,4 @@
-FROM --platform=linux/x86_64 python:3.10.8-slim
+FROM --platform=linux/x86_64 python:3.10.8-slim as base
 
 ARG PIPENV_EXTRA_ARGS
 
@@ -7,11 +7,9 @@ ENV PYTHONPATH="src/"
 
 WORKDIR /app/
 
-RUN apt-get update \
+RUN apt-get -y update && apt-get -y upgrade \
     # dependencies for building Python packages
-    && apt-get install -y build-essential curl \
-    # cleaning up unused files
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get install -y build-essential curl
 
 # Add local non-root user to avoid issue with files
 # created inside a container being owned by root.
@@ -42,3 +40,11 @@ RUN sed -i 's/\r$//g' /fastapi-start && chmod +x /fastapi-start
 
 # Select internal user
 USER code
+
+# worker instructions
+FROM base as worker
+
+USER root
+RUN apt-get install -y --no-install-recommends ffmpeg && apt-get install -y imagemagick
+
+USER code
diff --git a/compose/minio/create_bucket.sh b/compose/minio/create_bucket.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+#/usr/bin/mc alias set myminio http://minio:9000 minioaccess miniosecret;
+#/usr/bin/mc mb myminio/media;
+#/usr/bin/mc policy set public myminio/media;
+#exit 0;
+
+/usr/bin/mc config host add local http://minio:9000 minioaccess miniosecret;
+#/usr/bin/mc rm -r --force local/${CDN__BUCKET};
+/usr/bin/mc mb -p local/${CDN__BUCKET};
+/usr/bin/mc policy set download local/${CDN__BUCKET};
+/usr/bin/mc policy set public local/${CDN__BUCKET};
+/usr/bin/mc anonymous set upload local/${CDN__BUCKET};
+/usr/bin/mc anonymous set download local/${CDN__BUCKET};
+/usr/bin/mc anonymous set public local/${CDN__BUCKET};
+
+exit 0;
diff --git a/compose/postgres/Dockerfile b/compose/postgres/Dockerfile
@@ -0,0 +1,7 @@
+FROM postgres:15.4
+
+RUN apt-get update && apt-get install -y curl
+
+RUN apt-get -y install postgresql-15-cron
+
+COPY ./compose/postgres/init-db /docker-entrypoint-initdb.d
diff --git a/compose/postgres/init-db/002-pg-cron.sh b/compose/postgres/init-db/002-pg-cron.sh
@@ -0,0 +1,6 @@
+dbname="$POSTGRES_DB"
+
+echo "shared_preload_libraries = 'pg_cron'" >> /var/lib/postgresql/data/postgresql.conf
+echo "cron.database_name = '$dbname'" >> /var/lib/postgresql/data/postgresql.conf
+
+pg_ctl restart
diff --git a/compose/postgres/init-db/003-main.sql b/compose/postgres/init-db/003-main.sql
@@ -0,0 +1 @@
+CREATE EXTENSION pg_cron;
diff --git a/conftest.py b/conftest.py
@@ -1,9 +1,9 @@
-import asyncio
 import os
 
 import pytest
 from alembic import command
 from alembic.config import Config
+from pytest_asyncio import is_async_test
 
 alembic_configs = [Config("alembic.ini"), Config("alembic_arbitrary.ini")]
 
@@ -20,12 +20,11 @@ def after():
     os.environ.pop("PYTEST_APP_TESTING")
 
 
-@pytest.fixture(scope="session")
-def event_loop():
-    policy = asyncio.get_event_loop_policy()
-    loop = policy.new_event_loop()
-    yield loop
-    loop.close()
+def pytest_collection_modifyitems(items):
+    pytest_asyncio_tests = (item for item in items if is_async_test(item))
+    session_scope_marker = pytest.mark.asyncio(scope="session")
+    for async_test in pytest_asyncio_tests:
+        async_test.add_marker(session_scope_marker)
 
 
 def pytest_sessionstart():