From e40f33a4fd321ae51098b8530cdc446c25db930b Mon Sep 17 00:00:00 2001 From: Tudor Amariei Date: Tue, 24 Oct 2023 16:05:19 +0300 Subject: [PATCH] Change the build process - remove setup.cfg and use a pyproject.toml - refactor the docker files structure and process --- .github/dependabot.yml | 16 ----- .github/workflows/api_pipeline.yml | 2 +- .github/workflows/build.yml | 34 +++++++++ .github/workflows/client_pipeline.yml | 2 +- .github/workflows/codeql-analysis-js.yml | 2 +- .github/workflows/codeql-analysis-py.yml | 2 +- .github/workflows/deploy_staging.yml | 27 ------- .../workflows/local_deployment_pipeline.yml | 42 ----------- api/Dockerfile.old | 36 ---------- api/pyproject.toml | 71 +++++++++++++++++++ api/requirements-dev.txt | 2 +- api/requirements.in | 2 +- api/requirements.txt | 2 +- api/setup.cfg | 45 ------------ client/.dockerignore | 1 - deploy/pull.sh | 8 --- docker-compose.prod.yml | 5 +- docker-compose.test.yml | 41 ----------- docker-compose.yml | 8 +-- .../dockerfiles/Dockerfile.backend | 42 +++++++---- .../dockerfiles/Dockerfile.backend.dev | 10 +-- .../dockerfiles/Dockerfile.frontend | 0 .../dockerfiles/Dockerfile.frontend.dev | 6 +- docker/s6-rc.d/init/init.sh | 6 +- 24 files changed, 156 insertions(+), 256 deletions(-) delete mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/deploy_staging.yml delete mode 100644 .github/workflows/local_deployment_pipeline.yml delete mode 100644 api/Dockerfile.old create mode 100644 api/pyproject.toml delete mode 100644 api/setup.cfg delete mode 100755 deploy/pull.sh delete mode 100644 docker-compose.test.yml rename api/Dockerfile => docker/dockerfiles/Dockerfile.backend (66%) rename api/Dockerfile.dev => docker/dockerfiles/Dockerfile.backend.dev (87%) rename client/Dockerfile => docker/dockerfiles/Dockerfile.frontend (100%) rename client/Dockerfile.dev => docker/dockerfiles/Dockerfile.frontend.dev (79%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index ee0c442c..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 -updates: -- package-ecosystem: npm - directory: "/client" - schedule: - interval: monthly - timezone: Europe/Bucharest - open-pull-requests-limit: 5 - target-branch: develop - reviewers: - - tudoramariei - assignees: - - tudoramariei - labels: - - "dependencies" - - "client: ReactJS" diff --git a/.github/workflows/api_pipeline.yml b/.github/workflows/api_pipeline.yml index b7d223bb..a892b221 100644 --- a/.github/workflows/api_pipeline.yml +++ b/.github/workflows/api_pipeline.yml @@ -8,7 +8,7 @@ on: paths: - 'api/**.py' - 'api/requirements*.*' - - 'api/Dockerfile' + - 'docker/dockerfiles/Dockerfile.backend*' - 'docker-compose.yml' - '.github/workflows/api_pipeline.yml' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..db58bf3c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,34 @@ +on: + push: + branches: + - 'main' + tags: + - 'v*' + +name: Build Docker image + +jobs: + + build-backend-amd64: + name: backend/amd64 + uses: code4romania/.github/.github/workflows/build-push-image.yml@main + with: + images: ghcr.io/code4romania/seismic-risc-backend + context: ./ + dockerfile: ./docker/dockerfiles/Dockerfile.backend + registry: ghcr.io + secrets: + username: ${{ github.actor }} + token: ${{ secrets.GITHUB_TOKEN }} + + build-frontend-amd64: + name: frontend/amd64 + uses: code4romania/.github/.github/workflows/build-push-image.yml@main + with: + images: ghcr.io/code4romania/seismic-risc-frontend + context: ./client + dockerfile: ./docker/dockerfiles/Dockerfile.frontend + registry: ghcr.io + secrets: + username: ${{ github.actor }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/client_pipeline.yml b/.github/workflows/client_pipeline.yml index aff23e34..0a4441aa 100644 --- a/.github/workflows/client_pipeline.yml +++ b/.github/workflows/client_pipeline.yml @@ -8,7 +8,7 @@ on: paths: - 'client/**.js' - 'client/package*.json' - - 'client/Dockerfile' + - 'docker/dockerfiles/Dockerfile.frontend*' - 'docker-compose.yml' - '.github/workflows/client_pipeline.yml' diff --git a/.github/workflows/codeql-analysis-js.yml b/.github/workflows/codeql-analysis-js.yml index 764dcda7..0f0e2dfb 100644 --- a/.github/workflows/codeql-analysis-js.yml +++ b/.github/workflows/codeql-analysis-js.yml @@ -8,7 +8,7 @@ on: paths: - 'client/**.js' - 'client/package*.json' - - 'client/Dockerfile' + - 'docker/dockerfiles/Dockerfile.frontend*' - 'docker-compose.yml' - '.github/workflows/client_pipeline.yml' - '.github/workflows/code-analysis-js.yml' diff --git a/.github/workflows/codeql-analysis-py.yml b/.github/workflows/codeql-analysis-py.yml index 56bbb49b..f2c41980 100644 --- a/.github/workflows/codeql-analysis-py.yml +++ b/.github/workflows/codeql-analysis-py.yml @@ -8,7 +8,7 @@ on: paths: - 'api/**.py' - 'api/requirements*.*' - - 'api/Dockerfile' + - 'docker/dockerfiles/Dockerfile.backend*' - 'docker-compose.yml' - '.github/workflows/api_pipeline.yml' - '.github/workflows/code-analysis-py.yml' diff --git a/.github/workflows/deploy_staging.yml b/.github/workflows/deploy_staging.yml deleted file mode 100644 index cf5773e9..00000000 --- a/.github/workflows/deploy_staging.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Deploy to staging env - -on: - push: - branches: - - 'develop' - paths: - - 'api/**' - - '.github/workflows/*' - - './docker-compose.yml' - - './deploy/**' - - './.env*' - -jobs: - deploy: - name: Deploy to staging env - runs-on: ubuntu-22.04 - - steps: - - name: Executing remote SSH commands - if: github.ref == 'refs/heads/develop' - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.DEV_SERVER_IP }} - username: ${{ secrets.DEV_SERVER_SSH_USERNAME }} - key: ${{ secrets.DEV_SERVER_SSH_KEY }} - script: "/root/seismic-risc/deploy/pull.sh" diff --git a/.github/workflows/local_deployment_pipeline.yml b/.github/workflows/local_deployment_pipeline.yml deleted file mode 100644 index a23a1a9b..00000000 --- a/.github/workflows/local_deployment_pipeline.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Run first-time set-up for MacOS and Linux - -on: - push: - branches: - - 'main' - - 'develop' - pull_request: - branches: - - 'main' - - 'develop' - paths: - - 'api/**' - - 'client/**' - - '.github/workflows/*' - - './docker-compose.yml' - -jobs: - ubuntu-set-up: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - name: Set-up Docker - run: | - sudo apt-get update - sudo apt-get install ca-certificates curl gnupg - sudo install -m 0755 -d /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - sudo chmod a+r /etc/apt/keyrings/docker.gpg - echo \ - "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update - sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - - name: Create environment file - run: | - cp .env.example.dev .env - cp .env.example.dev .env.dev - - name: Initialize environment - run: | - make build diff --git a/api/Dockerfile.old b/api/Dockerfile.old deleted file mode 100644 index be59de4c..00000000 --- a/api/Dockerfile.old +++ /dev/null @@ -1,36 +0,0 @@ -FROM python:3.11.6-slim-bookworm - -ARG ENVIRONMENT - -ENV ENVIRONMENT ${ENVIRONMENT:-production} -ENV DJANGO_SETTINGS_MODULE seismic_site.settings.${ENVIRONMENT} -ENV DOCKERIZE_VERSION v0.6.1 - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - postgresql-client \ - libpq-dev \ - gettext \ - wget - -# install dockerize and update pip -RUN wget https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz && \ - tar -C /usr/local/bin -xzvf dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz && \ - rm dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz && \ - chown root:root /usr/local/bin/dockerize && \ - pip install "pip==23.*" -U - -# install and cache app dependencies -WORKDIR /opt/ - -COPY ./requirements*.txt ./ -RUN if [ "${ENVIRONMENT}" = "production" ]; then pip install -r requirements.txt; else pip install -r requirements-dev.txt; fi - -WORKDIR /code/ - -COPY ./docker-entrypoint / -COPY ./ /code/ - -ENTRYPOINT ["/docker-entrypoint"] -EXPOSE 8030 diff --git a/api/pyproject.toml b/api/pyproject.toml new file mode 100644 index 00000000..ba31277a --- /dev/null +++ b/api/pyproject.toml @@ -0,0 +1,71 @@ +[tool.ruff] +# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. +select = ["E", "F"] +ignore = [] + +# Allow autofix for all enabled rules (when `--fix`) is provided. +fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] +unfixable = [] + +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "migrations", +] + +# Same as Black. +line-length = 120 + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +# Assume Python 3.11. +target-version = "py311" + +[tool.ruff.mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 10 + +[tool.ruff.extend-per-file-ignores] +"settings/*.py" = ["F403", "F405"] +"*/__init__.py" = ["F401"] + +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "backend.test_settings" + +python_files = [ + "test.py", + "test*.py", + "*test.py", + "*tests.py", +] +filterwarnings = [ + "ignore:The django.utils.baseconv module is deprecated." +] + +[tool.black] +line-length = 120 +target-version = ["py311"] +extend-exclude = '''( + (.+/)?migrations/.+ +)''' diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt index ecab8c7c..f0f21760 100644 --- a/api/requirements-dev.txt +++ b/api/requirements-dev.txt @@ -183,7 +183,7 @@ prompt-toolkit==3.0.39 # via ipython psutil==5.9.6 # via -r requirements.txt -psycopg2==2.9.9 +psycopg2-binary==2.9.9 # via -r requirements.txt ptyprocess==0.7.0 # via pexpect diff --git a/api/requirements.in b/api/requirements.in index 90289ecd..dbbc62ef 100644 --- a/api/requirements.in +++ b/api/requirements.in @@ -20,7 +20,7 @@ djangorestframework==3.14.0 drf-spectacular==0.26.5 # database -psycopg2==2.9.9 +psycopg2-binary==2.9.9 dj-database-url==2.1.0 # file handling diff --git a/api/requirements.txt b/api/requirements.txt index 59c66efc..b62a3a3a 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -91,7 +91,7 @@ pillow==10.0.1 # via -r requirements.in psutil==5.9.6 # via -r requirements.in -psycopg2==2.9.9 +psycopg2-binary==2.9.9 # via -r requirements.in python-dateutil==2.8.2 # via diff --git a/api/setup.cfg b/api/setup.cfg deleted file mode 100644 index 51f11969..00000000 --- a/api/setup.cfg +++ /dev/null @@ -1,45 +0,0 @@ -[flake8] -max-line-length = 80 -extend-ignore = E203 -exclude = - */migrations/*.py - -[coverage:run] -branch = True -omit = - */migrations/* - */urls.py - */admin.py - seismic_site/* - manage.py - wait_for_db.py - -[coverage:report] -# fail_under = 90 -precision = 2 -sort = Name -exclude_lines = - def __repr__ - def __str__ -show_missing = True - -[tool:pytest] -env = - DJANGO_SETTINGS_MODULE=seismic_site.settings.test - ENVIRONMENT=test - -testpaths = - ./tests - -python_files = - tests.py - test_*.py - *_tests.py - -addopts = - -ra - --tb=native - --cov-config=.coveragerc --cov=. --cov-report=term --no-cov-on-fail - -filterwarnings = - ignore:Coverage disabled via --no-cov switch! diff --git a/client/.dockerignore b/client/.dockerignore index 210c47a9..3c3629e6 100644 --- a/client/.dockerignore +++ b/client/.dockerignore @@ -1,2 +1 @@ node_modules -Dockerfile diff --git a/deploy/pull.sh b/deploy/pull.sh deleted file mode 100755 index fc854f29..00000000 --- a/deploy/pull.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -e - -cd /root/seismic-risc -git reset --hard HEAD -git pull - -docker-compose build --build-arg ENVIRONMENT=production api -docker-compose up -d db api diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 1d03630c..8f9f8c78 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -15,8 +15,8 @@ services: file: dk-cp.base.yml service: api_base build: - context: . - dockerfile: ./api/Dockerfile + context: ./ + dockerfile: ./docker/dockerfiles/Dockerfile.backend args: - ENVIRONMENT=production env_file: @@ -31,7 +31,6 @@ services: service: client_base build: context: ./client - dockerfile: ./Dockerfile args: - ENVIRONMENT=production env_file: diff --git a/docker-compose.test.yml b/docker-compose.test.yml deleted file mode 100644 index 63c4d408..00000000 --- a/docker-compose.test.yml +++ /dev/null @@ -1,41 +0,0 @@ -version: "3.8" -name: seismic-test - -services: - - db: - container_name: seismic_db-test - extends: - file: docker-compose.dev.yml - service: db_base - - api: - container_name: seismic_api-test - extends: - file: docker-compose.dev.yml - service: api_base - build: - context: . - dockerfile: ./api/Dockerfile.dev - args: - - ENVIRONMENT=test - env_file: - - .env - - .env.test - environment: - ENVIRONMENT: "test" - - client: - container_name: seismic_client-test - extends: - file: docker-compose.dev.yml - service: client_base - env_file: - - .env - - .env.test - environment: - ENVIRONMENT: "test" - -volumes: - pgdata: - name: seismic_pgdata-test diff --git a/docker-compose.yml b/docker-compose.yml index 3fc46141..694be305 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,8 @@ services: file: dk-cp.base.yml service: api_base build: - context: . - dockerfile: ./api/Dockerfile.dev + context: ./ + dockerfile: ./docker/dockerfiles/Dockerfile.backend.dev args: - ENVIRONMENT=development env_file: @@ -35,8 +35,8 @@ services: file: dk-cp.base.yml service: client_base build: - context: ./client - dockerfile: ./Dockerfile.dev + context: ./ + dockerfile: ./docker/dockerfiles/Dockerfile.frontend.dev args: - ENVIRONMENT=development env_file: diff --git a/api/Dockerfile b/docker/dockerfiles/Dockerfile.backend similarity index 66% rename from api/Dockerfile rename to docker/dockerfiles/Dockerfile.backend index 47f1ba2b..0c223dc4 100644 --- a/api/Dockerfile +++ b/docker/dockerfiles/Dockerfile.backend @@ -1,9 +1,30 @@ +FROM python:3.11.6-slim-bookworm as build + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + gettext git gcc g++ && \ + python3 -m pip install --upgrade pip setuptools cython + +# Python virtualenv paths +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +WORKDIR /build +COPY ./api/requirements.txt ./api/pyproject.toml ./ +RUN python3 -m pip install --upgrade pip setuptools && \ + python3 -m pip install -r ./requirements.txt + + FROM python:3.11.6-slim-bookworm ENV PYTHONUNBUFFERED=1 ENV ENVIRONMENT=production -ENV DEBUG ${DEBUG:-False} +ENV DEBUG=${DEBUG:-False} ENV RUN_MIGRATION=True ENV RUN_COMPILE_MESSAGES=True @@ -36,28 +57,23 @@ COPY docker/nginx/nginx.conf /etc/nginx/sites-available/default COPY docker/s6-rc.d /etc/s6-overlay/s6-rc.d -# Python virtualenv paths +# Python virtualenv paths: ENV VIRTUAL_ENV=/opt/venv RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# install the backend libraries: -COPY ./api/setup.cfg ./api/requirements.txt /var/www/seismic/api/ -WORKDIR /var/www/seismic/api/ -RUN python3 -m pip install --upgrade pip setuptools && \ - python3 -m pip install -r ./requirements.txt - - -# copy the backend source code: -COPY ./api/ /var/www/seismic/api/ +# copy the backend source code and libraries from the build stage: +COPY --from=build "${VIRTUAL_ENV}" "${VIRTUAL_ENV}" +WORKDIR /var/www/seismic/api/ +COPY ./api . # Make sure scripts in .local are usable: ENV PATH=/root/.local/bin:$PATH - -WORKDIR /var/www/seismic/ +# activate the virtualenv: +RUN . "${VIRTUAL_ENV}/bin/activate" ENTRYPOINT ["/init"] diff --git a/api/Dockerfile.dev b/docker/dockerfiles/Dockerfile.backend.dev similarity index 87% rename from api/Dockerfile.dev rename to docker/dockerfiles/Dockerfile.backend.dev index cab4f663..9a93c87d 100644 --- a/api/Dockerfile.dev +++ b/docker/dockerfiles/Dockerfile.backend.dev @@ -36,23 +36,19 @@ COPY docker/nginx/nginx.conf /etc/nginx/sites-available/default COPY docker/s6-rc.d /etc/s6-overlay/s6-rc.d -# Python virtualenv paths +# Python virtualenv paths: ENV VIRTUAL_ENV=/opt/venv RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# install the backend libraries: -COPY ./api/setup.cfg ./api/requirements-dev.txt /var/www/seismic/api/ +# Copy the back-end source code and install the backend libraries: WORKDIR /var/www/seismic/api/ +COPY ./api/ . RUN python3 -m pip install --upgrade pip setuptools && \ python3 -m pip install -r ./requirements-dev.txt -# copy the backend source code: -COPY ./api/ /var/www/seismic/api/ - - # Make sure scripts in .local are usable: ENV PATH=/root/.local/bin:$PATH diff --git a/client/Dockerfile b/docker/dockerfiles/Dockerfile.frontend similarity index 100% rename from client/Dockerfile rename to docker/dockerfiles/Dockerfile.frontend diff --git a/client/Dockerfile.dev b/docker/dockerfiles/Dockerfile.frontend.dev similarity index 79% rename from client/Dockerfile.dev rename to docker/dockerfiles/Dockerfile.frontend.dev index f7b927e4..dd392ee0 100644 --- a/client/Dockerfile.dev +++ b/docker/dockerfiles/Dockerfile.frontend.dev @@ -13,11 +13,11 @@ WORKDIR /code ENV PATH /code/node_modules/.bin:${PATH} # install and cache app dependencies -COPY ./package*.json ./ +COPY ./client/package*.json ./ RUN npm i -COPY ./docker-entrypoint / -COPY ./ /code/ +COPY ./client/docker-entrypoint / +COPY ./client/ /code/ EXPOSE 3000 diff --git a/docker/s6-rc.d/init/init.sh b/docker/s6-rc.d/init/init.sh index d6d61dab..adfac65a 100755 --- a/docker/s6-rc.d/init/init.sh +++ b/docker/s6-rc.d/init/init.sh @@ -55,8 +55,8 @@ fi # Load the dummy data if is_enabled "${RUN_LOAD_INITIAL_DATA:-False}"; then - echo "Loading initial building data into the database" + echo "Loading initial building data into the database" - ./manage.py loaddata proximal_utilities - ./manage.py loaddata work_performed + ./manage.py loaddata proximal_utilities + ./manage.py loaddata work_performed fi