ref: remove dbt deps from ad-hoc commands and keep it at docker image…

… only (et/somenergia-jardiner!146) * ref: add dev target to dbt profiles and set it as default default was test, now is dev. test target is used by CI and we don't want to touch that during development. * ref: update docker and docker compose workflow * docker compose commands won't use .env file explicitly in Makefile * Split dbt-deps and dbt-docs services and expose them separatedly in Makefile * remove noisy documentation * set dbt environment variables explictly instead of refreshing them in every docker build stage * fix directory being used as dbt project dir from /main to /repo * ref: rename dirs and update names in CI - this is to match CI files in project mercat-balanc - renames container/app to docker/main and references all over CI - renames dbt-docs to dbt-deps - renames app image name to main - renames app-dev image name to just dev - also adds a docker tag for the current git SHA - updates Makefile and example.env * chore: constrain sqlalchemy even more because of long names issue this has been reviewed before, but somehow 1.40.49 started to fail again. See <python-poetry/poetry#8529> for details. * !ref: rename dbt-docs image to dbt-deps this matches the latest project's layout * ref: move docker-compose main to top directory first refactoring step toward matching version of mercat-balanç * ref: move mkdocs compose to top directory this reflects working environment in mercat-balanç * ci: fix download of empty artifact see https://gitlab.com/gitlab-org/gitlab/-/issues/29118 * ref: back to old alias for dbt arg it must be clashing with something * ref: remove dbt deps from ad-hoc commands and keep it at docker image only this sets the directory for installing packages through an environmental variable that defaults to dbt_packages, but is set at /home/somenergia/.dbt/dbt_packages when running inside a container
Som-Energia · Sep 2, 2024 · 0dae1cd · 0dae1cd
1 parent 85efeff
commit 0dae1cd
Show file tree

Hide file tree

Showing 19 changed files with 1,864 additions and 1,688 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -43,5 +43,5 @@ docker-compose.*
 # other
 *.log
 logs
-containers
+docker
 dags
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -21,23 +21,30 @@ variables:
 
   SOMENERGIA_REGISTRY: $SOM_HARBOR_DADES_URL
   DBT_PROJECT_DIR_NAME: dbt_jardiner
+  DBT_PACKAGES_INSTALL_PATH: /home/somenergia/.dbt/dbt_packages
   SOM_PROJECT_NAME: somenergia-jardiner
   DBT_SKIP_STATE: "false"
+
   IMAGE_NAME_PREFIX: $SOM_HARBOR_DADES_URL/$CI_PROJECT_NAME
-  IMAGE_NAME_APP_PRE_RELEASE: $IMAGE_NAME_PREFIX-app:pre-release
-  IMAGE_NAME_APP_RELEASE: $IMAGE_NAME_PREFIX-app:latest
-  IMAGE_NAME_DBT_DOCS_PRE_RELEASE: $IMAGE_NAME_PREFIX-dbt-docs:pre-release
-  IMAGE_NAME_DBT_DOCS_RELEASE: $IMAGE_NAME_PREFIX-dbt-docs:latest
+
+  IMAGE_NAME_MAIN_PRE_RELEASE: $IMAGE_NAME_PREFIX-main:pre-release
+  IMAGE_NAME_MAIN_RELEASE: $IMAGE_NAME_PREFIX-main:latest
+  IMAGE_NAME_MAIN_SHA_RELEASE: $IMAGE_NAME_PREFIX-main:$CI_COMMIT_SHA
+
+  IMAGE_NAME_DBT_DEPS_PRE_RELEASE: $IMAGE_NAME_PREFIX-dbt-deps:pre-release
+  IMAGE_NAME_DBT_DEPS_RELEASE: $IMAGE_NAME_PREFIX-dbt-deps:latest
+  IMAGE_NAME_DBT_DEPS_SHA_RELEASE: $IMAGE_NAME_PREFIX-dbt-deps:$CI_COMMIT_SHA
+
   GIT_CLONE_SSH_URL: git@$CI_SERVER_SHELL_SSH_HOST:et/$CI_PROJECT_NAME.git
-  MKDOCS_COMPOSE_FILE: containers/mkdocs/docker-compose.mkdocs.yml
-  APP_COMPOSE_FILE: containers/app/docker-compose.yml
+  MKDOCS_COMPOSE_FILE: docker-compose.mkdocs.yml
+  MAIN_COMPOSE_FILE: docker-compose.main.yml
   CI_TRIGGER_TASK_NAME: null
 
   # just run pre-commit hooks on the files that changed, but don't fix them
   PRE_COMMIT_AUTO_FIX: 0
 
 # environment variables to render compose files properly
-.compose_app_env: &compose_app_env
+.compose_dbt_env: &compose_dbt_env
   SOMENERGIA_PROJECT_ENVIRONMENT: "production"
   SOM_JARDINER_DB_USER: "$SOM_JARDINER_DB_USER"
   SOM_JARDINER_DB_PASSWORD: "$SOM_JARDINER_DB_PASSWORD"
@@ -56,15 +63,15 @@ pre-commit:
   tags:
     - somenergia-et
 
-# anchor with files to track when building app, builder and other images
+# anchor with files to track when building main, builder and other images
 .builder-build-changes-paths: &builder-build-changes-paths
-  - containers/app/Dockerfile
+  - docker/main/Dockerfile
   - pyproject.toml
   - poetry.lock
   - .gitlab-ci.yml
 
-.app-build-changes-paths: &app-build-changes-paths
-  - containers/app/Dockerfile
+.main-build-changes-paths: &main-build-changes-paths
+  - docker/main/Dockerfile
   - pyproject.toml
   - poetry.lock
   - dbt_jardiner/config/profiles.yml
@@ -73,7 +80,7 @@ pre-commit:
   - jardiner/*
   - .gitlab-ci.yml
 
-.dbt-build-changes-paths: &dbt-build-changes-paths
+.dbt-models-changes-paths: &dbt-models-changes-paths
   - dbt_jardiner/**/*.sql
   - dbt_jardiner/**/*.yaml
   - .gitlab-ci.yml
@@ -84,12 +91,12 @@ pre-commit:
   - docker login -u $SOM_HARBOR_DADES_USER -p $SOM_HARBOR_DADES_PASSWORD $SOM_HARBOR_DADES_URL
 
 build-builder-image-pre-release:
-  variables: *compose_app_env
+  variables: *compose_dbt_env
   stage: build
   before_script: *docker-login-before-script
   script:
-    - docker build --pull -t $IMAGE_NAME_APP_PRE_RELEASE --progress=plain --target=app -f containers/app/Dockerfile .
-    - docker push $IMAGE_NAME_APP_PRE_RELEASE
+    - docker build --pull -t $IMAGE_NAME_MAIN_PRE_RELEASE --progress=plain --target=main -f docker/main/Dockerfile .
+    - docker push $IMAGE_NAME_MAIN_PRE_RELEASE
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
       when: always
@@ -105,102 +112,118 @@ build-builder-image-pre-release:
   tags:
     - somenergia-et
 
-build-app-image-pre-release:
-  variables: *compose_app_env
+build-main-image-pre-release:
+  variables: *compose_dbt_env
   stage: build
   before_script: *docker-login-before-script
   script:
-    - docker build --pull -t $IMAGE_NAME_APP_PRE_RELEASE --progress=plain --target=app -f containers/app/Dockerfile .
-    - docker push $IMAGE_NAME_APP_PRE_RELEASE
+    - docker build --pull -t $IMAGE_NAME_MAIN_PRE_RELEASE --progress=plain --target=main -f docker/main/Dockerfile .
+    - docker push $IMAGE_NAME_MAIN_PRE_RELEASE
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
       when: always
       changes:
-        paths: *app-build-changes-paths
+        paths: *main-build-changes-paths
       allow_failure: false
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
       changes:
         compare_to: "refs/heads/main"
-        paths: *app-build-changes-paths
+        paths: *main-build-changes-paths
       allow_failure: false
   tags:
     - somenergia-et
 
-build-dbt-docs-image-pre-release:
-  variables: *compose_app_env
+build-dbt-deps-image-pre-release:
+  variables: *compose_dbt_env
   stage: build
   before_script: *docker-login-before-script
   script:
-    - docker build --pull -t $IMAGE_NAME_DBT_DOCS_PRE_RELEASE --progress=plain --target=dbt-docs -f containers/app/Dockerfile .
-    - docker push $IMAGE_NAME_DBT_DOCS_PRE_RELEASE
+    - docker build --pull -t $IMAGE_NAME_DBT_DEPS_PRE_RELEASE --progress=plain --target=dbt-deps -f docker/main/Dockerfile .
+    - docker push $IMAGE_NAME_DBT_DEPS_PRE_RELEASE
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
       when: always
       changes:
-        paths: *app-build-changes-paths
+        paths: *main-build-changes-paths
       allow_failure: false
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
       changes:
         compare_to: "refs/heads/main"
-        paths: *app-build-changes-paths
+        paths: *main-build-changes-paths
       allow_failure: false
   tags:
     - somenergia-et
 
 dbt-build:
   stage: test
+  resource_group: $DBT_BUILD_RESOURCE_GROUP
   variables:
-    DBT_TARGET_NAME: test
     DBHOST: $SOM_JARDINER_DB_HOST
     DBPORT: $SOM_JARDINER_DB_PORT
     DBUSER: $SOM_JARDINER_DB_USER
     DBPASSWORD: $SOM_JARDINER_DB_PASSWORD
     DBNAME: $SOM_JARDINER_DB_DBNAME
+    DBT_TARGET_NAME: test
+    DBT_BUILD_RESOURCE_GROUP: test
+    # https://docs.getdbt.com/best-practices/best-practice-workflows#run-only-modified-models-to-test-changes-slim-ci
     DBT_MODELS_SELECTOR: "--models state:modified+ --defer"
     DBT_STATE: "--state state/prod"
     DBT_MANIFEST_ARTIFACT_URL: "https://$CI_SERVER_HOST/api/v4/projects/$CI_PROJECT_ID/jobs/artifacts/${CI_DEFAULT_BRANCH}/download?job=pages&job_token=$CI_JOB_TOKEN"
-  image: ${IMAGE_NAME_DBT_DOCS_PRE_RELEASE}
+  image: ${IMAGE_NAME_DBT_DEPS_PRE_RELEASE}
   script:
-    - cd ${CI_PROJECT_DIR}/${DBT_PROJECT_DIR_NAME}
-    - dbt deps --target ${DBT_TARGET_NAME}
-    - echo "retrieving dbt artifacts from previous succesful job"
     # download artifacts from previous succesful job dbt-build at branch main
-    - curl --location --output /tmp/artifacts.zip ${DBT_MANIFEST_ARTIFACT_URL}
-    - unzip -o /tmp/artifacts.zip -d /tmp/artifacts
-    - cp /tmp/artifacts/public/dbt_docs/manifest.json ${CI_PROJECT_DIR}/${DBT_PROJECT_DIR_NAME}/state/prod/manifest.json
-    - dbt build --target ${DBT_TARGET_NAME} --store-failures --threads 4 ${DBT_MODELS_SELECTOR} ${DBT_STATE}
+    - echo "retrieving dbt artifacts from previous succesful job"
+    # gitlab sometimes fails silently when artifact is invalid or not found, https://gitlab.com/gitlab-org/gitlab/-/issues/29118
+    - export RESULT=$(curl --output /dev/null --silent --write-out "%{content_type}" "$DBT_MANIFEST_ARTIFACT_URL")
+    - |
+      echo "Artifact content type: ${RESULT}"
+      if [ "$RESULT" = "application/zip" ]; then
+          echo "Unzipping and copying manifest.json from artifacts"
+          curl --location --output /tmp/artifacts.zip ${DBT_MANIFEST_ARTIFACT_URL}
+          unzip -o /tmp/artifacts.zip -d /tmp/artifacts
+          cp /tmp/artifacts/public/dbt_docs/manifest.json ${CI_PROJECT_DIR}/${DBT_PROJECT_DIR}/state/prod/manifest.json
+          dbt build --target ${DBT_TARGET_NAME} --store-failures --threads 4 ${DBT_MODELS_SELECTOR} --state state/prod
+      else
+          echo "Artifact not found. Assuming this is the first run of the pipeline. Not using dbt states this time."
+          dbt build --target "${DBT_TARGET_NAME}" --store-failures --threads 4
+      fi
   tags:
     - somenergia-et
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
       when: always
       variables:
         DBT_TARGET_NAME: prod
+        DBT_BUILD_RESOURCE_GROUP: prod
       changes:
-        paths: *dbt-build-changes-paths
+        paths: *dbt-models-changes-paths
       allow_failure: false
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
       variables:
         DBT_MODELS_SELECTOR: "--select tag:dset_responses_fresh state:modified+ +state:modified+,config.materialized:incremental +state:modified+,config.materialized:table --defer"
         DBT_TARGET_NAME: test
+        DBT_BUILD_RESOURCE_GROUP: test
         DBT_FAIL_FAST: "True"
       changes:
         compare_to: "refs/heads/main"
-        paths: *dbt-build-changes-paths
+        paths: *dbt-models-changes-paths
       allow_failure: false
 
-release-app-image:
+release-main-image:
   stage: release
   before_script: *docker-login-before-script
   script:
-    - docker pull $IMAGE_NAME_APP_PRE_RELEASE
-    - docker tag $IMAGE_NAME_APP_PRE_RELEASE $IMAGE_NAME_APP_RELEASE
-    - docker push $IMAGE_NAME_APP_RELEASE
+    - docker pull $IMAGE_NAME_MAIN_PRE_RELEASE
+    - docker tag $IMAGE_NAME_MAIN_PRE_RELEASE $IMAGE_NAME_MAIN_RELEASE
+    # https://blog.container-solutions.com/tagging-docker-images-the-right-way
+    - docker tag $IMAGE_NAME_MAIN_RELEASE $IMAGE_NAME_MAIN_SHA_RELEASE
+    - docker push $IMAGE_NAME_MAIN_SHA_RELEASE
+    - docker push $IMAGE_NAME_MAIN_RELEASE
   rules:
-    # run only on main branch after build-app-image-pre-release is successful
+    # run only on main branch after build-main-image-pre-release is successful
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: manual
@@ -212,7 +235,7 @@ release-app-image:
 rsync-deploy-dags: # This job runs in the deploy stage and copies a file to a server
   stage: deploy # It only runs when *both* jobs in the test stage complete successfully.
   rules:
-    # run only on main branch after build-app-image-pre-release is successful
+    # run only on main branch after build-main-image-pre-release is successful
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: manual
@@ -311,21 +334,23 @@ rsync-deploy-dags: # This job runs in the deploy stage and copies a file to a se
     - somenergia-et
 
 # https://towardsdatascience.com/running-dbt-using-gitlab-ci-cd-8a2ef0f05af0
-release-dbt-docs-image:
+release-dbt-deps-image:
   stage: release
   before_script:
     - echo "Logging in to Docker Hub"
     - docker login -u $SOM_HARBOR_DADES_USER -p $SOM_HARBOR_DADES_PASSWORD $SOM_HARBOR_DADES_URL
   script:
-    - docker pull $IMAGE_NAME_DBT_DOCS_PRE_RELEASE
-    - docker tag $IMAGE_NAME_DBT_DOCS_PRE_RELEASE $IMAGE_NAME_DBT_DOCS_RELEASE
-    - docker push $IMAGE_NAME_DBT_DOCS_RELEASE
+    - docker pull $IMAGE_NAME_DBT_DEPS_PRE_RELEASE
+    - docker tag $IMAGE_NAME_DBT_DEPS_PRE_RELEASE $IMAGE_NAME_DBT_DEPS_RELEASE
+    - docker tag $IMAGE_NAME_DBT_DEPS_PRE_RELEASE $IMAGE_NAME_DBT_DEPS_SHA_RELEASE
+    - docker push $IMAGE_NAME_DBT_DEPS_SHA_RELEASE
+    - docker push $IMAGE_NAME_DBT_DEPS_RELEASE
   tags:
     - somenergia-et
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
       changes:
-        paths: *app-build-changes-paths
+        paths: *main-build-changes-paths
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: manual
       allow_failure: true
@@ -340,14 +365,14 @@ release-mkdocs-image:
     - docker login -u $SOM_HARBOR_DADES_USER -p $SOM_HARBOR_DADES_PASSWORD $SOM_HARBOR_DADES_URL
   script:
     - echo "Building mkdocs image"
-    - docker build --pull -t $MKDOCS_IMAGE_NAME -f containers/mkdocs/Dockerfile .
+    - docker build --pull -t $MKDOCS_IMAGE_NAME -f docker/mkdocs/Dockerfile .
     - echo "Pushing mkdocs image"
     - docker push $MKDOCS_IMAGE_NAME
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
       changes:
         paths:
-          - containers/mkdocs/Dockerfile
+          - docker/mkdocs/Dockerfile
           - mkdocs.yml
           - pyproject.toml
           - poetry.lock
@@ -390,13 +415,11 @@ generate-dbt-docs:
     DBUSER: $SOM_JARDINER_DB_USER
     DBPASSWORD: $SOM_JARDINER_DB_PASSWORD
     DBNAME: $SOM_JARDINER_DB_DBNAME
-  image: ${SOM_HARBOR_DADES_URL}/${SOM_PROJECT_NAME}-dbt-docs:latest
+  image: ${SOM_HARBOR_DADES_URL}/${SOM_PROJECT_NAME}-dbt-deps:latest
   script:
     - cd ${CI_PROJECT_DIR}/${DBT_PROJECT_DIR_NAME}
     - echo "Creating artifacts directory"
     - mkdir -p ${CI_PROJECT_DIR}/.artifacts/dbt_docs
-    - echo "Installing dbt dependencies"
-    - dbt deps
     - echo "Generating dbt docs"
     - dbt docs generate --target ${DBT_TARGET_NAME} --target-path ${CI_PROJECT_DIR}/.artifacts/dbt_docs --threads 4
   artifacts:
@@ -443,7 +466,7 @@ generate-mkdocs:
     paths:
       - .artifacts/docs
   rules:
-    # run only on main branch after build-app-image-pre-release is successful
+    # run only on main branch after build-main-image-pre-release is successful
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
       when: always
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
@@ -479,7 +502,7 @@ pages:
     paths:
       - public
   rules:
-    # run only on main branch after build-app-image-pre-release is successful
+    # run only on main branch after build-main-image-pre-release is successful
     - if: $CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "push"
       when: always
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-Original file line number
+Diff line change
@@ Expand Up / @@ -43,5 +43,5 @@ docker-compose.* @@
     # other
     *.log
     logs
-    containers
+    docker
     dags