SB- DBT Gatekeeper Submission

.circleci/config.yml

@@ -0,0 +1,82 @@
+version: 2.1
+
+executors:
+  standard:
+    docker:
+      - image: circleci/python:3.8
+    working_directory: "~/lib"
+
+# -----------------
+# Reusable commands
+# -----------------
+
+commands:
+  checkout_source:
+    steps:
+      - restore_cache:
+          keys:
+            - source-{{ .Branch }}-{{ .Revision }}
+            - source-{{ .Branch }}-
+            - source-
+      - checkout
+      - save_cache:
+          key: source-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - "./.git"
+
+  update_virtualenv:
+    steps:
+      - restore_cache:
+          keys:
+            - virtualenv-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
+            - virtualenv-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-
+            - virtualenv-{{ .Environment.CACHE_VERSION }}-
+      - run:
+          name: Install Python packages
+          command: pipenv install --dev --system
+      - save_cache:
+          key: virtualenv-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
+          paths:
+            - "./.venv"
+
+  prepare_dbt:
+    steps:
+      - run:
+          name: Installing Simba + UnixODBC
+          command: |
+            sudo mkdir -p /opt/simba/spark/lib/64/
+            sudo tar -xf driver/simba.tar.gz -C /opt/simba/spark/lib/64/
+            sudo apt-get install unixodbc-dev -y
+# --------------
+# Pipeline tasks
+# --------------
+
+jobs:
+  run-python-tests:
+    executor: standard
+    steps:
+      - checkout
+      - update_virtualenv
+      - run:
+          name: Run Python tests
+          command: make run-python-tests
+  run-dbt-project-evaluator:
+    executor: standard
+    environment:
+      SALT: himalayan
+      DBT_PROFILES_DIR: ~/lib/testing/
+      TESTING_PROFILE_NAME: jaffle_shop
+    steps:
+      - checkout
+      - update_virtualenv
+      - run:
+          name: Run dbt-project-evaluator package
+          command: make run-dbt-project-evaluator
+
+
+# Orchestrate our job run sequence
+workflows:
+  test:
+    jobs:
+      - run-python-tests
+      - run-dbt-project-evaluator

.gitignore

@@ -1,5 +1,11 @@
-
 target/
 dbt_modules/
+dbt_packages/
 logs/
+
 **/.DS_Store
+.env
+
+# Python temp files
+*.pyc
+__pycache__

Makefile

@@ -0,0 +1,40 @@
+init:
+	pipenv update
+	pipenv run dbt deps
+
+clean-env:
+	:> .env
+
+env-development-salt:
+	echo SALT=maldon >> .env
+
+env-target:
+	echo DATABRICKS_TARGET=$$(git symbolic-ref --short HEAD | tr /- _) >> .env
+
+package-project:
+	for PACKAGE in dbt_project_evaluator ; do \
+		cp package_projects/$$PACKAGE.yml dbt_packages/$$PACKAGE/dbt_project.yml ; \
+	done
+
+build-env: clean-env env-development-salt env-target package-project
+
+dbt-deps:
+	pipenv run dbt deps
+
+dbt-build: build-env
+	pipenv run dbt build --selector jaffle_shop
+
+run-dbt-project-evaluator: dbt-deps build-env
+	pipenv run dbt --warn-error build --select package:dbt_project_evaluator dbt_project_evaluator_exceptions
+
+lint: build-env
+	pipenv run sqlfluff lint
+
+format: build-env
+	pipenv run sqlfluff fix
+
+run-python-tests:
+	pipenv run pytest --quiet --show-capture=no --tb=no
+
+run-python-tests-detailed:
+	pipenv run pytest

Pipfile

@@ -0,0 +1,17 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+dbt-core = "==1.3.0"
+dbt-databricks = {extras = ["odbc"]}
+pytest-mock = "*"
+glob2 = "*"
+
+[dev-packages]
+pytest = "*"
+pytest-mock = "*"
+
+[requires]
+python_version = "3.8"