Improve development setup (project-koku#3914)

* Update macOS instructions
* Detect docker compose and prefer v2
  docker-compose v1 is deprecated
  https://www.docker.com/blog/announcing-compose-v2-general-availability/

* Ensure we are using buildkit
  The multi-arch image needs buildkit to work properly

* Remove M1 specific directions
  macOS specific directions are now incorporated in the main README

  The directions in the rosetta_on_m1.rst aren't really necessary
  and can lead to problems. Installing Python with brew is almost
  always a recipe for frustration.

  It's best to compile things natively for ARM rather than relying
  on emulation and maintaining separate copies of binaries.

* Automatically use copmose file for ARM systems
* Fix bad link
* Use correct tox environment in docs
* Use valid email address for pgAdmin login

Co-authored-by: Michael Skarbek <[email protected]>

.env.example

@@ -1,7 +1,7 @@
 DATABASE_SERVICE_NAME=POSTGRES_SQL
 DATABASE_ENGINE=postgresql
 DATABASE_NAME=postgres
-PGADMIN_EMAIL=postgres
+PGADMIN_EMAIL=postgres@local.dev
 PGADMIN_PASSWORD=postgres
 PGADMIN_PORT=8432
 POSTGRES_SQL_SERVICE_HOST=localhost
@@ -55,3 +55,5 @@ DEVELOPMENT_IDENTITY='{"identity": {"account_number": "10001", "org_id": "123456
 CACHED_VIEWS_DISABLED=False
 ACCOUNT_ENHANCED_METRICS=False
 ENHANCED_ORG_ADMIN=True
+
+DOCKER_BUILDKIT=1

Makefile

@@ -16,8 +16,22 @@ KOKU_SERVER_PORT = $(shell echo "${KOKU_API_PORT:-8000}")
 MASU_SERVER = $(shell echo "${MASU_SERVICE_HOST:-localhost}")
 MASU_SERVER_PORT = $(shell echo "${MASU_SERVICE_PORT:-5042}")
 DOCKER := $(shell which docker 2>/dev/null || which podman 2>/dev/null)
+DOCKER_BUILDKIT = 1
 scale = 1
 
+# Prefer Docker Compose v2
+DOCKER_COMPOSE_CHECK := $(shell $(DOCKER) compose version >/dev/null 2>&1 ; echo $$?)
+DOCKER_COMPOSE_BIN = $(DOCKER) compose
+ifneq ($(DOCKER_COMPOSE_CHECK), 0)
+	DOCKER_COMPOSE_BIN = $(DOCKER)-compose
+endif
+
+# Use ARM images on ARM systems
+DOCKER_COMPOSE = $(DOCKER_COMPOSE_BIN)
+ifeq (arm, $(findstring arm, $(shell uname -m)))
+	DOCKER_COMPOSE = $(DOCKER_COMPOSE_BIN) -f docker-compose.yml -f docker-compose.arm.yml
+endif
+
 # Testing directories
 TESTINGDIR = $(TOPDIR)/testing
 PROVIDER_TEMP_DIR = $(TESTINGDIR)/pvc_dir
@@ -26,13 +40,6 @@ OCP_PROVIDER_TEMP_DIR = $(PROVIDER_TEMP_DIR)/insights_local
 # How to execute Django's manage.py
 DJANGO_MANAGE = DJANGO_READ_DOT_ENV_FILE=True $(PYTHON) $(PYDIR)/manage.py
 
-# Docker compose specific file
-ifdef compose_file
-    DOCKER_COMPOSE = $(DOCKER)-compose -f docker-compose.yml -f $(compose_file)
-else
-	DOCKER_COMPOSE = $(DOCKER)-compose
-endif
-
 # Platform differences
 #
 # - Use 'sudo' on Linux
@@ -109,9 +116,9 @@ help:
 	@echo "  scan_project                          run security scan"
 	@echo ""
 	@echo "--- Commands using Docker Compose ---"
-	@echo "  docker-up                            run docker-compose up --build -d"
-	@echo "  docker-up-no-build                   run docker-compose up -d"
-	@echo "  docker-up-koku                       run docker-compose up -d koku-server"
+	@echo "  docker-up                            run docker compose up --build -d"
+	@echo "  docker-up-no-build                   run docker compose up -d"
+	@echo "  docker-up-koku                       run docker compose up -d koku-server"
 	@echo "                                         @param build : set to '--build' to build the container"
 	@echo "  docker-up-db                         run database only"
 	@echo "  docker-up-db-monitor                 run the database monitoring via grafana"
@@ -130,7 +137,6 @@ help:
 
 	@echo "  docker-shell                         run Django and database containers with shell access to server (for pdb)"
 	@echo "  docker-logs                          connect to console logs for all services"
-	@echo "  docker-test-all                      run unittests"
 	@echo "  docker-iqe-local-hccm                create container based off local hccm plugin. Requires env 'HCCM_PLUGIN_PATH'"
 	@echo "                                          @param iqe_cmd - (optional) Command to run. Defaults to 'bash'."
 	@echo "  docker-iqe-smokes-tests              run smoke tests"
@@ -315,7 +321,7 @@ KUSTOMIZE=$(shell which kustomize)
 endif
 
 ###############################
-### Docker-compose Commands ###
+### Docker compose Commands ###
 ###############################
 
 docker-down:
@@ -347,9 +353,6 @@ docker-reinitdb-with-sources-lite: docker-down-db remove-db docker-up-db run-mig
 docker-shell:
 	$(DOCKER_COMPOSE) run --service-ports koku-server
 
-docker-test-all:
-	docker-compose -f koku-test.yml up --build
-
 docker-restart-koku:
 	@if [ -n "$$($(DOCKER) ps -q -f name=koku_server)" ] ; then \
          $(DOCKER_COMPOSE) restart koku-server masu-server koku-worker koku-beat koku-listener ; \

README.rst

@@ -18,8 +18,6 @@ Getting Started
 
 This project is developed using Python 3.8. Make sure you have at least this version installed.
 
-| *Note*: M1 Mac users should follow the `M1 Mac Setup`_.
-
 Prerequisites
 -------------
 
@@ -55,15 +53,14 @@ This project is developed using the Django web framework. Many configuration set
     AWS_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_KEY
     AWS_RESOURCE_NAME=YOUR_COST_MANAGEMENT_AWS_ARN
 
-3. (Mac Only) If you are on Mac, do the following note that psycopg2 is a dependency of Django and installing the psycopg2 wheel will likely fail. The following steps should be taken to allow installation to succeed: ::
+3. (Mac Only) Install libraries for building wheels on ARM ::
 
-    brew install openssl
-    brew unlink openssl && brew link openssl --force
+    brew install openssl librdkafka
 
-4. (Mac Only) Also add the following to your ``.env``::
+4. (Mac Only) Also add the following to your ``.env`` or shell profile ::
 
-    LDFLAGS="-L/usr/local/opt/openssl/lib"
-    CPPFLAGS="-I/usr/local/opt/openssl/include"
+    LDFLAGS="-L$(brew --prefix openssl)/lib -L$(brew --prefix librdkafka)/lib"
+    CPPFLAGS="-I$(brew --prefix openssl)/include -I$(brew --prefix librdkafka)/include"
 
 5. Developing inside a virtual environment is recommended. A Pipfile is provided. Pipenv is recommended for combining virtual environment (virtualenv) and dependency management (pip). To install pipenv, use pip ::
 
@@ -94,17 +91,15 @@ This will explain how to start the server and its dependencies using Docker, cre
 Starting Koku using Docker Compose
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+.. note:: In order for the ``koku_base`` image to build correctly, buildkit must be enabled by setting ``DOCKER_BUILDKIT=1``. This is set in the ``.env`` file, but if you are having issues building the ``koku_base`` image, make sure buildkit is enabled.
+
 1. Start the docker containers::
 
     make docker-up
 
 2. Display log output from the docker containers. It is recommended that logs be kept in a second terminal ::
 
-    docker-compose logs -f koku-server koku-worker
-
-3. Install koku-nise::
-
-    pip install koku-nise
+    docker compose logs -f koku-server koku-worker
 
 Run AWS Scenario
 ^^^^^^^^^^^^^^^^
@@ -163,11 +158,11 @@ To bring down all the docker containers, run the following command::
 Database
 ^^^^^^^^
 
-PostgreSQL is used as the database backend for Koku. A docker-compose file is provided for creating a local database container. Assuming the default .env file values are used, to access the database directly using psql run ::
+PostgreSQL is used as the database backend for Koku. A docker compose file is provided for creating a local database container. Assuming the default .env file values are used, to access the database directly using psql run ::
 
     PGPASSWORD=postgres psql postgres -U postgres -h localhost -p 15432
 
-**Note:** There is a known limitation with docker-compose and Linux environments with SELinux enabled. You may see the following error during the postgres container deployment::
+**Note:** There is a known limitation with docker compose and Linux environments with SELinux enabled. You may see the following error during the postgres container deployment::
 
     "mkdir: cannot create directory '/var/lib/pgsql/data/userdata': Permission denied" can be resolved by granting ./pg_data ownership permissions to uid:26 (postgres user in centos/postgresql-96-centos7)
 
@@ -229,7 +224,7 @@ Information about Grafana dashboards can be found here: https://grafana.com/docs
 Using Trino and MinIO
 ^^^^^^^^^^^^^^^^^^^^^
 
-We have a special docker-compose file specifically for running Trino (formerly Presto) with MinIO for object storage. With the proper environment variables set the app will run circumventing our conventional Postgres processing in favor of using Trino.
+We have a special docker compose file specifically for running Trino (formerly Presto) with MinIO for object storage. With the proper environment variables set the app will run circumventing our conventional Postgres processing in favor of using Trino.
 
 Set the following environment variables ::
 
@@ -262,7 +257,7 @@ The MinIO UI will be available at http://127.0.0.1:9090/minio/. Use the S3_ACCES
 
 The Trinio UI will be available at http://127.0.0.1:8080/ui/. Login as `admin`. Details can be found there on queries. This is particularly useful for troubleshooting failures.
 
-For command line interactions with Trino install the CLI from https://trino.io/docs/current/installation/cli.html and follow instructions there. Use the following to login ::
+For command line interactions with Trino `install the CLI`_ and use the following to login ::
 
     trino --server localhost:8080 --catalog hive --schema acct10001 --user admin --debug
 
@@ -287,13 +282,13 @@ This will rebuild the tox virtual env and then run all tests.
 
 To run unit tests specifically::
 
-    tox -e py38
+    tox -e py39
 
 To run a specific subset of unit tests, you can pass a particular module path to tox. To do this, use positional args using the -- separator. For example::
 
-    tox -e py38 -- masu.test.external.downloader.azure.test_azure_services.AzureServiceTest
+    tox -e py39 -- masu.test.external.downloader.azure.test_azure_services.AzureServiceTest
 
-To run IQE Smoke, Vortex or API tests, while on the Red Hat network and koku deployed via docker-compose run::
+To run IQE Smoke, Vortex or API tests, while on the Red Hat network and koku deployed via docker compose run::
 
     make docker-iqe-smokes-tests
     make docker-iqe-vortex-tests
@@ -318,7 +313,7 @@ pgAdmin
 If you want to interact with the Postgres database from a GUI:
 
  1. Copy the `pgadmin_servers.json.example` into a `pgadmin_servers.json` file and if necessary, change any variables to match your database.
- 2. `docker-compose up` causes pgAdmin to run on http://localhost:8432
+ 2. `docker compose up` causes pgAdmin to run on http://localhost:8432
  3. In the login screen, the default login email is `postgres`
 
 Side note: The `pgadmin_servers.json` file uses [pgadmin servers.json syntax](https://www.pgadmin.org/docs/pgadmin4/development/import_export_servers.html#json-format)
@@ -345,7 +340,7 @@ Please refer to Contributing_.
 .. _pre-commit: https://pre-commit.com
 .. _Black: https://github.com/psf/black
 .. _Flake8: http://flake8.pycqa.org
-.. _`M1 Mac Setup`: https://github.com/project-koku/koku/blob/main/docs/koku_setup_on_m1.rst
+.. _`install the CLI`: https://trino.io/docs/current/client/cli.html
 
 .. |license| image:: https://img.shields.io/github/license/project-koku/koku.svg
    :target: https://github.com/project-koku/koku/blob/main/LICENSE

docker-compose.m1.yml → docker-compose.arm.yml

@@ -1,5 +1,4 @@
-# for the Makefile to use this file, set the `compose_file` env variable to:
-# compose_file=docker-compose.m1.yml
+# The Makefile will automatically use this file on ARM systems
 
 version: '3'
 

docker-compose.yml

@@ -488,7 +488,7 @@ services:
     container_name: pgAdmin
     image: dpage/pgadmin4
     environment:
-    - PGADMIN_DEFAULT_EMAIL=${PGADMIN_EMAIL-postgres}
+    - PGADMIN_DEFAULT_EMAIL=${PGADMIN_EMAIL-postgres@local.dev}
     - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_PASSWORD-postgres}
     ports:
       - "${PGADMIN_PORT-8432}:80"