From 9aff5243cc328b5586bc3f0c47eef935d3ff78bc Mon Sep 17 00:00:00 2001 From: Sam Doran Date: Fri, 7 Oct 2022 16:47:44 -0400 Subject: [PATCH] Improve development setup (#3914) * Update macOS instructions * Detect docker compose and prefer v2 docker-compose v1 is deprecated https://www.docker.com/blog/announcing-compose-v2-general-availability/ * Ensure we are using buildkit The multi-arch image needs buildkit to work properly * Remove M1 specific directions macOS specific directions are now incorporated in the main README The directions in the rosetta_on_m1.rst aren't really necessary and can lead to problems. Installing Python with brew is almost always a recipe for frustration. It's best to compile things natively for ARM rather than relying on emulation and maintaining separate copies of binaries. * Automatically use copmose file for ARM systems * Fix bad link * Use correct tox environment in docs * Use valid email address for pgAdmin login Co-authored-by: Michael Skarbek --- .env.example | 4 +- Makefile | 33 ++++--- README.rst | 39 ++++---- dev/scripts/m1_refresher.sh | 36 -------- ...r-compose.m1.yml => docker-compose.arm.yml | 3 +- docker-compose.yml | 2 +- docs/koku_setup_on_m1.rst | 66 -------------- docs/rosetta_on_m1.rst | 91 ------------------- 8 files changed, 40 insertions(+), 234 deletions(-) delete mode 100644 dev/scripts/m1_refresher.sh rename docker-compose.m1.yml => docker-compose.arm.yml (57%) delete mode 100644 docs/koku_setup_on_m1.rst delete mode 100644 docs/rosetta_on_m1.rst diff --git a/.env.example b/.env.example index 9e7b1dff1f..47d1f641cc 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,7 @@ DATABASE_SERVICE_NAME=POSTGRES_SQL DATABASE_ENGINE=postgresql DATABASE_NAME=postgres -PGADMIN_EMAIL=postgres +PGADMIN_EMAIL=postgres@local.dev PGADMIN_PASSWORD=postgres PGADMIN_PORT=8432 POSTGRES_SQL_SERVICE_HOST=localhost @@ -55,3 +55,5 @@ DEVELOPMENT_IDENTITY='{"identity": {"account_number": "10001", "org_id": "123456 CACHED_VIEWS_DISABLED=False ACCOUNT_ENHANCED_METRICS=False ENHANCED_ORG_ADMIN=True + +DOCKER_BUILDKIT=1 diff --git a/Makefile b/Makefile index e17df68878..57f9979dd6 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,22 @@ KOKU_SERVER_PORT = $(shell echo "${KOKU_API_PORT:-8000}") MASU_SERVER = $(shell echo "${MASU_SERVICE_HOST:-localhost}") MASU_SERVER_PORT = $(shell echo "${MASU_SERVICE_PORT:-5042}") DOCKER := $(shell which docker 2>/dev/null || which podman 2>/dev/null) +DOCKER_BUILDKIT = 1 scale = 1 +# Prefer Docker Compose v2 +DOCKER_COMPOSE_CHECK := $(shell $(DOCKER) compose version >/dev/null 2>&1 ; echo $$?) +DOCKER_COMPOSE_BIN = $(DOCKER) compose +ifneq ($(DOCKER_COMPOSE_CHECK), 0) + DOCKER_COMPOSE_BIN = $(DOCKER)-compose +endif + +# Use ARM images on ARM systems +DOCKER_COMPOSE = $(DOCKER_COMPOSE_BIN) +ifeq (arm, $(findstring arm, $(shell uname -m))) + DOCKER_COMPOSE = $(DOCKER_COMPOSE_BIN) -f docker-compose.yml -f docker-compose.arm.yml +endif + # Testing directories TESTINGDIR = $(TOPDIR)/testing PROVIDER_TEMP_DIR = $(TESTINGDIR)/pvc_dir @@ -26,13 +40,6 @@ OCP_PROVIDER_TEMP_DIR = $(PROVIDER_TEMP_DIR)/insights_local # How to execute Django's manage.py DJANGO_MANAGE = DJANGO_READ_DOT_ENV_FILE=True $(PYTHON) $(PYDIR)/manage.py -# Docker compose specific file -ifdef compose_file - DOCKER_COMPOSE = $(DOCKER)-compose -f docker-compose.yml -f $(compose_file) -else - DOCKER_COMPOSE = $(DOCKER)-compose -endif - # Platform differences # # - Use 'sudo' on Linux @@ -109,9 +116,9 @@ help: @echo " scan_project run security scan" @echo "" @echo "--- Commands using Docker Compose ---" - @echo " docker-up run docker-compose up --build -d" - @echo " docker-up-no-build run docker-compose up -d" - @echo " docker-up-koku run docker-compose up -d koku-server" + @echo " docker-up run docker compose up --build -d" + @echo " docker-up-no-build run docker compose up -d" + @echo " docker-up-koku run docker compose up -d koku-server" @echo " @param build : set to '--build' to build the container" @echo " docker-up-db run database only" @echo " docker-up-db-monitor run the database monitoring via grafana" @@ -130,7 +137,6 @@ help: @echo " docker-shell run Django and database containers with shell access to server (for pdb)" @echo " docker-logs connect to console logs for all services" - @echo " docker-test-all run unittests" @echo " docker-iqe-local-hccm create container based off local hccm plugin. Requires env 'HCCM_PLUGIN_PATH'" @echo " @param iqe_cmd - (optional) Command to run. Defaults to 'bash'." @echo " docker-iqe-smokes-tests run smoke tests" @@ -315,7 +321,7 @@ KUSTOMIZE=$(shell which kustomize) endif ############################### -### Docker-compose Commands ### +### Docker compose Commands ### ############################### docker-down: @@ -347,9 +353,6 @@ docker-reinitdb-with-sources-lite: docker-down-db remove-db docker-up-db run-mig docker-shell: $(DOCKER_COMPOSE) run --service-ports koku-server -docker-test-all: - docker-compose -f koku-test.yml up --build - docker-restart-koku: @if [ -n "$$($(DOCKER) ps -q -f name=koku_server)" ] ; then \ $(DOCKER_COMPOSE) restart koku-server masu-server koku-worker koku-beat koku-listener ; \ diff --git a/README.rst b/README.rst index ab73d89716..0ca02c85a0 100644 --- a/README.rst +++ b/README.rst @@ -18,8 +18,6 @@ Getting Started This project is developed using Python 3.8. Make sure you have at least this version installed. -| *Note*: M1 Mac users should follow the `M1 Mac Setup`_. - Prerequisites ------------- @@ -55,15 +53,14 @@ This project is developed using the Django web framework. Many configuration set AWS_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_KEY AWS_RESOURCE_NAME=YOUR_COST_MANAGEMENT_AWS_ARN -3. (Mac Only) If you are on Mac, do the following note that psycopg2 is a dependency of Django and installing the psycopg2 wheel will likely fail. The following steps should be taken to allow installation to succeed: :: +3. (Mac Only) Install libraries for building wheels on ARM :: - brew install openssl - brew unlink openssl && brew link openssl --force + brew install openssl librdkafka -4. (Mac Only) Also add the following to your ``.env``:: +4. (Mac Only) Also add the following to your ``.env`` or shell profile :: - LDFLAGS="-L/usr/local/opt/openssl/lib" - CPPFLAGS="-I/usr/local/opt/openssl/include" + LDFLAGS="-L$(brew --prefix openssl)/lib -L$(brew --prefix librdkafka)/lib" + CPPFLAGS="-I$(brew --prefix openssl)/include -I$(brew --prefix librdkafka)/include" 5. Developing inside a virtual environment is recommended. A Pipfile is provided. Pipenv is recommended for combining virtual environment (virtualenv) and dependency management (pip). To install pipenv, use pip :: @@ -94,17 +91,15 @@ This will explain how to start the server and its dependencies using Docker, cre Starting Koku using Docker Compose ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. note:: In order for the ``koku_base`` image to build correctly, buildkit must be enabled by setting ``DOCKER_BUILDKIT=1``. This is set in the ``.env`` file, but if you are having issues building the ``koku_base`` image, make sure buildkit is enabled. + 1. Start the docker containers:: make docker-up 2. Display log output from the docker containers. It is recommended that logs be kept in a second terminal :: - docker-compose logs -f koku-server koku-worker - -3. Install koku-nise:: - - pip install koku-nise + docker compose logs -f koku-server koku-worker Run AWS Scenario ^^^^^^^^^^^^^^^^ @@ -163,11 +158,11 @@ To bring down all the docker containers, run the following command:: Database ^^^^^^^^ -PostgreSQL is used as the database backend for Koku. A docker-compose file is provided for creating a local database container. Assuming the default .env file values are used, to access the database directly using psql run :: +PostgreSQL is used as the database backend for Koku. A docker compose file is provided for creating a local database container. Assuming the default .env file values are used, to access the database directly using psql run :: PGPASSWORD=postgres psql postgres -U postgres -h localhost -p 15432 -**Note:** There is a known limitation with docker-compose and Linux environments with SELinux enabled. You may see the following error during the postgres container deployment:: +**Note:** There is a known limitation with docker compose and Linux environments with SELinux enabled. You may see the following error during the postgres container deployment:: "mkdir: cannot create directory '/var/lib/pgsql/data/userdata': Permission denied" can be resolved by granting ./pg_data ownership permissions to uid:26 (postgres user in centos/postgresql-96-centos7) @@ -229,7 +224,7 @@ Information about Grafana dashboards can be found here: https://grafana.com/docs Using Trino and MinIO ^^^^^^^^^^^^^^^^^^^^^ -We have a special docker-compose file specifically for running Trino (formerly Presto) with MinIO for object storage. With the proper environment variables set the app will run circumventing our conventional Postgres processing in favor of using Trino. +We have a special docker compose file specifically for running Trino (formerly Presto) with MinIO for object storage. With the proper environment variables set the app will run circumventing our conventional Postgres processing in favor of using Trino. Set the following environment variables :: @@ -262,7 +257,7 @@ The MinIO UI will be available at http://127.0.0.1:9090/minio/. Use the S3_ACCES The Trinio UI will be available at http://127.0.0.1:8080/ui/. Login as `admin`. Details can be found there on queries. This is particularly useful for troubleshooting failures. -For command line interactions with Trino install the CLI from https://trino.io/docs/current/installation/cli.html and follow instructions there. Use the following to login :: +For command line interactions with Trino `install the CLI`_ and use the following to login :: trino --server localhost:8080 --catalog hive --schema acct10001 --user admin --debug @@ -287,13 +282,13 @@ This will rebuild the tox virtual env and then run all tests. To run unit tests specifically:: - tox -e py38 + tox -e py39 To run a specific subset of unit tests, you can pass a particular module path to tox. To do this, use positional args using the -- separator. For example:: - tox -e py38 -- masu.test.external.downloader.azure.test_azure_services.AzureServiceTest + tox -e py39 -- masu.test.external.downloader.azure.test_azure_services.AzureServiceTest -To run IQE Smoke, Vortex or API tests, while on the Red Hat network and koku deployed via docker-compose run:: +To run IQE Smoke, Vortex or API tests, while on the Red Hat network and koku deployed via docker compose run:: make docker-iqe-smokes-tests make docker-iqe-vortex-tests @@ -318,7 +313,7 @@ pgAdmin If you want to interact with the Postgres database from a GUI: 1. Copy the `pgadmin_servers.json.example` into a `pgadmin_servers.json` file and if necessary, change any variables to match your database. - 2. `docker-compose up` causes pgAdmin to run on http://localhost:8432 + 2. `docker compose up` causes pgAdmin to run on http://localhost:8432 3. In the login screen, the default login email is `postgres` Side note: The `pgadmin_servers.json` file uses [pgadmin servers.json syntax](https://www.pgadmin.org/docs/pgadmin4/development/import_export_servers.html#json-format) @@ -345,7 +340,7 @@ Please refer to Contributing_. .. _pre-commit: https://pre-commit.com .. _Black: https://github.com/psf/black .. _Flake8: http://flake8.pycqa.org -.. _`M1 Mac Setup`: https://github.com/project-koku/koku/blob/main/docs/koku_setup_on_m1.rst +.. _`install the CLI`: https://trino.io/docs/current/client/cli.html .. |license| image:: https://img.shields.io/github/license/project-koku/koku.svg :target: https://github.com/project-koku/koku/blob/main/LICENSE diff --git a/dev/scripts/m1_refresher.sh b/dev/scripts/m1_refresher.sh deleted file mode 100644 index f1edb5de47..0000000000 --- a/dev/scripts/m1_refresher.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -DEV_SCRIPTS_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -COMPOSE_FILE="testing/compose_files/docker-compose-m1.yml" -CFG_FILE="$DEV_SCRIPTS_PATH/../../.env" - -# export env variables -export $(grep -v -e "^#" -e "^DEVELOPMENT_IDENTITY" $CFG_FILE | xargs) - -# import common functions -source $DEV_SCRIPTS_PATH/common/logging.sh - -KOKU_CONTAINERS=("koku-worker", "koku_worker_1", "koku_worker_2", "koku_worker_3", "koku_worker_4") -RUNNING_SERVICES=() - -get_running_containers() { - for container in ${KOKU_CONTAINERS[@]}; do - if docker ps --format '{{ .Names }}' | grep $container > /dev/null 2>&1; then - docker_service=`echo $container | tr '_' '-'` - log-debug "Add $docker_service to restart list" - RUNNING_SERVICES+=($docker_service) - fi - done -} - -# docker-compose -f $COMPOSE_FILE up -d --force-recreate --no-deps --build $service -restart_services() { - for service in ${RUNNING_SERVICES[@]}; do - log-info "restarting: $service" - docker-compose -f $COMPOSE_FILE up -d --force-recreate --no-deps $service - done -} - -# exec -get_running_containers -restart_services diff --git a/docker-compose.m1.yml b/docker-compose.arm.yml similarity index 57% rename from docker-compose.m1.yml rename to docker-compose.arm.yml index ceced78287..ea3b20004f 100644 --- a/docker-compose.m1.yml +++ b/docker-compose.arm.yml @@ -1,5 +1,4 @@ -# for the Makefile to use this file, set the `compose_file` env variable to: -# compose_file=docker-compose.m1.yml +# The Makefile will automatically use this file on ARM systems version: '3' diff --git a/docker-compose.yml b/docker-compose.yml index 14852c4615..6d5c117e2c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -488,7 +488,7 @@ services: container_name: pgAdmin image: dpage/pgadmin4 environment: - - PGADMIN_DEFAULT_EMAIL=${PGADMIN_EMAIL-postgres} + - PGADMIN_DEFAULT_EMAIL=${PGADMIN_EMAIL-postgres@local.dev} - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_PASSWORD-postgres} ports: - "${PGADMIN_PORT-8432}:80" diff --git a/docs/koku_setup_on_m1.rst b/docs/koku_setup_on_m1.rst deleted file mode 100644 index 9c38f130ee..0000000000 --- a/docs/koku_setup_on_m1.rst +++ /dev/null @@ -1,66 +0,0 @@ -===================== -Koku Setup on M1 Mac -===================== - -About ------ - -This documentation is to guide M1 Mac users on how to successfully install and run Koku on their machines. - -Generally, in order to run Intel-based packages and programs on Mac with Apple silicon processors (M1), one can make `use of Rosetta`_ which simulates the Intel environment. - -However, as for running Koku on M1 Macs we can follow the general development documentation with a few modifications as shown below. - -Development ------------ - -From the current list of packages used by Koku, only ``confluent-kafka`` has no support on M1 Mac, but we can install it through ``brew`` with a few modifications. - -1. Install ``librdkafka`` through ``brew``: :: - - brew install librdkafka - -2. Add the following lines into ``~/.zshrc`` and run ``source ~/.zshrc`` :: - - export C_INCLUDE_PATH=$(brew --prefix)/include - export LIBRARY_PATH=$(brew --prefix)/lib - -After compoleting these steps, you should be able to follow the development on `Koku's README`_. Make sure to install the versions of Docker and Homebrew that are made for M1 Macs. - -Developing with Docker -^^^^^^^^^^^^^^^^^^^^^^ - -Alternatively, if you want to run the project in Docker environment, follow the below steps: - -1. Set ``compose_file`` in your ``.env`` file to point to docker-compose-m1.yml_ file specially created for M1 Mac: :: - - compose_file='testing/compose_files/docker-compose-m1.yml' - -2. docker-compose-m1.yml_ removes the use of ``watchdog`` package as at the time of creating this documentation, `watchdog` package is not supported on M1. - Instead, we use a VSCode extension to run a `bash script`_ on Save in order to re-build docker containers. - From VSCode Extensions, install `emeraldwalk.RunOnSave` and add the following to `settings.json`. More on `Run on Save`_: :: - - { - ... - "emeraldwalk.runonsave": { - "commands": [ - { - "cmd": "bash /koku/dev/scripts/m1_refresher.sh" - } - ] - } - } - -Once these steps are completed, follow the instructions on `Developing with docker-compose`_. - -References ----------- - -- https://github.com/jsbroks/coco-annotator/issues/493 - -.. _`use of Rosetta`: https://github.com/project-koku/koku/blob/main/docs/rosetta_on_m1.rst -.. _`Koku's README`: https://github.com/project-koku/koku/blob/main/README.rst -.. _`docker-compose-m1.yml`: https://github.com/project-koku/koku/blob/main/testing/compose_files/docker-compose-m1.yml -.. _`Run on Save`: https://betterprogramming.pub/automatically-execute-bash-commands-on-save-in-vs-code-7a3100449f63 -.. _`bash script`: https://github.com/project-koku/koku/tree/main/dev/scripts/m1_refresher.sh -.. _`Developing with docker-compose`: https://github.com/project-koku/koku/blob/main/README.rst#developing-with-docker-compose diff --git a/docs/rosetta_on_m1.rst b/docs/rosetta_on_m1.rst deleted file mode 100644 index 21da4f0a2a..0000000000 --- a/docs/rosetta_on_m1.rst +++ /dev/null @@ -1,91 +0,0 @@ -Using Rosetta -============= - -Rosetta enables a Mac with Apple silicon to use apps built for a Mac with an Intel processer. Follow the steps on `how to install Rosetta on your Mac`_. - -TL;DR: :: - - /usr/sbin/softwareupdate --install-rosetta --aggree-to-license - -1. Running Terminal with Rosetta --------------------------------- - -It is a good idea to have a separate terminal to run with Rosetta, while keeping the default terminal run on M1. - -1. Download and install `iTerm2`_ -2. Go to Applications in Finder -3. Find iTerm, right-click and select `Get Info` -4. Make sure `Open using Rosetta` is checked - -Alternatively, you can create a copy of your default terminal, rename the copy and have it run specifically with Rosetta. - -`Note`: We will be using this version of the terminal for all the steps that follow. - -2. Installing ``brew`` ----------------------- - -We also will want to have a separate ``brew`` version that runs Intel-based programs. When you install Homebrew on an Intel Mac, it installs it in the `/usr/local/homebrew` directory. - -1. Create a ``~/Downloads/homebrew`` and download Homebrew tarball and extract it to the ``~/Downloads/homebrew`` directory: :: - - cd ~/Downloads - mkdir homebrew - curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C homebrew - -2. Move the ``homebrew`` directory to ``/usr/local/homebrew``. You need the ``sudo`` command: :: - - sudo mv homebrew /usr/local/homebrew - -3. Add the following path and alias to the ``.zshrc`` file. *Note*: We are setting an alias ``brow`` (you could name it anything you'd like) to the version of ``homebrew`` that run on Intel-based machines. :: - - # need this for x86_64 brew - export PATH=$HOME/bin:/usr/local/bin:$PATH - export PATH="/usr/local/homebrew/bin:$PATH" - - # for intel x86_64 brew - alias brow='arch -x86_64 /usr/local/homebrew/bin/brew' - -4. Implement the new changes in ``.zshrc`` file: :: - - source ~/.zshrc - -Now you can install apps for Intel processors: :: - - brow install package-name - -And you can use all ``brew`` commands: :: - - brow list - brow reinstall package-name - brow --help - -3. Install Python ------------------ - -1. Install Python 3.9 using Intel-based ``brew``: :: - - brow install python@3.9 - brow list | grep python - -2. Assuming ``python`` and ``pip`` (comes with ``python`` installation above) currently do not point to any commands, add the following paths and aliases to ``~/.zshrc``: :: - - export PATH="/usr/local/homebrew/Cellar/python@3.9/3.9.13_1/bin/python3.9:$PATH" - export PATH="/usr/local/homebrew/Cellar/python@3.9/3.9.13_1/bin/pip3:$PATH" - - alias python='/usr/local/homebrew/Cellar/python@3.9/3.9.13_1/bin/python3.9' - alias pip='/usr/local/homebrew/Cellar/python@3.9/3.9.13_1/bin/pip3' - -3. Implement the new changes in ``.zshrc`` file: :: - - source ~/.zshrc - - -References ----------- - -- https://medium.com/mkdir-awesome/how-to-install-x86-64-homebrew-packages-on-apple-m1-macbook-54ba295230f -- https://til.simonwillison.net/macos/running-docker-on-remote-m1 - - -.. _`iTerm2`: https://iterm2.com/ -.. _`how to install Rosetta on your Mac`: https://support.apple.com/en-us/HT211861