.travis.yml

os: linux

dist: bionic

language: python

python:
  - '3.8'

cache: pip

env:
  global:
  - DEBIAN_FRONTEND=noninteractive
  - POSTGRES_HOST=localhost
  - USASPENDING_DB_HOST=localhost
  - USASPENDING_DB_PORT=5432
  - USASPENDING_DB_USER=usaspending
  - USASPENDING_DB_PASSWORD=usaspender
  - USASPENDING_DB_NAME=data_store_api
  - DATABASE_URL=postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/${USASPENDING_DB_NAME}
  - DOWNLOAD_DATABASE_URL=postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/${USASPENDING_DB_NAME}
  - DJANGO_SETTINGS_MODULE='usaspending_api.settings'
  - ES_SCHEME=http
  - ES_HOST=localhost
  - ES_PORT=9200
  - ES_HOSTNAME=${ES_SCHEME}://${ES_HOST}:${ES_PORT}
  - BROKER_DB_HOST=localhost
  - BROKER_DB_PORT=5432
  - BROKER_DB_USER=admin
  - BROKER_DB_PASSWORD=root
  - BROKER_DB_NAME=data_broker
  - DATA_BROKER_DATABASE_URL=postgres://${BROKER_DB_USER}:${BROKER_DB_PASSWORD}@${BROKER_DB_HOST}:${BROKER_DB_PORT}/${BROKER_DB_NAME}
  - DATA_BROKER_SRC_PATH="${TRAVIS_BUILD_DIR}/../data-act-broker-backend"  # Location in host machine where broker src code root can be found
  - BROKER_REPO_URL=https://github.com/fedspendingtransparency/data-act-broker-backend.git
  - BROKER_REPO_BRANCH=$(if [ "${TRAVIS_EVENT_TYPE}" = "pull_request" ] && [ ! -z "`git ls-remote --heads ${BROKER_REPO_URL} ${TRAVIS_BRANCH}`" ]; then echo "${TRAVIS_BRANCH}"; else echo "qat"; fi;)
  - BROKER_REPO_FOLDER=${DATA_BROKER_SRC_PATH}
  - BROKER_DOCKER_IMAGE=dataact-broker-backend
  - GRANTS_API_KEY=${GRANTS_API_KEY}
  - MINIO_DATA_DIR=${HOME}/Development/data/usaspending/docker/usaspending-s3  # needs to be same place docker-compose will look for it (based on .env file)
  - MINIO_HOST=localhost
  - PYTEST_XDIST_NUMPROCESSES=4
  - COLUMNS=240  # for wider terminal output
  - TRAVIS_JOB_INDEX="$(echo $TRAVIS_JOB_NUMBER | cut -d'.' -f2)"

jobs:
  include:
  - stage: Build
    name: pip install
    workspaces:
      create:
        name: pip
        paths:
          - $HOME/virtualenv/
          - $TRAVIS_BUILD_DIR/usaspending_api.egg_info/
    before_install: ""  # override default to no-op
    install:
    - travis_retry pip install setuptools==60.8.2
    - travis_retry pip install .[dev]
    - travis_retry pip install coveralls
    before_script: ""  # override default to no-op
    script: ""  # override default to no-op
  - name: docker build
    workspaces:
      create:
        name: docker
        paths:
          - docker_images/
    before_install: ""  # override default to no-op
    install:
    # Checkout dependent broker code used to spin up a broker integration test db. Put it in its own folder alongside this repo's code
    - echo "Using ${BROKER_REPO_BRANCH} branch from ${BROKER_REPO_URL}"
    - travis_retry git clone --branch ${BROKER_REPO_BRANCH} --single-branch --depth 1 ${BROKER_REPO_URL} ${BROKER_REPO_FOLDER}
    - docker build -t ${BROKER_DOCKER_IMAGE} ${BROKER_REPO_FOLDER}  # build image from which to call Broker scripts
    before_script: ""  # override default to no-op
    script: ""  # override default to no-op
    before_cache:
      # Save off docker images+layers used by this build into the `docker_images` cache dir
      - mkdir -p docker_images
      - docker save -o docker_images/${BROKER_DOCKER_IMAGE}.tar ${BROKER_DOCKER_IMAGE}
  - stage: Static Code Analysis
    name: flake8
    workspaces:
      use: [ pip ]
    before_install: ""  # override default to no-op
    install: ""  # override default to no-op
    before_script: ""  # override default to no-op
    script:
    - flake8
  - name: black
    workspaces:
      use: [ pip ]
    before_install: ""  # override default to no-op
    install: ""  # override default to no-op
    before_script: ""  # override default to no-op
    script:
    - black --check --diff .
  - name: API Docs
    workspaces:
      use: [ pip ]
    before_install: ""  # override default to no-op
    install: travis_retry npm install --global dredd@13.1.2  # add dredd for API contract testing
    before_script: ""  # override default to no-op
    script:
    - python manage.py check_for_endpoint_documentation
    - dredd > dredd-results.txt && echo '! grep -E "^[warn:|error:]" dredd-results.txt' | bash
  - stage: Automated Tests
    # NOTE: See conftest.py pytest_collection_modifyitems for how Marks are assigned to tests
    name: Spark Integration Tests - test_load_transactions_in_delta_fabs_fpds.py
    env:
    # Concurrent sessions does not seem to be efficient for this test. Keeping at 1 session
    - PYTEST_XDIST_NUMPROCESSES=0
    - PYTEST_SETUP_TEST_DATABASES=true
    - PYTEST_PRELOAD_SPARK_JARS=true
    - PYTEST_INCLUDE_GLOB='test_*.py *_test.py'
    - PYTEST_EXCLUDE_GLOB=
    - PYTEST_MATCH_EXPRESSION=test_load_transactions_in_delta_fabs_fpds.py
    - PYTEST_MARK_EXPRESSION=spark
    workspaces:
      use: [ pip,docker ]
      create:
        name: ws1
        paths:
          - coverage.*.xml
    # Inherits "global" job phases defined below (e.g. before_install, install, before_script, script, etc.)
  - name: Spark Integration Tests - test_load_transactions_in_delta_lookups.py
    env:
    # Concurrent sessions does not seem to be efficient for this test. Keeping at 1 session
    - PYTEST_XDIST_NUMPROCESSES=0
    - PYTEST_SETUP_TEST_DATABASES=true
    - PYTEST_PRELOAD_SPARK_JARS=true
    - PYTEST_INCLUDE_GLOB='test_*.py *_test.py'
    - PYTEST_EXCLUDE_GLOB=
    - PYTEST_MATCH_EXPRESSION=test_load_transactions_in_delta_lookups.py
    - PYTEST_MARK_EXPRESSION=spark
    workspaces:
      use: [ pip,docker ]
      create:
        name: ws2
        paths:
          - coverage.*.xml
    # Inherits "global" job phases defined below (e.g. before_install, install, before_script, script, etc.)
  - name: Spark Integration Tests - test_load_to_from_delta.py
    env:
    - PYTEST_XDIST_NUMPROCESSES=2
    - PYTEST_SETUP_TEST_DATABASES=true
    - PYTEST_PRELOAD_SPARK_JARS=true
    - PYTEST_INCLUDE_GLOB='test_*.py *_test.py'
    - PYTEST_EXCLUDE_GLOB=
    - PYTEST_MATCH_EXPRESSION=test_load_to_from_delta.py
    - PYTEST_MARK_EXPRESSION=spark
    workspaces:
      use: [ pip,docker ]
      create:
        name: ws3
        paths:
          - coverage.*.xml
      # Inherits "global" job phases defined below (e.g. before_install, install, before_script, script, etc.)
  - name: Spark Integration Tests - Other
    env:
    - PYTEST_XDIST_NUMPROCESSES=4
    - PYTEST_SETUP_TEST_DATABASES=true
    - PYTEST_PRELOAD_SPARK_JARS=true
    - PYTEST_INCLUDE_GLOB='test_*.py *_test.py'
    - PYTEST_EXCLUDE_GLOB=
    - PYTEST_MATCH_EXPRESSION='(not test_load_to_from_delta.py and not test_load_transactions_in_delta_lookups.py and not test_load_transactions_in_delta_fabs_fpds.py)'
    - PYTEST_MARK_EXPRESSION=spark
    workspaces:
      use: [ pip,docker ]
      create:
        name: ws4
        paths:
          - coverage.*.xml
    # Inherits "global" job phases defined below (e.g. before_install, install, before_script, script, etc.)
  - name: Non-Spark Integration Tests
    env:
    - PYTEST_SETUP_TEST_DATABASES=true
    - PYTEST_PRELOAD_SPARK_JARS=false
    - PYTEST_INCLUDE_GLOB=**/tests/integration/*
    - PYTEST_EXCLUDE_GLOB=
    - PYTEST_MATCH_EXPRESSION=
    - PYTEST_MARK_EXPRESSION='(not spark and not signal_handling)'
    workspaces:
      use: [ pip,docker ]
      create:
        name: ws5
        paths:
          - coverage.*.xml
    # Inherits "global" job phases defined below (e.g. before_install, install, before_script, script, etc.)
  - name: Non-Spark Integration Tests - Using Signal Handling
    env:
    - PYTEST_XDIST_NUMPROCESSES=0
    - PYTEST_SETUP_TEST_DATABASES=true
    - PYTEST_PRELOAD_SPARK_JARS=false
    - PYTEST_INCLUDE_GLOB=**/tests/integration/*
    - PYTEST_EXCLUDE_GLOB=
    - PYTEST_MATCH_EXPRESSION=
    - PYTEST_MARK_EXPRESSION='(signal_handling and not spark)'
    workspaces:
      use: [ pip,docker ]
      create:
        name: ws6
        paths:
          - coverage.*.xml
    # Inherits "global" job phases defined below (e.g. before_install, install, before_script, script, etc.)
  - name: Unit Tests
    env:
    - PYTEST_SETUP_TEST_DATABASES=false
    - PYTEST_PRELOAD_SPARK_JARS=false
    - PYTEST_INCLUDE_GLOB='test_*.py *_test.py'
    - PYTEST_EXCLUDE_GLOB=**/tests/integration/*
    - PYTEST_MATCH_EXPRESSION=
    - PYTEST_MARK_EXPRESSION='(not spark and not database and not elasticsearch and not signal_handling)'
    workspaces:
      use: [ pip,docker ]
      create:
        name: ws7
        paths:
          - coverage.*.xml
    # Inherits "global" job phases defined below (e.g. before_install, install, before_script, script, etc.)
  - name: Unit Tests - Using Signal Handling
    env:
    - PYTEST_XDIST_NUMPROCESSES=0
    - PYTEST_SETUP_TEST_DATABASES=false
    - PYTEST_PRELOAD_SPARK_JARS=false
    - PYTEST_INCLUDE_GLOB='test_*.py *_test.py'
    - PYTEST_EXCLUDE_GLOB=**/tests/integration/*
    - PYTEST_MATCH_EXPRESSION=
    - PYTEST_MARK_EXPRESSION='(signal_handling)'
    workspaces:
      use: [ pip,docker ]
      create:
        name: ws8
        paths:
          - coverage.*.xml
    # Inherits "global" job phases defined below (e.g. before_install, install, before_script, script, etc.)
  - stage: Code Coverage
    # NOTE: This stage will only run if ALL test stage jobs passed (there's no point to collect and report coverage if they did not)
    env:
    - IS_CODE_COVERAGE_REPORT=true
    workspaces:
      use: [ws1,ws2,ws3,ws4,ws5,ws6,ws7,ws8]
    before_install: ""  # override default to no-op
    install: ""  # override default to no-op
    before_script:
      # Get dependencies to report code coverage to code climate
      - travis_retry curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
      - chmod +x ./cc-test-reporter
    script:
    - ls -lh coverage*
    - for cf in coverage.*.xml; do ./cc-test-reporter format-coverage --prefix $TRAVIS_BUILD_DIR --input-type coverage.py --output coverage/codeclimate.$(echo "$cf" | cut -d'.' -f2).xml coverage.$(echo "$cf" | cut -d'.' -f2).xml; done
    - ls coverage/
    - ./cc-test-reporter sum-coverage --output - --parts $(find . -maxdepth 1 -name 'coverage.*.xml' | wc -l) ./coverage/codeclimate.*.xml | ./cc-test-reporter upload-coverage --input -

####
#### Below are the default job lifecycle phases if they are not overridden above on a per-job basis
####

before_install:
  # Reload any cached images and their layers from previous builds, to shorten docker pulls/builds
  - docker load -i docker_images/*.tar || true

install:
  # Checkout dependent broker code used to spin up a broker integration test db. Put it in its own folder alongside this repo's code
  - echo "Using ${BROKER_REPO_BRANCH} branch from ${BROKER_REPO_URL}"
  - travis_retry git clone --branch ${BROKER_REPO_BRANCH} --single-branch --depth 1 ${BROKER_REPO_URL} ${BROKER_REPO_FOLDER}

before_script:
  # Get dependencies to report code coverage to code climate
  - travis_retry curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
  - chmod +x ./cc-test-reporter
  - ./cc-test-reporter before-build
  - make docker-compose-up-usaspending args="-d usaspending-db usaspending-es"
  # Wait for services to be up
  - ttl=30; echo "Try DB conn from container for $ttl seconds"; until [ $ttl -le 0 ] || psql $DATABASE_URL -c 'select 1 where 1=1'; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
  - ttl=30; echo "Try ES conn from container for $ttl seconds"; until [ $ttl -le 0 ] || curl --silent -XGET --fail $ES_HOSTNAME; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
  # Our Postgres DB provided by Travis needs to have the (super) users specified by our env var DB URLs used
  - psql postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/postgres -c "ALTER USER ${USASPENDING_DB_USER} SET search_path TO public,raw,int,temp,rpt"
  - psql postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/postgres -c "CREATE USER ${BROKER_DB_USER} PASSWORD '${BROKER_DB_PASSWORD}' SUPERUSER"
  # Postgres DB also needs a readonly user, that is referenced in some code
  - psql postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/postgres -c "CREATE ROLE readonly;"
  - >
    psql postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/postgres -c "\copy (
        SELECT
          'GRANT USAGE ON SCHEMA ' || nspname || ' TO readonly; '
        || 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly; '
        || 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly; '
        FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')
      ) TO grant_to_readonly.sql;"
  - psql postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/postgres -c "\i grant_to_readonly.sql"
  # Trigger the setup of multiple test DBs that will be left for the next pytest run to reuse --numprocesses
  # Also, must manually set --numprocesses on Travis CI VMs; can't use auto (see: https://github.com/pytest-dev/pytest-xdist/pull/317)
  - if [ "${PYTEST_SETUP_TEST_DATABASES}" = true ]; then pytest --create-db --reuse-db --numprocesses ${PYTEST_XDIST_NUMPROCESSES} --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log 2> /dev/null 'usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup'; fi;
  # Trigger preloading of Spark dependent JARs to avoid Ivy (Maven) repo download race condition from multiple pytest-xdist worker test sessions
  - if [ "${PYTEST_PRELOAD_SPARK_JARS}" = true ]; then pytest --no-cov --disable-warnings -r=fEs --verbosity=3 'usaspending_api/tests/integration/test_setup_of_spark_dependencies.py::test_preload_spark_jars'; fi;
  - psql postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/postgres -c "\l"
  # Ensure MinIO can run as our s3 substitute
  - mkdir -p "${MINIO_DATA_DIR}"
  - mkdir -p "${HOME}/.ivy2"
  - make docker-compose-up-s3 args="-d"

script:
  - stty cols 240  # for wider terminal output
  - cd ${TRAVIS_BUILD_DIR}  # run build script out of repo dir
  # Check that no integration tests are outside **/tests/integration/ folders
  - return $(pytest --collect-only --quiet --ignore-glob='**/tests/integration/*' -m '(spark or database or elasticsearch)' --no-cov --disable-warnings | grep '^usaspending_api.*$' | wc -l)
  - test $? -gt 0 && echo 'Failing because integration tests would be improperly captured as unit tests. Run the previous pytest command locally to figure out which to move to a **/tests/integration/ folder'
  # Must manually set --numprocesses on Travis CI VMs; can't use auto (see: https://github.com/pytest-dev/pytest-xdist/pull/317)
  - pytest --override-ini=python_files="${PYTEST_INCLUDE_GLOB}" --ignore-glob="${PYTEST_EXCLUDE_GLOB}" -m "${PYTEST_MARK_EXPRESSION}" -k "${PYTEST_MATCH_EXPRESSION}" --cov=usaspending_api --cov-report term --cov-report xml:coverage.$TRAVIS_JOB_INDEX.xml --reuse-db -r=fEs --numprocesses ${PYTEST_XDIST_NUMPROCESSES} --dist worksteal --verbosity=1 --durations 50