From 6c49ab0510c36a3a20c3ed8d2139fa68de7f2fa6 Mon Sep 17 00:00:00 2001 From: Sam Doran Date: Fri, 4 Nov 2022 10:47:46 -0400 Subject: [PATCH] [CI] Update action versions, remove deprecated commands, improve organization (#3978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Make scripts executable * Update actions to use version that use Node.js 16 or later https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/ * Use environment file instead of deprecated command https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ * Switch to using and environment file This job didn’t define outputs so I’m don’t know how exactly it was working before. Using an environment variable is a bit simpler anyway. * Reformat run-tests env var and switch to enviorment file from output * Add names for steps and use latest action versions * Rename and consolidate workflows * Pin to ubuntu-20.04 to avoid spontaneous breakage * Move coverage upload into units job * Use the implicit expression syntax in if statements * Update method calls used in the github-script action Octokit context no longer has REST methods as of v5 of the github-script action --- .github/scripts/check_migrations.sh | 0 .github/scripts/get_description.sh | 0 .github/workflows/check-clowdapp.yml | 18 -- .github/workflows/ci.yml | 235 ++++++++++++++++++ .github/workflows/pre-commit.yml | 18 -- .github/workflows/pre-release.yaml | 72 +++--- .../workflows/required-smokes-labeler.yaml | 82 ------ .github/workflows/unittests.yml | 156 ------------ 8 files changed, 275 insertions(+), 306 deletions(-) mode change 100644 => 100755 .github/scripts/check_migrations.sh mode change 100644 => 100755 .github/scripts/get_description.sh delete mode 100644 .github/workflows/check-clowdapp.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/pre-commit.yml delete mode 100644 .github/workflows/required-smokes-labeler.yaml delete mode 100644 .github/workflows/unittests.yml diff --git a/.github/scripts/check_migrations.sh b/.github/scripts/check_migrations.sh old mode 100644 new mode 100755 diff --git a/.github/scripts/get_description.sh b/.github/scripts/get_description.sh old mode 100644 new mode 100755 diff --git a/.github/workflows/check-clowdapp.yml b/.github/workflows/check-clowdapp.yml deleted file mode 100644 index ffd0429b7f..0000000000 --- a/.github/workflows/check-clowdapp.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Check Clowdapp - -on: - pull_request: - branches: - - main - -jobs: - check-clowdapp: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Check manifest - run: bash .github/scripts/check_clowdapp.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..ad47ec72ad --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,235 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + sanity: + name: Sanity + runs-on: ubuntu-20.04 + + steps: + - name: Checkout + uses: actions/checkout@v3.1.0 + + - name: Install Python + uses: actions/setup-python@v4.3.0 + with: + python-version: '3.8' + + - name: Run pre-commit checks + uses: pre-commit/action@v3.0.0 + env: + SETUPTOOLS_USE_DISTUTILS: stdlib + + - name: Check clowdapp manifest + run: bash .github/scripts/check_clowdapp.sh + + smokes-labeler: + name: Smoke Test Label + runs-on: ubuntu-20.04 + + steps: + - name: Checkout + uses: actions/checkout@v3.1.0 + with: + fetch-depth: 0 + + - name: get Docker image files + id: docker-files + run: .github/scripts/files_require_smokes.sh >> docker-files.txt + + - name: add other required files + id: add-files + run: | + echo .dockerignore >> docker-files.txt; + echo Dockerfile >> docker-files.txt; + echo pr_check.sh >> docker-files.txt; + echo deploy/clowdapp.yaml >> docker-files.txt; + + - name: Show Dockerfiles + run: cat docker-files.txt + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v34.0.2 + with: + files_from_source_file: docker-files.txt + + - name: Set whether to run tests + id: check-files + run: | + if [ ! -z "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" ]; then + echo "RUN_TESTS=true" >> $GITHUB_ENV + fi + + - name: Setting smokes-required label + uses: actions/github-script@v6.3.3 + if: env.RUN_TESTS == 'true' + continue-on-error: true + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: [ 'smokes-required' ] + }) + github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: 'ok-to-skip-smokes' + }) + + - name: Remove smokes-required label + uses: actions/github-script@v6.3.3 + if: env.RUN_TESTS != 'true' + continue-on-error: true + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: [ 'ok-to-skip-smokes' ] + }) + github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: 'smokes-required' + }) + + changed-files: + name: Detect changed files + runs-on: ubuntu-20.04 + outputs: + run_tests: ${{ steps.check-files-or-fork.outputs.run_tests }} + + steps: + - name: Checkout + uses: actions/checkout@v3.1.0 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v34.0.2 + with: + files: | + db_functions/ + koku/ + .github/postgres + .github/scripts/check_migrations.sh + .github/workflows/unittests.yml + Pipfile.lock + + - name: Check files or fork + id: check-files-or-fork + run: | + if [ ! -z "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" ] || [ "${{ github.event.pull_request.head.repo.full_name }}" != "project-koku/koku" ]; then + echo "run_tests=true" >> $GITHUB_OUTPUT + fi + + units: + name: Units - ${{ matrix.python-version }} + needs: changed-files + if: needs.changed-files.outputs.run_tests == 'true' + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + python-version: + - '3.9' + env: + COMPOSE_FILE: .github/postgres/docker-compose.yaml + + steps: + - name: Checkout + uses: actions/checkout@v3.1.0 + with: + fetch-depth: 0 + + - name: Display build environment + run: printenv + + - name: Start the postgres DB + run: docker-compose up -d db + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4.3.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install pipenv + run: sudo python3 -m pip install pipenv + + - name: Cache dependencies + id: cache-dependencies + uses: actions/cache@v3.0.11 + with: + path: | + ~/.cache/pipenv + ~/.local/share/virtualenvs + key: ${{ runner.os }}-env-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}-${{ secrets.ACTIONS_CACHE_KEY_UUID }} + + - name: Install dependencies + if: steps.cache-dependencies.outputs.cache-hit != 'true' + run: | + pipenv install --dev --ignore-pipfile --python ${{ matrix.python-version }} + + - name: Check migrations + run: bash .github/scripts/check_migrations.sh + env: + DATABASE_SERVICE_NAME: POSTGRES_SQL + DATABASE_ENGINE: postgresql + DATABASE_NAME: postgres + DATABASE_USER: postgres + DATABASE_PASSWORD: postgres + POSTGRES_SQL_SERVICE_HOST: localhost + POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }} + PROMETHEUS_MULTIPROC_DIR: /tmp + + - name: Run unit tests + id: unit_tests_run + run: pipenv run coverage run ./koku/manage.py test --noinput --verbosity 2 ./koku/ + env: + DATABASE_SERVICE_NAME: POSTGRES_SQL + DATABASE_ENGINE: postgresql + DATABASE_NAME: postgres + DATABASE_USER: postgres + DATABASE_PASSWORD: postgres + POSTGRES_SQL_SERVICE_HOST: localhost + POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }} + HIVE_DATABASE_PASSWORD: hivedbpw + ACCOUNT_ENHANCED_METRICS: True + ENABLE_TRINO_SOURCE_TYPE: GCP + PROMETHEUS_MULTIPROC_DIR: /tmp + TRINO_DATE_STEP: 31 + MIDDLEWARE_TIME_TO_LIVE: 0 + ENHANCED_ORG_ADMIN: True + ENABLE_PARQUET_PROCESSING: True + + - name: Display build environment + run: | + printenv + echo "Unit tests and migration check were '${{ steps.unit_tests_run.outcome }}'" + + - name: Convert coverage report to XML + run: pipenv run coverage xml + if: steps.unit_tests_run.outcome == 'success' + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3.1.1 + if: steps.unit_tests_run.outcome == 'success' + with: + file: ./coverage.xml + flags: unittests + name: Python-${{ matrix.python-version}} + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 491cf61dfc..0000000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Linting and Formatting - -on: - pull_request: - branches: - - main - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - uses: pre-commit/action@v2.0.3 - env: - SETUPTOOLS_USE_DISTUTILS: stdlib diff --git a/.github/workflows/pre-release.yaml b/.github/workflows/pre-release.yaml index 1c5eb6e8cb..0702c6ee50 100644 --- a/.github/workflows/pre-release.yaml +++ b/.github/workflows/pre-release.yaml @@ -9,36 +9,44 @@ on: jobs: pre-release: - runs-on: ubuntu-latest + name: Pre-release + runs-on: ubuntu-20.04 + steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: determine minor version - id: tag-setter - run: | - DATE=$(date +"%Y.%m.%d") - PREV_RELEASE=$(git tag --list | tail -1) - PREV_DATE="${PREV_RELEASE%.*}" - MINOR_VERSION=0 - case $PREV_DATE in - *"$DATE"*) - MINOR_VERSION=${PREV_RELEASE##*.} - MINOR_VERSION=$((MINOR_VERSION+1)) - ;; - *) - MINOR_VERSION=0 - ;; - esac - echo "::set-output name=TAG_VERSION::r.$DATE.$MINOR_VERSION" - - name: set the release commit - run: echo "RELEASE_COMMIT=${{ github.event.inputs.commit }}" >> $GITHUB_ENV - - name: Create release body file - run: bash .github/scripts/get_description.sh - - uses: ncipollo/release-action@v1 - with: - bodyFile: release_body.md - commit: ${{ github.event.inputs.commit }} - prerelease: true - tag: ${{ steps.tag-setter.outputs.TAG_VERSION }} - token: ${{ secrets.GITHUB_TOKEN }} + - name: Checkout + uses: actions/checkout@v3.1.0 + with: + fetch-depth: 0 + + - name: Determine minor version + id: tag-setter + run: | + DATE=$(date +"%Y.%m.%d") + PREV_RELEASE=$(git tag --list | tail -1) + PREV_DATE="${PREV_RELEASE%.*}" + MINOR_VERSION=0 + case $PREV_DATE in + *"$DATE"*) + MINOR_VERSION=${PREV_RELEASE##*.} + MINOR_VERSION=$((MINOR_VERSION+1)) + ;; + *) + MINOR_VERSION=0 + ;; + esac + echo "TAG_VERSION=r.$DATE.$MINOR_VERSION" >> $GITHUB_ENV + + - name: Set the release commit + run: echo "RELEASE_COMMIT=${{ github.event.inputs.commit }}" >> $GITHUB_ENV + + - name: Create release body file + run: bash .github/scripts/get_description.sh + + - name: Set release body + uses: ncipollo/release-action@v1.11.1 + with: + bodyFile: release_body.md + commit: ${{ github.event.inputs.commit }} + prerelease: true + tag: ${{ env.TAG_VERSION }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/required-smokes-labeler.yaml b/.github/workflows/required-smokes-labeler.yaml deleted file mode 100644 index e0997c743f..0000000000 --- a/.github/workflows/required-smokes-labeler.yaml +++ /dev/null @@ -1,82 +0,0 @@ -name: Smoke Tests Required Labeler - -on: - pull_request: - push: - branches: - - main - -jobs: - smokes-labeler: - runs-on: ubuntu-20.04 - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: get Docker image files - id: docker-files - run: .github/scripts/files_require_smokes.sh >> docker-files.txt - - - name: add other required files - id: add-files - run: | - echo .dockerignore >> docker-files.txt; - echo Dockerfile >> docker-files.txt; - echo pr_check.sh >> docker-files.txt; - echo deploy/clowdapp.yaml >> docker-files.txt; - - - run: cat docker-files.txt - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v13 - with: - files_from_source_file: docker-files.txt - - - id: check-files - run: | - if [ ! -z "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" ]; then - echo "::set-output name=run-tests::true" - fi - - - name: "Setting smokes-required label" - uses: actions/github-script@v3 - if: ${{ steps.check-files.outputs.run-tests == 'true' }} - continue-on-error: true - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - github.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: [ 'smokes-required' ] - }) - github.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: 'ok-to-skip-smokes' - }) - - - name: "Remove smokes-required label" - uses: actions/github-script@v3 - if: ${{ steps.check-files.outputs.run-tests != 'true' }} - continue-on-error: true - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - github.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: [ 'ok-to-skip-smokes' ] - }) - github.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: 'smokes-required' - }) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml deleted file mode 100644 index 14252fe810..0000000000 --- a/.github/workflows/unittests.yml +++ /dev/null @@ -1,156 +0,0 @@ -name: Check Migrations and Run Unit Tests - -on: - pull_request: - push: - branches: - - main - -jobs: - - setup: - runs-on: ubuntu-20.04 - outputs: - run-tests: ${{ steps.check-files-or-fork.outputs.run-tests }} - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v13 - with: - files: | - db_functions/ - koku/ - .github/postgres - .github/scripts/check_migrations.sh - .github/workflows/unittests.yml - Pipfile.lock - - - id: check-files-or-fork - run: | - if [ ! -z "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" ] || [ "${{ github.event.pull_request.head.repo.full_name }}" != "project-koku/koku" ]; then - echo "::set-output name=run-tests::true" - fi - - - run: echo ${{ needs.setup.outputs.run-tests }} - - run: echo ${{ github.event.pull_request.head.repo.full_name }} - - - unit-tests: - needs: setup - if: ${{ needs.setup.outputs.run-tests == 'true' }} - runs-on: ubuntu-20.04 - # continue-on-error: ${{ matrix.experimental }} - strategy: - fail-fast: false - max-parallel: 4 - matrix: - python-version: [3.9] - # experimental: [false] - # include: - # - python-version: 3.10 - # experimental: true - env: - COMPOSE_FILE: .github/postgres/docker-compose.yaml - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Display build environment - run: printenv - - - name: Start the postgres DB - run: docker-compose up -d db - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install pipenv - run: sudo python3 -m pip install pipenv - - - name: Cache dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: | - ~/.cache/pipenv - ~/.local/share/virtualenvs - key: ${{ runner.os }}-env-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}-${{ secrets.ACTIONS_CACHE_KEY_UUID }} - - - name: Install dependencies - if: steps.cache-dependencies.outputs.cache-hit != 'true' - run: | - pipenv install --dev --ignore-pipfile --python ${{ matrix.python-version }} - - - name: Check migrations - run: bash .github/scripts/check_migrations.sh - env: - DATABASE_SERVICE_NAME: POSTGRES_SQL - DATABASE_ENGINE: postgresql - DATABASE_NAME: postgres - DATABASE_USER: postgres - DATABASE_PASSWORD: postgres - POSTGRES_SQL_SERVICE_HOST: localhost - POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }} - PROMETHEUS_MULTIPROC_DIR: /tmp - - - name: Run unit tests - run: pipenv run coverage run ./koku/manage.py test --noinput --verbosity 2 ./koku/ - env: - DATABASE_SERVICE_NAME: POSTGRES_SQL - DATABASE_ENGINE: postgresql - DATABASE_NAME: postgres - DATABASE_USER: postgres - DATABASE_PASSWORD: postgres - POSTGRES_SQL_SERVICE_HOST: localhost - POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }} - HIVE_DATABASE_PASSWORD: hivedbpw - ACCOUNT_ENHANCED_METRICS: True - ENABLE_TRINO_SOURCE_TYPE: GCP - PROMETHEUS_MULTIPROC_DIR: /tmp - TRINO_DATE_STEP: 31 - MIDDLEWARE_TIME_TO_LIVE: 0 - ENHANCED_ORG_ADMIN: True - ENABLE_PARQUET_PROCESSING: True - - - name: Convert coverage report to XML - run: pipenv run coverage xml - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - file: ./coverage.xml - flags: unittests - name: Python-${{ matrix.python-version}} - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true - - unit-tests-result: - needs: unit-tests - if: | - always() && - (needs.unit-tests.result == 'success' || needs.unit-tests.result == 'skipped') - runs-on: ubuntu-20.04 - steps: - - name: Display build environment - run: printenv - - - run: echo "Unit tests and migration check were '${{ needs.unit-tests.result }}'" - - - name: Override Codecov for skipped tests - # This is a hack to override the Codecov behavior for skipped tests. - if: always() && needs.unit-tests.result == 'skipped' && github.ref_name != 'main' - env: - GITHUB_TOKEN: "${{ github.token }}" - run: | - gh api "/repos/project-koku/koku/statuses/${{ github.event.pull_request.head.sha }}" -f description="codecov skipped because unit tests were not required" -f context="codecov/patch" -f state="success" - gh api "/repos/project-koku/koku/statuses/${{ github.event.pull_request.head.sha }}" -f description="codecov skipped because unit tests were not required" -f context="codecov/project" -f state="success"