[CI] Update action versions, remove deprecated commands, improve orga…

…nization (project-koku#3978)

* Make scripts executable
* Update actions to use version that use Node.js 16 or later
    https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/

* Use environment file instead of deprecated command
    https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/

* Switch to using and environment file
    This job didn’t define outputs so I’m don’t know how exactly it was working
    before. Using an environment variable is a bit simpler anyway.

* Reformat run-tests env var and switch to enviorment file from output
* Add names for steps and use latest action versions
* Rename and consolidate workflows
* Pin to ubuntu-20.04 to avoid spontaneous breakage
* Move coverage upload into units job
* Use the implicit expression syntax in if statements

* Update method calls used in the github-script action
    Octokit context no longer has REST methods as of v5 of the github-script action

.github/workflows/ci.yml

@@ -0,0 +1,235 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  sanity:
+    name: Sanity
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Install Python
+        uses: actions/[email protected]
+        with:
+          python-version: '3.8'
+
+      - name: Run pre-commit checks
+        uses: pre-commit/[email protected]
+        env:
+          SETUPTOOLS_USE_DISTUTILS: stdlib
+
+      - name: Check clowdapp manifest
+        run: bash .github/scripts/check_clowdapp.sh
+
+  smokes-labeler:
+    name: Smoke Test Label
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
+      - name: get Docker image files
+        id: docker-files
+        run: .github/scripts/files_require_smokes.sh >> docker-files.txt
+
+      - name: add other required files
+        id: add-files
+        run: |
+          echo .dockerignore >> docker-files.txt;
+          echo Dockerfile >> docker-files.txt;
+          echo pr_check.sh >> docker-files.txt;
+          echo deploy/clowdapp.yaml >> docker-files.txt;
+
+      - name: Show Dockerfiles
+        run: cat docker-files.txt
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/[email protected]
+        with:
+          files_from_source_file: docker-files.txt
+
+      - name: Set whether to run tests
+        id: check-files
+        run: |
+          if [ ! -z "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" ]; then
+            echo "RUN_TESTS=true" >> $GITHUB_ENV
+          fi
+
+      - name: Setting smokes-required label
+        uses: actions/[email protected]
+        if: env.RUN_TESTS == 'true'
+        continue-on-error: true
+        with:
+          script: |
+            github.rest.issues.addLabels({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: [ 'smokes-required' ]
+            })
+            github.rest.issues.removeLabel({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'ok-to-skip-smokes'
+            })
+
+      - name: Remove smokes-required label
+        uses: actions/[email protected]
+        if: env.RUN_TESTS != 'true'
+        continue-on-error: true
+        with:
+          script: |
+            github.rest.issues.addLabels({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: [ 'ok-to-skip-smokes' ]
+            })
+            github.rest.issues.removeLabel({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'smokes-required'
+            })
+
+  changed-files:
+    name: Detect changed files
+    runs-on: ubuntu-20.04
+    outputs:
+      run_tests: ${{ steps.check-files-or-fork.outputs.run_tests }}
+
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/[email protected]
+        with:
+          files: |
+            db_functions/
+            koku/
+            .github/postgres
+            .github/scripts/check_migrations.sh
+            .github/workflows/unittests.yml
+            Pipfile.lock
+
+      - name: Check files or fork
+        id: check-files-or-fork
+        run: |
+          if [ ! -z "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" ] || [ "${{ github.event.pull_request.head.repo.full_name }}" != "project-koku/koku" ]; then
+            echo "run_tests=true" >> $GITHUB_OUTPUT
+          fi
+
+  units:
+    name: Units - ${{ matrix.python-version }}
+    needs: changed-files
+    if: needs.changed-files.outputs.run_tests == 'true'
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        python-version:
+          - '3.9'
+    env:
+      COMPOSE_FILE: .github/postgres/docker-compose.yaml
+
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
+      - name: Display build environment
+        run: printenv
+
+      - name: Start the postgres DB
+        run: docker-compose up -d db
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install pipenv
+        run: sudo python3 -m pip install pipenv
+
+      - name: Cache dependencies
+        id: cache-dependencies
+        uses: actions/[email protected]
+        with:
+          path: |
+            ~/.cache/pipenv
+            ~/.local/share/virtualenvs
+          key: ${{ runner.os }}-env-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}-${{ secrets.ACTIONS_CACHE_KEY_UUID }}
+
+      - name: Install dependencies
+        if: steps.cache-dependencies.outputs.cache-hit != 'true'
+        run: |
+          pipenv install --dev --ignore-pipfile --python ${{ matrix.python-version }}
+
+      - name: Check migrations
+        run: bash .github/scripts/check_migrations.sh
+        env:
+          DATABASE_SERVICE_NAME: POSTGRES_SQL
+          DATABASE_ENGINE: postgresql
+          DATABASE_NAME: postgres
+          DATABASE_USER: postgres
+          DATABASE_PASSWORD: postgres
+          POSTGRES_SQL_SERVICE_HOST: localhost
+          POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }}
+          PROMETHEUS_MULTIPROC_DIR: /tmp
+
+      - name: Run unit tests
+        id: unit_tests_run
+        run: pipenv run coverage run ./koku/manage.py test --noinput --verbosity 2 ./koku/
+        env:
+          DATABASE_SERVICE_NAME: POSTGRES_SQL
+          DATABASE_ENGINE: postgresql
+          DATABASE_NAME: postgres
+          DATABASE_USER: postgres
+          DATABASE_PASSWORD: postgres
+          POSTGRES_SQL_SERVICE_HOST: localhost
+          POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }}
+          HIVE_DATABASE_PASSWORD: hivedbpw
+          ACCOUNT_ENHANCED_METRICS: True
+          ENABLE_TRINO_SOURCE_TYPE: GCP
+          PROMETHEUS_MULTIPROC_DIR: /tmp
+          TRINO_DATE_STEP: 31
+          MIDDLEWARE_TIME_TO_LIVE: 0
+          ENHANCED_ORG_ADMIN: True
+          ENABLE_PARQUET_PROCESSING: True
+
+      - name: Display build environment
+        run: |
+         printenv
+         echo "Unit tests and migration check were '${{ steps.unit_tests_run.outcome }}'"
+
+      - name: Convert coverage report to XML
+        run: pipenv run coverage xml
+        if: steps.unit_tests_run.outcome == 'success'
+
+      - name: Upload coverage to Codecov
+        uses: codecov/[email protected]
+        if: steps.unit_tests_run.outcome == 'success'
+        with:
+          file: ./coverage.xml
+          flags: unittests
+          name: Python-${{ matrix.python-version}}
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true

.github/workflows/pre-release.yaml

@@ -9,36 +9,44 @@ on:
 
 jobs:
   pre-release:
-    runs-on: ubuntu-latest
+    name: Pre-release
+    runs-on: ubuntu-20.04
+
     steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: determine minor version
-      id: tag-setter
-      run: |
-          DATE=$(date +"%Y.%m.%d")
-          PREV_RELEASE=$(git tag --list | tail -1)
-          PREV_DATE="${PREV_RELEASE%.*}"
-          MINOR_VERSION=0
-          case $PREV_DATE in
-            *"$DATE"*)
-              MINOR_VERSION=${PREV_RELEASE##*.}
-              MINOR_VERSION=$((MINOR_VERSION+1))
-              ;;
-            *)
-              MINOR_VERSION=0
-              ;;
-          esac
-          echo "::set-output name=TAG_VERSION::r.$DATE.$MINOR_VERSION"
-    - name: set the release commit
-      run: echo "RELEASE_COMMIT=${{ github.event.inputs.commit }}" >> $GITHUB_ENV
-    - name: Create release body file
-      run: bash .github/scripts/get_description.sh
-    - uses: ncipollo/release-action@v1
-      with:
-        bodyFile: release_body.md
-        commit: ${{ github.event.inputs.commit }}
-        prerelease: true
-        tag: ${{ steps.tag-setter.outputs.TAG_VERSION }}
-        token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
+      - name: Determine minor version
+        id: tag-setter
+        run: |
+            DATE=$(date +"%Y.%m.%d")
+            PREV_RELEASE=$(git tag --list | tail -1)
+            PREV_DATE="${PREV_RELEASE%.*}"
+            MINOR_VERSION=0
+            case $PREV_DATE in
+              *"$DATE"*)
+                MINOR_VERSION=${PREV_RELEASE##*.}
+                MINOR_VERSION=$((MINOR_VERSION+1))
+                ;;
+              *)
+                MINOR_VERSION=0
+                ;;
+            esac
+            echo "TAG_VERSION=r.$DATE.$MINOR_VERSION" >> $GITHUB_ENV
+
+      - name: Set the release commit
+        run: echo "RELEASE_COMMIT=${{ github.event.inputs.commit }}" >> $GITHUB_ENV
+
+      - name: Create release body file
+        run: bash .github/scripts/get_description.sh
+
+      - name: Set release body
+        uses: ncipollo/[email protected]
+        with:
+          bodyFile: release_body.md
+          commit: ${{ github.event.inputs.commit }}
+          prerelease: true
+          tag: ${{ env.TAG_VERSION }}
+          token: ${{ secrets.GITHUB_TOKEN }}