build #4657
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| versionBump: | |
| description: 'The part of the version to bump' | |
| required: true | |
| default: 'patch' | |
| type: choice | |
| options: | |
| - patch | |
| - minor | |
| - major | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'INTHEWILD.md' | |
| - 'README.md' | |
| - 'CHANGELOG.md' | |
| - '.github/**' | |
| - checkov/version.py | |
| - kubernetes/requirements.txt | |
| - coverage.svg | |
| - '.swm/**' | |
| - '.pre-commit-config.yaml' | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: 'build' | |
| cancel-in-progress: true | |
| jobs: | |
| security: | |
| uses: ./.github/workflows/security-shared.yml | |
| secrets: inherit | |
| integration-tests: | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python: ["3.10", "3.11", "3.12"] | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
| - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v3 | |
| - uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1 # v2 | |
| if: ${{ runner.os != 'windows' }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install pipenv | |
| run: | | |
| python -m pip install --no-cache-dir --upgrade pipenv | |
| - name: Build & install checkov package | |
| run: | | |
| # remove venv, if exists | |
| pipenv --rm || true | |
| pipenv --python ${{ matrix.python }} | |
| pipenv run pip install pytest pytest-xdist setuptools wheel | |
| pipenv run python setup.py sdist bdist_wheel | |
| bash -c 'pipenv run pip install dist/checkov-*.whl' | |
| - name: Clone Terragoat - vulnerable terraform | |
| run: git clone https://github.com/bridgecrewio/terragoat | |
| - name: Clone Cfngoat - vulnerable cloudformation | |
| run: git clone https://github.com/bridgecrewio/cfngoat | |
| - name: Clone Kubernetes-goat - vulnerable kubernetes | |
| run: git clone https://github.com/madhuakula/kubernetes-goat | |
| - name: Clone kustomize-goat - vulnerable kustomize | |
| run: git clone https://github.com/bridgecrewio/kustomizegoat | |
| - name: Create checkov reports | |
| run: | | |
| # Just making sure the API key tests don't run on PRs | |
| bash -c './integration_tests/prepare_data.sh "${{ matrix.os }}" "${{ matrix.python }}"' | |
| env: | |
| LOG_LEVEL: INFO | |
| BC_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
| PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
| TF_REGISTRY_TOKEN: ${{ secrets.TFC_TOKEN }} | |
| GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run integration tests | |
| run: | | |
| pipenv run pytest integration_tests | |
| integration-tests-old-python: | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python: ["3.8", "3.9"] | |
| os: [ubuntu-latest, macos-12, windows-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
| - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v3 | |
| - uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1 # v2 | |
| if: ${{ runner.os != 'windows' }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install pipenv | |
| run: | | |
| python -m pip install --no-cache-dir --upgrade pipenv | |
| - name: Build & install checkov package | |
| run: | | |
| # remove venv, if exists | |
| pipenv --rm || true | |
| pipenv --python ${{ matrix.python }} | |
| pipenv run pip install pytest pytest-xdist setuptools wheel | |
| pipenv run python setup.py sdist bdist_wheel | |
| bash -c 'pipenv run pip install dist/checkov-*.whl' | |
| - name: Clone Terragoat - vulnerable terraform | |
| run: git clone https://github.com/bridgecrewio/terragoat | |
| - name: Clone Cfngoat - vulnerable cloudformation | |
| run: git clone https://github.com/bridgecrewio/cfngoat | |
| - name: Clone Kubernetes-goat - vulnerable kubernetes | |
| run: git clone https://github.com/madhuakula/kubernetes-goat | |
| - name: Clone kustomize-goat - vulnerable kustomize | |
| run: git clone https://github.com/bridgecrewio/kustomizegoat | |
| - name: Create checkov reports | |
| run: | | |
| # Just making sure the API key tests don't run on PRs | |
| bash -c './integration_tests/prepare_data.sh "${{ matrix.os }}" "${{ matrix.python }}"' | |
| env: | |
| LOG_LEVEL: INFO | |
| BC_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
| PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
| TF_REGISTRY_TOKEN: ${{ secrets.TFC_TOKEN }} | |
| GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run integration tests | |
| run: | | |
| pipenv run pytest integration_tests | |
| prisma-tests: | |
| runs-on: [ self-hosted, public, linux, x64 ] | |
| env: | |
| PYTHON_VERSION: "3.8" | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
| - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install pipenv | |
| run: | | |
| python -m pip install --no-cache-dir --upgrade pipenv | |
| - name: Build & install checkov package | |
| run: | | |
| # remove venv, if exists | |
| pipenv --rm || true | |
| pipenv --python ${{ env.PYTHON_VERSION }} | |
| pipenv run pip install pytest pytest-xdist | |
| pipenv run python setup.py sdist bdist_wheel | |
| bash -c 'pipenv run pip install dist/checkov-*.whl' | |
| - name: Clone Terragoat - vulnerable terraform | |
| run: git clone https://github.com/bridgecrewio/terragoat | |
| - name: Run checkov with Prisma creds | |
| env: | |
| PRISMA_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
| PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
| run: | | |
| pipenv run checkov -s -d terragoat --bc-api-key "$PRISMA_KEY" --repo-id yuvalyacoby/terragoat > checkov_report_prisma.txt | |
| grep "prismacloud.io" checkov_report_prisma.txt | |
| exit $? | |
| sast-integration-tests: | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python: ["3.10", "3.11", "3.12"] | |
| os: [ubuntu-latest, macos-latest] | |
| runs-on: ${{ matrix.os }} | |
| continue-on-error: true # for now it is ok to fail | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
| - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install pipenv | |
| run: | | |
| python -m pip install --no-cache-dir --upgrade pipenv | |
| - name: Build & install checkov package | |
| run: | | |
| # remove venv, if exists | |
| pipenv --rm || true | |
| pipenv --python ${{ matrix.python }} | |
| pipenv run pip install pytest pytest-xdist setuptools wheel | |
| pipenv run python setup.py sdist bdist_wheel | |
| bash -c 'pipenv run pip install dist/checkov-*.whl' | |
| - name: Clone flask - Python repo for SAST | |
| run: git clone https://github.com/pallets/flask | |
| - name: Clone WebGoat - Java repo for SAST | |
| run: git clone https://github.com/WebGoat/WebGoat | |
| - name: Clone axios - JavaScript repo for SAST | |
| run: git clone https://github.com/axios/axios | |
| - name: Create checkov reports | |
| env: | |
| LOG_LEVEL: INFO | |
| BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
| PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
| run: bash -c './sast_integration_tests/prepare_data.sh' | |
| - name: Run integration tests | |
| run: | | |
| pipenv run pytest sast_integration_tests | |
| sast-integration-tests-old-python: | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python: ["3.8", "3.9"] | |
| os: [ubuntu-latest, macos-12] | |
| runs-on: ${{ matrix.os }} | |
| continue-on-error: true # for now it is ok to fail | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
| - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install pipenv | |
| run: | | |
| python -m pip install --no-cache-dir --upgrade pipenv | |
| - name: Build & install checkov package | |
| run: | | |
| # remove venv, if exists | |
| pipenv --rm || true | |
| pipenv --python ${{ matrix.python }} | |
| pipenv run pip install pytest pytest-xdist setuptools wheel | |
| pipenv run python setup.py sdist bdist_wheel | |
| bash -c 'pipenv run pip install dist/checkov-*.whl' | |
| - name: Clone flask - Python repo for SAST | |
| run: git clone https://github.com/pallets/flask | |
| - name: Clone WebGoat - Java repo for SAST | |
| run: git clone https://github.com/WebGoat/WebGoat | |
| - name: Clone axios - JavaScript repo for SAST | |
| run: git clone https://github.com/axios/axios | |
| - name: Create checkov reports | |
| env: | |
| LOG_LEVEL: INFO | |
| BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
| PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
| run: bash -c './sast_integration_tests/prepare_data.sh' | |
| - name: Run integration tests | |
| run: | | |
| pipenv run pytest sast_integration_tests | |
| unit-tests: | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-latest | |
| env: | |
| PYTHON_VERSION: "3.8" | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1 # v2 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install pipenv | |
| run: | | |
| python -m pip install --no-cache-dir --upgrade pipenv | |
| - name: Install dependencies | |
| run: | | |
| # remove venv, if exists | |
| pipenv --rm || true | |
| pipenv --python ${{ env.PYTHON_VERSION }} | |
| pipenv install --dev | |
| - name: Test with pytest | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| IS_TEST: true | |
| run: | | |
| pipenv run python -m pytest tests | |
| bump-version: | |
| needs: [integration-tests, unit-tests, prisma-tests, sast-integration-tests, integration-tests-old-python, sast-integration-tests-old-python] | |
| runs-on: [self-hosted, public, linux, x64] | |
| environment: release | |
| permissions: | |
| contents: write | |
| # IMPORTANT: this permission is mandatory for trusted publishing to pypi | |
| id-token: write | |
| timeout-minutes: 30 | |
| env: | |
| PYTHON_VERSION: "3.8" | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
| with: | |
| token: ${{ secrets.GH_PAT_SECRET }} | |
| - name: Import GPG key | |
| id: import_gpg | |
| uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4 # v5 | |
| with: | |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | |
| passphrase: ${{ secrets.PASSPHRASE }} | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install pipenv | |
| run: | | |
| python -m pip install --no-cache-dir --upgrade pipenv | |
| - name: Install dependencies | |
| run: | | |
| # remove venv, if exists | |
| pipenv --rm || true | |
| pipenv --python ${{ env.PYTHON_VERSION }} | |
| pipenv install | |
| - name: Calculate version | |
| run: | | |
| git fetch --tags --force | |
| latest_tag="$(git tag --sort=v:refname | tail -n 1)" | |
| echo "latest tag: $latest_tag" | |
| if [[ -z "${{ inputs.versionBump }}" ]] | |
| then | |
| version="patch" | |
| else | |
| version="${{ inputs.versionBump }}" | |
| fi | |
| case $version in | |
| minor) | |
| new_tag=$(echo "$latest_tag" | awk -F. -v a="$1" -v b="$2" -v c="$3" '{printf("%d.%d.%d", $1+a, $2+b+1 , 0)}') | |
| ;; | |
| major) | |
| new_tag=$(echo "$latest_tag" | awk -F. -v a="$1" -v b="$2" -v c="$3" '{printf("%d.%d.%d", $1+a+1, 0 , 0)}') | |
| ;; | |
| patch) | |
| new_tag=$(echo "$latest_tag" | awk -F. -v a="$1" -v b="$2" -v c="$3" '{printf("%d.%d.%d", $1+a, $2+b , $3+1)}') | |
| ;; | |
| esac | |
| echo "new tag: $new_tag" | |
| echo "version=$new_tag" >> "$GITHUB_OUTPUT" | |
| # grab major version for later image tag usage | |
| major_version=$(echo "${new_tag}" | head -c1) | |
| echo "major_version=$major_version" >> "$GITHUB_OUTPUT" | |
| id: calculateVersion | |
| - name: version | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }} | |
| run: | | |
| ## update docs | |
| export PYTHONPATH='.' | |
| # change the doc links to proper markdown versions | |
| export CHECKOV_CREATE_MARKDOWN_HYPERLINKS='True' | |
| git pull | |
| for i in cloudformation terraform kubernetes serverless arm dockerfile secrets github_configuration gitlab_configuration bitbucket_configuration github_actions gitlab_ci bicep openapi bitbucket_pipelines argo_workflows circleci_pipelines azure_pipelines ansible all | |
| do | |
| export scansdoc="docs/5.Policy Index/$i.md" | |
| echo "---" > "$scansdoc" | |
| echo "layout: default" >> "$scansdoc" | |
| echo "title: $i resource scans" >> "$scansdoc" | |
| echo "nav_order: 1" >> "$scansdoc" | |
| echo "---" >> "$scansdoc" | |
| echo "" >> "$scansdoc" | |
| echo "# $i resource scans (auto generated)" >> "$scansdoc" | |
| echo "" >> "$scansdoc" | |
| pipenv run python checkov/main.py --list --framework "$i" >> "$scansdoc" | |
| done | |
| #add cloudformation scans to serverless | |
| export scansdoc="docs/5.Policy Index/serverless.md" | |
| pipenv run python checkov/main.py --list --framework cloudformation >> "$scansdoc" | |
| git add "docs/5.Policy Index/*" | |
| git commit --reuse-message="HEAD@{1}" || echo "No changes to commit" | |
| git config --global user.name 'GitHub Actions Bot' | |
| git config --global user.email '[email protected]' | |
| new_tag=${{ steps.calculateVersion.outputs.version }} | |
| echo "new tag: $new_tag" | |
| ## update python version | |
| echo "version = '$new_tag'" > 'checkov/version.py' | |
| echo "checkov==$new_tag" > 'kubernetes/requirements.txt' | |
| git commit --reuse-message="HEAD@{1}" checkov/version.py kubernetes/requirements.txt || echo "No changes to commit" | |
| git push origin | |
| git tag $new_tag | |
| git push --tags | |
| id: version | |
| - name: create python package | |
| run: | | |
| pipenv run python setup.py sdist bdist_wheel | |
| - name: Publish a Python distribution to PyPI | |
| uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0 # v1 | |
| - name: sleep and wait for package to refresh | |
| run: | | |
| sleep 2m | |
| outputs: | |
| version: ${{ steps.calculateVersion.outputs.version }} | |
| major_version: ${{ steps.calculateVersion.outputs.major_version }} | |
| publish-checkov-dockerhub: | |
| needs: bump-version | |
| uses: bridgecrewio/gha-reusable-workflows/.github/workflows/publish-image.yaml@main | |
| permissions: | |
| contents: read | |
| id-token: write # Enable OIDC | |
| packages: write | |
| with: | |
| image_name_dockerhub: bridgecrew/checkov | |
| image_name_ghcr: ghcr.io/${{ github.repository }} | |
| image_tag_full: ${{ needs.bump-version.outputs.version }} | |
| image_tag_short: ${{ needs.bump-version.outputs.major_version }} | |
| runner: "['self-hosted', 'public', 'linux', 'x64']" | |
| secrets: | |
| BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
| PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| publish-checkov-k8s-dockerhub: | |
| needs: bump-version | |
| uses: bridgecrewio/gha-reusable-workflows/.github/workflows/publish-image.yaml@main | |
| permissions: | |
| contents: read | |
| id-token: write # Enable OIDC | |
| packages: write | |
| with: | |
| image_name_dockerhub: bridgecrew/checkov-k8s | |
| image_name_ghcr: ghcr.io/${{ github.repository }}-k8s | |
| image_tag_full: ${{ needs.bump-version.outputs.version }} | |
| image_tag_short: ${{ needs.bump-version.outputs.major_version }} | |
| dockerfile_path: kubernetes/Dockerfile | |
| runner: "['self-hosted', 'public', 'linux', 'x64']" | |
| secrets: | |
| BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
| PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| update-bridgecrew-projects: | |
| needs: publish-checkov-dockerhub | |
| runs-on: [self-hosted, public, linux, x64] | |
| environment: release | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
| - name: update checkov release | |
| run: | | |
| curl -X POST "https://jenkins-webhook.bridgecrew.cloud/buildByToken/build?job=Open-Source/upgrade-checkov&token=${{ secrets.BC_JENKINS_TOKEN }}" | |
| # trigger checkov-action update | |
| curl -XPOST -u "${{ secrets.GH_PAT_USER}}:${{secrets.GH_PAT_SECRET}}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/bridgecrewio/checkov-action/dispatches --data '{"event_type": "build"}' | |
| # trigger bridgecrew-py update | |
| curl -XPOST -u "${{ secrets.GH_PAT_USER}}:${{secrets.GH_PAT_SECRET}}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/bridgecrewio/bridgecrew-py/dispatches --data '{"event_type": "build"}' | |
| # trigger whorf update | |
| curl -XPOST -u "${{ secrets.GH_PAT_USER}}:${{secrets.GH_PAT_SECRET}}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/bridgecrewio/whorf/dispatches --data '{"event_type": "release"}' |