diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 49ec5eeb8..2068964cd 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,8 +1,11 @@ # Require admin approval for GitHub settings and workflow modifications /.github/ @usdigitalresponse/grants-admins -# Require admin approval for Terraform IAC modifications -/terraform/ @usdigitalresponse/grants-admins - # Require admin approval when Postgres root CA bundle is modified /packages/server/rds-combined-ca-bundle.pem @usdigitalresponse/grants-admins + +# Require admin approval for special doc modifications +README.md @usdigitalresponse/grants-admins +LICENSE @usdigitalresponse/grants-admins +CODE_OF_CONDUCT.md @usdigitalresponse/grants-admins +CONTRIBUTING.md @usdigitalresponse/grants-admins diff --git a/.github/next_release_version.bash b/.github/next_release_version.bash new file mode 100755 index 000000000..f6a8d1e48 --- /dev/null +++ b/.github/next_release_version.bash @@ -0,0 +1,53 @@ +#! /bin/bash + +# Defaults +next_version_release_year=$(TZ='UTC' date '+%Y') +next_version_release_number=1 + +if [[ $1 == 'test' ]]; then + echo 'Running tests...' >&2 + dotest() { + result=$(bash $0 "release/${1}" 2> /dev/null | tail -n 1) + expect="${2}" + if [[ $result != $expect ]]; then + printf "Test failed:\n Expected: $expect\n Received: $result\n" >&2 + exit 1 + fi + } + dotest 'release/1234.987' "$next_version_release_year.1" + dotest 'release/0.0' "$next_version_release_year.1" + dotest 'release/0' "$next_version_release_year.1" + dotest 'sometag' "$next_version_release_year.1" + dotest "release/$next_version_release_year.1" "$next_version_release_year.2" + dotest "release/$next_version_release_year.19" "$next_version_release_year.20" + dotest "release/$next_version_release_year.399" "$next_version_release_year.400" + echo 'Tests complete' >&2 + exit 0 +fi + +if [[ -z $1 ]]; then + # Ensure tag history is available + git fetch --prune --unshallow + tag=$(git describe --tags --match='release/[0-9][0-9][0-9][0-9].[0-9]*' refs/heads/main) +else + tag=$1 +fi + +regex='release\/([0-9]{4})\.([0-9]+)' +if [[ $tag =~ $regex ]]; then + echo "Found tag for previous release: $tag" >&2 + prev_version_release_number="${BASH_REMATCH[2]}" + echo "Previous version number: $prev_version_release_number" >&2 + if [[ $next_version_release_year == "${BASH_REMATCH[1]}" ]]; then + ((next_version_release_number=prev_version_release_number+1)) + else + echo "Ignoring previous version number because it pertains to a different year" >&2 + fi +else + echo "Could not locate a previous release version" >&2 +fi + +next_version="$next_version_release_year.$next_version_release_number" +echo "Next version: $next_version" >&2 +# Output result to stdout +printf "$next_version" diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 000000000..aae1a99ce --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,133 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/release-drafter/release-drafter/master/schema.json +name-template: 'v$RESOLVED_VERSION' +tag-template: 'release/$RESOLVED_VERSION' +tag-prefix: 'release/' +version-template: '2023.$MINOR' +version-resolver: + default: minor +prerelease: true +categories: + - title: ๐Ÿš€ New features and enhancements + collapse-after: 10 + labels: + - enhancement + - title: ๐Ÿ› Bug fixes + collapse-after: 10 + labels: + - bug + - title: ๐Ÿ“– Documentation improvements + collapse-after: 10 + labels: + - documentation + - title: ๐Ÿ”ง Dependency updates + collapse-after: 3 + labels: + - dependencies + - title: ๐Ÿ” Federal Grant Finder updates + collapse-after: 3 + labels: + - Grant Finder + - title: ๐Ÿงพ ARPA Reporter updates + collapse-after: 3 + labels: + - arpa validations + - arpa subrecipients + - arpa web tool + - arpa audit report + - arpa output templates + - arpa quarterly reporter + - performance reporter + - title: Other changes + labels: + - '*' +category-template: '### $TITLE' +exclude-labels: + - skip-changelog +exclude-contributors: + - dependabot + - 'dependabot[bot]' + - step-security-bot +autolabeler: + - label: javascript + files: + - '**/*.js' + - '**/package.json' + - 'packages/**' + - '**/yarn.lock' + - '**/.npmrc' + - '**/.nvmrc' + - '**/.nycrc' + - '**/.node-version' + - '**/.huskyrc.json' + - '**/lerna.json' + - '**/eslintrc.js' + - '**/.browserslistrc' + - label: database-changes + files: + - 'packages/server/migrations/**' + - 'packages/server/knexfile.js' + - 'packages/server/rds-combined-ca-bundle.pem' + - label: terraform + files: + - 'terraform/**' + - label: Infra + files: + - 'terraform/**' + - 'docker/**' + - '**/docker-compose.yml' + - '**/docker-compose.yaml' + - 'localstack/**' + - label: dependencies + files: + - '**/yarn.lock' + - '**/.terraform.lock.hcl' + branch: + - '/^dependabot\/.+$/i' + - label: documentation + files: + - README + - '**/doc/**' + - '**/docs/**' + - '**/*.md' + - .adr-dir + branch: + - '/^docs?\/.+$/' + - label: bug + branch: + - '/^fix\/.+$/i' + - '/^bug\/.+$/i' + title: + - '/\bfix(es)?\b/i' + - '/\bbug\b/i' + - '/\brevert(s)?\b/i' + - label: enhancement + branch: + - '/^feat(ures?)?\/.+$/i' + - '/^enhance(s|ments?)?\/.+$/i' + title: + - '/\b(?- + '*All changes in this release were crafted by robots (and reviewed by humans).*' +template: | + ## ๐Ÿ“š Summary + + The releaser should provide a high-level summary here (or remove this section). + + ## ๐Ÿ› ๏ธ Changes + + $CHANGES + + ## ๐Ÿค Contributors + + We would like to thank the following people who made this release possible: + + $CONTRIBUTORS + + ## Deployment History diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 000000000..98688b994 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,20 @@ +changelog: + exclude: + labels: + - skip-changelog + categories: + - title: ๐Ÿš€ New features and enhancements + labels: + - enhancement + - title: ๐Ÿ› Bug fixes + labels: + - bug + - title: ๐Ÿ“– Documentation improvements + labels: + - documentation + - title: ๐Ÿ”ง Dependency updates + labels: + - dependencies + - title: Other Changes + labels: + - "*" diff --git a/.github/workflows/aws-auth.yml b/.github/workflows/aws-auth.yml new file mode 100644 index 000000000..03d6221c4 --- /dev/null +++ b/.github/workflows/aws-auth.yml @@ -0,0 +1,69 @@ +name: Configure AWS Credentials + +on: + workflow_call: + inputs: + aws-region: + type: string + required: true + secrets: + role-to-assume: + required: true + gpg-passphrase: + required: true + outputs: + aws-access-key-id: + value: ${{ jobs.oidc-auth.outputs.aws-access-key-id }} + aws-secret-access-key: + value: ${{ jobs.oidc-auth.outputs.aws-secret-access-key }} + aws-session-token: + value: ${{ jobs.oidc-auth.outputs.aws-session-token }} + +permissions: + contents: read + id-token: write + +jobs: + oidc-auth: + name: OIDC Auth + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + outputs: + aws-access-key-id: ${{ steps.encrypt-aws-access-key-id.outputs.out }} + aws-secret-access-key: ${{ steps.encrypt-aws-secret-access-key.outputs.out }} + aws-session-token: ${{ steps.encrypt-aws-session-token.outputs.out }} + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + sts.us-west-2.amazonaws.com:443 + - id: auth + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + with: + aws-region: us-west-2 + role-to-assume: "${{ secrets.role-to-assume }}" + - name: Encrypt aws-access-key-id + id: encrypt-aws-access-key-id + run: | + encrypted=$(gpg --batch --yes --passphrase "$GPG_PASSPHRASE" -c --cipher-algo AES256 -o - <(echo "$AWS_ACCESS_KEY_ID") | base64 -w0) + echo "out=$encrypted" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + - name: Encrypt aws-secret-access-key + id: encrypt-aws-secret-access-key + run: | + encrypted=$(gpg --batch --yes --passphrase "$GPG_PASSPHRASE" -c --cipher-algo AES256 -o - <(echo "$AWS_SECRET_ACCESS_KEY") | base64 -w0) + echo "out=$encrypted" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + - name: Encrypt aws-session-token + id: encrypt-aws-session-token + run: | + encrypted=$(gpg --batch --yes --passphrase "$GPG_PASSPHRASE" -c --cipher-algo AES256 -o - <(echo "$AWS_SESSION_TOKEN") | base64 -w0) + echo "out=$encrypted" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml deleted file mode 100644 index e7625212d..000000000 --- a/.github/workflows/build-and-deploy.yml +++ /dev/null @@ -1,216 +0,0 @@ -name: Build and deploy GOST - -on: - push: - branches: - - _staging - - main - -concurrency: - group: ${{ github.workflow_ref }} - -permissions: - contents: read - id-token: write - packages: write - -jobs: - changes: - name: Detect changes - runs-on: ubuntu-latest - outputs: - api: ${{ steps.filter.outputs.api }} - website: ${{ steps.filter.outputs.website }} - terraform: ${{ steps.filter.outputs.terraform }} - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - base: ${{ github.ref }} - filters: | - api: - - ".github/workflows/build-api.yml" - - '.github/workflows/build-and-deploy.yml' - - 'packages/server/**' - - 'docker/production-api.Dockerfile' - - '.nvmrc' - - 'yarn.lock' - website: - - ".github/workflows/build-website.yml" - - '.github/workflows/build-and-deploy.yml' - - 'packages/client/**' - - '.node-version' - - '.nvmrc' - - 'yarn.lock' - terraform: - - 'terraform/**' - - build_api: - name: Build and push GOST API Docker image - needs: - - changes - if: needs.changes.outputs.api == 'true' - uses: "./.github/workflows/build-api.yml" - permissions: - contents: read - packages: write - - build_website: - name: Build website deployment artifact - needs: - - changes - if: needs.changes.outputs.website == 'true' - uses: "./.github/workflows/build-website.yml" - permissions: - contents: read - - select_target_environment: - name: Select target environment - uses: "./.github/workflows/select-target-environment.yml" - with: - ref_name: "${{ github.ref_name }}" - - deploy_terraform: - name: Deploy terraform - runs-on: ubuntu-latest - needs: - - select_target_environment - - changes - if: needs.changes.outputs.terraform == 'true' - concurrency: - group: run_terraform-${{ needs.select_target_environment.outputs.selected }} - cancel-in-progress: false - environment: ${{ needs.select_target_environment.outputs.selected }} - env: - TF_PLUGIN_CACHE_DIR: ~/.terraform.d/plugin-cache - TF_VAR_version_identifier: ${{ github.sha }} - TF_VAR_datadog_api_key: ${{ secrets.DATADOG_API_KEY }} - TF_VAR_datadog_app_key: ${{ secrets.DATADOG_APP_KEY }} - steps: - - uses: actions/checkout@v3 - - name: Get project TF version - id: get_version - run: echo "TF_VERSION=$(cat .terraform-version | tr -d '[:space:]')" | tee -a $GITHUB_OUTPUT - working-directory: terraform - - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: ${{ steps.get_version.outputs.TF_VERSION }} - - name: Ensure Terraform plugin cache exists - run: mkdir -p $TF_PLUGIN_CACHE_DIR - - name: Save/Restore Terraform plugin cache - uses: actions/cache@v3 - with: - path: ${{ env.TF_PLUGIN_CACHE_DIR }} - key: ${{ runner.os }}-terraform-${{ hashFiles('**/.terraform.lock.hcl') }} - restore-keys: | - ${{ runner.os }}-terraform- - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: us-west-2 - role-to-assume: "${{ secrets.AWS_ROLE_TO_ASSUME }}" - - name: Terraform Init - id: init - run: terraform init -backend-config="${{ needs.select_target_environment.outputs.selected }}.s3.tfbackend" - working-directory: terraform - - name: Terraform Validate - id: validate - run: terraform validate -no-color - working-directory: terraform - - name: Terraform Apply - if: steps.validate.outcome == 'success' - id: apply - run: terraform apply -auto-approve -input=false -no-color -var-file="${{ needs.select_target_environment.outputs.selected }}.tfvars" - working-directory: terraform - - deploy_website: - name: Deploy website - runs-on: ubuntu-latest - needs: - - build_website - - select_target_environment - - deploy_terraform - if: | - always() && - needs.build_website.result == 'success' && - (needs.deploy_terraform.result == 'success' || needs.deploy_terraform.result == 'skipped') - environment: ${{ needs.select_target_environment.outputs.selected }} - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: us-west-2 - role-to-assume: "${{ secrets.AWS_ROLE_TO_ASSUME }}" - - name: Get deployment parameters - uses: dkershner6/aws-ssm-getparameters-action@v1 - with: - withDecryption: "true" - parameterPairs: | - /gost/${{ needs.select_target_environment.outputs.selected }}/deploy-config/website/s3-uri = S3_DEPLOYMENT_URI, - /gost/${{ needs.select_target_environment.outputs.selected }}/deploy-config/website/distribution-id = CLOUDFRONT_DISTRIBUTION_ID - - name: Download build - uses: actions/download-artifact@v3 - with: - name: ${{ needs.build_website.outputs.artifact }} - path: dist - - name: Upload artifact to S3 - run: aws s3 sync ./dist ${{ env.S3_DEPLOYMENT_URI }} --sse --delete --no-progress - - name: Invalidate CloudFront cache - run: aws cloudfront create-invalidation --paths "/*" --distribution-id ${{ env.CLOUDFRONT_DISTRIBUTION_ID }} - - deploy_api: - name: Deploy API - runs-on: ubuntu-latest - needs: - - build_api - - select_target_environment - - deploy_terraform - if: | - always() && - needs.build_api.result == 'success' && - (needs.deploy_terraform.result == 'success' || needs.deploy_terraform.result == 'skipped') - environment: ${{ needs.select_target_environment.outputs.selected }} - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: us-west-2 - role-to-assume: "${{ secrets.AWS_ROLE_TO_ASSUME }}" - - name: Get deployment parameters - uses: dkershner6/aws-ssm-getparameters-action@v1 - with: - withDecryption: "true" - parameterPairs: | - /gost/${{ needs.select_target_environment.outputs.selected }}/deploy-config/api/cluster-name = ECS_CLUSTER_NAME, - /gost/${{ needs.select_target_environment.outputs.selected }}/deploy-config/api/service-name = ECS_SERVICE_NAME - - name: Update ECS service - run: aws ecs update-service --cluster ${{ env.ECS_CLUSTER_NAME }} --service ${{ env.ECS_SERVICE_NAME }} --force-new-deployment > /dev/null - - deploy_grants_consumer: - name: Deploy Grants Consumer - runs-on: ubuntu-latest - needs: - - build_api - - select_target_environment - - deploy_terraform - if: | - always() && - needs.build_api.result == 'success' && - (needs.deploy_terraform.result == 'success' || needs.deploy_terraform.result == 'skipped') - environment: ${{ needs.select_target_environment.outputs.selected }} - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: us-west-2 - role-to-assume: "${{ secrets.AWS_ROLE_TO_ASSUME }}" - - name: Get deployment parameters - uses: dkershner6/aws-ssm-getparameters-action@v1 - with: - withDecryption: "true" - parameterPairs: | - /gost/${{ needs.select_target_environment.outputs.selected }}/deploy-config/consume-grants/cluster-name = ECS_CLUSTER_NAME, - /gost/${{ needs.select_target_environment.outputs.selected }}/deploy-config/consume-grants/service-name = ECS_SERVICE_NAME - - name: Update ECS service - run: aws ecs update-service --cluster ${{ env.ECS_CLUSTER_NAME }} --service ${{ env.ECS_SERVICE_NAME }} --force-new-deployment > /dev/null diff --git a/.github/workflows/build-api.yml b/.github/workflows/build-api.yml deleted file mode 100644 index 1deccefab..000000000 --- a/.github/workflows/build-api.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Build Docker image for GOST API - -on: - pull_request: - paths: - - ".github/workflows/build-api.yml" - - "packages/server/**" - - "docker/production-api.Dockerfile" - - "yarn.lock" - workflow_call: {} - -permissions: - contents: read - packages: write - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }}-api - -jobs: - build-and-push: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Authenticate docker - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set date/time-based version string as env var - run: echo "DATETIME_VERSION=$(TZ=UTC date +'%Y%m%d.%H%M')" >> $GITHUB_ENV - - name: Extract metadata for Docker - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - type=raw,enable=true,priority=200,prefix=,suffix=,value=${{ env.DATETIME_VERSION }} - type=raw,enable=${{ github.event_name != 'pull_request' }},priority=300,value=latest - type=raw,enable=${{ github.ref == 'refs/heads/main' }},priority=300,value=stable - labels: | - org.opencontainers.image.title=${{ env.IMAGE_NAME }} - org.opencontainers.image.version=${{ env.DATETIME_VERSION }} - com.datadoghq.tags.service=gost - com.datadoghq.tags.version=${{ github.sha }} - - name: Build and push Docker image - uses: docker/build-push-action@v3 - with: - context: . - push: ${{ github.event_name != 'pull_request' }} - file: docker/production-api.Dockerfile - tags: ${{ steps.meta.outputs.tags }} - platforms: linux/amd64,linux/arm64 - labels: ${{ steps.meta.outputs.labels }} - build-args: | - GIT_COMMIT=${{ github.sha }} - GIT_REF=${{ github.ref }} - TIMESTAMP=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} diff --git a/.github/workflows/build-website.yml b/.github/workflows/build-website.yml deleted file mode 100644 index c637231a4..000000000 --- a/.github/workflows/build-website.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build GOST Website - -on: - pull_request: - paths: - - ".github/workflows/build-website.yml" - - "packages/client/**" - - ".node-version" - - ".nvmrc" - - "yarn.lock" - workflow_call: - outputs: - artifact: - description: "Identifier for the website build artifact" - value: ${{ jobs.build.outputs.artifact }} - -concurrency: - group: build-website-${{ github.workflow_ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -permissions: - contents: read - -jobs: - build: - name: Build website deployment artifact - runs-on: ubuntu-latest - permissions: - contents: read - outputs: - artifact: website-${{ github.sha }} - env: - BUILD_DIR: packages/client - steps: - - uses: actions/checkout@v3 - - name: Install Node.js - uses: actions/setup-node@v2 - with: - node-version-file: '.nvmrc' - cache: yarn - - name: Install dependencies - working-directory: ${{ env.BUILD_DIR }} - run: yarn install --frozen-lockfile - - name: Build the website - working-directory: ${{ env.BUILD_DIR }} - run: yarn build - - name: Upload build artifact - # if: ${{ github.event_name != 'pull_request' }} - uses: actions/upload-artifact@v3 - with: - name: website-${{ github.sha }} - path: ${{ env.BUILD_DIR }}/dist diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..5760bc8b1 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,283 @@ +name: Build + +permissions: + contents: read + packages: write + +on: + workflow_call: + inputs: + ref: + type: string + required: true + build-server-image: + description: Whether to build a `usdr-gost-api` Docker image using the `production-api.Dockerfile`. + type: boolean + default: true + build-website: + description: Whether to build website distribution artifacts for the client package. + type: boolean + default: true + server-image-args-ref: + description: Ref name for the build commit, like refs/heads/my-feature-branch-1. + type: string + required: true + server-image-artifacts-retention-days: + description: Number of days to store Docker attestation artifacts. + type: number + default: 90 + server-image-upload-attestations: + description: Whether to upload attestation files for the Docker build as artifacts. + type: boolean + default: false + server-image-push: + description: Whether to push Docker images to the registry after building. + type: boolean + default: true + server-image-name: + description: Name of the docker image. Use caution when setting a non-default value. + type: string + default: ${{ github.repository }}-api + server-image-registry: + description: The Docker image registry. Use caution when setting a non-default value. + type: string + default: ghcr.io + server-image-tag-latest: + description: Tags image builds with `latest`. + type: boolean + default: false + server-image-tag-production: + description: Tags image builds with `production`. + type: boolean + required: false + server-image-tag-pr: + description: A PR number to add as a Docker image tag (as `pr-`) when building for a pull request. + type: string + required: false + server-image-tag-release: + description: A tag value that, if provided, signifies the release version associated with the Docker image. + type: string + required: false + server-image-version: + description: Value to set for the `org.opencontainers.image.version label`. + type: string + default: "" + website-artifact-retention-days: + description: Number of days to retain website build artifacts. + type: number + default: 90 + outputs: + build-server-image-result: + value: ${{ jobs.server-docker-image.result }} + build-website-result: + value: ${{ jobs.website-bundle.result }} + server-image-digest: + value: ${{ jobs.server-docker-image.outputs.digest }} + server-attestation-artifacts-key: + value: ${{ jobs.server-docker-image.outputs.attestation-artifacts-key }} + server-attestation-artifacts-path: + value: ${{ jobs.server-docker-image.outputs.attestation-artifacts-path }} + website-artifacts-key: + value: ${{ jobs.website-bundle.outputs.artifacts-key }} + website-artifacts-path: + value: ${{ jobs.website-bundle.outputs.artifacts-path }} + website-checksums-sha256: + value: ${{ jobs.website-bundle.outputs.checksums-sha256 }} + +jobs: + server-docker-image: + name: Build server Docker image + if: inputs.build-server-image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + env: + ATTESTATION_ARTIFACTS_KEY: "server-image-attestations-${{ inputs.ref }}" + outputs: + commit-tag: ${{ inputs.ref }} + digest: ${{ steps.build-push.outputs.digest }} + attestation-artifacts-key: ${{ env.ATTESTATION_ARTIFACTS_KEY }} + attestation-artifacts-path: ${{ steps.store-attestations.outputs.path }} + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + id: checkout + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Set build info for the checked-out commit + id: commit-sha + run: echo "long=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Set up QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + with: + platforms: linux/amd64,linux/arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + with: + platforms: linux/amd64,linux/arm64 + - name: Authenticate docker + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ inputs.server-image-registry }}/${{ inputs.server-image-name }} + tags: | + type=raw,enable=true,priority=100,prefix=sha-,value=${{ steps.commit-sha.outputs.long }} + type=raw,enable=${{ inputs.server-image-tag-release != '' }},priority=200,value=${{ inputs.server-image-tag-release }} + type=raw,enable=${{ inputs.server-image-tag-latest }},priority=300,value=latest + type=raw,enable=${{ inputs.server-image-tag-pr != '' }},priority=600,prefix=pr-,value=${{ inputs.server-image-tag-pr }} + labels: | + org.opencontainers.image.title=${{ inputs.server-image-name }} + org.opencontainers.image.version=${{ inputs.server-image-version }} + org.opencontainers.image.revision=${{ steps.commit-sha.outputs.long }} + com.datadoghq.tags.service=gost + com.datadoghq.tags.version=${{ steps.commit-sha.outputs.long }} + - name: Set bake file definition as step output + id: bakefile + run: | + BAKEFILE_CONTENTS="$(cat $BAKEFILE_PATH)" + echo "result<> $GITHUB_OUTPUT + echo "$BAKEFILE_CONTENTS" >> $GITHUB_OUTPUT + echo "ENDOFBAKEFILE" >> $GITHUB_OUTPUT + env: + BAKEFILE_PATH: ${{ steps.meta.outputs.bake-file }} + - name: Build and push Docker image + id: build-push + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 + with: + context: . + github-token: ${{ secrets.GITHUB_TOKEN }} + push: ${{ inputs.server-image-push }} + file: docker/production-api.Dockerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: true + sbom: true + build-args: | + GIT_COMMIT=${{ inputs.ref }} + GIT_REF=${{ inputs.server-image-args-ref }} + TIMESTAMP=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} + - name: Publish build results + run: | + REPORT_FILE=$(mktemp -t summary.md.XXXXX) + cat >> $REPORT_FILE << 'ENDOFREPORT' + ## Docker Build Summary + + **Image ID:** `${{ steps.build-push.outputs.imageid }}` + **Image Digest:** `${{ steps.build-push.outputs.digest }}` + +
+ Bake File + + ```json + ${{ steps.bakefile.outputs.result }} + ``` + +
+
+ Build Metadata + + ```json + ${{ steps.build-push.outputs.metadata }} + ``` + +
+ ENDOFREPORT + cat "$REPORT_FILE" >> $GITHUB_STEP_SUMMARY + - name: Store attestations + id: store-attestations + if: inputs.server-image-upload-attestations + run: | + ATTESTATIONS_DIR=$(mktemp -d) + echo "path=$ATTESTATIONS_DIR" >> $GITHUB_OUTPUT + docker buildx imagetools inspect "$INSPECT_NAME" --format "{{ json .SBOM }}" > $ATTESTATIONS_DIR/sbom.sdpx.json + docker buildx imagetools inspect "$INSPECT_NAME" --format "{{ json .Provenance }}" > $ATTESTATIONS_DIR/provenance.json + env: + INSPECT_NAME: ${{ inputs.server-image-registry }}/${{ inputs.server-image-name }}@${{ steps.build-push.outputs.digest }} + - name: Upload attestations + if: steps.store-attestations.outcome == 'success' + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: ${{ env.ATTESTATION_ARTIFACTS_KEY }} + path: ${{ steps.store-attestations.outputs.path }} + if-no-files-found: error + retention-days: ${{ inputs.server-image-artifacts-retention-days }} + + website-bundle: + name: Build website deployment artifact + if: inputs.build-website + runs-on: ubuntu-latest + permissions: + contents: read + packages: none + env: + ARTIFACTS_KEY: website-${{ inputs.ref }} + ARTIFACTS_PATH: ${{ github.workspace }}/packages/client/dist + outputs: + artifacts-key: ${{ env.ARTIFACTS_KEY }} + artifacts-path: ${{ env.ARTIFACTS_PATH }} + checksums-sha256: ${{ steps.checksums.outputs.sha256 }} + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Setup Node + uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version-file: .nvmrc + cache: yarn + cache-dependency-path: yarn.lock + - name: Install dependencies + run: yarn workspace client install --frozen-lockfile + - name: Build the website + run: yarn workspace client build + - name: Compute build artifact checksums + id: checksums + run: | + DIST_CHECKSUMS=$(find "$ARTIFACTS_PATH" -type f -exec sha256sum {} \;) + echo "sha256<> $GITHUB_OUTPUT + echo "$DIST_CHECKSUMS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Publish build results + run: | + REPORT_FILE=$(mktemp -t summary.md.XXXXX) + cat >> $REPORT_FILE << 'ENDOFREPORT' + ## Build Website Summary + +
+ Checksums + + ``` + ${{ steps.checksums.outputs.sha256 }} + ``` + +
+ ENDOFREPORT + cat "$REPORT_FILE" >> $GITHUB_STEP_SUMMARY + - name: Upload build artifact + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: ${{ env.ARTIFACTS_KEY }} + path: ${{ env.ARTIFACTS_PATH }} + if-no-files-found: error + retention-days: ${{ inputs.website-artifact-retention-days }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 2bef50450..000000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: Continuous Integration - -on: - pull_request: {} - push: - branches: - - main - paths-ignore: - - "docs/**" - -jobs: - test-e2e-integration: - name: "Cypress Integration Tests" - runs-on: ubuntu-latest - services: - postgres: - image: "postgres:13" - env: - POSTGRES_DB: usdr_grants_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: password - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v2 - - name: Install Node.js - uses: actions/setup-node@v2 - with: - node-version: 16.14.0 - cache: yarn - - name: Install dependencies - run: yarn setup - - name: Run migrations - env: - POSTGRES_TEST_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" - # This is intentional to set POSTGRES_URL=POSTGRES_TEST_URL; ARPA Reporter test runner gates - # dev vs. CI differences based on whether these are the same. - POSTGRES_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" - run: yarn db:migrate - - name: Start Applications - env: - POSTGRES_TEST_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" - # This is intentional to set POSTGRES_URL=POSTGRES_TEST_URL; ARPA Reporter test runner gates - # dev vs. CI differences based on whether these are the same. - POSTGRES_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" - AWS_ACCESS_KEY_ID: "Fake AWS Key" - AWS_SECRET_ACCESS_KEY: "Fake AWS Secret" - NOTIFICATIONS_EMAIL: grants-identification@usdigitalresponse.org - DATA_DIR: './data' - run: yarn serve & - - name: Run e2e tests - uses: cypress-io/github-action@v5 - env: - CYPRESS_BASE_URL: 'http://localhost:8080' - with: - publish-summary: true - working-directory: packages/e2e - - test-server-client: - name: "Server and Client Unit Tests" - runs-on: ubuntu-latest - services: - postgres: - image: "postgres:13" - env: - POSTGRES_DB: usdr_grants_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: password - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v2 - - name: Install Node.js - uses: actions/setup-node@v2 - with: - node-version: 16.14.0 - cache: yarn - - name: Install dependencies - run: yarn setup - - name: Run Linter - run: yarn lint - - name: Run unit tests - env: - POSTGRES_TEST_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" - # This is intentional to set POSTGRES_URL=POSTGRES_TEST_URL; ARPA Reporter test runner gates - # dev vs. CI differences based on whether these are the same. - POSTGRES_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" - AWS_ACCESS_KEY_ID: "Fake AWS Key" - AWS_SECRET_ACCESS_KEY: "Fake AWS Secret" - NOTIFICATIONS_EMAIL: grants-identification@usdigitalresponse.org - run: | - # The .env file needs to be present; the example file is good enough. - cp packages/server/.env.example packages/server/.env - cp packages/client/.env.example packages/client/.env - cp packages/e2e/.env.example packages/e2e/.env - export CI=true; yarn coverage - - name: Publish coverage report to CodeClimate - uses: paambaati/codeclimate-action@v3.2.0 - env: - CC_TEST_REPORTER_ID: c0ab87c312e9ca57ec34d55ebec07ed396f96f039b3c725221918a75be71a0eb - with: - coverageLocations: | - ${{github.workspace}}/coverage/lcov.info:lcov diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..bf4eecba8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,117 @@ +name: Continuous Integration + +on: + pull_request_target: {} + +permissions: + contents: read + +jobs: + qa: + permissions: + contents: read + uses: ./.github/workflows/qa.yml + with: + ref: ${{ github.event.pull_request.head.sha }} + + publish-qa-results: + name: Publish QA Results + permissions: + contents: read + pull-requests: write + if: needs.qa.result != 'skipped' || needs.qa.result != 'cancelled' + needs: + - qa + uses: "./.github/workflows/publish-qa-results.yml" + with: + client-test-outcome: ${{ needs.qa.outputs.client-test-outcome }} + client-test-coverage-markdown-report: ${{ needs.qa.outputs.client-test-coverage-markdown-report }} + server-test-outcome: ${{ needs.qa.outputs.server-test-outcome }} + server-test-coverage-markdown-report: ${{ needs.qa.outputs.server-test-coverage-markdown-report }} + eslint-outcome: ${{ needs.qa.outputs.eslint-outcome }} + tflint-outcome: ${{ needs.qa.outputs.tflint-outcome }} + e2e-test-outcome: ${{ needs.qa.outputs.e2e-test-outcome }} + pr-number: ${{ github.event.pull_request.number }} + write-summary: true + write-comment: true + + build: + permissions: + contents: read + packages: write + name: Build deployment artifacts + uses: ./.github/workflows/build.yml + with: + ref: ${{ github.event.pull_request.head.sha }} + build-server-image: true + server-image-push: true + server-image-args-ref: ${{ github.ref }} + server-image-tag-latest: false + server-image-tag-pr: "${{ github.event.pull_request.number }}" + server-image-version: "rc-pr-${{ github.event.pull_request.number }}" + server-image-upload-attestations: true + server-image-artifacts-retention-days: 14 + build-website: true + website-artifact-retention-days: 14 + + aws-auth: + name: Configure AWS Credentials + permissions: + contents: read + id-token: write + uses: ./.github/workflows/aws-auth.yml + with: + aws-region: us-west-2 + secrets: + role-to-assume: ${{ secrets.CI_ROLE_ARN }} + gpg-passphrase: ${{ secrets.TFPLAN_SECRET }} + + tf-plan: + name: Plan Terraform + permissions: + contents: read + needs: + - aws-auth + - build + uses: ./.github/workflows/terraform-plan.yml + if: needs.build.outputs.build-server-image-result == 'success' && needs.build.outputs.build-website-result == 'success' && needs.aws-auth.result == 'success' + with: + ref: ${{ github.event.pull_request.head.sha }} + concurrency-group: run_terraform-staging + server-image-tag: ${{ github.event.pull_request.head.sha }} + server-image-digest: ${{ needs.build.outputs.server-image-digest }} + website-artifacts-key: ${{ needs.build.outputs.website-artifacts-key }} + website-artifacts-path: ${{ needs.build.outputs.website-artifacts-path }} + aws-region: us-west-2 + environment-key: staging + tf-backend-config-file: staging.s3.tfbackend + tf-var-file: staging.tfvars + upload-artifacts: false + artifacts-retention-days: 14 + secrets: + aws-access-key-id: ${{ needs.aws-auth.outputs.aws-access-key-id }} + aws-secret-access-key: ${{ needs.aws-auth.outputs.aws-secret-access-key }} + aws-session-token: ${{ needs.aws-auth.outputs.aws-session-token }} + datadog-api-key: ${{ secrets.DATADOG_API_KEY }} + datadog-app-key: ${{ secrets.DATADOG_APP_KEY }} + gpg-passphrase: ${{ secrets.TFPLAN_SECRET }} + + publish-tf-plan: + name: Publish Terraform Plan + permissions: + contents: read + pull-requests: write + if: needs.tf-plan.result != 'skipped' || needs.tf-plan.result != 'cancelled' + needs: + - tf-plan + uses: ./.github/workflows/publish-terraform-plan.yml + with: + write-summary: true + write-comment: true + pr-number: ${{ github.event.pull_request.number }} + tf-fmt-outcome: ${{ needs.tf-plan.outputs.fmt-outcome }} + tf-init-outcome: ${{ needs.tf-plan.outputs.init-outcome }} + tf-plan-outcome: ${{ needs.tf-plan.outputs.plan-outcome }} + tf-plan-output: ${{ needs.tf-plan.outputs.plan-output }} + tf-validate-outcome: ${{ needs.tf-plan.outputs.validate-outcome }} + tf-validate-output: ${{ needs.tf-plan.outputs.validate-output }} diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml new file mode 100644 index 000000000..3827edc8d --- /dev/null +++ b/.github/workflows/code-scanning.yml @@ -0,0 +1,80 @@ +name: "Code Scanning" + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '35 8 * * 1-5' + +permissions: + contents: read + +jobs: + dependency-review: + name: Dependency Review + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + show-progress: 'false' + persist-credentials: 'false' + - uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034 # v3.1.0 + + codeql: + name: CodeQL + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + show-progress: 'false' + persist-credentials: 'false' + - name: Initialize CodeQL + uses: github/codeql-action/init@8e0b1c74b1d5a0077b04d064c76ee714d3da7637 # v2.14.6 + with: + languages: javascript-typescript + queries: security-extended,security-and-quality + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@8e0b1c74b1d5a0077b04d064c76ee714d3da7637 # v2.14.6 + with: + category: "/language:javascript-typescript" + + gha-workflow-security: + name: GHA Workflow Security + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + permissions: + contents: read + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + show-progress: 'false' + persist-credentials: 'false' + - name: Ensure GitHub action versions are pinned to SHAs + uses: zgosalvez/github-actions-ensure-sha-pinned-actions@fdc1a0099560840b2be12459d0a61c7bc95b9db1 # v3.0.0 diff --git a/.github/workflows/dependabot-auto-approve.yml b/.github/workflows/dependabot-auto-approve.yml index 70485e64c..05344175b 100644 --- a/.github/workflows/dependabot-auto-approve.yml +++ b/.github/workflows/dependabot-auto-approve.yml @@ -13,21 +13,28 @@ jobs: runs-on: ubuntu-latest if: ${{ github.actor == 'dependabot[bot]' }} steps: + - name: Harden Runner + uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 - name: Dependabot metadata id: dependabot-metadata - uses: dependabot/fetch-metadata@v1 + uses: dependabot/fetch-metadata@c9c4182bf1b97f5224aee3906fd373f6b61b4526 # v1.6.0 - name: Approve a PR if dependency semver changes are minor or patch - if: ${{contains(fromJson('["version-update:semver-patch", "version-update:semver-minor"]'), steps.dependabot-metadata.outputs.update-type)}} + if: ${{ contains(fromJson('["version-update:semver-patch", "version-update:semver-minor"]'), steps.dependabot-metadata.outputs.update-type) }} run: gh pr review --approve "$PR_URL" env: - PR_URL: ${{github.event.pull_request.html_url}} - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + PR_URL: ${{ github.event.pull_request.html_url }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Enable auto-merge if dependency semver changes are minor or patch - if: ${{contains(fromJson('["version-update:semver-patch", "version-update:semver-minor"]'), steps.dependabot-metadata.outputs.update-type)}} + if: ${{ contains(fromJson('["version-update:semver-patch", "version-update:semver-minor"]'), steps.dependabot-metadata.outputs.update-type) }} run: | echo "Enabling auto-merge for Dependabot $UPDATE_TYPE" gh pr merge --auto --squash "$PR_URL" env: - PR_URL: ${{github.event.pull_request.html_url}} - GH_TOKEN: ${{secrets.GITHUB_TOKEN}} - UPDATE_TYPE: ${{steps.dependabot-metadata.outputs.update-type}} + PR_URL: ${{ github.event.pull_request.html_url }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + UPDATE_TYPE: ${{ steps.dependabot-metadata.outputs.update-type }} diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml new file mode 100644 index 000000000..9419a3349 --- /dev/null +++ b/.github/workflows/deploy-production.yml @@ -0,0 +1,189 @@ +name: Deploy to Production +run-name: Deploy ${{ github.ref }} + +on: + push: + tags: + - 'release/**' + workflow_dispatch: + +concurrency: + group: deploy-production + cancel-in-progress: false + +permissions: + contents: read + id-token: write + +jobs: + validate: + name: Validate commits for deployment + permissions: + contents: read + uses: ./.github/workflows/validate-deployment.yml + with: + protected-ref: main + deployment-ref: ${{ github.ref }} + + build: + name: Build deployment artifacts + permissions: + contents: read + packages: write + needs: + - validate + uses: ./.github/workflows/build.yml + with: + ref: ${{ github.sha }} + build-server-image: true + server-image-push: true + server-image-args-ref: ${{ github.ref }} + server-image-version: ${{ github.ref_name }} + server-image-tag-latest: false + server-image-tag-release: ${{ github.ref_name }} + server-image-upload-attestations: true + server-image-artifacts-retention-days: 90 + build-website: true + website-artifact-retention-days: 90 + + aws-auth: + name: Configure AWS Credentials + permissions: + contents: read + id-token: write + needs: + - validate + uses: ./.github/workflows/aws-auth.yml + with: + aws-region: us-west-2 + secrets: + gpg-passphrase: ${{ secrets.PRODUCTION_GPG_PASSPHRASE }} + role-to-assume: ${{ secrets.PRODUCTION_ROLE_ARN }} + + tf-plan: + name: Plan Terraform + permissions: + contents: read + needs: + - validate + - aws-auth + - build + uses: ./.github/workflows/terraform-plan.yml + with: + ref: ${{ github.sha }} + concurrency-group: run_terraform-production + server-image-tag: ${{ github.sha }} + server-image-digest: ${{ needs.build.outputs.server-image-digest }} + website-artifacts-key: ${{ needs.build.outputs.website-artifacts-key }} + website-artifacts-path: ${{ needs.build.outputs.website-artifacts-path }} + aws-region: us-west-2 + environment-key: production + tf-backend-config-file: prod.s3.tfbackend + tf-var-file: prod.tfvars + upload-artifacts: true + artifacts-retention-days: 90 + secrets: + aws-access-key-id: ${{ needs.aws-auth.outputs.aws-access-key-id }} + aws-secret-access-key: ${{ needs.aws-auth.outputs.aws-secret-access-key }} + aws-session-token: ${{ needs.aws-auth.outputs.aws-session-token }} + datadog-api-key: ${{ secrets.DATADOG_API_KEY }} + datadog-app-key: ${{ secrets.DATADOG_APP_KEY }} + gpg-passphrase: ${{ secrets.PRODUCTION_GPG_PASSPHRASE }} + + publish-tf-plan: + name: Publish Terraform Plan + permissions: + contents: read + pull-requests: write + if: needs.tf-plan.result != 'skipped' || needs.tf-plan.result != 'cancelled' + needs: + - tf-plan + uses: ./.github/workflows/publish-terraform-plan.yml + with: + write-summary: true + write-comment: false + tf-fmt-outcome: ${{ needs.tf-plan.outputs.fmt-outcome }} + tf-init-outcome: ${{ needs.tf-plan.outputs.init-outcome }} + tf-plan-outcome: ${{ needs.tf-plan.outputs.plan-outcome }} + tf-plan-output: ${{ needs.tf-plan.outputs.plan-output }} + tf-validate-outcome: ${{ needs.tf-plan.outputs.validate-outcome }} + tf-validate-output: ${{ needs.tf-plan.outputs.validate-output }} + + tf-apply: + name: Deploy to Production + needs: + - build + - aws-auth + - tf-plan + if: needs.tf-plan.outputs.plan-exitcode == 2 + uses: ./.github/workflows/terraform-apply.yml + with: + website-artifacts-key: ${{ needs.build.outputs.website-artifacts-key }} + website-artifacts-path: ${{ needs.build.outputs.website-artifacts-path }} + tf-plan-artifacts-key: ${{ needs.tf-plan.outputs.artifacts-key }} + aws-region: us-west-2 + concurrency-group: run_terraform-production + tf-backend-config-file: prod.s3.tfbackend + environment-name: production + secrets: + aws-access-key-id: ${{ needs.aws-auth.outputs.aws-access-key-id }} + aws-secret-access-key: ${{ needs.aws-auth.outputs.aws-secret-access-key }} + aws-session-token: ${{ needs.aws-auth.outputs.aws-session-token }} + datadog-api-key: ${{ secrets.DATADOG_API_KEY }} + datadog-app-key: ${{ secrets.DATADOG_APP_KEY }} + gpg-passphrase: ${{ secrets.PRODUCTION_GPG_PASSPHRASE }} + + update-release: + name: Update release + runs-on: ubuntu-latest + if: startsWith('refs/tags/release/', github.ref) + permissions: + contents: write + needs: + - build + - tf-apply + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + steps: + - name: Harden Runner + uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Download website build artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: ${{ needs.build.outputs.website-artifacts-key }} + path: ${{ needs.build.outputs.website-artifacts-path }} + - name: Download docker build attestation artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: ${{ needs.build.outputs.server-attestation-artifacts-key }} + path: ${{ needs.build.outputs.server-attestation-artifacts-path }} + - name: Upload release assets + run: | + WEBSITE_TAR_FILE="client-dist.tar.gz" + tar -czf "$WEBSITE_TAR_FILE" -C $(dirname "$WEBSITE_DIST_PATH") . + gh release upload --clobber "$RELEASE_TAG" \ + "$WEBSITE_TAR_FILE#Client website bundle" \ + "$PROVENANCE_FILE#Server Docker image provenance attestations" \ + "$SBOM_FILE#Server Docker image SBOM attestations" + env: + WEBSITE_DIST_PATH: ${{ needs.build.outputs.website-artifacts-path }} + PROVENANCE_FILE: ${{ needs.build.outputs.server-attestation-artifacts-path }}/provenance.json + SBOM_FILE: ${{ needs.build.outputs.server-attestation-artifacts-path }}/sbom.sdpx.json + - name: Get release notes + id: get + continue-on-error: true + run: gh release view "$RELEASE_TAG" --json body --jq .body > release_notes.md + - name: Add deployment history to release notes + if: always() && steps.get.outcome == 'success' + run: printf "\n- Deployed at $(date --iso-8601=seconds)\n" >> release_notes.md + - name: Update release notes and status + if: always() && steps.get.outcome == 'success' + run: gh release edit "$RELEASE_TAG" --draft=false --prerelease=false --latest --verify-tag -F release_notes.md diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 000000000..3af3a6413 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,129 @@ +name: Deploy to Staging + +on: + push: + branches: + - main + +concurrency: + group: deploy-staging + cancel-in-progress: false + +permissions: + contents: read + id-token: write + +jobs: + validate: + name: Validate commits for deployment + permissions: + contents: read + uses: ./.github/workflows/validate-deployment.yml + with: + protected-ref: main + deployment-ref: ${{ github.ref }} + + build: + name: Build deployment artifacts + permissions: + contents: read + packages: write + needs: + - validate + uses: ./.github/workflows/build.yml + with: + ref: ${{ github.sha }} + build-server-image: true + server-image-push: true + server-image-args-ref: ${{ github.ref }} + server-image-version: "rc-${{ github.sha }}" + server-image-tag-latest: true + build-website: true + website-artifact-retention-days: 14 + + aws-auth: + name: Configure AWS Credentials + permissions: + contents: read + id-token: write + needs: + - validate + uses: ./.github/workflows/aws-auth.yml + with: + aws-region: us-west-2 + secrets: + gpg-passphrase: ${{ secrets.STAGING_GPG_PASSPHRASE }} + role-to-assume: ${{ secrets.STAGING_ROLE_ARN }} + + tf-plan: + name: Plan Terraform + permissions: + contents: read + needs: + - validate + - aws-auth + - build + uses: ./.github/workflows/terraform-plan.yml + with: + ref: ${{ github.sha }} + concurrency-group: run_terraform-staging + server-image-tag: ${{ github.sha }} + server-image-digest: ${{ needs.build.outputs.server-image-digest }} + website-artifacts-key: ${{ needs.build.outputs.website-artifacts-key }} + website-artifacts-path: ${{ needs.build.outputs.website-artifacts-path }} + aws-region: us-west-2 + environment-key: staging + tf-backend-config-file: staging.s3.tfbackend + tf-var-file: staging.tfvars + upload-artifacts: true + artifacts-retention-days: 30 + secrets: + aws-access-key-id: ${{ needs.aws-auth.outputs.aws-access-key-id }} + aws-secret-access-key: ${{ needs.aws-auth.outputs.aws-secret-access-key }} + aws-session-token: ${{ needs.aws-auth.outputs.aws-session-token }} + datadog-api-key: ${{ secrets.DATADOG_API_KEY }} + datadog-app-key: ${{ secrets.DATADOG_APP_KEY }} + gpg-passphrase: ${{ secrets.STAGING_GPG_PASSPHRASE }} + + publish-tf-plan: + name: Publish Terraform Plan + permissions: + contents: read + pull-requests: write + if: needs.tf-plan.result != 'skipped' || needs.tf-plan.result != 'cancelled' + needs: + - tf-plan + uses: ./.github/workflows/publish-terraform-plan.yml + with: + write-summary: true + write-comment: false + tf-fmt-outcome: ${{ needs.tf-plan.outputs.fmt-outcome }} + tf-init-outcome: ${{ needs.tf-plan.outputs.init-outcome }} + tf-plan-outcome: ${{ needs.tf-plan.outputs.plan-outcome }} + tf-plan-output: ${{ needs.tf-plan.outputs.plan-output }} + tf-validate-outcome: ${{ needs.tf-plan.outputs.validate-outcome }} + tf-validate-output: ${{ needs.tf-plan.outputs.validate-output }} + + tf-apply: + name: Deploy to Staging + needs: + - build + - aws-auth + - tf-plan + if: needs.tf-plan.outputs.plan-exitcode == 2 + uses: ./.github/workflows/terraform-apply.yml + with: + website-artifacts-key: ${{ needs.build.outputs.website-artifacts-key }} + website-artifacts-path: ${{ needs.build.outputs.website-artifacts-path }} + tf-plan-artifacts-key: ${{ needs.tf-plan.outputs.artifacts-key }} + aws-region: us-west-2 + concurrency-group: run_terraform-staging + tf-backend-config-file: staging.s3.tfbackend + environment-name: staging + secrets: + aws-access-key-id: ${{ needs.aws-auth.outputs.aws-access-key-id }} + aws-secret-access-key: ${{ needs.aws-auth.outputs.aws-secret-access-key }} + aws-session-token: ${{ needs.aws-auth.outputs.aws-session-token }} + datadog-api-key: ${{ secrets.DATADOG_API_KEY }} + datadog-app-key: ${{ secrets.DATADOG_APP_KEY }} + gpg-passphrase: ${{ secrets.STAGING_GPG_PASSPHRASE }} diff --git a/.github/workflows/pr-tidying.yml b/.github/workflows/pr-tidying.yml new file mode 100644 index 000000000..5516923bc --- /dev/null +++ b/.github/workflows/pr-tidying.yml @@ -0,0 +1,211 @@ +name: Pull Request Tidying + +on: + pull_request_target: + types: + - opened + - reopened + - ready_for_review + - unassigned + - edited + - unlocked + +permissions: + contents: read + pull-requests: write + +jobs: + default-assignee: + # Disabled for now + if: false + name: Assign contributor + continue-on-error: true + permissions: + pull-requests: write + runs-on: ubuntu-latest + outputs: + check-passed: ${{ steps.check.outputs.result == 'passed' }} + fixed: ${{ steps.fix.outputs.result == 'fixed' }} + env: + DEFAULT_ASSIGNEE: ${{ github.actor }} + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - name: Check if PR already has assignee + id: check + run: | + if [[ -n $CURRENT_ASSIGNEE ]]; then + echo "result=passed" >> $GITHUB_OUTPUT + fi + env: + CURRENT_ASSIGNEE: ${{ github.event.pull_request.assignee.login || '' }} + - id: skip-edited-event + if: github.event.action == 'edited' + run: | + echo "Fix will be skipped because the triggering action is an edit event" + exit 1 + - id: prevent-reassignment + if: github.event.action == 'unassigned' && env.DEFAULT_ASSIGNEE == github.event.assignee.login + run: | + echo "Fix will be skipped to avoid reassigning the unassigned user" + exit 1 + - name: Add assignee + id: fix + if: steps.check.outputs.result != 'passed' + run: | + level=$(gh api /repos/$REPO/collaborators/$AUTHOR/permission --jq .permission) + if [[ $level = "write" || $level = "admin" ]]; then + gh pr edit "$PR_NUMBER" --repo "$REPO" --add-assignee "$AUTHOR" + echo "result=fixed" >> $GITHUB_OUTPUT + fi + env: + REPO: ${{ github.repository }} + AUTHOR: ${{ env.DEFAULT_ASSIGNEE }} + PR_NUMBER: ${{ github.event.pull_request.number }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + issue-in-title: + # Disabled for now + if: false + name: Add issue references to title + continue-on-error: true + runs-on: ubuntu-latest + outputs: + check-passed: ${{ steps.check.outputs.result == 'passed' }} + fixed: ${{ steps.fix.outputs.result == 'fixed' }} + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - name: Check the current PR title + id: check + run: | + regex='\s\([#][0-9]+\s?\)$' + if [[ $PR_TITLE =~ $regex ]]; then + echo "result=passed" >> $GITHUB_OUTPUT + fi + - id: skip-unassigned-event + if: github.event.action == 'unassigned' + run: | + echo "Fix will be skipped because the triggering action is an unassignment event" + exit 1 + - id: skip-non-title-edits + if: github.event.action == 'edited' && github.event.changes.title.from == github.event.pull_request.title + run: | + echo "Fix will be skipped because the triggering edit action did not modify the PR title" + exit 1 + - name: Parse issue(s) from Ticket heading + id: parse + if: steps.check.outputs.result != 'passed' + run: | + regex='^[#][#][#] Ticket (([#][0-9]+\s?)+)' + line=$(gh pr view "$PR_NUMBER" --json body --jq .body | head -n 1) + issues='' + if [[ $line =~ $regex ]]; then + issues="${BASH_REMATCH[1]}" + fi + echo "issue-numbers=$(tr -s '[:blank:]' <<< $issues)" >> $GITHUB_OUTPUT + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - id: skip-empty-issues + if: steps.parse.outputs.issue-numbers == '' + run: | + echo "Fix will be skipped because no issue numbers were parsed" + exit 1 + - name: Format the new title + id: new-title + run: echo result="$ORIGINAL_TITLE (issue $ISSUE_NUMBERS)" >> $GITHUB_OUTPUT + env: + ORIGINAL_TITLE: ${{ github.event.pull_request.title }} + ISSUE_NUMBERS: ${{ steps.parse.outputs.issue-numbers }} + - id: skip-prevent-revert + if: github.event.action == 'edited' && github.event.changes.title.from == steps.new-title.outputs.result + run: | + echo "Fix will be skipped to avoid setting the title back to its previous value" + exit 1 + - name: Update PR title + id: fix + if: steps.check.outputs.result != 'passed' && steps.parse.outputs.issue-numbers != '' + run: | + gh pr edit "$PR_NUMBER" --title "$ORIGINAL_TITLE (issue $ISSUE_NUMBERS)" + echo "result=fixed" >> $GITHUB_OUTPUT + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + ORIGINAL_TITLE: ${{ github.event.pull_request.title }} + ISSUE_NUMBERS: ${{ steps.parse.outputs.issue-numbers }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + comment: + name: Create/update PR comment + # Disabled for now + if: always() && github.actor != 'dependabot[bot]' && false + needs: + - default-assignee + - issue-in-title + runs-on: ubuntu-latest + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - name: Create comment file + run: | + COMMENT_FILE=$(mktemp -t comment.md.XXXXX) + echo "COMMENT_FILE=$COMMENT_FILE" >> $GITHUB_ENV + cat >> $COMMENT_FILE << 'ENDOFCOMMENT' + ## PR Helper Bot ๐Ÿค– + + **Thanks for your pull request!** + My job is to check for any missing conventions and automatically tidy things up (if I can). + You'll find the results below, along with any suggested actions for you to take. + +
+ Result key + โœ… = Already passing + ๐Ÿ› ๏ธ = Fixed (now passing) + โš ๏ธ = Consider fixing +
+ + | Convention | Result | + |:-------------------------------------|:------:| + | PR has assignee (defaults to author) | ${{ env.DEFAULT_ASSIGNEE }} | + | PR title ends with `(#issue)` | ${{ env.ISSUE_IN_TITLE }} | + + *Pusher: @${{ env.GH_ACTOR }}, Action: `${{ env.GH_ACTION }}`, Workflow: [`${{ env.GH_WORKFLOW }}`](${{ env.GH_SERVER}}/${{ env.GH_REPO }}/actions/runs/${{ env.GH_RUN_ID }})* + ENDOFCOMMENT + env: + DEFAULT_ASSIGNEE: ${{ (needs.default-assignee.outputs.check-passed && 'โœ…' ) || (needs.default-assignee.outputs.fixed && '๐Ÿ› ๏ธ') || 'โš ๏ธ' }} + ISSUE_IN_TITLE: ${{ (needs.issue-in-title.outputs.check-passed && 'โœ…' ) || (needs.issue-in-title.outputs.fixed && '๐Ÿ› ๏ธ') || 'โš ๏ธ' }} + GH_ACTOR: ${{ github.actor }} + GH_ACTION: ${{ github.event_name }} + GH_WORKFLOW: ${{ github.workflow }} + GH_SERVER: ${{ github.server_url }} + GH_REPO: ${{ github.repository }} + GH_RUN_ID: ${{ github.run_id }} + - name: Output comment body to step summary + run: cat $COMMENT_FILE >> $GITHUB_STEP_SUMMARY + - name: Write the comment body + id: comment-body + run: | + CONTENT=$(cat $COMMENT_FILE) + echo "COMMENT_CONTENT<> $GITHUB_OUTPUT + echo "$CONTENT" >> $GITHUB_OUTPUT + echo "ENDOFREPORT" >> $GITHUB_OUTPUT + - name: Find previous report comment + id: find-comment + uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: 'PR Helper Bot ๐Ÿค–' + - name: Create or update comment + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3.1.0 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: ${{ steps.comment-body.outputs.COMMENT_CONTENT }} + edit-mode: replace diff --git a/.github/workflows/publish-qa-results.yml b/.github/workflows/publish-qa-results.yml new file mode 100644 index 000000000..8b4a3e48b --- /dev/null +++ b/.github/workflows/publish-qa-results.yml @@ -0,0 +1,128 @@ +name: Publish QA Results + +on: + workflow_call: + inputs: + client-test-outcome: + type: string + required: true + client-test-coverage-markdown-report: + type: string + required: true + server-test-outcome: + type: string + required: true + server-test-coverage-markdown-report: + type: string + required: true + e2e-test-outcome: + type: string + required: true + eslint-outcome: + type: string + required: true + tflint-outcome: + type: string + required: true + pr-number: + type: string + required: false + write-summary: + type: boolean + default: true + write-comment: + type: boolean + default: false + +permissions: + contents: read + pull-requests: write + +jobs: + publish: + name: Publish QA Results + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - name: Write the report markdown file + run: | + REPORT_FILE=$(mktemp -t summary.md.XXXXX) + echo "REPORT_FILE=$REPORT_FILE" >> $GITHUB_ENV + cat >> $REPORT_FILE << 'ENDOFREPORT' + ## QA Summary + + | QA Check | Result | + |:----------------|:-------:| + | ๐ŸŒ Client Tests | ${{ (env.CLIENT_TEST_OUTCOME == 'success' && 'โœ…') || (env.CLIENT_TEST_OUTCOME == 'skipped' && 'โž–') || 'โŒ' }} | + | ๐Ÿ”— Server Tests | ${{ (env.SERVER_TEST_OUTCOME == 'success' && 'โœ…') || (env.SERVER_TEST_OUTCOME == 'skipped' && 'โž–') || 'โŒ' }} | + | ๐Ÿค E2E Tests | ${{ (env.E2E_TEST_OUTCOME == 'success' && 'โœ…') || (env.E2E_TEST_OUTCOME == 'skipped' && 'โž–') || 'โŒ' }} | + | ๐Ÿ“ ESLint | ${{ (env.ESLINT_OUTCOME == 'success' && 'โœ…') || (env.ESLINT_OUTCOME == 'skipped' && 'โž–') || 'โŒ' }} | + | ๐Ÿงน TFLint | ${{ (env.TFLINT_OUTCOME == 'success' && 'โœ…') || (env.TFLINT_OUTCOME == 'skipped' && 'โž–') || 'โŒ' }} | + + ### Test Coverage + +
+ Coverage report for `packages/client` + + ${{ env.CLIENT_COVERAGE_REPORT }} + +
+ +
+ Coverage report for `packages/server` + + ${{ env.SERVER_COVERAGE_REPORT }} + +
+ + *Pusher: @${{ env.GH_ACTOR }}, Action: `${{ env.GH_ACTION }}`, Workflow: [`${{ env.GH_WORKFLOW }}`](${{ env.GH_SERVER}}/${{ env.GH_REPO }}/actions/runs/${{ env.GH_RUN_ID }})* + ENDOFREPORT + env: + CLIENT_TEST_OUTCOME: ${{ inputs.client-test-outcome }} + CLIENT_COVERAGE_REPORT: ${{ inputs.client-test-coverage-markdown-report }} + SERVER_TEST_OUTCOME: ${{ inputs.server-test-outcome }} + SERVER_COVERAGE_REPORT: ${{ inputs.server-test-coverage-markdown-report }} + E2E_TEST_OUTCOME: ${{ inputs.e2e-test-outcome }} + ESLINT_OUTCOME: ${{ inputs.eslint-outcome }} + TFLINT_OUTCOME: ${{ inputs.tflint-outcome }} + GH_ACTOR: ${{ github.actor }} + GH_ACTION: ${{ github.event_name }} + GH_WORKFLOW: ${{ github.workflow }} + GH_SERVER: ${{ github.server_url }} + GH_REPO: ${{ github.repository }} + GH_RUN_ID: ${{ github.run_id }} + - name: Write the step summary + if: inputs.write-summary + run: cat $REPORT_FILE | head -c 65500 >> $GITHUB_STEP_SUMMARY # Observe GitHub's 65535 character limit + - name: Write the comment body + id: comment-body + run: | + CONTENT=$(cat $REPORT_FILE) + echo "REPORT_CONTENT<> $GITHUB_OUTPUT + echo "$CONTENT" >> $GITHUB_OUTPUT + echo "ENDOFREPORT" >> $GITHUB_OUTPUT + - name: Warn on missing comment requirements + if: inputs.write-comment && inputs.pr-number == '' + run: "echo 'WARNING: Cannot write a comment because pr-number is not set'" + - name: Find previous report comment + id: find-comment + if: inputs.write-comment && inputs.pr-number != '' + uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 + with: + issue-number: ${{ inputs.pr-number }} + comment-author: 'github-actions[bot]' + body-includes: QA Summary + - name: Create or update comment + if: inputs.write-comment && inputs.pr-number != '' + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3.1.0 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: ${{ steps.comment-body.outputs.REPORT_CONTENT }} + edit-mode: replace diff --git a/.github/workflows/publish-terraform-plan.yml b/.github/workflows/publish-terraform-plan.yml new file mode 100644 index 000000000..9bb93a39a --- /dev/null +++ b/.github/workflows/publish-terraform-plan.yml @@ -0,0 +1,139 @@ +name: Publish Terraform Plan + +on: + workflow_call: + inputs: + tf-fmt-outcome: + type: string + required: true + tf-init-outcome: + type: string + required: true + tf-plan-outcome: + type: string + required: true + tf-plan-output: + type: string + required: true + tf-validate-outcome: + type: string + required: true + tf-validate-output: + type: string + required: true + pr-number: + type: string + required: false + write-summary: + type: boolean + default: true + write-comment: + type: boolean + default: false + +permissions: + contents: read + pull-requests: write + +jobs: + publish: + name: Publish Terraform Plan + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + - name: Reformat Plan + run: | + PLAN=$(echo "$PLAN_RAW_OUTPUT" | sed -E 's/^([[:space:]]+)([-+])/\2\1/g') + echo "PLAN_REFORMATTED<> $GITHUB_ENV + echo "$PLAN" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + env: + PLAN_RAW_OUTPUT: ${{ inputs.tf-plan-output }} + - name: Write the report markdown file + run: | + REPORT_FILE=$(mktemp -t summary.md.XXXXX) + echo "REPORT_FILE=$REPORT_FILE" >> $GITHUB_ENV + cat >> $REPORT_FILE << 'ENDOFREPORT' + ## Terraform Summary + + | Step | Result | + |:-----------------------------|:-------:| + | ๐Ÿ–Œ Terraform Format & Style | ${{ (env.TF_FMT_OUTCOME == 'success' && 'โœ…') || (env.TF_FMT_OUTCOME == 'skipped' && 'โž–') || 'โŒ' }} | + | โš™๏ธ Terraform Initialization | ${{ (env.TF_INIT_OUTCOME == 'success' && 'โœ…') || (env.TF_INIT_OUTCOME == 'skipped' && 'โž–') || 'โŒ' }} | + | ๐Ÿค– Terraform Validation | ${{ (env.TF_VALIDATE_OUTCOME == 'success' && 'โœ…') || (env.TF_VALIDATE_OUTCOME == 'skipped' && 'โž–') || 'โŒ' }} | + | ๐Ÿ“– Terraform Plan | ${{ (env.TF_PLAN_OUTCOME == 'success' && 'โœ…') || (env.TF_PLAN_OUTCOME == 'skipped' && 'โž–') || 'โŒ' }} | + + _**Hint:** If "Terraform Format & Style" failed, run `terraform fmt -recursive` from the `terraform/` directory and commit the results._ + + ### Output + +
+ Validation Output + + ``` + ${{ env.TF_VALIDATE_OUTPUT }} + ``` + +
+ +
+ Plan Output + + ```diff + ${{ env.TF_PLAN_OUTPUT }} + ``` + +
+ + *Pusher: @${{ env.GH_ACTOR }}, Action: `${{ env.GH_ACTION }}`, Workflow: [`${{ env.GH_WORKFLOW }}`](${{ env.GH_SERVER}}/${{ env.GH_REPO }}/actions/runs/${{ env.GH_RUN_ID }})* + ENDOFREPORT + env: + TF_FMT_OUTCOME: ${{ inputs.tf-fmt-outcome }} + TF_INIT_OUTCOME: ${{ inputs.tf-init-outcome }} + TF_VALIDATE_OUTCOME: ${{ inputs.tf-validate-outcome }} + TF_VALIDATE_OUTPUT: ${{ inputs.tf-validate-output }} + TF_PLAN_OUTCOME: ${{ inputs.tf-plan-outcome }} + TF_PLAN_OUTPUT: ${{ env.PLAN_REFORMATTED }} + GH_ACTOR: ${{ github.actor }} + GH_ACTION: ${{ github.event_name }} + GH_WORKFLOW: ${{ github.workflow }} + GH_SERVER: ${{ github.server_url }} + GH_REPO: ${{ github.repository }} + GH_RUN_ID: ${{ github.run_id }} + - name: Write the step summary + if: inputs.write-summary + run: cat $REPORT_FILE | head -c 65500 >> $GITHUB_STEP_SUMMARY # Observe GitHub's 65535 character limit + - name: Write the comment body + id: comment-body + run: | + CONTENT=$(cat $REPORT_FILE) + echo "REPORT_CONTENT<> $GITHUB_OUTPUT + echo "$CONTENT" >> $GITHUB_OUTPUT + echo "ENDOFREPORT" >> $GITHUB_OUTPUT + - name: Warn on missing comment requirements + if: inputs.write-comment && inputs.pr-number == '' + run: "echo 'WARNING: Cannot write a comment because pr-number is not set'" + - name: Find previous report comment + id: find-comment + if: inputs.write-comment && inputs.pr-number != '' + uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 + with: + issue-number: ${{ inputs.pr-number }} + comment-author: 'github-actions[bot]' + body-includes: Terraform Summary + - name: Create or update comment + if: inputs.write-comment && inputs.pr-number != '' + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3.1.0 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: ${{ steps.comment-body.outputs.REPORT_CONTENT }} + edit-mode: replace diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml new file mode 100644 index 000000000..5701c769d --- /dev/null +++ b/.github/workflows/qa.yml @@ -0,0 +1,282 @@ +name: QA Checks + +on: + workflow_call: + inputs: + ref: + type: string + required: true + outputs: + server-test-outcome: + value: ${{ jobs.test-server.result }} + server-test-coverage-markdown-report: + value: ${{ jobs.test-server.outputs.coverage-markdown-report }} + client-test-outcome: + value: ${{ jobs.test-client.result }} + client-test-coverage-markdown-report: + value: ${{ jobs.test-client.outputs.coverage-markdown-report }} + e2e-test-outcome: + value: ${{ jobs.test-e2e.result }} + eslint-outcome: + value: ${{ jobs.eslint.result }} + tflint-outcome: + value: ${{ jobs.tflint.result }} + +permissions: + contents: read + +jobs: + prepare-qa: + name: Prepare for QA + runs-on: ubuntu-latest + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Setup Node + uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version-file: .nvmrc + cache: yarn + cache-dependency-path: yarn.lock + - name: Install dependencies + run: yarn install --frozen-lockfile + env: + CI: "true" + + test-server: + name: Test server-side code + runs-on: ubuntu-latest + needs: + - prepare-qa + outputs: + coverage-markdown-report: ${{ steps.coverage-markdown.outputs.markdownReport }} + services: + postgres: + image: "postgres:13" + env: + POSTGRES_DB: usdr_grants_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Setup Node + uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version-file: .nvmrc + cache: yarn + cache-dependency-path: yarn.lock + - name: Install dependencies + run: yarn workspace server install -D --frozen-lockfile + env: + CI: "true" + - name: Prepare test execution environment + run: cp packages/server/.env.example packages/server/.env + - name: Run unit tests with coverage + run: yarn workspace server coverage + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + CI: "true" + POSTGRES_TEST_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" + POSTGRES_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" + NOTIFICATIONS_EMAIL: grants-identification@usdigitalresponse.org + - name: Generate coverage text report + run: npx nyc report --reporter text --cwd packages/server > packages/server/coverage.txt + - name: Generate coverage markdown report + id: coverage-markdown + uses: fingerprintjs/action-coverage-report-md@b7fcda0d2891d215c6808d32ca249e43f55abe3f # v1.0.6 + with: + textReportPath: 'packages/server/coverage.txt' + srcBasePath: 'packages/server/' + + test-client: + name: Test client-side code + runs-on: ubuntu-latest + needs: + - prepare-qa + outputs: + coverage-markdown-report: ${{ steps.coverage-markdown.outputs.markdownReport }} + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Setup Node + uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version-file: .nvmrc + cache: yarn + cache-dependency-path: yarn.lock + - name: Install dependencies + run: yarn workspace client install -D --frozen-lockfile + env: + CI: "true" + - name: Prepare test execution environment + run: cp packages/client/.env.example packages/client/.env + - name: Run unit tests with coverage + run: yarn workspace client coverage + env: + CI: "true" + - name: Generate coverage text report + run: npx nyc report --reporter text --cwd packages/client > packages/client/coverage.txt + - name: Generate coverage markdown report + id: coverage-markdown + uses: fingerprintjs/action-coverage-report-md@b7fcda0d2891d215c6808d32ca249e43f55abe3f # v1.0.6 + with: + textReportPath: 'packages/client/coverage.txt' + srcBasePath: 'packages/client/' + + test-e2e: + name: Test end-to-end integration + runs-on: ubuntu-latest + needs: + - prepare-qa + services: + postgres: + image: "postgres:13" + env: + POSTGRES_DB: usdr_grants_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Install Node.js + uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version-file: .nvmrc + cache: yarn + cache-dependency-path: yarn.lock + - name: Install dependencies + run: yarn install --frozen-lockfile + env: + CI: "1" + - name: Run migrations + env: + POSTGRES_TEST_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" + # This is intentional to set POSTGRES_URL=POSTGRES_TEST_URL; ARPA Reporter test runner gates + # dev vs. CI differences based on whether these are the same. + POSTGRES_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" + run: yarn db:migrate + - name: Start Applications + env: + POSTGRES_TEST_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" + # This is intentional to set POSTGRES_URL=POSTGRES_TEST_URL; ARPA Reporter test runner gates + # dev vs. CI differences based on whether these are the same. + POSTGRES_URL: "postgresql://postgres:password@localhost:${{ job.services.postgres.ports[5432] }}/usdr_grants_test" + AWS_ACCESS_KEY_ID: "Fake AWS Key" + AWS_SECRET_ACCESS_KEY: "Fake AWS Secret" + NOTIFICATIONS_EMAIL: grants-identification@usdigitalresponse.org + DATA_DIR: './data' + run: yarn serve & + - name: Run e2e tests + uses: cypress-io/github-action@ebe8b24c4428922d0f793a5c4c96853a633180e3 # v6.6.0 + env: + CYPRESS_BASE_URL: 'http://localhost:8080' + with: + publish-summary: true + working-directory: packages/e2e + wait-on: ${{ env.CYPRESS_BASE_URL }} + + eslint: + name: Lint JavaScript + runs-on: ubuntu-latest + needs: + - prepare-qa + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Setup Node + uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version-file: .nvmrc + cache: yarn + cache-dependency-path: yarn.lock + - name: Install dependencies + run: yarn install -D --frozen-lockfile + env: + CI: "true" + - name: Run linter + run: yarn lint + + tflint: + name: Lint terraform + runs-on: ubuntu-latest + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + name: Cache plugin dir + with: + path: .tflint.d/plugins + key: ${{ runner.os }}-tflint-${{ hashFiles('terraform/.tflint.hcl') }} + - uses: terraform-linters/setup-tflint@19a52fbac37dacb22a09518e4ef6ee234f2d4987 # v4.0.0 + name: Setup TFLint + with: + tflint_version: latest + tflint_wrapper: true + - name: Show TFLint version + run: tflint --version + - name: Init TFLint + run: tflint --init + working-directory: terraform + env: + GITHUB_TOKEN: ${{ github.token }} + - name: Run TFLint + run: tflint --format compact --recursive --minimum-failure-severity=error diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml new file mode 100644 index 000000000..cfbf08fd6 --- /dev/null +++ b/.github/workflows/release-drafter.yml @@ -0,0 +1,82 @@ +name: Release Drafter + +on: + push: + branches: + - main + pull_request_target: + types: + - opened + - synchronize + - reopened + workflow_dispatch: + +permissions: + contents: read + +jobs: + label_pull_requests: + name: "Label pull requests" + if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - name: "Run auto-labeler" + uses: release-drafter/release-drafter@09c613e259eb8d4e7c81c2cb00618eb5fc4575a7 # v5.25.0 + with: + disable-releaser: true + disable-autolabeler: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + draft_release: + name: Create or update next release draft + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + permissions: + contents: write + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + show-progress: false + persist-credentials: 'false' + - name: Get tag of "latest" release + id: latest_release + run: echo "tag=$(gh release view --json tagName --jq .tagName)" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: "Determine next release version" + id: next_release + run: | + chmod +x .github/next_release_version.bash + echo "version=$(bash .github/next_release_version.bash $LATEST_TAG)" >> $GITHUB_OUTPUT + env: + LATEST_TAG: ${{ steps.latest_release.outputs.tag || '' }} + - name: "Generate release notes and label pull requests" + uses: release-drafter/release-drafter@09c613e259eb8d4e7c81c2cb00618eb5fc4575a7 # v5.25.0 + with: + version: ${{ steps.next_release.outputs.version }} + publish: false + disable-releaser: false + disable-autolabeler: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/select-target-environment.yml b/.github/workflows/select-target-environment.yml deleted file mode 100644 index c98531a0b..000000000 --- a/.github/workflows/select-target-environment.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Select target environment - -on: - workflow_call: - inputs: - ref_name: - type: string - required: true - outputs: - selected: - description: Name of the environment to target - value: ${{ jobs.select.outputs.selected }} - -jobs: - select: - runs-on: ubuntu-latest - steps: - - uses: kanga333/variable-mapper@master - with: - key: "${{ inputs.ref_name }}" - map: | - { - "_staging": { - "selected": "staging" - }, - "main": { - "selected": "prod" - }, - ".*": { - "selected": "sandbox" - } - } - export_to: log,env - mode: first_match - outputs: - selected: ${{ env.selected }} diff --git a/.github/workflows/terraform-apply.yml b/.github/workflows/terraform-apply.yml new file mode 100644 index 000000000..f92ba0ab3 --- /dev/null +++ b/.github/workflows/terraform-apply.yml @@ -0,0 +1,153 @@ +name: Terraform Apply + +permissions: + contents: read + +on: + workflow_call: + inputs: + website-artifacts-key: + type: string + required: true + website-artifacts-path: + type: string + required: true + clear-web-cache: + description: Whether to invalidate the CloudFront distribution cache as a post-deployment step. + type: boolean + default: true + tf-plan-artifacts-key: + type: string + required: true + tf-backend-config-file: + type: string + required: true + aws-region: + type: string + required: true + environment-name: + type: string + required: true + concurrency-group: + description: Name of the concurrency group (avoids simultaneous Terraform execution against the same environment) + type: string + default: run_terraform + secrets: + aws-access-key-id: + required: true + aws-secret-access-key: + required: true + aws-session-token: + required: true + datadog-api-key: + required: true + datadog-app-key: + required: true + gpg-passphrase: + required: true + +jobs: + do: + name: Apply Terraform from Plan + runs-on: ubuntu-latest + permissions: + contents: read + defaults: + run: + working-directory: terraform + env: + AWS_DEFAULT_REGION: ${{ inputs.aws-region }} + AWS_REGION: ${{ inputs.aws-region }} + TF_CLI_ARGS: "-no-color" + TF_IN_AUTOMATION: "true" + TF_INPUT: 0 + TF_PLUGIN_CACHE_DIR: ~/.terraform.d/plugin-cache + environment: ${{ inputs.environment-name }} + concurrency: + group: ${{ inputs.concurrency-group }} + cancel-in-progress: false + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + *.amazonaws.com:443 + actions-results-receiver-production.githubapp.com:443 + api.datadoghq.com:443 + checkpoint-api.hashicorp.com:443 + github.com:443 + objects.githubusercontent.com:443 + registry.terraform.io:443 + releases.hashicorp.com:443 + - name: Download Terraform artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: ${{ inputs.tf-plan-artifacts-key }} + path: ${{ github.workspace }}/terraform + - name: Clear any cached provider plugins in artifact + run: rm -rf "$TF_PLUGIN_CACHE_DIR" + - name: Get project TF version + id: get_tf_version + run: echo "TF_VERSION=$(cat .terraform-version | tr -d '[:space:]')" | tee -a $GITHUB_OUTPUT + - uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0 + with: + terraform_version: ${{ steps.get_tf_version.outputs.TF_VERSION }} + - name: Download website artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: ${{ inputs.website-artifacts-key }} + path: ${{ inputs.website-artifacts-path }} + - name: Decrypt plan file + run: gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o tfplan tfplan.gpg + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + - name: Get AWS access key ID + id: decrypt-aws-access-key-id + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-access-key-id }} + - name: Get AWS secret access key + id: decrypt-aws-secret-access-key + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-secret-access-key }} + - name: Get AWS session token + id: decrypt-aws-session-token + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-session-token }} + - name: Terraform Init + run: terraform init + env: + AWS_ACCESS_KEY_ID: "${{ steps.decrypt-aws-access-key-id.outputs.out }}" + AWS_SECRET_ACCESS_KEY: "${{ steps.decrypt-aws-secret-access-key.outputs.out }}" + AWS_SESSION_TOKEN: "${{ steps.decrypt-aws-session-token.outputs.out }}" + TF_CLI_ARGS_init: "-backend-config=${{ inputs.tf-backend-config-file }}" + - name: Terraform Apply + run: terraform apply tfplan + env: + AWS_ACCESS_KEY_ID: "${{ steps.decrypt-aws-access-key-id.outputs.out }}" + AWS_SECRET_ACCESS_KEY: "${{ steps.decrypt-aws-secret-access-key.outputs.out }}" + AWS_SESSION_TOKEN: "${{ steps.decrypt-aws-session-token.outputs.out }}" + - name: Invalidate CloudFront distribution cache + if: success() && inputs.clear-web-cache + run: | + DISTRIBUTION_ID=$(terraform output -json | jq --raw-output .website_cloudfront_distribution_id.value) + aws cloudfront create-invalidation --paths '/*' --distribution-id $DISTRIBUTION_ID + env: + AWS_ACCESS_KEY_ID: "${{ steps.decrypt-aws-access-key-id.outputs.out }}" + AWS_SECRET_ACCESS_KEY: "${{ steps.decrypt-aws-secret-access-key.outputs.out }}" + AWS_SESSION_TOKEN: "${{ steps.decrypt-aws-session-token.outputs.out }}" diff --git a/.github/workflows/terraform-ci.yml b/.github/workflows/terraform-ci.yml deleted file mode 100644 index feb619f28..000000000 --- a/.github/workflows/terraform-ci.yml +++ /dev/null @@ -1,153 +0,0 @@ -name: Terraform CI - -on: - pull_request: - paths: - - 'terraform/**' - - '.github/workflows/terraform-ci.yml' - -permissions: - pull-requests: write - contents: read - id-token: write - -concurrency: - group: ${{ github.workflow_ref }} - -jobs: - lint: - name: Lint terraform - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - uses: hashicorp/setup-terraform@v2 - - name: Terraform fmt - id: fmt - run: terraform fmt -check -diff -recursive ./terraform - - select_target_environment: - uses: "./.github/workflows/select-target-environment.yml" - with: - ref_name: "${{ github.base_ref }}" - - validate_plan_report: - name: Validate and plan terraform - runs-on: ubuntu-latest - needs: - - lint - - select_target_environment - environment: ${{ needs.select_target_environment.outputs.selected }} - env: - TF_PLUGIN_CACHE_DIR: ~/.terraform.d/plugin-cache - TF_VAR_version_identifier: ${{ github.sha }} - TF_VAR_datadog_api_key: ${{ secrets.DATADOG_API_KEY }} - TF_VAR_datadog_app_key: ${{ secrets.DATADOG_APP_KEY }} - concurrency: - group: run_terraform-${{ needs.select_target_environment.outputs.selected }} - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: us-west-2 - role-to-assume: "${{ secrets.AWS_ROLE_TO_ASSUME }}" - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Get project TF version - id: get_version - run: echo "TF_VERSION=$(cat .terraform-version | tr -d '[:space:]')" | tee -a $GITHUB_OUTPUT - working-directory: terraform - - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: ${{ steps.get_version.outputs.TF_VERSION }} - - name: Ensure Terraform plugin cache exists - run: mkdir -p $TF_PLUGIN_CACHE_DIR - - name: Save/Restore Terraform plugin cache - uses: actions/cache@v3 - with: - path: ${{ env.TF_PLUGIN_CACHE_DIR }} - key: ${{ runner.os }}-terraform-${{ hashFiles('**/.terraform.lock.hcl') }} - restore-keys: | - ${{ runner.os }}-terraform- - - name: Ensure Terraform plugin cache still exists - run: mkdir -p $TF_PLUGIN_CACHE_DIR - - name: Terraform Init - id: init - run: terraform init -backend-config="${{ needs.select_target_environment.outputs.selected }}.s3.tfbackend" - working-directory: terraform - - name: Terraform Validate - id: validate - run: terraform validate -no-color - working-directory: terraform - - name: Terraform Plan - if: steps.validate.outcome == 'success' - id: plan - run: terraform plan -input=false -no-color -out=tfplan -var-file="${{ needs.select_target_environment.outputs.selected }}.tfvars" && terraform show -no-color tfplan - working-directory: terraform - - name: Reformat Plan - if: steps.plan.outcome != 'cancelled' && steps.plan.outcome != 'skipped' - run: | - echo '${{ steps.plan.outputs.stdout || steps.plan.outputs.stderr }}' \ - | sed -E 's/^([[:space:]]+)([-+])/\2\1/g' > plan.txt - - name: Put Plan in Env Var - if: steps.plan.outcome != 'cancelled' && steps.plan.outcome != 'skipped' - run: | - PLAN=$(cat plan.txt) - echo "PLAN<> $GITHUB_ENV - echo "$PLAN" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - uses: actions/github-script@v6 - if: github.event_name == 'pull_request' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - // 1. Retrieve existing bot comments for the PR - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }) - const botComment = comments.find(comment => { - return comment.user.type === 'Bot' && comment.body.includes('Report for project: \`terraform\`') - }) - - // 2. Prepare format of the comment - const output = `### Report for project: \`terraform\` - #### Terraform Initialization โš™๏ธ\`${{ steps.init.outcome }}\` - #### Terraform Validation ๐Ÿค–\`${{ steps.validate.outcome }}\` -
Validation Output - - \`\`\`\n - ${{ steps.validate.outputs.stdout }} - \`\`\` - -
- - #### Terraform Plan ๐Ÿ“–\`${{ steps.plan.outcome }}\` - -
Show Plan - - \`\`\`diff\n - ${process.env.PLAN} - \`\`\` - -
- - *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Workflow: \`${{ github.workflow }}\`*`; - - // 3. If we have a comment, update it, otherwise create a new one - if (botComment) { - github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - body: output - }) - } else { - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: output - }) - } diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml new file mode 100644 index 000000000..10f084047 --- /dev/null +++ b/.github/workflows/terraform-plan.yml @@ -0,0 +1,211 @@ +name: Terraform Plan + +permissions: + contents: read + +on: + workflow_call: + inputs: + ref: + type: string + required: true + environment-key: + type: string + required: true + aws-region: + type: string + required: true + tf-backend-config-file: + type: string + required: true + tf-var-file: + type: string + required: true + artifacts-retention-days: + description: Number of days to retain build artifacts + type: number + default: 90 + upload-artifacts: + type: boolean + default: false + concurrency-group: + description: Name of the concurrency group (avoids simultaneous Terraform execution against the same environment) + type: string + default: run_terraform + server-image-tag: + type: string + required: true + server-image-digest: + type: string + required: true + website-artifacts-key: + type: string + required: true + website-artifacts-path: + type: string + required: true + secrets: + aws-access-key-id: + required: true + aws-secret-access-key: + required: true + aws-session-token: + required: true + datadog-api-key: + required: true + datadog-app-key: + required: true + gpg-passphrase: + required: true + outputs: + artifacts-key: + value: ${{ jobs.do.outputs.artifacts-key }} + fmt-outcome: + value: ${{ jobs.do.outputs.fmt_outcome }} + init-outcome: + value: ${{ jobs.do.outputs.init_outcome }} + validate-outcome: + value: ${{ jobs.do.outputs.validate_outcome }} + validate-output: + value: ${{ jobs.do.outputs.validate_output }} + plan-exitcode: + value: ${{ jobs.do.outputs.plan_exitcode }} + plan-outcome: + value: ${{ jobs.do.outputs.plan_outcome }} + plan-output: + value: ${{ jobs.do.outputs.plan_output }} + +jobs: + do: + name: Validate and plan terraform + runs-on: ubuntu-latest + permissions: + contents: read + defaults: + run: + working-directory: terraform + outputs: + artifacts-key: ${{ env.ARTIFACTS_KEY }} + fmt_outcome: ${{ steps.fmt.outcome }} + init_outcome: ${{ steps.init.outcome }} + validate_outcome: ${{ steps.validate.outcome }} + validate_output: ${{ steps.validate.outputs.stdout }} + plan_exitcode: ${{ steps.plan.outputs.exitcode }} + plan_outcome: ${{ steps.plan.outcome }} + plan_output: ${{ steps.show_plan.outputs.stdout || steps.show_plan.outputs.stderr }} + env: + ARTIFACTS_KEY: terraform-${{ inputs.environment-key }}-${{ inputs.ref }} + AWS_DEFAULT_REGION: ${{ inputs.aws-region }} + AWS_REGION: ${{ inputs.aws-region }} + TF_CLI_ARGS: "-no-color" + TF_IN_AUTOMATION: "true" + TF_INPUT: 0 + concurrency: + group: ${{ inputs.concurrency-group }} + cancel-in-progress: false + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Validate workflow configuration + if: inputs.upload-artifacts && (env.GPG_PASSPHRASE == '') + run: | + echo 'gpg-passphrase is required when upload-artifacts is true' + exit 1 + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + - name: Download website artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: ${{ inputs.website-artifacts-key }} + path: ${{ inputs.website-artifacts-path }} + - name: Get project TF version + id: get_tf_version + run: echo "TF_VERSION=$(cat .terraform-version | tr -d '[:space:]')" | tee -a $GITHUB_OUTPUT + - uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0 + with: + terraform_version: ${{ steps.get_tf_version.outputs.TF_VERSION }} + - name: Terraform fmt + id: fmt + run: terraform fmt -check -diff -recursive + - name: Get AWS access key ID + id: decrypt-aws-access-key-id + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-access-key-id }} + - name: Get AWS secret access key + id: decrypt-aws-secret-access-key + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-secret-access-key }} + - name: Get AWS session token + id: decrypt-aws-session-token + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-session-token }} + - name: Terraform Init + id: init + run: terraform init + env: + AWS_ACCESS_KEY_ID: "${{ steps.decrypt-aws-access-key-id.outputs.out }}" + AWS_SECRET_ACCESS_KEY: "${{ steps.decrypt-aws-secret-access-key.outputs.out }}" + AWS_SESSION_TOKEN: "${{ steps.decrypt-aws-session-token.outputs.out }}" + TF_CLI_ARGS_init: "-backend-config=${{ inputs.tf-backend-config-file }}" + - name: Terraform Validate + id: validate + run: terraform validate -no-color + - name: Terraform Plan + if: always() && steps.validate.outcome == 'success' + id: plan + run: terraform plan -out="tfplan" -detailed-exitcode + env: + AWS_ACCESS_KEY_ID: "${{ steps.decrypt-aws-access-key-id.outputs.out }}" + AWS_SECRET_ACCESS_KEY: "${{ steps.decrypt-aws-secret-access-key.outputs.out }}" + AWS_SESSION_TOKEN: "${{ steps.decrypt-aws-session-token.outputs.out }}" + GPG_PASSPHRASE: "" # Just in case + TF_CLI_ARGS_plan: "-var-file=${{ inputs.tf-var-file }}" + TF_VAR_version_identifier: ${{ inputs.ref }} + TF_VAR_git_commit_sha: ${{ inputs.ref }} + TF_VAR_datadog_api_key: ${{ secrets.datadog-api-key }} + TF_VAR_datadog_app_key: ${{ secrets.datadog-app-key }} + TF_VAR_api_container_image_tag: "${{ inputs.server-image-tag }}@${{ inputs.server-image-digest }}" + TF_VAR_website_origin_artifacts_dist_path: "${{ inputs.website-artifacts-path }}" + - name: Generate plaintext plan + id: show_plan + run: terraform show tfplan + - name: Encrypt terraform plan file + id: encrypt_plan + if: success() && inputs.upload-artifacts + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + run: | + gpg --batch --yes --passphrase "$GPG_PASSPHRASE" -c --cipher-algo AES256 tfplan + rm tfplan + - name: Store terraform artifacts + if: success() && inputs.upload-artifacts + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: ${{ env.ARTIFACTS_KEY }} + path: | + ${{ github.workspace }}/terraform + !${{ github.workspace }}/terraform/.terraform + if-no-files-found: error + retention-days: ${{ inputs.artifacts-retention-days }} diff --git a/.github/workflows/validate-deployment.yml b/.github/workflows/validate-deployment.yml new file mode 100644 index 000000000..6ee042860 --- /dev/null +++ b/.github/workflows/validate-deployment.yml @@ -0,0 +1,76 @@ +name: Validate commits for deployment + +permissions: + contents: read + +on: + workflow_call: + inputs: + protected-ref: + type: string + required: true + deployment-ref: + type: string + required: true + outputs: + valid: + value: ${{ jobs.validate.result == 'skipped' || jobs.validate.result == 'success' }} + workflow_dispatch: + inputs: + protected-ref: + type: string + required: true + deployment-ref: + type: string + required: true + +jobs: + validate: + name: Validate + runs-on: ubuntu-latest + if: inputs.deployment-ref != inputs.protected-ref + env: + PROTECTED_REF: ${{ inputs.protected-ref }} + DEPLOYMENT_REF: ${{ inputs.deployment-ref }} + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ env.PROTECTED_REF }} + show-progress: 'false' + persist-credentials: 'false' + - name: Count commits in candidate ref that are not in protected ref + id: all-cherries + run: | + CHERRY_FILE=$(mktemp -t cherries.XXXXX) + git cherry -v "$PROTECTED_REF" "$DEPLOYMENT_REF" > $CHERRY_FILE + echo "count=$(cat $CHERRY_FILE | wc -l)" >> $GITHUB_OUTPUT + echo "file=$CHERRY_FILE" >> $GITHUB_OUTPUT + - name: Count commits that were not cherry-picked from protected ref + id: missing-cherries + run: echo "count=$(cat $CHERRY_FILE | grep -E '^[+]' | wc -l)" >> $GITHUB_OUTPUT + env: + CHERRY_FILE: ${{ steps.all-cherries.outputs.file }} + - name: Report outcome to step summary + run: | + echo "Result: $VALIDATION_RESULT" >> $GITHUB_STEP_SUMMARY + echo "Found $TOTAL_CHERRY_COUNT commits in $DEPLOYMENT_REF not in $PROTECTED_REF." >> $GITHUB_STEP_SUMMARY + echo "Of these, $INVALID_CHERRY_COUNT were not cherry-picked from $PROTECTED_REF." >> $GITHUB_STEP_SUMMARY + echo "Only commits from pull requests that were approved and merged to $PROTECTED_REF are eligible for deployment." >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat "$CHERRY_FILE" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + env: + VALIDATION_RESULT: ${{ fromJson(steps.missing-cherries.outputs.count) == 0 && 'SUCCESS' || 'FAIL' }} + TOTAL_CHERRY_COUNT: ${{ steps.all-cherries.outputs.count }} + INVALID_CHERRY_COUNT: ${{ steps.missing-cherries.outputs.count }} + CHERRY_FILE: ${{ steps.all-cherries.outputs.file }} + - name: Fail validation + if: fromJson(steps.missing-cherries.outputs.count) > 0 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + core.setFailed('Invalid commits detected') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 746cadee5..6f21b8fb2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,140 +1,101 @@ # How to contribute to U.S. Digital Response projects -Everyone is welcome to contribute, and we value everybody's contribution. Code -is not the only way to help the community. Answering questions, helping -others, reaching out, and improving the documentation are all immensely valuable -to the community. +Everyone is welcome to contribute, and we value everybody's contribution. +Code is not the only way to help the community. +Answering questions, helping others, reaching out, and improving documentation are all immensely valuable to the community. ## You can contribute in so many ways! -There are 3 ways you can contribute to this project: +There are several ways you can contribute to this repository: -- Fixing outstanding issues with the existing code; -- Contributing to the examples or to the documentation; -- Submitting issues related to bugs or desired new features. +- Enhancing and/or fixing outstanding issues with the existing code +- Contributing useful improvements, clarifications, and updates to existing documentation +- Submitting [issues](https://github.com/usdigitalresponse/usdr-gost/issues/new/choose) related to bugs or desired new features +- Reporting security vulnerabilities _All are equally valuable to the community._ -We also onboard new volunteers to USDR projects in general at [https://www.usdigitalresponse.org/raisingyourhand](https://www.usdigitalresponse.org/raisingyourhand). Please sign up if youโ€™d like to help on other projects. +We also onboard new volunteers to USDR projects in general at https://www.usdigitalresponse.org/volunteer. +Please sign up if youโ€™d like to help on other projects. ## Submitting a new issue or feature request -Do your best to follow these guidelines when submitting an issue or a feature -request. It will make it easier for us to come back to you quickly and with good -feedback. +Do your best to follow these guidelines when submitting an issue or a feature request. +It will make it easier for us to come back to you quickly and with good feedback. +First, please check whether your issue or feature request has already been submitted by searching our [open issues](https://github.com/usdigitalresponse/usdr-gost/issues?q=is%3Aopen+is%3Aissue). -### Did you find a bug? -Open source code is robust and reliable thanks to the users who notify us of -the problems they encounter, so thank you for reporting an issue. +### Bugs -First, we would really appreciate it if you could **make sure the bug was not -already reported** (use the search bar on GitHub under the โ€œIssuesโ€ tab). +If you find an [open bug](https://github.com/usdigitalresponse/usdr-gost/issues?q=is%3Aopen+is%3Aissue+label%3Abug) that sounds like your issue, please consider adding a comment to the existing issue with details that will help us better understand the problem. -Did not find it? :( So we can act quickly on it, please include the following: +To submit something new, please fill out and submit [this form](https://github.com/usdigitalresponse/usdr-gost/issues/new?template=default_issue.yml&title=%5BBug%5D%3A+). -- Your **operating system**. -- If the problem is with the API, your **client (e.g. cURL, Python Requests)**. -- If the problem was with the demo UI or docs pages, your **browser (e.g. Firefox, Chrome, Edge, Internet Explorer)**. -- Any code errors that you have access to. +### Security vulnerabilities -### Do you want a new feature? +Please fill out and submit [this form](https://github.com/usdigitalresponse/usdr-gost/security/advisories/new) +if you believe you have discovered a security-related problem. -A world-class feature request addresses the following points: - -1. Motivation first: - -- Is it related to a problem/frustration with the current system? If so, please explain why. -- Is it related to something you would need for a project? We'd love to hear about it! -- Is it something you worked on and think could benefit the community? Awesome! Tell us what problem it solved for you. -2. Write a _full paragraph_ describing the feature; -3. Provide a **code snippet** or **design mockup or sketch** if possible, to demonstrate its future use. +### New features -If your issue is well written, weโ€™re already 80% of the way there by the time you -post it. - - -## Start contributing! (Pull Requests) - -Before writing code, we strongly advise you to search through the exising PRs or -issues to make sure that nobody is already working on the same thing. If you are -unsure, it is always a good idea to open an issue to get some feedback. +A world-class feature request addresses the following points: -You will need basic `git` proficiency to be able to contribute to -this project. `git` is not the easiest tool to use but it has the greatest -manual. Type `git --help` in a shell and enjoy. +1. Motivation first: + - Is it related to a problem/frustration with the current system? If so, ease explain why. + - Is it related to something you would need for a project? We'd love to hear about it! + - Is it something you worked on and think could benefit the community? + Awesome! + Tell us what problem it solved for you. + - Do you think you have a neat idea and want to share it? + We love those too! + Tell us your use-case and what value it might bring to the project. +2. Write a _full paragraph_ describing why you think the feature is important. +3. Provide a **code snippet** or **design mockup or sketch** if possible, to demonstrate its +future use. + +By the way, your suggestions do not need to be restricted to changes to how this project works in a production environment! +We would love to hear your ideas for improving all types of things, including: +- Ways to enhance the developer experience. +- Testing and other maintainability strategies. +- Automated CI/CD workflows. + + +## Start contributing! (pull requests) + +Before writing code, we strongly advise you to search through the existing PRs or issues to make sure that nobody is already working on the same thing. +If you are unsure, it is always a good idea to open an issue to get some feedback. + +Things you will need: +- Basic `git` proficiency. +- Knowledge of one or more of the languages used in this project, such as JavaScript and/or Terraform. +- A development environment. +See the [Development](https://github.com/usdigitalresponse/usdr-gost/blob/main/README.md#development) section of our README for more information. +- A positive attitude. Follow these steps to start contributing: -1. Fork the repository by - clicking on the 'Fork' button on the repository's page. This creates a copy of the code - under your GitHub user account. - -2. Clone your fork to your local disk - -3. Create a new branch to hold your development changes: - - ```bash - $ git checkout -b a-descriptive-name-for-my-changes - ``` - - please avoid working on the `main` or `gh-pages` branch directly. - -4. Develop the features on your branch. - - Once you're happy with your changes, add changed files using `git add` and - make a commit with `git commit` to record your changes locally: - - ```bash - $ git add modified_file.py - $ git commit - ``` - - Commit messages must follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) style. - - It is a good idea to sync your copy of the code with the original - repository regularly. This way you can quickly account for changes: - - ```bash - $ git fetch origin - $ git rebase origin/_staging - ``` - - Push the changes to your account using: - - ```bash - $ git push -u origin a-descriptive-name-for-my-changes - ``` - -5. Once you are satisfied (**and the checklist below is happy, too**), go to the - webpage of your fork on GitHub. Click on 'Pull request' to send your changes - to the project maintainers for review. - -6. It's ok if maintainers ask you for changes. It happens to core contributors - too! So everyone can see the changes in the Pull request, work in your local - branch and push the changes to your fork. They will automatically appear in - the pull request. - - -### Checklist - -1. The title of your pull request should be a summary of its contribution; - -2. If your pull request adresses an issue, please [mention the issue number in - the pull request description to make sure they are linked](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) (and so people consulting the issue know you are working on it); - -3. To indicate a work in progress, please submit your pull request as a draft. - This is useful to avoid duplicated work, and to differentiate it from PRs - ready to be merged. - - ![Creating a draft pull request](./docs/_assets/draft-pr.png) - -4. If there are any tests, make sure that they pass and cover your new features and bugfixes. - - -#### This guide was based on [HuggingFace/transformers](https://github.com/huggingface/transformers/blob/master/CONTRIBUTING.md) which was itself based on [SciKit-Learn](https://github.com/scikit-learn/scikit-learn/blob/master/CONTRIBUTING.md) +1. Fork the repository by clicking on the 'Fork' button on the repository's page. +This creates a copy of the code under your GitHub user account. + - **Note:** If you are an active volunteer with contributor access to this repository, + you may alternatively clone and make branches against this repository directly. +2. Clone your fork to your development environment. +3. Create a new branch to hold your development changes. +We recommend that you use the following conventions when naming your branch: + | **Branch Prefix** | **Type of Contribution** | **Example** | + |-------------------|------------------------------------------------------------------------|-------------------------------------| + | `feat/` | New features and enhancements. | `feat/generate-jetpacks` | + | `bug/` or `fix/` | Changes that resolve or mitigate a problem. | `fix/jetpack-fuel-gauge-inaccuracy` | + | `docs/` | Improvements, clarifications, and/or updates to existing documentation | `docs/clarify-landing-instructions` | + +4. Develop, commit, and push the features on your branch. +5. Once you are satisfied, submit your pull request! + - Make sure the title of your PR represents a short summary of the contribution. + - Fill out the different sections that appear in the pull request template. + - Please include unit tests for all JavaScript code! + - If your work isn't quite ready but you want early feedback, submit your pull request as a draft. +6. Don't worry if maintainers ask you for changes - it's all part of the collaborative process! diff --git a/README.md b/README.md index e1e6d1781..acbb5a0f9 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,13 @@ This repository falls under [U.S. Digital Responseโ€™s Code of Conduct](./CODE_O This project wouldnโ€™t exist without the hard work of many people. Please see [`CONTRIBUTING.md`](./CONTRIBUTING.md) to find out how you can help. +## Release Management + +Releases are versioned using a `YYYY.inc` scheme that represents the year of the release, and the incremental release number for that year. +You can view a list of all historical releases on the [Releases page](https://github.com/usdigitalresponse/usdr-gost/releases). + +For details on deploying releases to Production, see our [Release Process](./docs/releasing.md) documentation. + ## License & Copyright Copyright (C) 2020-2021 U.S. Digital Response (USDR) diff --git a/docker-compose.yml b/docker-compose.yml index ec8e32e75..938e752a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,6 +73,8 @@ services: - POSTGRESQL_PASSWORD=password123 - POSTGRESQL_DATABASE=usdr_grants - POSTGRESQL_USER=postgres + ports: + - 5432:5432 volumes: - ./docker/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d - ./docker/postgres/docker-entrypoint-preinitdb.d:/docker-entrypoint-preinitdb.d diff --git a/docker/README.md b/docker/README.md index 2e238771d..9ea38a0cb 100644 --- a/docker/README.md +++ b/docker/README.md @@ -163,3 +163,33 @@ any files within this directory will be ignored by git. Simply update the `postg definition in the `docker-compose.yml` file to mount this volume โ€“ย you can do so by adding/uncommenting the corresponding `services.postgres.volumes` item in that file so that the `./docker/postgres/persistence:/bitnami/postgresql` volume mount is enabled. + + +### Create SQS queues in localstack +SQS queues should automatically be setup from the `localstack/entrypoint/init-aws.sh` script. +This script gets run by localstack after the container is setup. + +To confirm evreything is working well, run the following locally (you may need the `aws cli` locally) +``` +aws --region us-west-2 --endpoint http://localhost:4566 sqs list-queues +``` + +If the queue is not setup, first check that you have an updated version of localstack. + +If you have an updated version, but still need to setup the queuse, run the below commands. +``` +docker ps +# assuming the localstack container name is gost-localstack_main +docker exec -it gost-localstack_main /bin/bash +bash-5.0# awslocal sqs --queue-name arpa-queue +bash-5.0# awslocal sqs list-queues +``` +Now we can use that queue name and set it as the environment variable `ARPA_AUDIT_REPORT_SQS_QUEUE_URL`. + +To confirm that you can access the queues locally, use the `aws cli` again +``` +aws --region us-west-2 --endpoint http://localhost:4566 sqs list-queues +``` +For this to work, you must have `AWS_ACCESS_KEY_ID=test` and `AWS_SECRET_ACCESS_KEY=test`. +This is part of the standard localstack setup. + diff --git a/docs/deployment.md b/docs/deployment.md deleted file mode 100644 index a950eb168..000000000 --- a/docs/deployment.md +++ /dev/null @@ -1,64 +0,0 @@ -# Deployment - -(Note, as of Mar 12, 2023, information in this file needs updating to account for the new AWS-based deployment process) - -## Render - -1. Create web service - - ![create-web-service](./img/create-web-service.png) - -2. Create database - - ![create-database](./img/create-database.png) - -3. Update web service environment variables - - ![update-web-env-vars](./img/update-web-env-vars.png) - - **NOTE:** Don't set `NODE_ENV=production` else NPM dev deps will not be installed and prod deployments will fail [(source)](https://github.com/vuejs/vue-cli/issues/5107#issuecomment-586701382) - - ![prod-env-error](./img/prod-env-error.png) - - ```sh - POSTGRES_URL= # Render Internal connection string ie postgres://cares_opportunity_user:@/cares_opportunity_1e53 - - COOKIE_SECRET= - - WEBSITE_DOMAIN= # Render web service url ie. https://cares-grant-opportunities-qi8i.onrender.com - - NODE_ENV=development or production or test - - NOTIFICATIONS_EMAIL="grants-identification@usdigitalresponse.org" - AWS_ACCESS_KEY_ID= - AWS_SECRET_ACCESS_KEY= - - ENABLE_GRANTS_SCRAPER=true - GRANTS_SCRAPER_DATE_RANGE=7 # date range of grants that will be scraped - GRANTS_SCRAPER_DELAY=1000 # delay in milliseconds for scraper - - NODE_OPTIONS=--max_old_space_size=1024 # increase node max memory, had problems with node not using all of renders server memory. This will depend on the plan - ``` - -## DB Migrations - -1. Get the postgres external connection string from render. Set it as an environment variable - - `export POSTGRES_URL="postgres://user:{pass}@{domain}/{db}?ssl=true"` - - NOTE: must add `?ssl=true` - -1. Change directory to packages/server - -1. Update seeds/dev files accordingly - - seeds/dev/ref/agencies.js - list of agencies to be created. You can update this with the state provided agency. Note: We add a special USDR agency for our accounts in the system - - seeds/dev/index.js - Update the admin list variable accordingly - -1. Run the following commands - - ```sh - npx knex migrate:latest - npx knex seed:run - ``` - - After that you should be able to access the site and login with the users set in the migration. diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 000000000..cff98c2f9 --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,174 @@ +# Release Process + +## Release changes to Staging + +_"As a contributor whose pull request has been been approved, I want to deploy my changes to Staging."_ + +Simple โ€“ย just merge your pull request! + +Any pull request merged to the `main` branch will automatically activate the deployment pipeline that targets the Staging environment. +You can check the status of a Staging deployment by viewing the ["Deploy to Staging" workflow history](https://github.com/usdigitalresponse/usdr-gost/actions/workflows/deploy-staging.yml) + +## Release changes to Production + +_"As a team member with sufficient access to manage GitHub releases, I want to deploy a set of changes to Production."_ + +### Scenario 1: All changes deployed to Staging have been successfully QA'd + +> [!TIP] +> This is the simplest scenario, and our **preferred way** to deploy changes to Production. + +1. Navigate to the repository [Releases page](https://github.com/usdigitalresponse/usdr-gost/releases). +2. Locate the current draft release. This will displayed at the top, along with a "Draft" label. +3. Click the pencil icon for the draft release to edit it. +4. In the "Write" tab for the release notes: + - Edit the placeholder text under the **Summary** heading to provide a brief description of the release as a benefit to stakeholders. Summaries are encouraged ("bug fixes and dependency updates" is fine), but if you don't want to provide one, be sure to delete the heading and its placeholder text. + - Optional: check for any uncategorized changes under the **Other changes** heading (this heading will only be present if there are uncategorized changes), and move/copy them to more appropriate category headings. +5. Switch to the "Preview" tab and review the release notes to confirm that the release notes look suitable for publishing. +6. Ensure that the "Set as pre-release" box is checked (it should be preselected), and then click "Publish Release". + +At this point, the deployment will start preparing to ship the changes associated with the published release. +Once the changes are ready to ship, administrators will be notified to review the deployment plan created by Terraform and give final approval. +For more information, refer to **What happens when a release is published?** + +### Scenario 2: Not all changes on Staging have been successfully QA'd, but we still want to deploy _some_ things to Production + +#### Scenario 2.1: Desired changes to deploy are contained in sequential commits directly following the latest release + +> [!TIP] +> Use this workflow when there are consecutive commits that should be deployed to Production, up to a certain point. +> That is, if the latest Production deployment/release tag point to commit A, and commits B, C, and D have since been added to `main`, +> this workflow can be used to deploy commits B and C only. +> +> If non-sequential commits to `main` need to be deployed, refer to Scenario 2.2 below. + +Example scenario (all commits on `main`): +``` +commit-A <- Deployed to Production, tagged as the latest release +commit-B <- Successfully QA'd, ready for Production +commit-C <- Successfully QA'd, ready for Production +commit-D <- NOT successfully QA'd +``` + +1. Follow Steps 1-4 in **Scenario 1** to prepare the release notes. +2. While editing the next draft release, select the "Target" dropdown menu, switch to the "Recent commits" tab, and select the final/latest commit in the series that you want to deploy. +3. Edit the release notes to remove documented changes that come after the selected target commit (i.e. remove changes that will not be deployed in this release). +4. Follow the remaining steps in **Scenario 1** to deploy the release. + +#### Scenario 2.2: Desired changes to deploy are contained in nonsequential commits + +> [!TIP] +> Use this workflow when there are **nonconsecutive** commits that should be deployed to Production. +> That is, if the latest Production deployment/release tag point to commit A, and commits B, C, D, and E have since been added to `main`, +> this workflow can be used to deploy commits B and D only. +> +> If non-sequential commits to `main` need to be deployed, refer to Scenario 2.2 below. + +Example scenario (all commits on `main`): +``` +commit-A <- Deployed to Production, tagged as the latest release +commit-B <- Successfully QA'd, ready for Production +commit-C <- NOT successfully QA'd +commit-D <- Successfully QA'd, ready for Production +commit-E <- NOT successfully QA'd +``` + +> [!WARNING] +> Check with a repository admin first to ensure that database migrations will not be affected and are compatible with the desired changes. + +> [!NOTE] +> This use-case requires working knowledge of `git cherry-pick`. + +> [!IMPORTANT] +> - Only commits that exist on `main` may cherry-picked for a release. +> The deployment workflow checks that the contents of the specified release tag satisfy this requirement; +> if the release tag contains any commit that is not also on `main` _or_ was not cherry-picked from a commit on `main`, the workflow execution will fail. +> - Always be sure to provide the `-x` flag when running `git cherry-pick` so that the resulting commit message [includes a reference to the source commit](https://git-scm.com/docs/git-cherry-pick#Documentation/git-cherry-pick.txt--x). + +1. Check out a new branch based on the revision currently deployed to Production (i.e. the latest release tag). +2. Cherry-pick the commit(s) onto the new branch from `main` that you want to include in the release. +3. Push your branch that now contains the cherry-picked commits that you want to release. +4. Follow Steps 1-4 in **Scenario 1** to prepare the release notes. +5. While editing the next draft release, select the "Target" dropdown menu and use the "Branches" tab to select the branch that contains the cherry-picked commits that you want to release. +6. Edit the release notes to remove documented changes were not cherry-picked (i.e. remove changes that will not be deployed in this release). +7. Follow the remaining steps in **Scenario 1** to deploy the release. +8. Optional: Delete the release branch (the release commit is now tagged; the branch is no longer needed) +9. Optional: Generate the next draft release. + - Using the GitHub web UI: + 1. Navigate to the ["Release Drafter" workflow](https://github.com/usdigitalresponse/usdr-gost/actions/workflows/release-drafter.yml) + 2. Click "Run workflow" + 3. Select the "Branch" dropdown menu and choose the `main` branch + 4. Click the "Run workflow" button located beneath the selected branch. + - Using the GitHub CLI: + ```cli + gh workflow run "Release Drafter" --ref main + ``` + +> [!TIP] +> You can skip the final step if you want to wait for a new draft release to be created automatically the next time `main` is updated. + +### Scenario 3: Retry a failed deployment without making code changes. + +1. Use the ["Deploy to Production" workflow history](https://github.com/usdigitalresponse/usdr-gost/actions/workflows/deploy-production.yml) to locate and navigate to the failed workflow run. +2. Click the "Re-run jobs" button. If prompted, select "Re-run all jobs". +3. Wait for the new workflow run to generate a deployment plan and prompt administrators for review and approval. + +### Scenario 4: Revert the Production environment to a previous deployment. + +> [!WARNING] +> Check with a repository admin first to ensure that database migrations will not be affected! + +- Using the GitHub web UI, using a recent deployment workflow: + 1. Use the ["Deploy to Production" workflow history](https://github.com/usdigitalresponse/usdr-gost/actions/workflows/deploy-production.yml) to locate and navigate to the workflow run that you want to re-execute. + 2. Click "Re-run all jobs". + 3. Wait for the new workflow to generate a deployment plan and prompt administrators for review and approval. +- Using the GitHub web UI, as a new workflow run: + 1. Navigate to the ["Deploy to Production" workflow](https://github.com/usdigitalresponse/usdr-gost/actions/workflows/deploy-production.yml). + 2. Click "Run workflow" + 3. Select the "Branch" dropdown menu, switch to the "Tags" tab, and select the release tag that you want to deploy. + 4. Wait for the new workflow to generate a deployment plan and prompt administrators for review and approval. +- Using the GitHub CLI: + ```cli + gh workflow run "Deploy to Production" --ref release/1979.33 + ``` + +## What happens when a release is published? + +When a release is published, its configured tag is pushed based on the configured "Target" (usually the `main` branch). +A push to any tag prefixed with `release/` will trigger the "Deploy to Production" workflow to execute for that tag, which performs the following steps: + +1. Validates the contents of the release tag to ensure that it only contains commits (and/or cherry-picked commits) from `main`. +2. Builds and publishes deployment artifacts for the release. +3. Generates a deployment plan for the changes associated with the release. +4. Once a deployment plan has been generated and is ready to ship, repository administrators will be notified with a request to review and approve the deployment plan. +5. Once approved, the pipeline execution will proceed with deploying the changes to Production. +6. Following successful deployment, the deployment pipeline will update the release to: + - Remove its "Pre-release" label + - Add the "Latest" label + - Append the date and time of deployment under the "Release History" heading of the release notes. + - Upload build artifacts as release assets. + +## Useful Information + +### What is a tag, and how do we use them for releases? + +A tag is an alias that (for the purposes of this document) provides an alias that points to a specific commit. There are many different tagging conventions, but for the purposes of our release process, we are specifically talking about tags formatted as `release/`, where `` is comprised of two delimited elements: the year of the release, and a number that increments for each subsequent release for that year (i.e. `yyyy.N`). For example, `2023.5` would represent the fifth release published in 2023; `2024.16` would represent the 16th release published in 2024, etc. + +#### Why not semantic versioning? + +Many software projects (especially libraries) follow versioning schemes like [semantic versioning](https://semver.org/) because it standardizes an explicit way of communicating compatibility changes from one version to the next. You probably recognize semantic versions, like `v1.2.3`. However, because the usdsr-gost repository provides multiple services and offers no single, concrete contract (i.e. an API) about which downstream consumers contend with compatibility decisions, a semantic versioning scheme is less relevant and would likely create ambiguilties rather than clarifying them. + +### What is a release? + +[GitHub releases](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository) are a repository-level feature in GitHub that add documentation ("release notes") and static file assets (a repo snapshot, and optionally platform-specific build outputs) for a specific tag, which generally serves as a version identifier for the release. + +GitHub releases can be in either a **draft** or **published** state. When a release is a **draft**, it is still being assembled and not generally visible to the public. Furthermore, drafts do not have to be associated with a tag that exists in the repository. As mentioned elsewhere, we will use automation to do most of the work around creating release drafts and keeping their contents up-to-date, although humans are responsible for applying final edits to release notes and saving the final outcome as a **published** release. The most-recently published release to successfully deploy to Production is automatically marked as "latest" in GitHub. + +Ideally, the repository will always have exactly 1 draft release at any moment (provided there are pull requests that have been merged since the last published release), which can be thought of as the "next release". While no release tag will exist for a draft, we can determine what that tag will be at the time the draft is created by looking at the current year and the tag associated with the latest published release. For example, if the latest published release associated with a tag of `release/2023.4` and the current year is still 2023, we know that the tag associated with the next release will be associated with a tag of `release/2023.5`; if the year is 2024, the next release tag will be `release/2024.1`. + +#### Release Terminology + +- **Draft release:** A release which has been created in GitHub but is not yet available for public viewing. Ideally, exactly one release draft exists as long as there are changes on `main` that have not yet been deployed to Production. +- **Published release:** Any release that is not a draft, i.e. is publicly viewable, and is considered to be a finalized representation of at least one change to Production. +- **Latest release:** A label indicating the last (published) release that has successfully deployed to Production. +- **Pre-release:** A label indicating that a published release has not yet fully and/or successfully deployed to Production. diff --git a/localstack/entrypoint/init-aws.sh b/localstack/entrypoint/init-aws.sh index fcb85e9b2..39c332009 100755 --- a/localstack/entrypoint/init-aws.sh +++ b/localstack/entrypoint/init-aws.sh @@ -1,5 +1,7 @@ -#! /bin/bash +#!/bin/bash +export LOCALSTACK_HOST=localhost +export HOSTNAME_EXTERNAL=localstack export AWS_ACCESS_KEY_ID="test" export AWS_SECRET_ACCESS_KEY="test" @@ -15,3 +17,4 @@ done awslocal s3api create-bucket --bucket arpa-audit-reports --region us-west-2 --create-bucket-configuration '{"LocationConstraint": "us-west-2"}' awslocal sqs create-queue --queue-name grants-ingest-events +awslocal sqs create-queue --queue-name arpa-queue diff --git a/package.json b/package.json index cee4669d8..1cfb945c4 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,9 @@ "start": "yarn serve", "test:client": "yarn workspace client test", "test:server": "yarn workspace server test", - "test": "yarn workspaces run test" + "test": "yarn workspaces run test", + "start:arpa-audit-worker": "yarn workspace server start:arpa-audit-worker", + "debug:arpa-audit-worker": "yarn workspace server debug:arpa-audit-worker" }, "dependencies": {}, "devDependencies": { diff --git a/packages/client/public/deploy-config.js b/packages/client/public/deploy-config.js index 1f6284675..8d0b32431 100644 --- a/packages/client/public/deploy-config.js +++ b/packages/client/public/deploy-config.js @@ -20,7 +20,6 @@ window.APP_CONFIG.overrideFeatureFlag = (flagName, overrideValue) => { // For local development only. Register feature flags for Staging and Production by // configuring the `website_feature_flags` input variable in Terraform (see `terraform/*.tfvars`). window.APP_CONFIG.featureFlags = { - useNewTable: true, myProfileEnabled: true, newTerminologyEnabled: true, }; diff --git a/packages/client/public/errors/403.html b/packages/client/public/errors/403.html deleted file mode 100644 index 43f4c9034..000000000 --- a/packages/client/public/errors/403.html +++ /dev/null @@ -1,10 +0,0 @@ - - - -
-

403

-

Unauthorized

-

You are not authorized to view the page. Click here login and try again.

-
- - diff --git a/packages/client/public/errors/404.html b/packages/client/public/errors/404.html deleted file mode 100644 index d518751a0..000000000 --- a/packages/client/public/errors/404.html +++ /dev/null @@ -1,10 +0,0 @@ - - - -
-

404

-

Page not found

-

The page you are looking for does not exist. Click here to go to the home page.

-
- - diff --git a/packages/client/src/assets/adjust-vue-select.css b/packages/client/src/assets/adjust-vue-select.css new file mode 100644 index 000000000..3ac6f84a8 --- /dev/null +++ b/packages/client/src/assets/adjust-vue-select.css @@ -0,0 +1,11 @@ +/* Make vue-select dropdowns match the height for other form controls */ +.vs__dropdown-toggle { + padding-top: 4px; + padding-bottom: 7px; + padding-left: 4px; + padding-right: 4px; +} + +.vs__open-indicator { + cursor: pointer; +} diff --git a/packages/client/src/components/GrantsTable.vue b/packages/client/src/components/GrantsTable.vue index 2e3e83516..4ef439527 100644 --- a/packages/client/src/components/GrantsTable.vue +++ b/packages/client/src/components/GrantsTable.vue @@ -1,64 +1,74 @@