Azure returns #2529
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Deploy to AWS | |
| on: | |
| push: | |
| branches: | |
| - dev | |
| pull_request: | |
| branches: | |
| - dev | |
| env: | |
| PROJECT_NAME: wri-odp | |
| BRANCH_NAME: dev | |
| ENABLE_TRIVY_SCANNING: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| security-events: write | |
| jobs: | |
| buildandtest: | |
| name: Build and Scan Image with Integration Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.OIDC_ROLE }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| with: | |
| mask-password: 'true' | |
| - name: Add url(s) to hosts | |
| run: | | |
| sudo echo "127.0.0.1 ckan-dev" | sudo tee -a /etc/hosts | |
| sudo echo "127.0.0.1 minio" | sudo tee -a /etc/hosts | |
| sudo echo "127.0.0.1 frontend" | sudo tee -a /etc/hosts | |
| sudo echo "127.0.0.1 prefect" | sudo tee -a /etc/hosts | |
| - name: Build and push CKAN image to ECR | |
| env: | |
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| CKAN_REPO: ${{ secrets.ECR_CKAN_REPO}} | |
| IMAGE_TAG: dev-${{ github.sha }} | |
| run: | | |
| mv ckan-backend-dev/src/ckanext-wri deployment/ckan/ | |
| docker build -t $REGISTRY/$CKAN_REPO:$IMAGE_TAG deployment/ckan | |
| docker push $REGISTRY/$CKAN_REPO:$IMAGE_TAG | |
| - name: Build and push Frontend image to ECR | |
| env: | |
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| FRONTEND_REPO: ${{ secrets.ECR_FRONTEND_REPO}} | |
| IMAGE_TAG: dev-${{ github.sha }} | |
| run: | | |
| docker build -t $REGISTRY/$FRONTEND_REPO:$IMAGE_TAG \ | |
| --build-arg NEXTAUTH_SECRET=${{ secrets.DEV_FRONTEND_NEXTAUTH_SECRET }} \ | |
| --build-arg NEXTAUTH_URL=${{ secrets.DEV_FRONTEND_NEXTAUTH_URL }} \ | |
| --build-arg CKAN_URL=${{ secrets.DEV_FRONTEND_CKAN_URL }} \ | |
| --build-arg NEXT_PUBLIC_CKAN_URL=${{ secrets.DEV_FRONTEND_CKAN_URL }} \ | |
| --build-arg NEXT_PUBLIC_NEXTAUTH_URL=${{ secrets.DEV_FRONTEND_NEXTAUTH_URL }} \ | |
| --build-arg NEXT_PUBLIC_GTM_ID=${{ secrets.DEV_NEXT_PUBLIC_GTM_ID }} \ | |
| --build-arg NEXT_PUBLIC_HOTJAR_ID=${{ secrets.DEV_NEXT_PUBLIC_HOTJAR_ID }} \ | |
| --build-arg NEXT_PUBLIC_GFW_API_KEY=${{ secrets.NEXT_PUBLIC_GFW_API_KEY }} \ | |
| --build-arg OSANO_URL=${{ secrets.DEV_OSANO_URL }} \ | |
| --build-arg NEXT_PUBLIC_DEPLOYMENT_TYPE="dev" \ | |
| deployment/frontend | |
| docker push $REGISTRY/$FRONTEND_REPO:$IMAGE_TAG | |
| - name: Build and push Datapusher image to ECR | |
| env: | |
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO}} | |
| IMAGE_TAG: dev-${{ github.sha }} | |
| run: | | |
| docker build -t $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG \ | |
| datapusher | |
| docker push $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG | |
| - name: Build and push Migration image to ECR | |
| env: | |
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO}} | |
| IMAGE_TAG: dev-${{ github.sha }} | |
| run: | | |
| docker build -t $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG-migration \ | |
| migration | |
| docker push $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG-migration | |
| - name: Set up Docker Containers | |
| env: | |
| CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}' | |
| SCRIPT_NAME: '/private-admin' | |
| run: | | |
| docker compose -f docker-compose.test.yml --env-file .env.example up --build -d | |
| working-directory: ./ckan-backend-dev | |
| - name: Initialize the Extensions | |
| run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh | |
| - name: Cypress Install and CKAN setup | |
| uses: cypress-io/github-action@v6 | |
| with: | |
| wait-on: 'http://localhost:5000/private-admin/en' | |
| wait-on-timeout: 120 | |
| node-version: 18 | |
| runTests: false | |
| working-directory: ./integration-tests | |
| - name: Create sysadmin API for Authorization | |
| run: bash ./ckan-backend-dev/ckan/scripts/cypress_setup.sh | |
| - name: Set up Frontend Docker Container | |
| env: | |
| CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}' | |
| run: docker compose -f docker-compose.test.yml --env-file .env.example up --build -d frontend | |
| working-directory: ./ckan-backend-dev | |
| - name: Initialize the Extensions | |
| run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh | |
| - name: Print Logs | |
| env: | |
| CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}' | |
| run: docker compose -f docker-compose.test.yml --env-file .env.example logs | |
| working-directory: ./ckan-backend-dev | |
| - name: Cypress Install and Frontend setup | |
| uses: cypress-io/github-action@v6 | |
| with: | |
| wait-on: 'http://localhost:3000' | |
| wait-on-timeout: 60 | |
| node-version: 18 | |
| runTests: false | |
| working-directory: ./e2e-tests | |
| - name: Run Integration tests 🧪 | |
| uses: cypress-io/github-action@v6 | |
| with: | |
| command: node test.js | |
| working-directory: ./integration-tests | |
| - name: Check prefect url | |
| uses: jtalk/url-health-check-action@v4 | |
| with: | |
| # Check the following URLs one by one sequentially | |
| url: http://prefect:4200 | |
| # Follow redirects, or just report success on 3xx status codes | |
| follow-redirect: false # Optional, defaults to "false" | |
| # Fail this action after this many failed attempts | |
| max-attempts: 3 # Optional, defaults to 1 | |
| - name: Run frontend tests 🧪 | |
| uses: cypress-io/github-action@v6 | |
| with: | |
| command: npm run test | |
| working-directory: ./e2e-tests | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: cypress-screenshots | |
| path: /home/runner/work/wri-odp/wri-odp/e2e-tests/cypress/screenshots | |
| if-no-files-found: ignore | |
| - name: Print Error Logs | |
| if: failure() | |
| env: | |
| CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}' | |
| run: docker compose -f docker-compose.test.yml --env-file .env.example logs | |
| working-directory: ./ckan-backend-dev | |
| - name: Copy run_unit_tests.sh | |
| run: docker cp ./ckan/scripts/run_unit_tests.sh ckan-wri:/srv/app/run_unit_tests.sh | |
| working-directory: ./ckan-backend-dev | |
| - name: Copy s3filestore test.ini fix script for minio | |
| run: | | |
| docker cp ./ckan/scripts/fix_s3filestore_test_ini.sh ckan-wri:/srv/app/fix_s3filestore_test_ini.sh | |
| working-directory: ./ckan-backend-dev | |
| - name: Fix s3filestore test.ini for minio | |
| env: | |
| CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}' | |
| run: docker compose -f docker-compose.test.yml --env-file .env.example exec -T ckan-dev /bin/bash -c "/srv/app/fix_s3filestore_test_ini.sh" | |
| working-directory: ./ckan-backend-dev | |
| - name: Initialize the pending datasets table | |
| run: docker exec ckan-wri sh -c "ckan -c production.ini pendingdatasetsdb" | |
| - name: Run Unit Tests 🧪 | |
| env: | |
| CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}' | |
| run: docker compose -f docker-compose.test.yml --env-file .env.example exec -T ckan-dev /bin/bash -c "/srv/app/run_unit_tests.sh" | |
| working-directory: ./ckan-backend-dev | |
| - name: Run Trivy Vulnerability Scanner for CKAN Container 🧪 | |
| if: ${{ env.ENABLE_TRIVY_SCANNING == 'true' }} | |
| uses: aquasecurity/trivy-action@master | |
| env: | |
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| REPOSITORY: ${{ secrets.ECR_CKAN_REPO }} | |
| TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db | |
| with: | |
| image-ref: '${{ env.REGISTRY }}/${{ env.REPOSITORY }}:dev-${{ github.sha }}' | |
| format: 'sarif' | |
| output: ckan-trivy-results.sarif | |
| exit-code: '0' | |
| ignore-unfixed: true | |
| vuln-type: 'os,library' | |
| severity: 'CRITICAL,HIGH' | |
| - name: Run Trivy Vulnerability Scanner for Frontend Container 🧪 | |
| if: ${{ env.ENABLE_TRIVY_SCANNING == 'true' }} | |
| uses: aquasecurity/trivy-action@master | |
| env: | |
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| REPOSITORY: ${{ secrets.ECR_FRONTEND_REPO }} | |
| TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db | |
| with: | |
| image-ref: '${{ env.REGISTRY }}/${{ env.REPOSITORY }}:dev-${{ github.sha }}' | |
| format: 'sarif' | |
| output: frontend-trivy-results.sarif | |
| exit-code: '0' | |
| ignore-unfixed: true | |
| vuln-type: 'os,library' | |
| severity: 'CRITICAL,HIGH' | |
| - name: Upload CKAN container Trivy scan results to GitHub Code scanning | |
| if: ${{ env.ENABLE_TRIVY_SCANNING == 'true' }} | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: ckan-trivy-results.sarif | |
| - name: Upload Frontend container Trivy scan results to GitHub Code scanning | |
| if: ${{ env.ENABLE_TRIVY_SCANNING == 'true' }} | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| category: frontend_container_trivy_results | |
| sarif_file: frontend-trivy-results.sarif | |
| - name: Tear down containers | |
| env: | |
| CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}' | |
| if: failure() || success() | |
| run: docker compose -f docker-compose.test.yml --env-file .env.example down -v --remove-orphans | |
| working-directory: ./ckan-backend-dev | |
| deploy: | |
| name: Deploy To AWS | |
| runs-on: ubuntu-latest | |
| needs: | |
| - buildandtest | |
| if: github.event_name != 'pull_request' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ secrets.OIDC_ROLE }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| with: | |
| mask-password: 'true' | |
| - name: Run Templater and update values.yaml | |
| env: | |
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| CKAN_REPO: ${{ secrets.ECR_CKAN_REPO}} | |
| FRONTEND_REPO: ${{ secrets.ECR_FRONTEND_REPO}} | |
| DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO }} | |
| IMAGE_TAG: ${{ github.sha }} | |
| run: | | |
| cd deployment | |
| curl https://raw.githubusercontent.com/datopian/devops-tools/master/scripts/templater.sh > /tmp/templater.sh | |
| bash /tmp/templater.sh helm-templates/values.yaml.$BRANCH_NAME.template > helm-templates/values.yaml | |
| - name: Configure Kubeconfig | |
| run: | | |
| echo $BRANCH_NAME | |
| mkdir -p /home/runner/.kube | |
| aws eks --region ${{ secrets.AWS_REGION }} update-kubeconfig --name ${{ secrets.CLUSTER_NAME }} --role-arn ${{ secrets.KUBEROLE }} | |
| chmod 600 ~/.kube/config | |
| env: | |
| GITHUB_SHA: '${{ github.sha }}' | |
| - name: Install Helm | |
| uses: azure/setup-helm@v3 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| id: install | |
| - name: 'Deploy using Helm Upgrade' | |
| run: | | |
| set -e | |
| helm upgrade -i dx-helm-wri-$BRANCH_NAME-release ./deployment/helm-templates -f ./deployment/helm-templates/values.yaml -n $PROJECT_NAME-$BRANCH_NAME --create-namespace --wait |