-
Notifications
You must be signed in to change notification settings - Fork 0
269 lines (268 loc) · 11.9 KB
/
main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
name: Build and Deploy to AWS
on:
push:
branches:
- dev
pull_request:
branches:
- dev
env:
PROJECT_NAME: wri-odp
BRANCH_NAME: dev
ENABLE_TRIVY_SCANNING: true
permissions:
id-token: write
contents: read
security-events: write
jobs:
buildandtest:
name: Build and Scan Image with Integration Tests
runs-on: ubuntu-latest
steps:
- name: Git clone the repository
uses: actions/checkout@v4
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.OIDC_ROLE }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
with:
mask-password: 'true'
- name: Add url(s) to hosts
run: |
sudo echo "127.0.0.1 ckan-dev" | sudo tee -a /etc/hosts
sudo echo "127.0.0.1 minio" | sudo tee -a /etc/hosts
sudo echo "127.0.0.1 frontend" | sudo tee -a /etc/hosts
sudo echo "127.0.0.1 prefect" | sudo tee -a /etc/hosts
- name: Build and push CKAN image to ECR
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
CKAN_REPO: ${{ secrets.ECR_CKAN_REPO}}
IMAGE_TAG: dev-${{ github.sha }}
run: |
mv ckan-backend-dev/src/ckanext-wri deployment/ckan/
docker build -t $REGISTRY/$CKAN_REPO:$IMAGE_TAG deployment/ckan
docker push $REGISTRY/$CKAN_REPO:$IMAGE_TAG
- name: Build and push Frontend image to ECR
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
FRONTEND_REPO: ${{ secrets.ECR_FRONTEND_REPO}}
IMAGE_TAG: dev-${{ github.sha }}
run: |
docker build -t $REGISTRY/$FRONTEND_REPO:$IMAGE_TAG \
--build-arg NEXTAUTH_SECRET=${{ secrets.DEV_FRONTEND_NEXTAUTH_SECRET }} \
--build-arg NEXTAUTH_URL=${{ secrets.DEV_FRONTEND_NEXTAUTH_URL }} \
--build-arg CKAN_URL=${{ secrets.DEV_FRONTEND_CKAN_URL }} \
--build-arg NEXT_PUBLIC_CKAN_URL=${{ secrets.DEV_FRONTEND_CKAN_URL }} \
--build-arg NEXT_PUBLIC_NEXTAUTH_URL=${{ secrets.DEV_FRONTEND_NEXTAUTH_URL }} \
--build-arg NEXT_PUBLIC_GTM_ID=${{ secrets.DEV_NEXT_PUBLIC_GTM_ID }} \
--build-arg NEXT_PUBLIC_HOTJAR_ID=${{ secrets.DEV_NEXT_PUBLIC_HOTJAR_ID }} \
--build-arg NEXT_PUBLIC_GFW_API_KEY=${{ secrets.NEXT_PUBLIC_GFW_API_KEY }} \
--build-arg OSANO_URL=${{ secrets.DEV_OSANO_URL }} \
--build-arg NEXT_PUBLIC_DEPLOYMENT_TYPE="dev" \
deployment/frontend
docker push $REGISTRY/$FRONTEND_REPO:$IMAGE_TAG
- name: Build and push Datapusher image to ECR
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO}}
IMAGE_TAG: dev-${{ github.sha }}
run: |
docker build -t $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG \
datapusher
docker push $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG
- name: Build and push Migration image to ECR
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO}}
IMAGE_TAG: dev-${{ github.sha }}
run: |
docker build -t $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG-migration \
migration
docker push $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG-migration
- name: Set up Docker Containers
env:
CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
SCRIPT_NAME: '/private-admin'
run: |
docker compose -f docker-compose.test.yml --env-file .env.example up --build -d
working-directory: ./ckan-backend-dev
- name: Initialize the Extensions
run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
- name: Cypress Install and CKAN setup
uses: cypress-io/github-action@v6
with:
wait-on: 'http://localhost:5000/private-admin/en'
wait-on-timeout: 120
node-version: 18
runTests: false
working-directory: ./integration-tests
- name: Create sysadmin API for Authorization
run: bash ./ckan-backend-dev/ckan/scripts/cypress_setup.sh
- name: Set up Frontend Docker Container
env:
CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
run: docker compose -f docker-compose.test.yml --env-file .env.example up --build -d frontend
working-directory: ./ckan-backend-dev
- name: Initialize the Extensions
run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
- name: Print Logs
env:
CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
run: docker compose -f docker-compose.test.yml --env-file .env.example logs
working-directory: ./ckan-backend-dev
- name: Cypress Install and Frontend setup
uses: cypress-io/github-action@v6
with:
wait-on: 'http://localhost:3000'
wait-on-timeout: 60
node-version: 18
runTests: false
working-directory: ./e2e-tests
- name: Run Integration tests 🧪
uses: cypress-io/github-action@v6
with:
command: node test.js
working-directory: ./integration-tests
- name: Check prefect url
uses: jtalk/url-health-check-action@v4
with:
# Check the following URLs one by one sequentially
url: http://prefect:4200
# Follow redirects, or just report success on 3xx status codes
follow-redirect: false # Optional, defaults to "false"
# Fail this action after this many failed attempts
max-attempts: 3 # Optional, defaults to 1
- name: Run frontend tests 🧪
uses: cypress-io/github-action@v6
with:
command: npm run test
working-directory: ./e2e-tests
- uses: actions/upload-artifact@v4
if: failure()
with:
name: cypress-screenshots
path: /home/runner/work/wri-odp/wri-odp/e2e-tests/cypress/screenshots
if-no-files-found: ignore
- name: Print Error Logs
if: failure()
env:
CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
run: docker compose -f docker-compose.test.yml --env-file .env.example logs
working-directory: ./ckan-backend-dev
- name: Copy run_unit_tests.sh
run: docker cp ./ckan/scripts/run_unit_tests.sh ckan-wri:/srv/app/run_unit_tests.sh
working-directory: ./ckan-backend-dev
- name: Copy s3filestore test.ini fix script for minio
run: |
docker cp ./ckan/scripts/fix_s3filestore_test_ini.sh ckan-wri:/srv/app/fix_s3filestore_test_ini.sh
working-directory: ./ckan-backend-dev
- name: Fix s3filestore test.ini for minio
env:
CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}'
run: docker compose -f docker-compose.test.yml --env-file .env.example exec -T ckan-dev /bin/bash -c "/srv/app/fix_s3filestore_test_ini.sh"
working-directory: ./ckan-backend-dev
- name: Initialize the pending datasets table
run: docker exec ckan-wri sh -c "ckan -c production.ini pendingdatasetsdb"
- name: Run Unit Tests 🧪
env:
CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}'
run: docker compose -f docker-compose.test.yml --env-file .env.example exec -T ckan-dev /bin/bash -c "/srv/app/run_unit_tests.sh"
working-directory: ./ckan-backend-dev
- name: Run Trivy Vulnerability Scanner for CKAN Container 🧪
if: ${{ env.ENABLE_TRIVY_SCANNING == 'true' }}
uses: aquasecurity/trivy-action@master
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
REPOSITORY: ${{ secrets.ECR_CKAN_REPO }}
TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db
with:
image-ref: '${{ env.REGISTRY }}/${{ env.REPOSITORY }}:dev-${{ github.sha }}'
format: 'sarif'
output: ckan-trivy-results.sarif
exit-code: '0'
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
- name: Run Trivy Vulnerability Scanner for Frontend Container 🧪
if: ${{ env.ENABLE_TRIVY_SCANNING == 'true' }}
uses: aquasecurity/trivy-action@master
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
REPOSITORY: ${{ secrets.ECR_FRONTEND_REPO }}
TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db
with:
image-ref: '${{ env.REGISTRY }}/${{ env.REPOSITORY }}:dev-${{ github.sha }}'
format: 'sarif'
output: frontend-trivy-results.sarif
exit-code: '0'
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
- name: Upload CKAN container Trivy scan results to GitHub Code scanning
if: ${{ env.ENABLE_TRIVY_SCANNING == 'true' }}
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: ckan-trivy-results.sarif
- name: Upload Frontend container Trivy scan results to GitHub Code scanning
if: ${{ env.ENABLE_TRIVY_SCANNING == 'true' }}
uses: github/codeql-action/upload-sarif@v3
with:
category: frontend_container_trivy_results
sarif_file: frontend-trivy-results.sarif
- name: Tear down containers
env:
CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}'
if: failure() || success()
run: docker compose -f docker-compose.test.yml --env-file .env.example down -v --remove-orphans
working-directory: ./ckan-backend-dev
deploy:
name: Deploy To AWS
runs-on: ubuntu-latest
needs:
- buildandtest
if: github.event_name != 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.OIDC_ROLE }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
with:
mask-password: 'true'
- name: Run Templater and update values.yaml
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
CKAN_REPO: ${{ secrets.ECR_CKAN_REPO}}
FRONTEND_REPO: ${{ secrets.ECR_FRONTEND_REPO}}
DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO }}
IMAGE_TAG: ${{ github.sha }}
run: |
cd deployment
curl https://raw.githubusercontent.com/datopian/devops-tools/master/scripts/templater.sh > /tmp/templater.sh
bash /tmp/templater.sh helm-templates/values.yaml.$BRANCH_NAME.template > helm-templates/values.yaml
- name: Configure Kubeconfig
run: |
echo $BRANCH_NAME
mkdir -p /home/runner/.kube
aws eks --region ${{ secrets.AWS_REGION }} update-kubeconfig --name ${{ secrets.CLUSTER_NAME }} --role-arn ${{ secrets.KUBEROLE }}
chmod 600 ~/.kube/config
env:
GITHUB_SHA: '${{ github.sha }}'
- name: Install Helm
uses: azure/setup-helm@v3
with:
token: ${{ secrets.GITHUB_TOKEN }}
id: install
- name: 'Deploy using Helm Upgrade'
run: |
set -e
helm upgrade -i dx-helm-wri-$BRANCH_NAME-release ./deployment/helm-templates -f ./deployment/helm-templates/values.yaml -n $PROJECT_NAME-$BRANCH_NAME --create-namespace --wait