Merge pull request #199 from wri/merge/dev-staging

Merge dev into staging

.codeql-config.yml

@@ -0,0 +1,4 @@
+name: "CodeQL config"
+
+paths-ignore:
+  - 'e2e/**'

.github/workflows/codeql.yml

@@ -51,6 +51,7 @@ jobs:
       uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
+        config-file: .codeql-config.yml
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.

.github/workflows/main.yml

@@ -10,9 +10,9 @@ env:
   PROJECT_NAME: wri-odp
   BRANCH_NAME: staging
 permissions:
-      id-token: write
-      contents: read
-      security-events: write
+  id-token: write
+  contents: read
+  security-events: write
 jobs:
   buildandtest:
     name: Build and Scan Image with Integration Tests
@@ -55,17 +55,17 @@ jobs:
           --build-arg NEXTAUTH_SECRET=${{ secrets.STAGING_FRONTEND_NEXTAUTH_SECRET }} \
           --build-arg NEXTAUTH_URL=${{ secrets.STAGING_FRONTEND_NEXTAUTH_URL }} \
           --build-arg CKAN_URL=${{ secrets.STAGING_FRONTEND_CKAN_URL }} \
+          --build-arg NEXT_PUBLIC_CKAN_URL=${{ secrets.STAGING_FRONTEND_CKAN_URL }} \
           deployment/frontend
           docker push $REGISTRY/$FRONTEND_REPO:$IMAGE_TAG
       - name: Set up Docker Containers
         env:
           CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}'
         run: |
-          docker compose -f docker-compose.test.yml --env-file .env.example up --build -d \
-          ckan-dev datapusher db solr redis minio minio-setup
+          docker compose -f docker-compose.test.yml --env-file .env.example up --build -d
         working-directory: ./ckan-backend-dev
-      - name: Initialize the ckanext-issues table
-        run: docker exec ckan-wri sh -c "ckan -c production.ini issuesdb"
+      - name: Initialize the Extensions
+        run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
       - name: Cypress Install and CKAN setup
         uses: cypress-io/github-action@v6
         with:
@@ -81,6 +81,11 @@ jobs:
           CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}'
         run: docker compose -f docker-compose.test.yml --env-file .env.example up --build -d frontend
         working-directory: ./ckan-backend-dev
+      - name: Initialize the Extensions
+        run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
+      - name: Print Logs
+        run: docker-compose -f docker-compose.test.yml --env-file .env.example logs
+        working-directory: ./ckan-backend-dev
       - name: Cypress Install and Frontend setup
         uses: cypress-io/github-action@v6
         with:
@@ -104,7 +109,11 @@ jobs:
         with:
           name: cypress-screenshots
           path: /home/runner/work/wri-odp/wri-odp/e2e-tests/cypress/screenshots
-          if-no-files-found: ignore 
+          if-no-files-found: ignore
+      - name: Print Error Logs
+        if: failure()
+        run: docker-compose -f docker-compose.test.yml --env-file .env.example logs
+        working-directory: ./ckan-backend-dev
       - name: Copy run_unit_tests.sh
         run: docker cp ./ckan/scripts/run_unit_tests.sh ckan-wri:/srv/app/run_unit_tests.sh
         working-directory: ./ckan-backend-dev
@@ -188,7 +197,7 @@ jobs:
           FRONTEND_REPO: ${{ secrets.ECR_FRONTEND_REPO}}
           DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO }}
           IMAGE_TAG: ${{ github.sha }}
-        run: |    
+        run: |
           cd deployment
           curl https://raw.githubusercontent.com/datopian/devops-tools/master/scripts/templater.sh > /tmp/templater.sh
           bash /tmp/templater.sh helm-templates/values.yaml.$BRANCH_NAME.template > helm-templates/values.yaml

.gitignore

@@ -168,4 +168,6 @@ integration-tests/cypress/screenshots
 integration-tests/cypress.json
 
 ckan-backend-dev/src/ckan*
-!ckan-backend-dev/src/ckanext-wri
+!ckan-backend-dev/src/ckanext-wri
+
+test_tests.sh

LICENSE

@@ -1,5 +1,6 @@
 MIT License
 
+
 Copyright (c) 2023 World Resources Institute
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

README.md

@@ -30,3 +30,5 @@ The table of contents in the READMEs are generated using generated using [DocToc
 Then, to update the table of contents in a markdown file, for example, `README.md`, run:
 
     doctoc README.md
+
+

ckan-backend-dev/.env.example

@@ -8,6 +8,7 @@ CKAN_CONTAINER_NAME=ckan-wri
 WORKER_CONTAINER_NAME=ckan-worker
 MINIO_CONTAINER_NAME=minio-wri
 CKAN_IMAGE=${CKAN_IMAGE}
+SMTP_CONTAINER_NAME=smtp-wri
 
 # UWSGI
 UWSGI_HARAKIRI=50
@@ -33,6 +34,7 @@ DATASTORE_DB=datastore
 CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb
 CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
+CKAN__DATASTORE__SQLSEARCH__ENABLED=True
 
 # Test database connections
 TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
@@ -64,7 +66,7 @@ CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=True
 CKAN__AUTH__ALLOW_COLLABORATORS_TO_CHANGE_OWNER_ORG=True
 
 # Solr
-SOLR_IMAGE_VERSION=2.10-solr8
+SOLR_IMAGE_VERSION=2.10-solr8-spatial
 CKAN_SOLR_URL=http://solr:8983/solr/ckan
 TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
 
@@ -104,16 +106,36 @@ CKAN___SCHEMING__PRESETS=ckanext.wri.schema:presets.json
 
 # auth
 CKANEXT__AUTH__INCLUDE_FRONTEND_LOGIN_TOKEN=True
+CKANEXT__AUTH__AZURE_TENANT_ID=ffffffff-ffff-ffff-ffff-ffffffffffff
+CKANEXT__AUTH__AZURE_CLIENT_ID=ffffffff-ffff-ffff-ffff-ffffffffffff
 
 # custom auth
 CKANEXT__WRI__ODP_URL=http://frontend:3000
 
+# DataAPI
+#CKANEXT__DATA_API__HASURA_URL=http://hasura-svc:80
+#CKANEXT__DATA_API__HASURA_ADMIN_KEY=test123546789
+#CKANEXT__DATA_API__HASURA_DB="WRI Datastore"
+
+
 # Frontend Creds
 NEXTAUTH_SECRET=secret
 NEXTAUTH_URL=http://frontend:3000
 CKAN_URL=http://ckan-dev:5000
+NEXT_PUBLIC_CKAN_URL=http://ckan-dev:5000
 S3_ACCESS_KEY_ID=${MINIO_ROOT_USER}
 S3_SECRET_KEY_ID=${MINIO_ROOT_PASSWORD}
 S3_BUCKET_NAME=ckan
 S3_BUCKET_REGION=us-east-1
-SYS_ADMIN_API_KEY=CKAN_API_TOKEN
+SYS_ADMIN_API_KEY=CKAN_API_TOKEN
+RW_API_KEY=ffffffff-ffff-ffff-ffff-ffffffffffff
+AZURE_AD_CLIENT_ID=ffffffff-ffff-ffff-ffff-ffffffffffff
+AZURE_AD_CLIENT_SECRET=ffffffff-ffff-ffff-ffff-ffffffffffff
+AZURE_AD_TENANT_ID=ffffffff-ffff-ffff-ffff-ffffffffffff
+SYS_ADMIN_API_KEY=CKAN_API_TOKEN
+SMTP_SERVER="smtp.ssss.com"
+SMTP_PORT="587"
+SMTP_USER="SMTP_Injection"
+SMTP_PASSWORD="b07f"
+SMTP_FROM="[email protected]"
+

ckan-backend-dev/.env.frontend.example

@@ -8,4 +8,9 @@ S3_ACCESS_KEY_ID=${MINIO_ROOT_USER}
 S3_SECRET_KEY_ID=${MINIO_ROOT_PASSWORD}
 S3_BUCKET_NAME=ckan
 S3_BUCKET_REGION=us-east-1
-SYS_ADMIN_API_KEY=CKAN_API_TOKEN
+SYS_ADMIN_API_KEY=CKAN_API_TOKEN
+SMTP_SERVER="smtp.ssss.com"
+SMTP_PORT="587"
+SMTP_USER="SMTP_Injection"
+SMTP_PASSWORD="b07f"
+SMTP_FROM="[email protected]"

ckan-backend-dev/ckan/Dockerfile.dev

@@ -26,10 +26,30 @@ RUN pip3 install -e 'git+https://github.com/datopian/ckanext-scheming.git@ckan-2
     pip3 install -e 'git+https://github.com/datopian/ckanext-s3filestore.git@wri/cost-splitting-orgs#egg=ckanext-s3filestore' && \
     pip3 install -r 'https://raw.githubusercontent.com/datopian/ckanext-s3filestore/wri/cost-splitting-orgs/requirements.txt' && \
     pip3 install -r 'https://raw.githubusercontent.com/datopian/ckanext-s3filestore/wri/cost-splitting-orgs/dev-requirements.txt' && \
-    pip3 install -e 'git+https://github.com/datopian/ckanext-auth.git@auth-object-return-token#egg=ckanext-auth' && \
+    pip3 install -e 'git+https://github.com/datopian/ckanext-auth.git@fff325e0238334b54f23a392bb0682a5abc3da3f#egg=ckanext-auth' && \
     pip3 install -e 'git+https://github.com/ckan/ckanext-hierarchy.git@master#egg=ckanext-hierarchy' && \
     pip3 install -e 'git+https://github.com/datopian/[email protected]#egg=ckanext-issues'
 
+# Required by shapely dependency
+RUN apk --update add build-base libxslt-dev python3-dev 
+
+RUN apk add --virtual .build-deps \
+        --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing \
+        --repository http://dl-cdn.alpinelinux.org/alpine/edge/main \
+        gcc libc-dev geos-dev geos && \
+    runDeps="$(scanelf --needed --nobanner --recursive /usr/local \
+    | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \
+    | xargs -r apk info --installed \
+    | sort -u)" && \
+    apk add --virtual .rundeps $runDeps
+
+RUN geos-config --cflags
+
+RUN pip install --disable-pip-version-check Shapely==2.0.1
+
+RUN apk del build-base python3-dev && \
+    rm -rf /var/cache/apk/*
+
 # Update ckanext-s3filestore test.ini with minio credentials
 RUN sed -i "s|ckanext.s3filestore.aws_access_key_id = test-access-key|ckanext.s3filestore.aws_access_key_id = ${AWS_ACCESS_KEY_ID}|g" src/ckanext-s3filestore/test.ini && \
     sed -i "s|ckanext.s3filestore.aws_secret_access_key = test-secret-key|ckanext.s3filestore.aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}|g" src/ckanext-s3filestore/test.ini && \
@@ -74,4 +94,6 @@ RUN ckan config-tool ${CKAN_INI} "api_token.jwt.algorithm = RS256" && \
     ckan config-tool ${CKAN_INI} "api_token.jwt.encode.secret = file:${APP_DIR}/jwtRS256.key" && \
     ckan config-tool ${CKAN_INI} "api_token.jwt.decode.secret = file:${APP_DIR}/jwtRS256.key.pub"
 
+COPY docker-entrypoint.d/* /docker-entrypoint.d/
+
 CMD ["sh", "-c", "${APP_DIR}/start_ckan_development.sh"]

ckan-backend-dev/ckan/docker-entrypoint.d/init_tables.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+echo "Initializing custom tables..."
+
+ckan -c production.ini notificationdb
+
+EXIT_CODE=$?
+
+if [ $EXIT_CODE -ne 0 ]; then
+    echo "Failed to initialize custom tables"
+    exit $EXIT_CODE
+fi
+
+echo "Custom tables initialized successfully"

ckan-backend-dev/ckan/scripts/init-extensions.sh

@@ -0,0 +1,4 @@
+
+# Initialize Issues and Notification DB
+docker exec ckan-wri sh -c "ckan -c production.ini issuesdb"
+docker exec ckan-wri sh -c "ckan -c production.ini notificationdb"

ckan-backend-dev/ckan/setup/schema.xml

@@ -96,6 +96,14 @@ attribute with the form `ckan-X.Y` -->
       </analyzer>
     </fieldType>
 
+    <fieldType name="location_rpt"   class="solr.SpatialRecursivePrefixTreeFieldType"
+        spatialContextFactory="JTS"
+        autoIndex="true"
+        validationRule="repairBuffer0"
+        distErrPct="0.025"
+        maxDistErr="0.001"
+        distanceUnits="kilometers" />
+
 </types>
 
 
@@ -193,6 +201,8 @@ attribute with the form `ckan-X.Y` -->
     <field name="methodology" type="text" indexed="true" stored="true"/>
     <fiend name="featured_dataset" type="boolean" indexed="true" stored="true"/>
     <field name="wri_data" type="boolean" indexed="true" stored="true"/>
+    <field name="spatial_geom" type="location_rpt" indexed="true" multiValued="true" />
+    <field name="spatial_address" type="string" indexed="true" stored="true" multiValued="true"/>
 </fields>
 
 <uniqueKey>index_id</uniqueKey>

ckan-backend-dev/docker-compose.dev.yml

@@ -32,21 +32,18 @@ services:
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
+      interval: 40s
+      timeout: 30s
+      retries: 3
+      start_period: 60s
     image: ${CKAN_IMAGE}
 
   frontend:
     container_name: wri-frontend
     build:
       context: ../deployment/frontend/
       args:
-        - NEXTAUTH_SECRET=secret
-        - NEXTAUTH_URL=http://localhost:3000
-        - CKAN_URL=http://ckan-dev:5000
-        - S3_ACCESS_KEY_ID=${MINIO_ROOT_USER}
-        - S3_SECRET_KEY_ID=${MINIO_ROOT_PASSWORD}
-        - S3_BUCKET_NAME=ckan
-        - S3_BUCKET_REGION=us-east-1
-        - SYS_ADMIN_API_KEY=${SYS_ADMIN_API_KEY}
+        - NEXT_PUBLIC_CKAN_URL=http://ckan-dev:5000
     environment:
       - NEXTAUTH_SECRET=secret
       - NEXTAUTH_URL=http://localhost:3000
@@ -55,7 +52,19 @@ services:
       - S3_SECRET_KEY_ID=${MINIO_ROOT_PASSWORD}
       - S3_BUCKET_NAME=ckan
       - S3_BUCKET_REGION=us-east-1
+      - RW_API_KEY=${RW_API_KEY}
+      - AZURE_AD_TENANT_ID=${AZURE_AD_TENANT_ID}
+      - AZURE_AD_CLIENT_ID=${AZURE_AD_CLIENT_ID}
+      - AZURE_AD_CLIENT_SECRET=${AZURE_AD_CLIENT_SECRET}
       - SYS_ADMIN_API_KEY=${SYS_ADMIN_API_KEY}
+      - SMTP_SERVER=${SMTP_SERVER}
+      - SMTP_PORT=${SMTP_PORT}
+      - SMTP_USER=${SMTP_USER}
+      - SMTP_PASSWORD=${SMTP_PASSWORD}
+      - SMTP_FROM=${SMTP_FROM}
+    depends_on:
+      ckan-dev:
+        condition: service_healthy
     ports:
       - "0.0.0.0:3000:3000"
     healthcheck:
@@ -137,3 +146,13 @@ services:
     volumes:
       - ./ckan/setup/create_minio_bucket.sh:/srv/app/create_minio_bucket.sh
     command: ["-c", "/srv/app/create_minio_bucket.sh"]
+
+  smtp:
+    container_name: ${SMTP_CONTAINER_NAME}
+    image: mailhog/mailhog
+    ports:
+      - "1025:1025"
+      - "8025:8025"
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8025/"]

ckan-backend-dev/docker-compose.test.yml

@@ -41,15 +41,30 @@ services:
     build:
       context: ../deployment/frontend/
       args:
-        - NEXTAUTH_SECRET=secret
-        - NEXTAUTH_URL=http://localhost:3000
-        - CKAN_URL=http://ckan-dev:5000
+        - NEXT_PUBLIC_CKAN_URL=http://ckan-dev:5000
+    environment:
+      - NEXTAUTH_SECRET=secret
+      - NEXTAUTH_URL=http://localhost:3000
+      - CKAN_URL=http://ckan-dev:5000
+      - S3_ACCESS_KEY_ID=${MINIO_ROOT_USER}
+      - S3_SECRET_KEY_ID=${MINIO_ROOT_PASSWORD}
+      - S3_BUCKET_NAME=ckan
+      - S3_BUCKET_REGION=us-east-1
+      - RW_API_KEY=${RW_API_KEY}
+      - AZURE_AD_TENANT_ID=${AZURE_AD_TENANT_ID}
+      - AZURE_AD_CLIENT_ID=${AZURE_AD_CLIENT_ID}
+      - AZURE_AD_CLIENT_SECRET=${AZURE_AD_CLIENT_SECRET}
+      - SYS_ADMIN_API_KEY=${SYS_ADMIN_API_KEY}
+      - SMTP_SERVER=${SMTP_SERVER}
+      - SMTP_PORT=${SMTP_PORT}
+      - SMTP_USER=${SMTP_USER}
+      - SMTP_PASSWORD=${SMTP_PASSWORD}
+      - SMTP_FROM=${SMTP_FROM}
+    depends_on:
+      ckan-dev:
+        condition: service_healthy
     ports:
       - "0.0.0.0:3000:3000"
-    depends_on:
-      - ckan-dev
-    env_file:
-      - ".env.example"
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:3000"]