Merge pull request #2063 from vrk-kpa/AV-2067_create_test_bucket_and_…

…role

AV-2067: create test bucket and role

.github/workflows/test.yml

@@ -197,7 +197,20 @@ jobs:
 
       - run: npm ci
 
-      - name: configure environment
+      - name: configure AWS credentials for running tests
+        uses: aws-actions/configure-aws-credentials@v4
+        id: aws-credentials
+        with:
+          role-to-assume: ${{ secrets.AWS_TEST_ROLE }}
+          role-session-name: github-actions
+          aws-region: eu-west-1
+          output-credentials: true
+
+      - name: get caller identity
+        run: |
+          aws sts get-caller-identity
+
+      - name: configure Cypress
         shell: bash
         run: |
           # configure cypress
@@ -226,24 +239,37 @@ jobs:
             },
           })
           EOT
-          # configure docker
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: configure .env
+        shell: bash
+        run: |
           cp -f docker/.env.template docker/.env
-          sed -i.bak -E 's/^(REGISTRY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${REGISTRY}"'\"/' docker/.env
-          sed -i.bak -E 's/^(REPOSITORY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${REPOSITORY}"'\"/' docker/.env
-          sed -i.bak -E 's/^(MATOMO_ENABLED[[:blank:]]*=[[:blank:]]*).*/\1false/' docker/.env
-          sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_ENABLED[[:blank:]]*=[[:blank:]]*).*/\1\"'"${CKAN_CLOUDSTORAGE_ENABLED}"'\"/' docker/.env.ckan.local
-          sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_DRIVER_OPTIONS[[:blank:]]*=[[:blank:]]*).*/\1\"'"{'key': '${AWS_ACCESS_KEY_ID}', 'secret': '${AWS_SECRET_ACCESS_KEY}', 'token': ''}"'\"/' docker/.env.ckan.local
-          sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_CONTAINER_NAME[[:blank:]]*=[[:blank:]]*).*/\1\"'"${CKAN_CLOUDSTORAGE_CONTAINER_NAME}"'\"/' docker/.env.ckan.local
-          sed -i.bak -E 's/^(AWS_ACCESS_KEY_ID[[:blank:]]*=[[:blank:]]*).*/\1\"'"${AWS_ACCESS_KEY_ID}"'\"/' docker/.env.ckan.local
-          sed -i.bak -E 's/^(AWS_SECRET_ACCESS_KEY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${AWS_SECRET_ACCESS_KEY}"'\"/' docker/.env.ckan.local
+          pip install "python-dotenv[cli]"
+          dotenv -f docker/.env set REGISTRY ${REGISTRY}
+          dotenv -f docker/.env set REPOSITORY ${REPOSITORY}
+          dotenv -f docker/.env set MATOMO_ENABLED false
         env:
           REGISTRY: ${{ secrets.REGISTRY }}
           REPOSITORY: ${{ secrets.REPOSITORY }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          CKAN_CLOUDSTORAGE_ENABLED: ${{ secrets.CKAN_CLOUDSTORAGE_ENABLED }}
-          CKAN_CLOUDSTORAGE_CONTAINER_NAME: ${{ secrets.CKAN_CLOUDSTORAGE_CONTAINER_NAME }}
 
+      - name: configure .env.ckan.local
+        shell: bash
+        run: |
+          dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_ENABLED ${CKAN_CLOUDSTORAGE_ENABLED}
+          dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_DRIVER_OPTIONS "{'key': '${AWS_ACCESS_KEY_ID}', 'secret': '${AWS_SECRET_ACCESS_KEY}', 'token': '${AWS_SESSION_TOKEN}' }"
+          dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_CONTAINER_NAME ${CKAN_CLOUDSTORAGE_CONTAINER_NAME}
+          dotenv -f docker/.env.ckan.local set AWS_ACCESS_KEY_ID ${AWS_ACCESS_KEY_ID}
+          dotenv -f docker/.env.ckan.local set AWS_SECRET_ACCESS_KEY ${AWS_SECRET_ACCESS_KEY}
+        env:
+          CKAN_CLOUDSTORAGE_ENABLED: true
+          CKAN_CLOUDSTORAGE_CONTAINER_NAME: ${{ secrets.CKAN_CLOUDSTORAGE_CONTAINER_NAME }}
+          AWS_ACCESS_KEY_ID: ${{ steps.aws-credentials.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws-credentials.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws-credentials.outputs.aws-session-token }}
 
       - name: configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1-node16

cdk/bin/opendata.ts

@@ -16,6 +16,7 @@ import {CertificateStack} from "../lib/certificate-stack";
 import {BypassCdnStack} from "../lib/bypass-cdn-stack";
 import {MonitoringStack} from "../lib/monitoring-stack";
 import {LambdaStack} from "../lib/lambda-stack";
+import {CiTestStack} from "../lib/ci-test-stack";
 
 // load .env file, shared with docker setup
 // mainly for ECR repo and image tag information
@@ -969,3 +970,14 @@ const monitoringStackProd = new MonitoringStack(app, 'MonitoringStack-prod', {
   domainName: prodProps.domainName,
   secondaryDomainName: prodProps.secondaryDomainName,
 });
+
+
+const ciTestStackBeta = new CiTestStack(app, 'CiTestStack-beta', {
+  env: {
+    account: betaProps.account,
+    region: betaProps.region
+  },
+  githubOrg: "vrk-kpa",
+  githubRepo: "opendata",
+  testBucketName: "avoindata-ci-test-bucket"
+})

cdk/lib/ci-test-stack-props.ts

@@ -0,0 +1,7 @@
+import {StackProps} from "aws-cdk-lib";
+
+export interface CiTestStackProps extends StackProps {
+  testBucketName: string,
+  githubOrg: string,
+  githubRepo: string
+}

cdk/lib/ci-test-stack.ts

@@ -0,0 +1,39 @@
+import {aws_iam, aws_s3, Duration, Stack} from "aws-cdk-lib";
+import {Construct} from "constructs";
+import {CiTestStackProps} from "./ci-test-stack-props";
+
+export class CiTestStack extends Stack {
+  constructor(scope: Construct, id: string, props: CiTestStackProps) {
+    super(scope, id, props);
+
+    const testBucket = new aws_s3.Bucket(this,'TestBucket', {
+      bucketName: props.testBucketName,
+      blockPublicAccess: aws_s3.BlockPublicAccess.BLOCK_ALL,
+      lifecycleRules: [
+        {
+          expiration: Duration.days(1)
+        }
+      ]
+    })
+
+    const oidcProviderArn = Stack.of(this).formatArn({
+        region: "",
+        partition: "aws",
+        resource: "oidc-provider",
+        service: "iam",
+        resourceName: "token.actions.githubusercontent.com"
+      })
+
+    const testRole = new aws_iam.Role(this, 'TestRole', {
+      assumedBy: new aws_iam.WebIdentityPrincipal(oidcProviderArn, {
+          StringLike: {
+            "token.actions.githubusercontent.com:sub": `repo:${props.githubOrg}/${props.githubRepo}:*`
+          }
+      })
+    })
+
+    testBucket.grantWrite(testRole)
+    testBucket.grantRead(testRole)
+  }
+
+}

ckan/Dockerfile

@@ -1,4 +1,4 @@
-# build args
+# build args 
 ARG SECRET_NPMRC
 
 #

ckan/ckanext/ckanext-ytp_main/ckanext/ytp/resourcestatusplugin.py

@@ -1,4 +1,5 @@
 import ckan.plugins as p
+import yaml
 from ckan.plugins.toolkit import config
 from ckan.lib.plugins import DefaultTranslation
 import ckan.logic as logic
@@ -78,7 +79,15 @@ def __init__(self, resource):
         filename = os.path.basename(urlparse(resource.get('url')).path)
         object_key = 'resources/%s/%s' % (resource_id, filename)
 
-        s3 = boto3.client('s3')
+        driver_options = config.get('ckanext.cloudstorage.driver_options')
+        if driver_options:
+            driver_options = yaml.safe_load(driver_options)
+            s3 = boto3.client('s3',
+                aws_access_key_id=driver_options.get('key'),
+                aws_secret_access_key=driver_options.get('secret'),
+                aws_session_token=driver_options.get('token'))
+        else:
+            s3 = boto3.client('s3')
         tags_response = s3.get_object_tagging(Bucket=aws_bucket_name, Key=object_key)
         self.tags = {item['Key']: item['Value'] for item in tags_response['TagSet']}