Merge branch 'master' into AV-2064_remove-resource-proxy

vrk-kpa · Oct 3, 2023 · e90ffac · e90ffac
2 parents 0adef75 + 2b4e931
commit e90ffac
Show file tree

Hide file tree

Showing 6 changed files with 109 additions and 300 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -197,7 +197,20 @@ jobs:
 
       - run: npm ci
 
-      - name: configure environment
+      - name: configure AWS credentials for running tests
+        uses: aws-actions/configure-aws-credentials@v4
+        id: aws-credentials
+        with:
+          role-to-assume: ${{ secrets.AWS_TEST_ROLE }}
+          role-session-name: github-actions
+          aws-region: eu-west-1
+          output-credentials: true
+
+      - name: get caller identity
+        run: |
+          aws sts get-caller-identity
+
+      - name: configure Cypress
         shell: bash
         run: |
           # configure cypress
@@ -226,24 +239,37 @@ jobs:
             },
           })
           EOT
-          # configure docker
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: configure .env
+        shell: bash
+        run: |
           cp -f docker/.env.template docker/.env
-          sed -i.bak -E 's/^(REGISTRY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${REGISTRY}"'\"/' docker/.env
-          sed -i.bak -E 's/^(REPOSITORY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${REPOSITORY}"'\"/' docker/.env
-          sed -i.bak -E 's/^(MATOMO_ENABLED[[:blank:]]*=[[:blank:]]*).*/\1false/' docker/.env
-          sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_ENABLED[[:blank:]]*=[[:blank:]]*).*/\1\"'"${CKAN_CLOUDSTORAGE_ENABLED}"'\"/' docker/.env.ckan.local
-          sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_DRIVER_OPTIONS[[:blank:]]*=[[:blank:]]*).*/\1\"'"{'key': '${AWS_ACCESS_KEY_ID}', 'secret': '${AWS_SECRET_ACCESS_KEY}', 'token': ''}"'\"/' docker/.env.ckan.local
-          sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_CONTAINER_NAME[[:blank:]]*=[[:blank:]]*).*/\1\"'"${CKAN_CLOUDSTORAGE_CONTAINER_NAME}"'\"/' docker/.env.ckan.local
-          sed -i.bak -E 's/^(AWS_ACCESS_KEY_ID[[:blank:]]*=[[:blank:]]*).*/\1\"'"${AWS_ACCESS_KEY_ID}"'\"/' docker/.env.ckan.local
-          sed -i.bak -E 's/^(AWS_SECRET_ACCESS_KEY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${AWS_SECRET_ACCESS_KEY}"'\"/' docker/.env.ckan.local
+          pip install "python-dotenv[cli]"
+          dotenv -f docker/.env set REGISTRY ${REGISTRY}
+          dotenv -f docker/.env set REPOSITORY ${REPOSITORY}
+          dotenv -f docker/.env set MATOMO_ENABLED false
         env:
           REGISTRY: ${{ secrets.REGISTRY }}
           REPOSITORY: ${{ secrets.REPOSITORY }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          CKAN_CLOUDSTORAGE_ENABLED: ${{ secrets.CKAN_CLOUDSTORAGE_ENABLED }}
-          CKAN_CLOUDSTORAGE_CONTAINER_NAME: ${{ secrets.CKAN_CLOUDSTORAGE_CONTAINER_NAME }}
 
+      - name: configure .env.ckan.local
+        shell: bash
+        run: |
+          dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_ENABLED ${CKAN_CLOUDSTORAGE_ENABLED}
+          dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_DRIVER_OPTIONS "{'key': '${AWS_ACCESS_KEY_ID}', 'secret': '${AWS_SECRET_ACCESS_KEY}', 'token': '${AWS_SESSION_TOKEN}' }"
+          dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_CONTAINER_NAME ${CKAN_CLOUDSTORAGE_CONTAINER_NAME}
+          dotenv -f docker/.env.ckan.local set AWS_ACCESS_KEY_ID ${AWS_ACCESS_KEY_ID}
+          dotenv -f docker/.env.ckan.local set AWS_SECRET_ACCESS_KEY ${AWS_SECRET_ACCESS_KEY}
+        env:
+          CKAN_CLOUDSTORAGE_ENABLED: true
+          CKAN_CLOUDSTORAGE_CONTAINER_NAME: ${{ secrets.CKAN_CLOUDSTORAGE_CONTAINER_NAME }}
+          AWS_ACCESS_KEY_ID: ${{ steps.aws-credentials.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws-credentials.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws-credentials.outputs.aws-session-token }}
 
       - name: configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1-node16

diff --git a/cdk/bin/opendata.ts b/cdk/bin/opendata.ts
@@ -16,6 +16,7 @@ import {CertificateStack} from "../lib/certificate-stack";
 import {BypassCdnStack} from "../lib/bypass-cdn-stack";
 import {MonitoringStack} from "../lib/monitoring-stack";
 import {LambdaStack} from "../lib/lambda-stack";
+import {CiTestStack} from "../lib/ci-test-stack";
 
 // load .env file, shared with docker setup
 // mainly for ECR repo and image tag information
@@ -39,290 +40,6 @@ const envProps: EnvProps = {
   FUSEKI_IMAGE_TAG: parseEnv('FUSEKI_IMAGE_TAG'),
 };
 
-//
-// infratest env
-//
-
-const infratestProps = {
-  account: '156418131626',
-  region: 'eu-west-1',
-  environment: 'infratest',
-  fqdn: 'betaavoindata.fi',
-  secondaryFqdn: 'betaopendata.fi',
-  domainName: 'infratest.betaavoindata.fi',
-  secondaryDomainName: 'infratest.betaopendata.fi',
-};
-
-const clusterStackInfratest = new ClusterStack(app, 'ClusterStack-infratest', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName,
-});
-
-const backupStackInfratest = new BackupStack(app, 'BackupStack-infratest', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  domainName: infratestProps.domainName,
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryDomainName: infratestProps.secondaryDomainName,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  backups: false,
-  importVault: false
-})
-
-const fileSystemStackInfratest = new FileSystemStack(app, 'FileSystemStack-infratest', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName,
-  vpc: clusterStackInfratest.vpc,
-  backups: false,
-  backupPlan: backupStackInfratest.backupPlan,
-  importMigrationFs: true,
-});
-
-const databaseStackInfratest = new DatabaseStack(app, 'DatabaseStack-infratest', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName,
-  vpc: clusterStackInfratest.vpc,
-  backups: false,
-  backupPlan: backupStackInfratest.backupPlan,
-  multiAz: false
-});
-
-const lambdaStackInfratest = new LambdaStack(app, 'LambdaStack-infra', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName,
-  datastoreInstance: databaseStackInfratest.datastoreInstance,
-  datastoreCredentials: databaseStackInfratest.datastoreCredentials,
-  vpc: clusterStackInfratest.vpc
-})
-
-
-const certificateStackInfratest = new CertificateStack(app, 'CertificateStack-infra', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName
-})
-
-const certificateStackForCLoudfrontInfratest = new CertificateStack(app, 'CertificateStackForCloudfront-infra', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: 'us-east-1',
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName
-})
-
-const loadBalancerStackInfratest = new LoadBalancerStack(app, 'LoadBalancerStackInfratest-infratest', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName,
-  vpc: clusterStackInfratest.vpc
-});
-
-const cacheStackInfratest = new CacheStack(app, 'CacheStack-infratest', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName,
-  vpc: clusterStackInfratest.vpc,
-  cacheNodeType: 'cache.t3.small',
-  cacheEngineVersion: '6.x',
-  cacheNumNodes: 1,
-});
-
-const ckanStackInfratest = new CkanStack(app, 'CkanStack-infratest', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName,
-  vpc: clusterStackInfratest.vpc,
-  cluster: clusterStackInfratest.cluster,
-  namespace: clusterStackInfratest.namespace,
-  fileSystems: {
-    'ckan': fileSystemStackInfratest.ckanFs,
-    'solr': fileSystemStackInfratest.solrFs,
-    'fuseki': fileSystemStackInfratest.fusekiFs,
-  },
-  databaseSecurityGroup: databaseStackInfratest.databaseSecurityGroup,
-  databaseInstance: databaseStackInfratest.databaseInstance,
-  datastoreInstance: databaseStackInfratest.datastoreInstance,
-  datastoreCredentials: databaseStackInfratest.datastoreCredentials,
-  datastoreJobsCredentials: lambdaStackInfratest.datastoreJobsCredentials,
-  datastoreReadCredentials: lambdaStackInfratest.datastoreReadCredentials,
-  datastoreUserCredentials: lambdaStackInfratest.datastoreUserCredentials,
-  datastoreSecurityGroup: databaseStackInfratest.datastoreSecurityGroup,
-  cachePort: cacheStackInfratest.cachePort,
-  cacheSecurityGroup: cacheStackInfratest.cacheSecurityGroup,
-  cacheCluster: cacheStackInfratest.cacheCluster,
-  captchaEnabled: false,
-  analyticsEnabled: false,
-  ckanTaskDef: {
-    taskCpu: 512,
-    taskMem: 1024,
-    taskMinCapacity: 1,
-    taskMaxCapacity: 2,
-  },
-  ckanCronTaskDef: {
-    taskCpu: 512,
-    taskMem: 1024,
-    taskMinCapacity: 0,
-    taskMaxCapacity: 1,
-  },
-  datapusherTaskDef: {
-    taskCpu: 512,
-    taskMem: 1024,
-    taskMinCapacity: 1,
-    taskMaxCapacity: 2,
-  },
-  solrTaskDef: {
-    taskCpu: 512,
-    taskMem: 1024,
-    taskMinCapacity: 0,
-    taskMaxCapacity: 1,
-  },
-  fusekiTaskDef: {
-    taskCpu: 512,
-    taskMem: 1024,
-    taskMinCapacity: 0,
-    taskMaxCapacity: 1,
-  },
-  ckanCronEnabled: false,
-  archiverSendNotificationEmailsToMaintainers: false,
-  archiverExemptDomainsFromBrokenLinkNotifications: [],
-  cloudstorageEnabled: true
-});
-
-const drupalStackInfratest = new DrupalStack(app, 'DrupalStack-infratest', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName,
-  vpc: clusterStackInfratest.vpc,
-  cluster: clusterStackInfratest.cluster,
-  namespace: clusterStackInfratest.namespace,
-  fileSystems: {
-    'drupal': fileSystemStackInfratest.drupalFs,
-  },
-  databaseSecurityGroup: databaseStackInfratest.databaseSecurityGroup,
-  databaseInstance: databaseStackInfratest.databaseInstance,
-  cachePort: cacheStackInfratest.cachePort,
-  cacheSecurityGroup: cacheStackInfratest.cacheSecurityGroup,
-  cacheCluster: cacheStackInfratest.cacheCluster,
-  captchaEnabled: false,
-  analyticsEnabled: false,
-  drupalTaskDef: {
-    taskCpu: 512,
-    taskMem: 1024,
-    taskMinCapacity: 1,
-    taskMaxCapacity: 2,
-  }
-});
-
-const webStackInfratest = new WebStack(app, 'WebStack-infratest', {
-  envProps: envProps,
-  env: {
-    account: infratestProps.account,
-    region: infratestProps.region,
-  },
-  environment: infratestProps.environment,
-  fqdn: infratestProps.fqdn,
-  secondaryFqdn: infratestProps.secondaryFqdn,
-  domainName: infratestProps.domainName,
-  secondaryDomainName: infratestProps.secondaryDomainName,
-  vpc: clusterStackInfratest.vpc,
-  cluster: clusterStackInfratest.cluster,
-  namespace: clusterStackInfratest.namespace,
-  fileSystems: {
-    'drupal': fileSystemStackInfratest.drupalFs,
-  },
-  databaseSecurityGroup: databaseStackInfratest.databaseSecurityGroup,
-  databaseInstance: databaseStackInfratest.databaseInstance,
-  cachePort: cacheStackInfratest.cachePort,
-  cacheSecurityGroup: cacheStackInfratest.cacheSecurityGroup,
-  cacheCluster: cacheStackInfratest.cacheCluster,
-  certificate: certificateStackInfratest.certificate,
-  loadBalancer: loadBalancerStackInfratest.loadBalancer,
-  nginxTaskDef: {
-    taskCpu: 512,
-    taskMem: 1024,
-    taskMinCapacity: 1,
-    taskMaxCapacity: 2,
-  },
-  drupalService: drupalStackInfratest.drupalService,
-  ckanService: ckanStackInfratest.ckanService,
-  allowRobots: 'false',
-});
 
 //
 // beta env
@@ -969,3 +686,14 @@ const monitoringStackProd = new MonitoringStack(app, 'MonitoringStack-prod', {
   domainName: prodProps.domainName,
   secondaryDomainName: prodProps.secondaryDomainName,
 });
+
+
+const ciTestStackBeta = new CiTestStack(app, 'CiTestStack-beta', {
+  env: {
+    account: betaProps.account,
+    region: betaProps.region
+  },
+  githubOrg: "vrk-kpa",
+  githubRepo: "opendata",
+  testBucketName: "avoindata-ci-test-bucket"
+})
diff --git a/cdk/lib/ci-test-stack-props.ts b/cdk/lib/ci-test-stack-props.ts
@@ -0,0 +1,7 @@
+import {StackProps} from "aws-cdk-lib";
+
+export interface CiTestStackProps extends StackProps {
+  testBucketName: string,
+  githubOrg: string,
+  githubRepo: string
+}