diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca4114e13c..5890e20538 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -197,7 +197,20 @@ jobs: - run: npm ci - - name: configure environment + - name: configure AWS credentials for running tests + uses: aws-actions/configure-aws-credentials@v4 + id: aws-credentials + with: + role-to-assume: ${{ secrets.AWS_TEST_ROLE }} + role-session-name: github-actions + aws-region: eu-west-1 + output-credentials: true + + - name: get caller identity + run: | + aws sts get-caller-identity + + - name: configure Cypress shell: bash run: | # configure cypress @@ -226,24 +239,37 @@ jobs: }, }) EOT - # configure docker + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: configure .env + shell: bash + run: | cp -f docker/.env.template docker/.env - sed -i.bak -E 's/^(REGISTRY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${REGISTRY}"'\"/' docker/.env - sed -i.bak -E 's/^(REPOSITORY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${REPOSITORY}"'\"/' docker/.env - sed -i.bak -E 's/^(MATOMO_ENABLED[[:blank:]]*=[[:blank:]]*).*/\1false/' docker/.env - sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_ENABLED[[:blank:]]*=[[:blank:]]*).*/\1\"'"${CKAN_CLOUDSTORAGE_ENABLED}"'\"/' docker/.env.ckan.local - sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_DRIVER_OPTIONS[[:blank:]]*=[[:blank:]]*).*/\1\"'"{'key': '${AWS_ACCESS_KEY_ID}', 'secret': '${AWS_SECRET_ACCESS_KEY}', 'token': ''}"'\"/' docker/.env.ckan.local - sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_CONTAINER_NAME[[:blank:]]*=[[:blank:]]*).*/\1\"'"${CKAN_CLOUDSTORAGE_CONTAINER_NAME}"'\"/' docker/.env.ckan.local - sed -i.bak -E 's/^(AWS_ACCESS_KEY_ID[[:blank:]]*=[[:blank:]]*).*/\1\"'"${AWS_ACCESS_KEY_ID}"'\"/' docker/.env.ckan.local - sed -i.bak -E 's/^(AWS_SECRET_ACCESS_KEY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${AWS_SECRET_ACCESS_KEY}"'\"/' docker/.env.ckan.local + pip install "python-dotenv[cli]" + dotenv -f docker/.env set REGISTRY ${REGISTRY} + dotenv -f docker/.env set REPOSITORY ${REPOSITORY} + dotenv -f docker/.env set MATOMO_ENABLED false env: REGISTRY: ${{ secrets.REGISTRY }} REPOSITORY: ${{ secrets.REPOSITORY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - CKAN_CLOUDSTORAGE_ENABLED: ${{ secrets.CKAN_CLOUDSTORAGE_ENABLED }} - CKAN_CLOUDSTORAGE_CONTAINER_NAME: ${{ secrets.CKAN_CLOUDSTORAGE_CONTAINER_NAME }} + - name: configure .env.ckan.local + shell: bash + run: | + dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_ENABLED ${CKAN_CLOUDSTORAGE_ENABLED} + dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_DRIVER_OPTIONS "{'key': '${AWS_ACCESS_KEY_ID}', 'secret': '${AWS_SECRET_ACCESS_KEY}', 'token': '${AWS_SESSION_TOKEN}' }" + dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_CONTAINER_NAME ${CKAN_CLOUDSTORAGE_CONTAINER_NAME} + dotenv -f docker/.env.ckan.local set AWS_ACCESS_KEY_ID ${AWS_ACCESS_KEY_ID} + dotenv -f docker/.env.ckan.local set AWS_SECRET_ACCESS_KEY ${AWS_SECRET_ACCESS_KEY} + env: + CKAN_CLOUDSTORAGE_ENABLED: true + CKAN_CLOUDSTORAGE_CONTAINER_NAME: ${{ secrets.CKAN_CLOUDSTORAGE_CONTAINER_NAME }} + AWS_ACCESS_KEY_ID: ${{ steps.aws-credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.aws-credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.aws-credentials.outputs.aws-session-token }} - name: configure AWS credentials uses: aws-actions/configure-aws-credentials@v1-node16 diff --git a/cdk/bin/opendata.ts b/cdk/bin/opendata.ts index f3e0967689..5513487915 100644 --- a/cdk/bin/opendata.ts +++ b/cdk/bin/opendata.ts @@ -16,6 +16,7 @@ import {CertificateStack} from "../lib/certificate-stack"; import {BypassCdnStack} from "../lib/bypass-cdn-stack"; import {MonitoringStack} from "../lib/monitoring-stack"; import {LambdaStack} from "../lib/lambda-stack"; +import {CiTestStack} from "../lib/ci-test-stack"; // load .env file, shared with docker setup // mainly for ECR repo and image tag information @@ -39,290 +40,6 @@ const envProps: EnvProps = { FUSEKI_IMAGE_TAG: parseEnv('FUSEKI_IMAGE_TAG'), }; -// -// infratest env -// - -const infratestProps = { - account: '156418131626', - region: 'eu-west-1', - environment: 'infratest', - fqdn: 'betaavoindata.fi', - secondaryFqdn: 'betaopendata.fi', - domainName: 'infratest.betaavoindata.fi', - secondaryDomainName: 'infratest.betaopendata.fi', -}; - -const clusterStackInfratest = new ClusterStack(app, 'ClusterStack-infratest', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName, -}); - -const backupStackInfratest = new BackupStack(app, 'BackupStack-infratest', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - domainName: infratestProps.domainName, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryDomainName: infratestProps.secondaryDomainName, - secondaryFqdn: infratestProps.secondaryFqdn, - backups: false, - importVault: false -}) - -const fileSystemStackInfratest = new FileSystemStack(app, 'FileSystemStack-infratest', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName, - vpc: clusterStackInfratest.vpc, - backups: false, - backupPlan: backupStackInfratest.backupPlan, - importMigrationFs: true, -}); - -const databaseStackInfratest = new DatabaseStack(app, 'DatabaseStack-infratest', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName, - vpc: clusterStackInfratest.vpc, - backups: false, - backupPlan: backupStackInfratest.backupPlan, - multiAz: false -}); - -const lambdaStackInfratest = new LambdaStack(app, 'LambdaStack-infra', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName, - datastoreInstance: databaseStackInfratest.datastoreInstance, - datastoreCredentials: databaseStackInfratest.datastoreCredentials, - vpc: clusterStackInfratest.vpc -}) - - -const certificateStackInfratest = new CertificateStack(app, 'CertificateStack-infra', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName -}) - -const certificateStackForCLoudfrontInfratest = new CertificateStack(app, 'CertificateStackForCloudfront-infra', { - envProps: envProps, - env: { - account: infratestProps.account, - region: 'us-east-1', - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName -}) - -const loadBalancerStackInfratest = new LoadBalancerStack(app, 'LoadBalancerStackInfratest-infratest', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName, - vpc: clusterStackInfratest.vpc -}); - -const cacheStackInfratest = new CacheStack(app, 'CacheStack-infratest', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName, - vpc: clusterStackInfratest.vpc, - cacheNodeType: 'cache.t3.small', - cacheEngineVersion: '6.x', - cacheNumNodes: 1, -}); - -const ckanStackInfratest = new CkanStack(app, 'CkanStack-infratest', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName, - vpc: clusterStackInfratest.vpc, - cluster: clusterStackInfratest.cluster, - namespace: clusterStackInfratest.namespace, - fileSystems: { - 'ckan': fileSystemStackInfratest.ckanFs, - 'solr': fileSystemStackInfratest.solrFs, - 'fuseki': fileSystemStackInfratest.fusekiFs, - }, - databaseSecurityGroup: databaseStackInfratest.databaseSecurityGroup, - databaseInstance: databaseStackInfratest.databaseInstance, - datastoreInstance: databaseStackInfratest.datastoreInstance, - datastoreCredentials: databaseStackInfratest.datastoreCredentials, - datastoreJobsCredentials: lambdaStackInfratest.datastoreJobsCredentials, - datastoreReadCredentials: lambdaStackInfratest.datastoreReadCredentials, - datastoreUserCredentials: lambdaStackInfratest.datastoreUserCredentials, - datastoreSecurityGroup: databaseStackInfratest.datastoreSecurityGroup, - cachePort: cacheStackInfratest.cachePort, - cacheSecurityGroup: cacheStackInfratest.cacheSecurityGroup, - cacheCluster: cacheStackInfratest.cacheCluster, - captchaEnabled: false, - analyticsEnabled: false, - ckanTaskDef: { - taskCpu: 512, - taskMem: 1024, - taskMinCapacity: 1, - taskMaxCapacity: 2, - }, - ckanCronTaskDef: { - taskCpu: 512, - taskMem: 1024, - taskMinCapacity: 0, - taskMaxCapacity: 1, - }, - datapusherTaskDef: { - taskCpu: 512, - taskMem: 1024, - taskMinCapacity: 1, - taskMaxCapacity: 2, - }, - solrTaskDef: { - taskCpu: 512, - taskMem: 1024, - taskMinCapacity: 0, - taskMaxCapacity: 1, - }, - fusekiTaskDef: { - taskCpu: 512, - taskMem: 1024, - taskMinCapacity: 0, - taskMaxCapacity: 1, - }, - ckanCronEnabled: false, - archiverSendNotificationEmailsToMaintainers: false, - archiverExemptDomainsFromBrokenLinkNotifications: [], - cloudstorageEnabled: true -}); - -const drupalStackInfratest = new DrupalStack(app, 'DrupalStack-infratest', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName, - vpc: clusterStackInfratest.vpc, - cluster: clusterStackInfratest.cluster, - namespace: clusterStackInfratest.namespace, - fileSystems: { - 'drupal': fileSystemStackInfratest.drupalFs, - }, - databaseSecurityGroup: databaseStackInfratest.databaseSecurityGroup, - databaseInstance: databaseStackInfratest.databaseInstance, - cachePort: cacheStackInfratest.cachePort, - cacheSecurityGroup: cacheStackInfratest.cacheSecurityGroup, - cacheCluster: cacheStackInfratest.cacheCluster, - captchaEnabled: false, - analyticsEnabled: false, - drupalTaskDef: { - taskCpu: 512, - taskMem: 1024, - taskMinCapacity: 1, - taskMaxCapacity: 2, - } -}); - -const webStackInfratest = new WebStack(app, 'WebStack-infratest', { - envProps: envProps, - env: { - account: infratestProps.account, - region: infratestProps.region, - }, - environment: infratestProps.environment, - fqdn: infratestProps.fqdn, - secondaryFqdn: infratestProps.secondaryFqdn, - domainName: infratestProps.domainName, - secondaryDomainName: infratestProps.secondaryDomainName, - vpc: clusterStackInfratest.vpc, - cluster: clusterStackInfratest.cluster, - namespace: clusterStackInfratest.namespace, - fileSystems: { - 'drupal': fileSystemStackInfratest.drupalFs, - }, - databaseSecurityGroup: databaseStackInfratest.databaseSecurityGroup, - databaseInstance: databaseStackInfratest.databaseInstance, - cachePort: cacheStackInfratest.cachePort, - cacheSecurityGroup: cacheStackInfratest.cacheSecurityGroup, - cacheCluster: cacheStackInfratest.cacheCluster, - certificate: certificateStackInfratest.certificate, - loadBalancer: loadBalancerStackInfratest.loadBalancer, - nginxTaskDef: { - taskCpu: 512, - taskMem: 1024, - taskMinCapacity: 1, - taskMaxCapacity: 2, - }, - drupalService: drupalStackInfratest.drupalService, - ckanService: ckanStackInfratest.ckanService, - allowRobots: 'false', -}); // // beta env @@ -969,3 +686,14 @@ const monitoringStackProd = new MonitoringStack(app, 'MonitoringStack-prod', { domainName: prodProps.domainName, secondaryDomainName: prodProps.secondaryDomainName, }); + + +const ciTestStackBeta = new CiTestStack(app, 'CiTestStack-beta', { + env: { + account: betaProps.account, + region: betaProps.region + }, + githubOrg: "vrk-kpa", + githubRepo: "opendata", + testBucketName: "avoindata-ci-test-bucket" +}) diff --git a/cdk/lib/ci-test-stack-props.ts b/cdk/lib/ci-test-stack-props.ts new file mode 100644 index 0000000000..e91532a071 --- /dev/null +++ b/cdk/lib/ci-test-stack-props.ts @@ -0,0 +1,7 @@ +import {StackProps} from "aws-cdk-lib"; + +export interface CiTestStackProps extends StackProps { + testBucketName: string, + githubOrg: string, + githubRepo: string +} diff --git a/cdk/lib/ci-test-stack.ts b/cdk/lib/ci-test-stack.ts new file mode 100644 index 0000000000..2f00ef40a0 --- /dev/null +++ b/cdk/lib/ci-test-stack.ts @@ -0,0 +1,39 @@ +import {aws_iam, aws_s3, Duration, Stack} from "aws-cdk-lib"; +import {Construct} from "constructs"; +import {CiTestStackProps} from "./ci-test-stack-props"; + +export class CiTestStack extends Stack { + constructor(scope: Construct, id: string, props: CiTestStackProps) { + super(scope, id, props); + + const testBucket = new aws_s3.Bucket(this,'TestBucket', { + bucketName: props.testBucketName, + blockPublicAccess: aws_s3.BlockPublicAccess.BLOCK_ALL, + lifecycleRules: [ + { + expiration: Duration.days(1) + } + ] + }) + + const oidcProviderArn = Stack.of(this).formatArn({ + region: "", + partition: "aws", + resource: "oidc-provider", + service: "iam", + resourceName: "token.actions.githubusercontent.com" + }) + + const testRole = new aws_iam.Role(this, 'TestRole', { + assumedBy: new aws_iam.WebIdentityPrincipal(oidcProviderArn, { + StringLike: { + "token.actions.githubusercontent.com:sub": `repo:${props.githubOrg}/${props.githubRepo}:*` + } + }) + }) + + testBucket.grantWrite(testRole) + testBucket.grantRead(testRole) + } + +} diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 9bde570c0c..8a0d630163 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -1,4 +1,4 @@ -# build args +# build args ARG SECRET_NPMRC # diff --git a/ckan/ckanext/ckanext-ytp_main/ckanext/ytp/resourcestatusplugin.py b/ckan/ckanext/ckanext-ytp_main/ckanext/ytp/resourcestatusplugin.py index d2290dc85e..c3b821463f 100644 --- a/ckan/ckanext/ckanext-ytp_main/ckanext/ytp/resourcestatusplugin.py +++ b/ckan/ckanext/ckanext-ytp_main/ckanext/ytp/resourcestatusplugin.py @@ -1,4 +1,5 @@ import ckan.plugins as p +import yaml from ckan.plugins.toolkit import config from ckan.lib.plugins import DefaultTranslation import ckan.logic as logic @@ -78,7 +79,15 @@ def __init__(self, resource): filename = os.path.basename(urlparse(resource.get('url')).path) object_key = 'resources/%s/%s' % (resource_id, filename) - s3 = boto3.client('s3') + driver_options = config.get('ckanext.cloudstorage.driver_options') + if driver_options: + driver_options = yaml.safe_load(driver_options) + s3 = boto3.client('s3', + aws_access_key_id=driver_options.get('key'), + aws_secret_access_key=driver_options.get('secret'), + aws_session_token=driver_options.get('token')) + else: + s3 = boto3.client('s3') tags_response = s3.get_object_tagging(Bucket=aws_bucket_name, Key=object_key) self.tags = {item['Key']: item['Value'] for item in tags_response['TagSet']}