Merge branch 'master' into AV-2060_suomifi-brand-name

cdk/lib/ckan-stack.ts

@@ -38,6 +38,10 @@ export class CkanStack extends Stack {
       parameterName: `/${props.environment}/opendata/ckan/harvester_instruction_url`,
     });
 
+    const pCkanOrganizationapprovalEmail = ssm.StringParameter.fromStringParameterAttributes(this, 'pCkanOrganizationapprovalEmail', {
+      parameterName: `/${props.environment}/opendata/ckan/organizationapproval_email`,
+    });
+
     const host = props.databaseInstance.instanceEndpoint;
 
     const datastoreHost = props.datastoreInstance.instanceEndpoint;
@@ -258,6 +262,7 @@ export class CkanStack extends Stack {
       SENTRY_ENV: props.environment,
       CKAN_SYSADMIN_NAME: pSysadminUser.stringValue,
       CKAN_SYSADMIN_EMAIL: pSysadminEmail.stringValue,
+      ORGANIZATIONAPPROVAL_EMAIL: pCkanOrganizationapprovalEmail.stringValue,
       // fuseki
       FUSEKI_HOST: `fuseki.${props.namespace.namespaceName}`,
       FUSEKI_PORT: '3030',

cdk/lib/lambda-stack.ts

@@ -32,8 +32,8 @@ export class LambdaStack extends Stack {
 
     const sendToZulip = new SendToZulip(this, 'send-to-zulip', {
       zulipApiUser: '[email protected]',
-      zulipApiUrl: 'https://turina.dvv.fi',
-      zulipStream: 'avoindata.fi',
+      zulipApiUrl: 'turina.dvv.fi',
+      zulipStream: 'Avoindata.fi',
       zulipTopic: 'Container restarts',
       envProps: props.envProps,
       env: props.env,

cdk/lib/monitoring-stack.ts

@@ -26,7 +26,8 @@ export class MonitoringStack extends Stack {
       eventPattern: {
         source: ['aws.ecs'],
         detail: {
-          message: [{suffix: 'failed container health checks.'}]
+          desiredStatus: ['STOPPED'],
+          stoppedReason: [{wildcard: '*health*'}]
         }
       },
       targets: [sendToDeveloperZulipTarget, taskHealthCheckFailLogGroupTarget],

cdk/lib/send-to-zulip.function.ts

@@ -1,41 +1,76 @@
-import {Handler, APIGatewayEvent} from 'aws-lambda';
+import {Handler} from 'aws-lambda';
 import {GetSecretValueCommand, SecretsManagerClient} from "@aws-sdk/client-secrets-manager";
 import * as https from 'https';
+import FormData = require('form-data');
 
 const { ZULIP_API_URL, ZULIP_API_USER, ZULIP_API_KEY_SECRET, ZULIP_STREAM, ZULIP_TOPIC } = process.env;
 
-export const handler: Handler = async (event: APIGatewayEvent) => {
+function eventMessage(event: any) {
+  const {detail} = event;
+  if(detail?.eventName) {
+    // Generic event
+    const {resources} = event;
+    return `${detail?.eventName}: ${resources?.join(', ')}`;
+  } else if(detail?.stoppedReason) {
+    // Container stopped event
+    const {taskArn, group, stoppedReason} = detail;
+    return `${taskArn} (${group}): ${stoppedReason}`;
+  } else {
+    return 'Unknown message type';
+  }
+}
+export const handler: Handler = async (event: any) => {
+  if(!ZULIP_API_URL || !ZULIP_API_USER || !ZULIP_API_KEY_SECRET ||
+     !ZULIP_STREAM || !ZULIP_TOPIC) {
+    return {
+      statusCode: 500,
+      body: 'Missing configuration values',
+    }
+  }
+
   const secretsManagerClient = new SecretsManagerClient({region: "eu-west-1"});
   const command = new GetSecretValueCommand({
     SecretId: ZULIP_API_KEY_SECRET
   });
   const response = await secretsManagerClient.send(command);
   const zulipApiKey = response.SecretString;
 
+  const message = eventMessage(event);
+
+  const data = new FormData();
+  data.append('type', 'stream');
+  data.append('to', ZULIP_STREAM);
+  data.append('topic', ZULIP_TOPIC);
+  data.append('content', message);
+
   const options: https.RequestOptions = {
     hostname: ZULIP_API_URL,
     port: 443,
     path: '/api/v1/messages',
     method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      Authorization: `Basic ${Buffer.from(`${ZULIP_API_USER}:${zulipApiKey}`).toString('base64')}`,
-    },
+    auth: `${ZULIP_API_USER}:${zulipApiKey}`,
+    headers: data.getHeaders(),
   };
 
-  const { message } = JSON.parse(event.body || '');
-  const data = {
-    type: 'stream',
-    to: ZULIP_STREAM,
-    topic: ZULIP_TOPIC,
-    content: message,
-  };
-
-  https.request(options, (res: any) => {
-    console.log('Response from Zulip API:', res.statusCode);
-  }).on('error', (error: any) => {
-    console.error('Error sending message to Zulip:', error);
-  }).end(JSON.stringify(data));
+  await new Promise((resolve, reject) => {
+    const req = https.request(options, (res: any) => {
+      if(res.statusCode != 200) {
+        console.log('Response from Zulip API:', res.statusCode);
+        res.on('data', (chunk: any) => {
+          console.log(chunk.toString());
+        }).on('end', () => {
+          resolve(res);
+        });
+      } else {
+        resolve(res);
+      }
+    }).on('error', (error: any) => {
+      console.error('Error sending message to Zulip:', error);
+      reject(error);
+    });
+    data.pipe(req);
+    req.end();
+  });
 
   return {
     statusCode: 200,

cdk/lib/send-to-zulip.ts

@@ -9,7 +9,8 @@ export class SendToZulip extends Construct {
     super(scope, id);
 
     // Task restart zulip reporting
-    const zulipSecret = new sm.Secret(this, `/${props.environment}/opendata/common/zulip_api_key`);
+    const zulipSecret = sm.Secret.fromSecretNameV2(this, 'sZulipSecret', `/${props.environment}/zulip_api_key`);
+
     this.lambda = new NodejsFunction(this, 'function', {
       environment: {
         ZULIP_API_USER: props.zulipApiUser,

cdk/package.json

@@ -25,6 +25,7 @@
     "aws-cdk-lib": "^2.83.1",
     "constructs": "^10.0.0",
     "dotenv": "^10.0.0",
+    "form-data": "^4.0.0",
     "knex": "^2.4.2",
     "pg": "^8.11.0",
     "source-map-support": "^0.5.16"

ckan/templates/production.ini.j2

@@ -120,7 +120,7 @@ ckan.harvest.mq.hostname = {{ environ('REDIS_HOST') }}
 ckan.harvest.mq.port = {{ environ('REDIS_PORT') }}
 ckan.harvest.mq.redis_db = {{ environ('REDIS_DB') }}
 
-ckanext.organizationapproval.admin_email = {{ environ('SYSADMIN_EMAIL') }}
+ckanext.organizationapproval.admin_email = {{ environ('ORGANIZATIONAPPROVAL_EMAIL') }}
 ckanext.ytp_request.admin_email = {{ environ('SYSADMIN_EMAIL') }}
 
 ckanext-archiver.archive_dir = {{ environ('CKAN_ARCHIVER_PATH') }}

docker/.env.nginx.local

@@ -4,6 +4,6 @@ NGINX_ROOT=/var/www/html
 NGINX_MAX_BODY_SIZE=5000M
 NGINX_EXPIRES=1h
 NGINX_CSP_DEFAULT_SRC=""
-NGINX_CSP_SCRIPT_SRC="platform.twitter.com syndication.twitter.com cdn.syndication.twimg.com https://www.google.com/recaptcha/ https://www.gstatic.com/ https://www.google.com cdn.matomo.cloud suomi.matomo.cloud browser.sentry-cdn.com"
+NGINX_CSP_SCRIPT_SRC="platform.twitter.com syndication.twitter.com cdn.syndication.twimg.com https://www.google.com/recaptcha/ https://www.gstatic.com/ https://www.google.com cdn.matomo.cloud suomi.matomo.cloud browser.sentry-cdn.com https://unpkg.com/@ckeditor/ckeditor5-inspector/build/inspector.js"
 NGINX_CSP_STYLE_SRC="https://fonts.googleapis.com https://platform.twitter.com https://ton.twimg.com https://www.google.com https://ajax.googleapis.com https://www.gstatic.com"
 NGINX_CSP_FRAME_SRC="syndication.twitter.com https://platform.twitter.com https://www.google.com/recaptcha/"

docker/.env.template

@@ -3,8 +3,8 @@ REGISTRY=""
 REPOSITORY=""
 
 # opendata images
-CKAN_IMAGE_TAG="8dc72f94000aea3be4fb11962d7d7885380336ad"
-DRUPAL_IMAGE_TAG="8dc72f94000aea3be4fb11962d7d7885380336ad"
+CKAN_IMAGE_TAG="dea73d1b66e194ccd535709e99d6753f31faa320"
+DRUPAL_IMAGE_TAG="95386bfb4bd93d3d5506827f91dace1a2ab03088"
 SOLR_IMAGE_TAG="acedaa3c1d35875178b6d104fe413bef0bbdd155"
 NGINX_IMAGE_TAG="cc823d149f1f80af4354a6f38973cec9fb994cc1"
 DATAPUSHER_IMAGE_TAG="37245d2426ed6218a7352115d4147139defc9d8e"
@@ -50,21 +50,16 @@ SENTRY_ENV="local"
 
 # drupal + ckan roles
 ROLES_CKAN_ADMIN=ckan_admin
-ROLES_EDITOR=editor
 ROLES_PUBLISHER=publisher
 
 # initial users
-USERS_0_USER=test-editor
-USERS_0_PASS=test-editor
-USERS_0_EMAIL=editor@localhost
-USERS_0_ROLES="ckan_admin editor"
-USERS_1_USER=test-publisher
-USERS_1_PASS=test-publisher
-USERS_1_EMAIL=publisher@localhost
-USERS_1_ROLES="publisher"
-USERS_2_USER=test-user
-USERS_2_PASS=test-user
-USERS_2_EMAIL=user@localhost
+USERS_0_USER=test-publisher
+USERS_0_PASS=test-publisher
+USERS_0_EMAIL=publisher@localhost
+USERS_0_ROLES="publisher"
+USERS_1_USER=test-user
+USERS_1_PASS=test-user
+USERS_1_EMAIL=user@localhost
 
 # postgres
 DB_CKAN_HOST=postgres
@@ -101,6 +96,7 @@ SOLR_PATH=solr/ckan
 # ckan
 CKAN_HOST=ckan
 CKAN_PORT=5000
+ORGANIZATIONAPPROVAL_EMAIL=admin@localhost
 
 # datapusher
 DATAPUSHER_HOST=datapusher