diff --git a/.env.defaults b/.env.defaults index 85039165..63c8a167 100644 --- a/.env.defaults +++ b/.env.defaults @@ -28,6 +28,9 @@ AWS_DEFAULT_REGION=us-west-2 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test +# Localstack environment defaults (needed for SQS routing) +LOCALSTACK_HOSTNAME=localhost + # Datadog environment defaults (note: RUM vars are not secret) DD_ENABLED=false DD_ENV=sandbox diff --git a/.gitignore b/.gitignore index be28e2a3..c685a7b4 100644 --- a/.gitignore +++ b/.gitignore @@ -146,6 +146,7 @@ terraform/.terraform .tool-versions localstack/volume/ +volume/ # Python virtual environments .venv/ @@ -165,3 +166,6 @@ python/dist/ # Generated Type files web/types/graphql.d.ts api/types/graphql.d.ts + +# Generated zips for local testing +scripts/local_treasury_file_email/*.zip diff --git a/api/src/lib/aws.ts b/api/src/lib/aws.ts index d58df665..b25517f8 100644 --- a/api/src/lib/aws.ts +++ b/api/src/lib/aws.ts @@ -196,9 +196,7 @@ function getSQSClient() { let sqs: SQSClient if (process.env.LOCALSTACK_HOSTNAME) { console.log('------------ USING LOCALSTACK FOR SQS ------------') - const endpoint = `http://${process.env.LOCALSTACK_HOSTNAME}:${ - process.env.EDGE_PORT || 4566 - }` + const endpoint = `http://sqs.us-west-2.localhost.localstack.cloud:4566/000000000000/treasury-email-queue` sqs = new SQSClient({ endpoint, region: process.env.AWS_DEFAULT_REGION }) } else { sqs = new SQSClient() diff --git a/api/src/services/uploads/uploads.test.ts b/api/src/services/uploads/uploads.test.ts index 11a8d4a9..05160119 100644 --- a/api/src/services/uploads/uploads.test.ts +++ b/api/src/services/uploads/uploads.test.ts @@ -4,7 +4,7 @@ import { deleteUploadFile, s3UploadFilePutSignedUrl, getSignedUrl, - startStepFunctionExecution, + sendSqsMessage, } from 'src/lib/aws' import { db } from 'src/lib/db' import { logger } from 'src/lib/logger' @@ -23,9 +23,6 @@ import { getUploadsByExpenditureCategory, getValidUploadsInCurrentPeriod, sendTreasuryReport, - SubrecipientLambdaPayload, - ProjectLambdaPayload, - CreateArchiveLambdaPayload, EmailLambdaPayload, } from './uploads' import type { StandardScenario } from './uploads.scenarios' @@ -42,7 +39,7 @@ jest.mock('src/lib/aws', () => ({ deleteUploadFile: jest.fn(), s3UploadFilePutSignedUrl: jest.fn(), getSignedUrl: jest.fn(), - startStepFunctionExecution: jest.fn(), + sendSqsMessage: jest.fn(), })) jest.mock('uuid', () => ({ v4: () => '00000000-0000-0000-0000-000000000000', @@ -379,6 +376,8 @@ describe('treasury report', () => { beforeEach(() => { jest.resetAllMocks() process.env.TREASURY_STEP_FUNCTION_ARN = 'test-arn' + process.env.TREASURY_EMAIL_SQS_URL = + 'https://sqs.us-east-1.amazon.com/fake_aws_account_key/fake_queue' }) scenario( @@ -387,124 +386,27 @@ describe('treasury report', () => { mockCurrentUser(scenario.user.one) const mockOrganization = scenario.organization.one const mockReportingPeriod = scenario.reportingPeriod.one - const mockUpload = scenario.upload.two const mockUser = scenario.user.one - const projectPayload: ProjectLambdaPayload = { - '1A': { - organization: { - id: mockOrganization.id, - preferences: { - current_reporting_period_id: mockReportingPeriod.id, - }, - }, - user: { - email: mockUser.email, - id: mockUser.id, - }, - outputTemplateId: mockReportingPeriod.outputTemplateId, - uploadsToAdd: { - [mockUpload.agencyId]: { - objectKey: `uploads/${mockOrganization.id}/${mockUpload.agencyId}/${mockReportingPeriod.id}/${mockUpload.id}/${mockUpload.filename}`, - createdAt: mockUpload.createdAt, - filename: mockUpload.filename, - }, - }, - uploadsToRemove: {}, - ProjectType: '1A', - }, - '1B': { - organization: { - id: mockOrganization.id, - preferences: { - current_reporting_period_id: mockReportingPeriod.id, - }, - }, - user: { - email: mockUser.email, - id: mockUser.id, - }, - outputTemplateId: mockReportingPeriod.outputTemplateId, - uploadsToAdd: {}, - uploadsToRemove: {}, - ProjectType: '1B', - }, - '1C': { - organization: { - id: mockOrganization.id, - preferences: { - current_reporting_period_id: mockReportingPeriod.id, - }, - }, - user: { - email: mockUser.email, - id: mockUser.id, - }, - outputTemplateId: mockReportingPeriod.outputTemplateId, - uploadsToAdd: {}, - uploadsToRemove: {}, - ProjectType: '1C', - }, - } - const subrecipientPayload: SubrecipientLambdaPayload = { - Subrecipient: { - organization: { - id: mockOrganization.id, - preferences: { - current_reporting_period_id: mockReportingPeriod.id, - }, - }, - user: { - email: mockUser.email, - id: mockUser.id, - }, - outputTemplateId: mockReportingPeriod.outputTemplateId, - }, - } - - const zipPayload: CreateArchiveLambdaPayload = { - zip: { - organization: { - id: mockOrganization.id, - preferences: { - current_reporting_period_id: mockReportingPeriod.id, - }, - }, - }, - } const emailPayload: EmailLambdaPayload = { - email: { - organization: { - id: mockOrganization.id, - preferences: { - current_reporting_period_id: mockReportingPeriod.id, - }, - }, - user: { - email: mockUser.email, - id: mockUser.id, + organization: { + id: mockOrganization.id, + preferences: { + current_reporting_period_id: mockReportingPeriod.id, }, }, + user: { + email: mockUser.email, + id: mockUser.id, + }, } - const input = JSON.stringify({ - '1A': {}, - '1B': {}, - '1C': {}, - Subrecipient: {}, - zip: {}, - email: {}, - ...projectPayload, - ...subrecipientPayload, - ...zipPayload, - ...emailPayload, - }) + const input = emailPayload const result = await sendTreasuryReport() expect(result).toBe(true) - expect(startStepFunctionExecution).toHaveBeenCalledWith( - 'test-arn', - `Force-kick-off-00000000-0000-0000-0000-000000000000`, + expect(sendSqsMessage).toHaveBeenCalledWith( + 'https://sqs.us-east-1.amazon.com/fake_aws_account_key/fake_queue', input ) } diff --git a/api/src/services/uploads/uploads.ts b/api/src/services/uploads/uploads.ts index 5de1f2d5..4e515402 100644 --- a/api/src/services/uploads/uploads.ts +++ b/api/src/services/uploads/uploads.ts @@ -6,7 +6,6 @@ import type { MutationResolvers, UploadRelationResolvers, } from 'types/graphql' -import { v4 as uuidv4 } from 'uuid' import { RedwoodError } from '@redwoodjs/api' @@ -16,7 +15,7 @@ import { s3UploadFilePutSignedUrl, getSignedUrl, getS3UploadFileKey, - startStepFunctionExecution, + sendSqsMessage, } from 'src/lib/aws' import { ROLES } from 'src/lib/constants' import { db } from 'src/lib/db' @@ -212,7 +211,7 @@ export type SubrecipientLambdaPayload = Record< InfoForSubrecipient > export type CreateArchiveLambdaPayload = Record<'zip', InfoForArchive> -export type EmailLambdaPayload = Record<'email', InfoForEmail> +export type EmailLambdaPayload = InfoForEmail export const getUploadsByExpenditureCategory = async ( organization: Organization, @@ -348,19 +347,17 @@ export const getEmailLambdaPayload = async ( user: CurrentUser ): Promise => { return { - email: { - organization: { - id: organization.id, - preferences: { - current_reporting_period_id: - organization.preferences['current_reporting_period_id'], - }, - }, - user: { - email: user.email, - id: user.id, + organization: { + id: organization.id, + preferences: { + current_reporting_period_id: + organization.preferences['current_reporting_period_id'], }, }, + user: { + email: user.email, + id: user.id, + }, } } @@ -370,41 +367,12 @@ export const sendTreasuryReport: MutationResolvers['sendTreasuryReport'] = const organization = await db.organization.findFirst({ where: { id: context.currentUser.agency.organizationId }, }) - const reportingPeriod = await db.reportingPeriod.findFirst({ - where: { id: organization.preferences['current_reporting_period_id'] }, - }) - const projectLambdaPayload: ProjectLambdaPayload = - await getUploadsByExpenditureCategory(organization, reportingPeriod) - const subrecipientLambdaPayload: SubrecipientLambdaPayload = - await getSubrecipientLambdaPayload( - organization, - context.currentUser, - reportingPeriod - ) - const createArchiveLambdaPayload: CreateArchiveLambdaPayload = - await getCreateArchiveLambdaPayload(organization) - const emailLambdaPayload: EmailLambdaPayload = await getEmailLambdaPayload(organization, context.currentUser) - const input = { - '1A': {}, - '1B': {}, - '1C': {}, - Subrecipient: {}, - zip: {}, - email: {}, - ...projectLambdaPayload, - ...subrecipientLambdaPayload, - ...createArchiveLambdaPayload, - ...emailLambdaPayload, - } - - await startStepFunctionExecution( - process.env.TREASURY_STEP_FUNCTION_ARN, - `Force-kick-off-${uuidv4()}`, - JSON.stringify(input) - ) + const input = emailLambdaPayload + + await sendSqsMessage(process.env.TREASURY_EMAIL_SQS_URL, input) return true } catch (error) { logger.error(error, 'Error sending Treasury Report') diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ddee704e..9e5e5b01 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -17,6 +17,10 @@ services: - db networks: - redwood + - ls + dns: + # Set the DNS server to be the LocalStack container + - 10.0.2.20 environment: - DATABASE_URL=postgresql://redwood:redwood@db:5432/redwood - TEST_DATABASE_URL=postgresql://redwood:redwood@db:5432/redwood_test @@ -91,7 +95,10 @@ services: - DOCKER_HOST=unix:///var/run/docker.sock - AWS_DEFAULT_REGION=${AWS_REGION:-us-west-2} networks: - - redwood + redwood: + ls: + # Set the container IP address in the 10.0.2.0/24 subnet + ipv4_address: 10.0.2.20 volumes: - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - "/var/run/docker.sock:/var/run/docker.sock" @@ -100,6 +107,11 @@ services: networks: redwood: driver: bridge + ls: + ipam: + config: + # Specify the subnet range for IP address allocation + - subnet: 10.0.2.0/24 volumes: node_modules: diff --git a/localstack/entrypoint/init-aws.sh b/localstack/entrypoint/init-aws.sh index 88f11dfd..8e09bedf 100755 --- a/localstack/entrypoint/init-aws.sh +++ b/localstack/entrypoint/init-aws.sh @@ -13,3 +13,4 @@ for email in "${VALID_EMAILS[@]}"; do done awslocal s3api create-bucket --bucket cpf-reporter --region us-west-2 --create-bucket-configuration '{"LocationConstraint": "us-west-2"}' +awslocal sqs create-queue --queue-name treasury-email-queue diff --git a/python/src/functions/generate_presigned_url_and_send_email.py b/python/src/functions/generate_presigned_url_and_send_email.py index 78d98837..085c32bd 100644 --- a/python/src/functions/generate_presigned_url_and_send_email.py +++ b/python/src/functions/generate_presigned_url_and_send_email.py @@ -1,5 +1,6 @@ +import json import os -from typing import Any, Optional, Tuple +from typing import Any, Dict, Optional, Tuple import boto3 import chevron @@ -28,7 +29,7 @@ class SendTreasuryEmailLambdaPayload(BaseModel): @reset_contextvars -def handle(event: SendTreasuryEmailLambdaPayload, context: Context) -> dict[str, Any]: +def handle(event: Dict[str, Any], context: Context) -> dict[str, Any]: """Lambda handler for emailing Treasury reports Given a user and organization object- send an email to the user that @@ -37,18 +38,29 @@ def handle(event: SendTreasuryEmailLambdaPayload, context: Context) -> dict[str, If the object does not exist then raise an exception. Args: - event: S3 Lambda event of type `s3:ObjectCreated:*` + event: S3 Lambda event of type `s3:ObjectCreated:*` or a single SQS message + with a `Records` field context: Lambda context """ - structlog.contextvars.bind_contextvars(lambda_event={"step_function": event}) logger = get_logger() - logger.info("received new invocation event from step function") + logger.info("received a new inovcation event...") try: + # Lambda payload payload = SendTreasuryEmailLambdaPayload.model_validate(event) + structlog.contextvars.bind_contextvars(lambda_event={"step_function": event}) + logger.info("parsed event from step function") except Exception: - logger.exception("Exception parsing Send Treasury Email event payload") - return {"statusCode": 400, "body": "Bad Request"} + try: + # SQS event + payload = SendTreasuryEmailLambdaPayload.model_validate( + json.loads(event["Records"][0]["body"]) + ) + structlog.contextvars.bind_contextvars(lambda_event={"event_source": event}) + logger.info("parsed event from SQS") + except Exception: + logger.exception("Exception parsing Send Treasury Email event payload") + return {"statusCode": 400, "body": "Bad Request"} try: process_event(payload, logger) diff --git a/scripts/local_treasury_file_email/README.md b/scripts/local_treasury_file_email/README.md new file mode 100644 index 00000000..a24ad499 --- /dev/null +++ b/scripts/local_treasury_file_email/README.md @@ -0,0 +1,49 @@ +# Treasury Report Emailing testing in localstack + +## Create a definition file that contains the lambda function definition +As found in `lambda_function-email-presigned-url`, defined in `treasury_generation_lambda_functions.tf` + +1. create a zip with the lambda you want to test +``` +./setup.sh +``` + +2. create the lambda in awslocal +``` +awslocal lambda create-function \ + --function-name treasury-file-email \ + --runtime python3.12 \ + --zip-file fileb://sendTreasuryReportLambda.zip \ + --handler generate_presigned_url_and_send_email.handle \ + --role arn:aws:iam::000000000000:role/lambda-role +``` + 1. If you need to update the lambda function after the fact... + ``` + awslocal lambda update-function-code \ + --function-name treasury-file-email \ + --zip-file fileb://sendTreasuryReportLambda.zip + ``` + +3. update the environment variables +``` +awslocal lambda update-function-configuration \ + --function-name treasury-file-email \ + --environment "{\"Variables\": {\"AWS_REGION\":\"us-west-2\",\"AWS_ACCESS_KEY_ID\":\"test\",\"AWS_SECRET_ACCESS_KEY\":\"test\"}}" +``` + +4. Create the event source mapping +``` +awslocal lambda create-event-source-mapping \ + --function-name treasury-file-email \ + --batch-size 1 \ + --event-source-arn arn:aws:sqs:us-west-2:000000000000:treasury-email-queue +``` + +5. Trigger the SQS message on the UI locally + +6. Check the logs +``` +LOG_STREAM_NAME=$(awslocal --region us-west-2 logs describe-log-streams --log-group-name "/aws/lambda/treasury-file-email" --order-by "LastEventTime" --descending | jq -r '.logStreams | first | .logStreamName') + +awslocal logs get-log-events --log-group-name "/aws/lambda/treasury-file-email" --log-stream-name "$LOG_STREAM_NAME" +``` diff --git a/scripts/local_treasury_file_email/setup.sh b/scripts/local_treasury_file_email/setup.sh new file mode 100755 index 00000000..b565e161 --- /dev/null +++ b/scripts/local_treasury_file_email/setup.sh @@ -0,0 +1,18 @@ +# Copy the file locally +mkdir tmp +cp -r ../../python/* tmp/ +cp tmp/src/functions/generate_presigned_url_and_send_email.py tmp/ + +# Install requirements locally +cd tmp +poetry install --only main --sync +cp --recursive .venv/lib/python*/site-packages/* . + + +# Zip the file +zip -r sendTreasuryReportLambda.zip . +mv sendTreasuryReportLambda.zip .. +cd .. + +# Clean up +rm -rf tmp diff --git a/terraform/functions.tf b/terraform/functions.tf index ffe7f620..c8449b3e 100644 --- a/terraform/functions.tf +++ b/terraform/functions.tf @@ -385,6 +385,7 @@ module "lambda_function-graphql" { PASSAGE_API_KEY_SECRET_ARN = data.aws_ssm_parameter.passage_api_key_secret_arn.value AUTH_PROVIDER = "passage" TREASURY_STEP_FUNCTION_ARN = module.treasury_generation_step_function.state_machine_arn + TREASURY_EMAIL_SQS_URL = aws_sqs_queue.email_queue.id PASSAGE_APP_ID = var.passage_app_id }) diff --git a/terraform/treasury_generation_lambda_functions.tf b/terraform/treasury_generation_lambda_functions.tf index 576a44fb..de920940 100644 --- a/terraform/treasury_generation_lambda_functions.tf +++ b/terraform/treasury_generation_lambda_functions.tf @@ -443,5 +443,20 @@ module "lambda_function-email-presigned-url" { principal = "states.amazonaws.com" source_arn = module.treasury_generation_step_function.state_machine_arn } + } } + +// SQS queue for email triggers +resource "aws_sqs_queue" "email_queue" { + name = "${var.namespace}-treasury-email-queue" +} + + +// Event source from SQS +resource "aws_lambda_event_source_mapping" "email_event" { + event_source_arn = aws_sqs_queue.email_queue.arn + enabled = true + function_name = module.lambda_function-email-presigned-url.lambda_function_arn + batch_size = 1 +}