From ef319d14a6b8480a436d02693dcf795ed6714152 Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Fri, 22 Nov 2024 12:13:14 +0000 Subject: [PATCH] Add CDK config for image package source --- .env | 3 + docs/cloudquery-implementation.md | 3 +- docs/getting-started.md | 1 + .../service-catalogue.test.ts.snap | 869 +++++++++++++++++- packages/cdk/lib/cloudquery/config.ts | 27 +- packages/cdk/lib/cloudquery/index.ts | 52 +- packages/cdk/lib/cloudquery/policies.ts | 29 + packages/cdk/lib/cloudquery/versions.ts | 1 + packages/dev-environment/docker-compose.yaml | 1 + 9 files changed, 980 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 7f4f24563..3559aa33b 100644 --- a/.env +++ b/.env @@ -32,6 +32,9 @@ CQ_GITHUB_LANGUAGES=0.0.5 # See https://github.com/guardian/cq-source-ns1 CQ_NS1=0.1.2 +# See https://github.com/guardian/cq-image-packages +CQ_IMAGE_PACKAGES=1.0.0 + # --- FOR LOCAL DEVELOPMENT ONLY --- STAGE=DEV DATABASE_USER=postgres diff --git a/docs/cloudquery-implementation.md b/docs/cloudquery-implementation.md index 41129aefb..e2d3c0de6 100644 --- a/docs/cloudquery-implementation.md +++ b/docs/cloudquery-implementation.md @@ -10,6 +10,7 @@ We are using CloudQuery to collect data from: - Snyk - Fastly - Galaxies of the Guardian +- Image packages As it is relatively easy collect data with CloudQuery, we have, largely, opted to collect _all_ the data. By collecting this extra data, we enable others to answer their questions. For example: @@ -20,7 +21,7 @@ By collecting this extra data, we enable others to answer their questions. For e We have implemented CloudQuery to run on AWS ECS, writing data to [Postgres](https://www.cloudquery.io/docs/plugins/destinations/postgresql/overview). -This is all [defined using GuCDK](../packages/cdk/lib/cloudquery.ts). +This is all [defined using GuCDK](../packages/cdk/lib/cloudquery/index.ts). We have more interest in some data sources than others. For example, we'd like to know more about AWS Lambdas across the estate than AWS Elastic Beanstalk deployments. diff --git a/docs/getting-started.md b/docs/getting-started.md index a1e82f3f8..ba3910d26 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -9,6 +9,7 @@ We are using CloudQuery to collect data about how The Guardian uses: - Snyk - Fastly - Galaxies of the Guardian +- Image packages CloudQuery generally collects metadata about these resources. That means that we have information like the name of a repository, and when it was created, but not the contents of the files in it. diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 7a5b9d25d..6e7b720d9 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -10,6 +10,7 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` "GuSecurityGroup", "GuLoggingStreamNameParameter", "GuStringParameter", + "GuStringParameter", "GuAnghammaradTopicParameter", "GuDistributionBucketParameter", "GuLambdaFunction", @@ -62,8 +63,674 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` "Description": "Virtual Private Cloud to run EC2 instances within. Should NOT be the account default VPC.", "Type": "AWS::SSM::Parameter::Value", }, - }, - "Resources": { + "packagesBucketNameParam": { + "Default": "/TEST/deploy/amigo/amigo.data.bucket", + "Type": "AWS::SSM::Parameter::Value", + }, + }, + "Resources": { + "CloudquerySourceAmigoBakePackagesScheduledEventRule3FDBCEB5": { + "Properties": { + "ScheduleExpression": "rate(1 day)", + "State": "ENABLED", + "Targets": [ + { + "Arn": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + "EcsParameters": { + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsVpcConfiguration": { + "AssignPublicIp": "DISABLED", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "PostgresAccessSecurityGroupServicecatalogue03C78F14", + "GroupId", + ], + }, + ], + "Subnets": { + "Ref": "PrivateSubnets", + }, + }, + }, + "PropagateTags": "TASK_DEFINITION", + "TaskCount": 1, + "TaskDefinitionArn": { + "Ref": "CloudquerySourceAmigoBakePackagesTaskDefinitionF04CFC72", + }, + }, + "Id": "Target0", + "Input": "{}", + "RoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAmigoBakePackagesTaskDefinitionEventsRoleB18B35DF", + "Arn", + ], + }, + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionCloudquerySourceAmigoBakePackagesFirelensLogGroupA4EF0BA2": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AmigoBakePackages", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionEventsRoleB18B35DF": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "events.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AmigoBakePackages", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionEventsRoleDefaultPolicy145F68FA": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ecs:RunTask", + "Condition": { + "ArnEquals": { + "ecs:cluster": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + }, + }, + "Effect": "Allow", + "Resource": { + "Ref": "CloudquerySourceAmigoBakePackagesTaskDefinitionF04CFC72", + }, + }, + { + "Action": "ecs:TagResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":ecs:", + { + "Ref": "AWS::Region", + }, + ":*:task/", + { + "Ref": "servicecatalogueCluster5FC34DC5", + }, + "/*", + ], + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleD495DC33", + "Arn", + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAmigoBakePackagesE3F44845", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceAmigoBakePackagesTaskDefinitionEventsRoleDefaultPolicy145F68FA", + "Roles": [ + { + "Ref": "CloudquerySourceAmigoBakePackagesTaskDefinitionEventsRoleB18B35DF", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleD495DC33": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AmigoBakePackages", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleDefaultPolicyBD5A8255": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "cloudqueryapikeyCCF82F53", + }, + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceAmigoBakePackagesTaskDefinitionCloudquerySourceAmigoBakePackagesFirelensLogGroupA4EF0BA2", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleDefaultPolicyBD5A8255", + "Roles": [ + { + "Ref": "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleD495DC33", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionF04CFC72": { + "Properties": { + "ContainerDefinitions": [ + { + "Command": [ + "/bin/sh", + "-c", + { + "Fn::Join": [ + "", + [ + "printf 'kind: source +spec: + name: image-packages + registry: github + path: guardian/image-packages + version: v1.0.0 + destinations: + - postgresql + tables: + - amigo_bake_packages + spec: + base_images_table: amigo-TEST-base-images + recipes_table: amigo-TEST-recipes + bakes_table: amigo-TEST-bakes + bucket: ", + { + "Ref": "packagesBucketNameParam", + }, + " +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination +spec: + name: postgresql + registry: github + path: cloudquery/postgresql + version: v7.2.0 + migrate_mode: forced + spec: + connection_string: >- + user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 + dbname=postgres sslmode=verify-full +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", + ], + ], + }, + ], + "DependsOn": [ + { + "Condition": "HEALTHY", + "ContainerName": "CloudquerySource-AmigoBakePackagesAWSOTELCollector", + }, + ], + "DockerLabels": { + "App": "service-catalogue", + "Name": "AmigoBakePackages", + "Stack": "deploy", + "Stage": "TEST", + }, + "EntryPoint": [ + "", + ], + "Environment": [ + { + "Name": "GOMEMLIMIT", + "Value": "409MiB", + }, + ], + "Essential": true, + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:stable", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, + }, + }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], + "Name": "CloudquerySource-AmigoBakePackagesContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ + { + "Name": "DB_USERNAME", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":username::", + ], + ], + }, + }, + { + "Name": "DB_HOST", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":host::", + ], + ], + }, + }, + { + "Name": "DB_PASSWORD", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":password::", + ], + ], + }, + }, + { + "Name": "CLOUDQUERY_API_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "cloudqueryapikeyCCF82F53", + }, + ":api-key::", + ], + ], + }, + }, + ], + }, + { + "Command": [ + "--config=/etc/ecs/ecs-xray.yaml", + ], + "Essential": true, + "HealthCheck": { + "Command": [ + "CMD", + "/healthcheck", + ], + "Interval": 5, + "Retries": 3, + "Timeout": 5, + }, + "Image": "public.ecr.aws/aws-observability/aws-otel-collector:v0.35.0", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, + }, + }, + "Name": "CloudquerySource-AmigoBakePackagesAWSOTELCollector", + "PortMappings": [ + { + "ContainerPort": 4318, + "Protocol": "tcp", + }, + ], + "ReadonlyRootFilesystem": true, + }, + { + "Command": [ + "/bin/sh", + "-c", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('amigo_bake_packages', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + ], + "DockerLabels": { + "App": "service-catalogue", + "Name": "AmigoBakePackages", + "Stack": "deploy", + "Stage": "TEST", + }, + "EntryPoint": [ + "", + ], + "Essential": false, + "Image": "public.ecr.aws/docker/library/postgres:16-alpine", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, + }, + }, + "Name": "CloudquerySource-AmigoBakePackagesPostgresContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ + { + "Name": "PGUSER", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":username::", + ], + ], + }, + }, + { + "Name": "PGHOST", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":host::", + ], + ], + }, + }, + { + "Name": "PGPASSWORD", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":password::", + ], + ], + }, + }, + ], + }, + { + "Environment": [ + { + "Name": "STACK", + "Value": "deploy", + }, + { + "Name": "STAGE", + "Value": "TEST", + }, + { + "Name": "APP", + "Value": "service-catalogue", + }, + { + "Name": "GU_REPO", + "Value": "guardian/service-catalogue", + }, + ], + "Essential": true, + "FirelensConfiguration": { + "Type": "fluentbit", + }, + "Image": "ghcr.io/guardian/devx-logs:2", + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "CloudquerySourceAmigoBakePackagesTaskDefinitionCloudquerySourceAmigoBakePackagesFirelensLogGroupA4EF0BA2", + }, + "awslogs-region": { + "Ref": "AWS::Region", + }, + "awslogs-stream-prefix": "deploy/TEST/service-catalogue", + }, + }, + "MountPoints": [ + { + "ContainerPath": "/init", + "ReadOnly": false, + "SourceVolume": "firelens-volume", + }, + ], + "Name": "CloudquerySource-AmigoBakePackagesFirelens", + "ReadonlyRootFilesystem": true, + }, + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleD495DC33", + "Arn", + ], + }, + "Family": "ServiceCatalogueCloudquerySourceAmigoBakePackagesTaskDefinition07388B36", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE", + ], + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AmigoBakePackages", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + "TaskRoleArn": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAmigoBakePackagesE3F44845", + "Arn", + ], + }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + { + "Name": "firelens-volume", + }, + ], + }, + "Type": "AWS::ECS::TaskDefinition", + }, "CloudquerySourceAwsCostExplorerScheduledEventRule85BE97F8": { "Properties": { "ScheduleExpression": "rate(7 days)", @@ -23630,6 +24297,204 @@ spec: }, "Type": "AWS::ECS::ClusterCapacityProviderAssociations", }, + "servicecatalogueTESTtaskAmigoBakePackagesDefaultPolicy1351FB30": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "kinesis:Describe*", + "kinesis:Put*", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":kinesis:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":stream/", + { + "Ref": "LoggingStreamName", + }, + ], + ], + }, + }, + { + "Action": [ + "dynamodb:GetItem", + "dynamodb:BatchGetItem", + "dynamodb:Query", + "dynamodb:Scan", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:aws:dynamodb:", + { + "Ref": "AWS::Region", + }, + ":000000000018:table/amigo-TEST-base-images", + ], + ], + }, + { + "Fn::Join": [ + "", + [ + "arn:aws:dynamodb:", + { + "Ref": "AWS::Region", + }, + ":000000000018:table/amigo-TEST-recipes", + ], + ], + }, + { + "Fn::Join": [ + "", + [ + "arn:aws:dynamodb:", + { + "Ref": "AWS::Region", + }, + ":000000000018:table/amigo-TEST-bakes", + ], + ], + }, + ], + }, + { + "Action": "s3:GetObject", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":s3:::", + { + "Ref": "packagesBucketNameParam", + }, + "/packagelists/*", + ], + ], + }, + }, + { + "Action": "rds-db:connect", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":rds-db:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":dbuser:", + { + "Fn::GetAtt": [ + "PostgresInstance16DE4286E", + "DbiResourceId", + ], + }, + "/{{resolve:secretsmanager:", + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":SecretString:username::}}", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "servicecatalogueTESTtaskAmigoBakePackagesDefaultPolicy1351FB30", + "Roles": [ + { + "Ref": "servicecatalogueTESTtaskAmigoBakePackagesE3F44845", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "servicecatalogueTESTtaskAmigoBakePackagesE3F44845": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/AWSXrayWriteOnlyAccess", + ], + ], + }, + ], + "RoleName": "service-catalogue-TEST-task-AmigoBakePackages", + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "servicecatalogueTESTtaskAwsCostExplorer78777A06": { "Properties": { "AssumeRolePolicyDocument": { diff --git a/packages/cdk/lib/cloudquery/config.ts b/packages/cdk/lib/cloudquery/config.ts index eb10109eb..364b42bfa 100644 --- a/packages/cdk/lib/cloudquery/config.ts +++ b/packages/cdk/lib/cloudquery/config.ts @@ -99,7 +99,7 @@ export function awsSourceConfigForOrganisation( * * @param accountNumber The AWS account to query. ServiceCatalogue will assume the role `cloudquery-access` in this account. * @param tableConfig Which tables to include or exclude. - * + * @param extraConfig Extra spec fields. * @see https://www.cloudquery.io/docs/plugins/sources/aws/configuration#account */ export function awsSourceConfigForAccount( @@ -301,6 +301,31 @@ export function githubLanguagesConfig(): CloudqueryConfig { }; } +export function amigoBakePackagesConfig( + baseImagesTableName: string, + recipesTableName: string, + bakesTableName: string, + packagesBucketName: string, +): CloudqueryConfig { + return { + kind: 'source', + spec: { + name: 'image-packages', + registry: 'github', + path: 'guardian/image-packages', + version: `v${Versions.CloudqueryImagePackages}`, + destinations: ['postgresql'], + tables: ['amigo_bake_packages'], + spec: { + base_images_table: baseImagesTableName, + recipes_table: recipesTableName, + bakes_table: bakesTableName, + bucket: packagesBucketName, + }, + }, + }; +} + // Tables we are skipping because they are slow and or uninteresting to us. export const skipTables = [ 'aws_ec2_vpc_endpoint_services', // this resource includes services that are available from AWS as well as other AWS Accounts diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index a016aa772..834fac90f 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -3,7 +3,7 @@ import { GuStringParameter } from '@guardian/cdk/lib/constructs/core'; import { GuSecurityGroup } from '@guardian/cdk/lib/constructs/ec2'; import { GuS3Bucket } from '@guardian/cdk/lib/constructs/s3'; import { GuardianAwsAccounts } from '@guardian/private-infrastructure-config'; -import { Duration } from 'aws-cdk-lib'; +import { Aws, Duration } from 'aws-cdk-lib'; import type { IVpc } from 'aws-cdk-lib/aws-ec2'; import { Secret } from 'aws-cdk-lib/aws-ecs'; import { Schedule } from 'aws-cdk-lib/aws-events'; @@ -14,6 +14,7 @@ import { StringParameter } from 'aws-cdk-lib/aws-ssm'; import type { CloudquerySource } from './cluster'; import { CloudqueryCluster } from './cluster'; import { + amigoBakePackagesConfig, awsSourceConfigForAccount, awsSourceConfigForOrganisation, fastlySourceConfig, @@ -27,7 +28,12 @@ import { snykSourceConfig, } from './config'; import { Images } from './images'; -import { cloudqueryAccess, listOrgsPolicy, readBucketPolicy } from './policies'; +import { + cloudqueryAccess, + listOrgsPolicy, + readBucketPolicy, + readDynamoDbTablePolicy, +} from './policies'; interface CloudqueryEcsClusterProps { vpc: IVpc; @@ -588,6 +594,47 @@ export function addCloudqueryEcsCluster( config: ns1SourceConfig(), }; + const packagesBucketName = new GuStringParameter( + scope, + 'packagesBucketNameParam', + { + fromSSM: true, + default: `/${stage}/deploy/amigo/amigo.data.bucket`, + }, + ).valueAsString; + + const packagesBucket = GuS3Bucket.fromBucketName( + scope, + 'packagesBucket', + packagesBucketName, + ); + + const baseImagesTableName = `amigo-${stage}-base-images`; + const recipesTableName = `amigo-${stage}-recipes`; + const bakesTableName = `amigo-${stage}-bakes`; + + const amigoBakePackagesSource: CloudquerySource = { + name: 'AmigoBakePackages', + description: 'Packages installed in Amigo bakes.', + schedule: nonProdSchedule ?? Schedule.rate(Duration.days(1)), + config: amigoBakePackagesConfig( + baseImagesTableName, + recipesTableName, + bakesTableName, + packagesBucket.bucketName, + ), + policies: [ + readDynamoDbTablePolicy( + GuardianAwsAccounts.DeployTools, + Aws.REGION, + baseImagesTableName, + recipesTableName, + bakesTableName, + ), + readBucketPolicy(`${packagesBucket.bucketArn}/packagelists/*`), + ], + }; + return new CloudqueryCluster(scope, `${app}Cluster`, { app, vpc, @@ -605,6 +652,7 @@ export function addCloudqueryEcsCluster( riffRaffSources, githubLanguagesSource, ns1Source, + amigoBakePackagesSource, ], }); } diff --git a/packages/cdk/lib/cloudquery/policies.ts b/packages/cdk/lib/cloudquery/policies.ts index 7b1d85657..5f591c428 100644 --- a/packages/cdk/lib/cloudquery/policies.ts +++ b/packages/cdk/lib/cloudquery/policies.ts @@ -36,6 +36,35 @@ export const readBucketPolicy = (...resources: string[]): PolicyStatement => { }); }; +/** + * Create a policy statement allowing read access to the given DynamoDB tables. + * + * @param accountId the AWS account ID + * @param region the AWS region + * @param tableNames a list of DynamoDB table names + * @returns a policy statement allowing read access to the given DynamoDB tables. + */ +export const readDynamoDbTablePolicy = ( + accountId: string, + region: string, + ...tableNames: string[] +): PolicyStatement => { + return new PolicyStatement({ + effect: Effect.ALLOW, + // for each table name, create a resource ARN + resources: tableNames.map( + (tableName) => + `arn:aws:dynamodb:${region}:${accountId}:table/${tableName}`, + ), + actions: [ + 'dynamodb:GetItem', + 'dynamodb:BatchGetItem', + 'dynamodb:Query', + 'dynamodb:Scan', + ], + }); +}; + export function singletonPolicy(cluster: Cluster) { return new PolicyStatement({ effect: Effect.ALLOW, diff --git a/packages/cdk/lib/cloudquery/versions.ts b/packages/cdk/lib/cloudquery/versions.ts index f9baba548..be544e3a5 100644 --- a/packages/cdk/lib/cloudquery/versions.ts +++ b/packages/cdk/lib/cloudquery/versions.ts @@ -27,4 +27,5 @@ export const Versions = { CloudquerySnyk: envOrError('CQ_SNYK'), CloudqueryGithubLanguages: envOrError('CQ_GITHUB_LANGUAGES'), CloudqueryNs1: envOrError('CQ_NS1'), + CloudqueryImagePackages: envOrError('CQ_IMAGE_PACKAGES'), }; diff --git a/packages/dev-environment/docker-compose.yaml b/packages/dev-environment/docker-compose.yaml index 8193d2dd4..4056453c8 100644 --- a/packages/dev-environment/docker-compose.yaml +++ b/packages/dev-environment/docker-compose.yaml @@ -30,6 +30,7 @@ services: - github_languages - aws_ec2_instances - aws_cloudformation_stacks + - amigo_bake_packages postgres: image: postgres:14.6 ports: