Merge pull request #61 from ASFHyP3/develop

Release v0.1.0

.github/actions/deploy/action.yml

@@ -1,5 +1,5 @@
 name: Deploy to AWS
-description: Deploys a copy of the application to AWS via CloudFormation
+description: Deploys the OpenData bucket stack, and the seperate application stack, to AWS via CloudFormation
 
 inputs:
   AWS_ACCESS_KEY_ID:
@@ -10,14 +10,26 @@ inputs:
     required: true
   TEMPLATE_BUCKET:
     required: true
-  STACK_NAME:
+  AWS_APPLICATION_ACCOUNT_ID:
     required: true
-  BUCKET_NAME:
+  OPENDATA_AWS_ACCESS_KEY_ID:
     required: true
-  DOMAIN_NAME:
+  OPENDATA_AWS_SECRET_ACCESS_KEY:
+    required: true
+  OPENDATA_CLOUDFORMATION_ROLE_ARN:
+    required: true
+  OPENDATA_TEMPLATE_BUCKET:
     required: true
   CERTIFICATE_ARN:
     required: true
+  APP_STACK_NAME:
+    required: true
+  BUCKET_STACK_NAME:
+    required: true
+  BUCKET_NAME:
+    required: true
+  DOMAIN_NAME:
+    required: true
   CDSE_USERNAME:
     required: true
   CDSE_PASSWORD:
@@ -26,6 +38,28 @@ inputs:
 runs:
   using: composite
   steps:
+
+    - uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ inputs.OPENDATA_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ inputs.OPENDATA_AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-west-2
+
+    - name: package and deploy the bucket stack
+      shell: bash
+      run: |
+        aws cloudformation package \
+          --template-file apps/bucket/cloudformation.yml \
+          --s3-bucket ${{ inputs.OPENDATA_TEMPLATE_BUCKET }} \
+          --output-template-file bucket_packaged.yml
+        aws cloudformation deploy \
+          --stack-name ${{ inputs.BUCKET_STACK_NAME }} \
+          --template-file bucket_packaged.yml \
+          --role-arn ${{ inputs.OPENDATA_CLOUDFORMATION_ROLE_ARN }} \
+          --parameter-overrides \
+              BucketName='${{ inputs.BUCKET_NAME }}' \
+              AwsApplicationAccountId='${{ inputs.AWS_APPLICATION_ACCOUNT_ID }}'
+
     - uses: aws-actions/configure-aws-credentials@v4
       with:
         aws-access-key-id: ${{ inputs.AWS_ACCESS_KEY_ID }}
@@ -43,21 +77,21 @@ runs:
         python -m pip install -r requirements-api.txt -t apps/api/src
         python -m pip install -r requirements-fetcher.txt -t apps/fetcher/src
 
-    - name: Package and deploy
+    - name: package and deploy the application stack
       shell: bash
       run: |
         aws cloudformation package \
           --template-file cloudformation.yml \
           --s3-bucket ${{ inputs.TEMPLATE_BUCKET }} \
-          --output-template-file packaged.yml
+          --output-template-file app_packaged.yml
         aws cloudformation deploy \
-          --stack-name ${{ inputs.STACK_NAME }} \
-          --template-file packaged.yml \
+          --stack-name ${{ inputs.APP_STACK_NAME }} \
+          --template-file app_packaged.yml \
           --role-arn ${{ inputs.CLOUDFORMATION_ROLE_ARN }} \
           --capabilities CAPABILITY_IAM \
           --parameter-overrides \
               BucketName='${{ inputs.BUCKET_NAME }}' \
               DomainName='${{ inputs.DOMAIN_NAME }}' \
               CertificateArn='${{ inputs.CERTIFICATE_ARN }}' \
               CdseUsername='${{ inputs.CDSE_USERNAME }}' \
-              CdsePassword='${{ inputs.CDSE_PASSWORD }}'
+              CdsePassword='${{ inputs.CDSE_PASSWORD }}'

.github/workflows/deploy-prod.yml

@@ -4,6 +4,24 @@ on:
   push:
     branches:
       - main
+  workflow_dispatch:
+    inputs:
+      APPLICATION_STACK_NAME:
+        type: string
+        description: The name to give to the application stack.
+        default: s1-orbits
+      BUCKET_STACK_NAME:
+        type: string
+        description: The name to give to the bucket stack.
+        default: s1-orbits-bucket
+      BUCKET_NAME:
+        type: string
+        description: The name to give to the bucket.
+        default: s1-orbits     
+      DOMAIN_NAME:
+        type: string
+        description: The domain name for the API.
+        default: s1-orbits.asf.alaska.edu
 
 concurrency: deploy-prod
 
@@ -21,10 +39,16 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
-          TEMPLATE_BUCKET: cf-templates-1hz9ldhhl4ahu-us-west-2
-          STACK_NAME: s1-orbits
-          BUCKET_NAME: s1-orbits
-          DOMAIN_NAME: s1-orbits.asf.alaska.edu
+          OPENDATA_AWS_ACCESS_KEY_ID: ${{ secrets.OPENDATA_AWS_ACCESS_KEY_ID }}
+          OPENDATA_AWS_SECRET_ACCESS_KEY: ${{ secrets.OPENDATA_AWS_SECRET_ACCESS_KEY }}
+          OPENDATA_CLOUDFORMATION_ROLE_ARN: ${{ secrets.OPENDATA_CLOUDFORMATION_ROLE_ARN }}
+          AWS_APPLICATION_ACCOUNT_ID: ${{ secrets.AWS_APPLICATION_ACCOUNT_ID }}
           CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }}
           CDSE_USERNAME: ${{ secrets.CDSE_USERNAME }}
           CDSE_PASSWORD: ${{ secrets.CDSE_PASSWORD }}
+          BUCKET_STACK_NAME: ${{ github.event.inputs.BUCKET_STACK_NAME || 's1-orbits-bucket' }}
+          APP_STACK_NAME: ${{ github.event.inputs.APPLICATION_STACK_NAME || 's1-orbits' }}
+          BUCKET_NAME: ${{ github.event.inputs.BUCKET_NAME || 's1-orbits' }}
+          DOMAIN_NAME: ${{ github.event.inputs.DOMAIN_NAME || 's1-orbits.asf.alaska.edu' }}
+          OPENDATA_TEMPLATE_BUCKET: cf-templates-250jrbt1c7xh-us-west-2
+          TEMPLATE_BUCKET: cf-templates-aubvn3i9olmk-us-west-2

.github/workflows/deploy-test.yml

@@ -4,6 +4,24 @@ on:
   push:
     branches:
       - develop
+  workflow_dispatch:
+    inputs:
+      APPLICATION_STACK_NAME:
+        type: string
+        description: The name to give to the application stack.
+        default: s1-orbits-test
+      BUCKET_STACK_NAME:
+        type: string
+        description: The name to give to the bucket stack.
+        default: s1-orbits-bucket-test
+      BUCKET_NAME:
+        type: string
+        description: The name to give to the bucket.
+        default: s1-orbits-test 
+      DOMAIN_NAME:
+        type: string
+        description: The domain name for the API.
+        default: s1-orbits-test.asf.alaska.edu
 
 concurrency: deploy-test
 
@@ -21,10 +39,16 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
-          TEMPLATE_BUCKET: cf-templates-1hz9ldhhl4ahu-us-west-2
-          STACK_NAME: s1-orbits-test
-          BUCKET_NAME: s1-orbits-test
-          DOMAIN_NAME: s1-orbits-test.asf.alaska.edu
+          OPENDATA_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          OPENDATA_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          OPENDATA_CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
+          AWS_APPLICATION_ACCOUNT_ID: ${{ secrets.AWS_APPLICATION_ACCOUNT_ID }}
           CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }}
           CDSE_USERNAME: ${{ secrets.CDSE_USERNAME }}
           CDSE_PASSWORD: ${{ secrets.CDSE_PASSWORD }}
+          BUCKET_STACK_NAME: ${{ github.event.inputs.BUCKET_STACK_NAME || 's1-orbits-bucket-test' }}
+          APP_STACK_NAME: ${{ github.event.inputs.APPLICATION_STACK_NAME || 's1-orbits-test' }}
+          BUCKET_NAME: ${{ github.event.inputs.BUCKET_NAME || 's1-orbits-test' }}
+          DOMAIN_NAME: ${{ github.event.inputs.DOMAIN_NAME || 's1-orbits-test.asf.alaska.edu' }}
+          OPENDATA_TEMPLATE_BUCKET: cf-templates-aubvn3i9olmk-us-west-2
+          TEMPLATE_BUCKET: cf-templates-aubvn3i9olmk-us-west-2

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.0]
+
+* The bucket stack has been separated from the application stack for deployment into a separate account.
+
 ## [0.0.5]
 
 * Automated creation of a log bucket for the data bucket.

apps/api/cloudformation.yml

@@ -1,5 +1,7 @@
-Parameters:
+AWSTemplateFormatVersion: '2010-09-09'
+Description: Creates an API stack with a custom domain and lambda handler for retrieving s1 orbit files.
 
+Parameters:
   BucketName:
     Type: String
 
@@ -10,7 +12,6 @@ Parameters:
     Type: String
 
 Resources:
-
   Api:
     Type: AWS::ApiGatewayV2::Api
     Properties:

apps/bucket/cloudformation.yml

@@ -1,21 +1,22 @@
-# adapted from https://s3-us-west-2.amazonaws.com/opendata.aws/pds-bucket-cf.yml
----
 AWSTemplateFormatVersion: '2010-09-09'
-Description: This template creates the AWS infrastructure to publish a public data set on S3. It creates a publicly-accessible S3 bucket for the dataset, enables CloudWatch Metrics for the dataset bucket, and creates a public SQS and Lambda subscribable SNS Topic.
+Description: Creates a publicly-accessible OpenData S3 bucket for s1-orbits, with Cloudwatch metrics, as well as a public SQS and Lambda subscribable SNS Topic.
 
 Parameters:
-  DataSetName:
+  BucketName:
     AllowedPattern: "[a-z0-9\\.\\-]*"
     ConstraintDescription: may only contain lowercase letters, numbers, and ., or - characters
     Description: "The name of the dataset's S3 bucket. This will be used to create the dataset S3 bucket."
     MaxLength: '250'
     MinLength: '1'
     Type: String
 
+  AwsApplicationAccountId:
+    Type: String
+
 Resources:
   SNSTopic:
     Properties:
-      TopicName: !Join [ "", [ !Join [ "", !Split [ ".", !Ref DataSetName ] ], "-object_created" ] ]
+      TopicName: !Join [ "", [ !Join [ "", !Split [ ".", !Ref BucketName ] ], "-object_created" ] ]
     Type: AWS::SNS::Topic
 
   SNSTopicPolicy:
@@ -34,7 +35,7 @@ Resources:
             Service: s3.amazonaws.com
           Condition:
             ArnLike:
-              aws:SourceArn: !Sub arn:aws:s3:::${DataSetName}
+              aws:SourceArn: !Sub arn:aws:s3:::${BucketName}
         - Sid: allowOnlySQSandLambdaSubscription
           Effect: Allow
           Action:
@@ -77,7 +78,7 @@ Resources:
     Properties:
       Bucket: !Ref LogBucket
       PolicyDocument:
-        Version: 2012-10-17
+        Version: '2012-10-17'
         Statement:
           - Effect: Allow
             Principal:
@@ -96,7 +97,7 @@ Resources:
     DependsOn:
       - SNSTopicPolicy
     Properties:
-      BucketName: !Ref DataSetName
+      BucketName: !Ref BucketName
       LoggingConfiguration:
         DestinationBucketName: !Ref LogBucket
         TargetObjectKeyFormat:
@@ -154,12 +155,10 @@ Resources:
           Resource:
           - !Sub arn:aws:s3:::${DataSetBucket}/*
           - !Sub arn:aws:s3:::${DataSetBucket}
+        Statement:
+        - Action: s3:PutObject
+          Effect: Allow
+          Principal:
+            AWS: !Ref AwsApplicationAccountId
+          Resource: !Sub arn:aws:s3:::${DataSetBucket}/*
     Type: AWS::S3::BucketPolicy
-
-Outputs:
-  DataBucket:
-    Description: "S3 data bucket name"
-    Value: !Sub ${DataSetBucket}
-  SNSTopic:
-    Description: "SQS and Lambda subscribable SNS Topic"
-    Value: !Ref SNSTopic

apps/fetcher/cloudformation.yml

@@ -1,7 +1,7 @@
-AWSTemplateFormatVersion: 2010-09-09
+AWSTemplateFormatVersion: '2010-09-09'
+Description: Creates a lambda handler capable of pushing new s1 orbit data to the OpenData bucket every 11 minutes.
 
 Parameters:
-
   BucketName:
     Type: String
 
@@ -14,7 +14,6 @@ Parameters:
     NoEcho: true
 
 Resources:
-
   Secret:
     Type: AWS::SecretsManager::Secret
     Properties:

cicd/opendata-cicd.yml

@@ -0,0 +1,61 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Description: Deploys IAM resources supporting automated deployments of the bucket stack in the Open Data account
+
+Resources:
+  CloudformationDeploymentRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          Action: sts:AssumeRole
+          Principal:
+            Service: cloudformation.amazonaws.com
+          Effect: Allow
+      Policies:
+        - PolicyName: cloud-formation-deployment-policy
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Effect: Allow
+                Action:
+                  - s3:*
+                  - sns:*
+                Resource: "*"
+
+  GithubActionsUser:
+    Type: AWS::IAM::User
+    Properties:
+      UserName: github-actions
+      Policies:
+        - PolicyName: github-actions-user-policy
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Effect: Allow
+                Action:
+                  - cloudformation:ValidateTemplate
+                  - cloudformation:DescribeStacks
+                  - ssm:GetParameters
+                  - s3:PutObject
+                Resource: "*"
+
+              - Effect: Allow
+                Action: iam:PassRole
+                Resource: !GetAtt CloudformationDeploymentRole.Arn
+                Condition:
+                  StringLike:
+                    iam:AssociatedResourceArn: !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/*"
+
+              - Effect: Allow
+                Action:
+                  - cloudformation:SetStackPolicy
+                  - cloudformation:CreateStack
+                  - cloudformation:UpdateStack
+                  - cloudformation:CreateChangeSet
+                  - cloudformation:DescribeChangeSet
+                  - cloudformation:ExecuteChangeSet
+                  - cloudformation:DeleteChangeSet
+                  - cloudformation:GetTemplateSummary
+                Resource: !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/*"