Merge pull request #42 from GSA/moto-test

adds integration testing with moto

.circleci/config.yml

@@ -36,6 +36,56 @@ jobs:
       - run:
           name: Lint Handler
           command: make lint_handler
+  integration_test:
+    environment:
+      AWS_REGION: us-east-1
+    machine:
+      # https://circleci.com/docs/2.0/configuration-reference/#available-machine-images
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      - run:
+          name: Install make, curl, unzip
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y make curl unzip software-properties-common
+      - run:
+          name: Install Terraform
+          command: |
+            TERRAFORM_LATEST_URL=$(curl --silent --location --head --output /dev/null --write-out '%{url_effective}' -- https://github.com/hashicorp/terraform/releases/latest)
+            TERRAFORM_VERSION=${TERRAFORM_LATEST_URL##*/}
+            TERRAFORM_VERSION=${TERRAFORM_VERSION:1:${#TERRAFORM_VERSION}}
+            curl -o terraform.zip -sSL "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip"
+            sudo unzip terraform.zip -d /bin
+            sudo chmod +x /bin/terraform
+      - run:
+          name: Install Moto Server
+          command: |
+            sudo add-apt-repository ppa:deadsnakes/ppa
+            sudo apt-get update
+            sudo apt install -y python3.7-dev python3.7-distutils python3-pip
+            sudo python3.7 -m pip install --upgrade pip setuptools flask requests six distro
+            git clone https://github.com/brady-gsa/moto.git
+            cd moto
+            git pull origin fix_lambda_config
+            git checkout fix_lambda_config
+            cd ..
+            sudo python3.7 -m pip install -e moto --ignore-installed PyYAML
+      - run:
+          name: Install golang
+          command: |
+            curl -o go.tar.gz -sSL https://dl.google.com/go/go1.14.linux-amd64.tar.gz
+            sudo tar -xvf go.tar.gz
+            sudo rm -rf /usr/local/go
+            sudo mv -f go /usr/local
+            mkdir ~/go
+            echo 'export GOROOT=/usr/local/go' >> $BASH_ENV
+            echo 'export GOPATH=$HOME/go' >> $BASH_ENV
+            echo 'export PATH=$GOPATH/bin:$GOROOT/bin:$PATH' >> $BASH_ENV
+      - run:
+          name: Test Terraform
+          command: |
+            make integration_test
   test_handler:
     docker:
       - image: circleci/golang:latest
@@ -77,6 +127,7 @@ workflows:
       - tfsec
       - lint_handler
       - test_handler
+      - integration_test
       - release_handler:
           filters:
             tags:

Makefile

@@ -43,7 +43,7 @@ release_handler: precommit
 	make -C handler release
 
 integration_test: precommit
-	make -C handler integration_test
+	make -C tests integration_test
 
 plan_terraform: validate_terraform
 	terraform plan

README.md

@@ -208,6 +208,7 @@ provider "aws" {
 | source\_file | \(optional\) full or relative path to zipped binary of lambda handler | string | `"../release/grace-inventory-lambda.zip"` | no |
 | appenv | \(optional\) The environment in which the script is running \(development \| test \| production\) | string | `"development"` | no |
 | project_name | \(required\) project name \(e.g. grace, fcs, fas, etc.\). Used as prefix for AWS S3 bucket name | string | `"grace"` | yes |
+| access\_logging\_bucket | \(optional\) the S3 bucket that will receiving on-access logs for the inventory bucket | string | `""` | no |
 | accounts\_info | \(optional\) Determines which accounts to parse.  Can be "self", comma delimited list of Account IDs or an S3 URI containing JSON output of `aws organizations list-accounts`.  If empty, tries to query accounts with `organizations:ListAccounts` | string | `"self"` | no |
 | master\_account\_id | \(optional\) Account ID of AWS Master Payer Account | string | `""` | no |
 | master\_role\_name | \(optional\) Role assumed by lambda function to query organizations in Master Payer account | string | `""` | no |

handler/inv/inv_test.go

@@ -137,11 +137,25 @@ func TestNew(t *testing.T) {
 					t.Fatalf("error setting environment variable: %v", err)
 				}
 			}
+
 			actual, err := New()
+
+			envBucket := os.Getenv("s3_bucket")
+			envKmsKey := os.Getenv("kms_key_id")
+			// maps are dynamically randomized in memory
+			// we must cleanup the ENV before running the
+			// next test
+			for k := range tc.env {
+				enverr := os.Unsetenv(k)
+				if enverr != nil {
+					t.Fatalf("error removing environment variable: %v", enverr)
+				}
+			}
+
 			if tc.expectedErr == "" {
 				assert.NilError(t, err)
-				assert.Equal(t, actual.bucketID, os.Getenv("s3_bucket"))
-				assert.Equal(t, actual.kmsKeyID, os.Getenv("kms_key_id"))
+				assert.Equal(t, actual.bucketID, envBucket)
+				assert.Equal(t, actual.kmsKeyID, envKmsKey)
 			} else {
 				assert.ErrorContains(t, err, tc.expectedErr)
 			}

main.tf

@@ -2,8 +2,7 @@ data "aws_caller_identity" "current" {
 }
 
 locals {
-  app_name       = "${var.project_name}-${var.appenv}-inventory"
-  account_id     = data.aws_caller_identity.current.account_id
-  logging_bucket = var.appenv == "integration-testing" ? "grace-development-access-logs" : "${var.project_name}-${var.appenv}-access-logs"
+  app_name   = "${var.project_name}-${var.appenv}-inventory"
+  account_id = data.aws_caller_identity.current.account_id
 }
 

s3.tf

@@ -1,3 +1,7 @@
+locals {
+  useAccessLogging = length(var.access_logging_bucket) > 0 ? [1] : []
+}
+
 resource "aws_s3_bucket" "bucket" {
   bucket        = local.app_name
   acl           = "private"
@@ -7,9 +11,13 @@ resource "aws_s3_bucket" "bucket" {
     enabled = true
   }
 
-  logging {
-    target_bucket = local.logging_bucket
-    target_prefix = "${local.app_name}-logs/"
+  #tfsec:ignore:AWS002
+  dynamic "logging" {
+    for_each = local.useAccessLogging
+    content {
+      target_bucket = var.access_logging_bucket
+      target_prefix = "${local.app_name}-logs/"
+    }
   }
 
   server_side_encryption_configuration {

tests/Makefile

@@ -1,41 +1,22 @@
-environment := integration-testing
-export appenv := $(shell echo "$(environment)" | tr '[:upper:]' '[:lower:]')
-export TF_VAR_appenv := $(appenv)
-undefine TF_VAR_master_account_id
-undefine TF_VAR_master_role_name
-export backend_key := grace_integration_testing_inventory_lambda.tfstate
+default: integration_test
 
-.PHONY: check apply plan validate init destroy
-check:
-ifeq ($(strip $(backend_bucket)),)
-	@echo "backend_bucket must be provided"
-	@exit 1
-endif
-ifeq ($(strip $(TF_VAR_appenv)),)
-	@echo "TF_VAR_appenv must be provided"
-	@exit 1
-else
-	@echo "appenv: $(TF_VAR_appenv)"
-endif
-ifeq ($(strip $(backend_key)),)
-	@echo "backend_key must be provided"
-	@exit 1
-endif
+integration_test: lint
+	go test -v tests_test.go
 
-destroy: init
-	terraform destroy -auto-approve
+lint: dependencies
+	golangci-lint run ./...
 
-apply: plan
-	terraform apply -auto-approve
+dependencies: golangci go.sum init
 
-plan: validate
-	terraform plan
+golangci:
+	go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
 
-validate: init
-	terraform validate
-	terrascan --location . --tests all
+go.sum: go.mod
+	go mod tidy
 
-init: check
-	[[ -d ../release ]] || mkdir ../release
-	[[ -e ../release/grace-inventory-lambda.zip ]] || touch ../release/grace-inventory-lambda.zip
-	terraform init -backend-config="bucket=$(backend_bucket)" -backend-config="key=$(backend_key)"
+go.mod:
+	go mod init
+
+init:
+	mkdir -p ../release
+	touch ../release/grace-inventory-lambda.zip

tests/go.mod

@@ -0,0 +1,8 @@
+module github.com/GSA/grace-inventory/tests
+
+go 1.13
+
+require (
+	github.com/GSA/grace-tftest v0.0.4
+	github.com/aws/aws-sdk-go v1.29.23
+)

tests/go.sum

@@ -0,0 +1,66 @@
+github.com/GSA/grace-tftest v0.0.4-0.20200313023112-26cef98c0666 h1:+bqmj8HQOWJMZlqAtQcBWXWTMoNtz+xZtTojBnrmh1M=
+github.com/GSA/grace-tftest v0.0.4-0.20200313023112-26cef98c0666/go.mod h1:NELBSCI6T7rBeraqb75demHAgDyT/qHGgMaj2eqxM9g=
+github.com/GSA/grace-tftest v0.0.4-0.20200313151616-6dd791099f13 h1:sZaRa5EQ74lDaXRWQxc67PiMliUxYZzCgbdQ/PcF/6Q=
+github.com/GSA/grace-tftest v0.0.4-0.20200313151616-6dd791099f13/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/GSA/grace-tftest v0.0.4-0.20200313154439-d0ab50ce5ab8 h1:QvZNnCtyL7teO1FmT9tMh4KbLar4awOfTNQVdnFWrgk=
+github.com/GSA/grace-tftest v0.0.4-0.20200313154439-d0ab50ce5ab8/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/GSA/grace-tftest v0.0.4-0.20200313185701-bbb6dc6bfa63 h1:gPLym6iymzfmkEPxgxQbkuD4uXoXBYxngIGtJvtgGGU=
+github.com/GSA/grace-tftest v0.0.4-0.20200313185701-bbb6dc6bfa63/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/GSA/grace-tftest v0.0.4-0.20200313191055-fcd14e033f52 h1:Dm7IbIZVSDfW+w2ns0VP4dmUA+JMkyUBSXQRlMh0i+M=
+github.com/GSA/grace-tftest v0.0.4-0.20200313191055-fcd14e033f52/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/GSA/grace-tftest v0.0.4-0.20200313193400-ac6a51587207 h1:5W2DWQ1pVYJdZ26nI4X+P3EzxvPUlcebgXUxfRZkh/U=
+github.com/GSA/grace-tftest v0.0.4-0.20200313193400-ac6a51587207/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/GSA/grace-tftest v0.0.4-0.20200313210050-cd7ae1bde7a0 h1:O+Ee5HTvHFwQH6Pb4bELbkTqI0hRFR+6tS3+jBNlPCE=
+github.com/GSA/grace-tftest v0.0.4-0.20200313210050-cd7ae1bde7a0/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/GSA/grace-tftest v0.0.4-0.20200314034831-b29bbc3e63ee h1:ovyS2KeXUvjzY1HRVLFcs7yJN6BKgVWUxV1j7Hg+DVo=
+github.com/GSA/grace-tftest v0.0.4-0.20200314034831-b29bbc3e63ee/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/GSA/grace-tftest v0.0.4-0.20200314035056-5aa045927f00 h1:LFJYxlKb5ciTmRDreq1Oc0MSlJTaqtwZRA9tM37LMRA=
+github.com/GSA/grace-tftest v0.0.4-0.20200314035056-5aa045927f00/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/GSA/grace-tftest v0.0.4-0.20200314040620-c42727ade922 h1:+bkjxJwgmd+mZPfqwlf35mpKckjixFUSKkvoZSTmrUU=
+github.com/GSA/grace-tftest v0.0.4-0.20200314040620-c42727ade922/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/GSA/grace-tftest v0.0.4-0.20200314042054-55643204b309 h1:CA+3QwdFcoy49eHUIvNvbpLEPtiW9+CoeKoB7glZ8cw=
+github.com/GSA/grace-tftest v0.0.4-0.20200314042054-55643204b309/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/GSA/grace-tftest v0.0.4 h1:AQgnd8muvQaByTJ1e0hHXnTCBM//m5bd8oZeZa8YOPg=
+github.com/GSA/grace-tftest v0.0.4/go.mod h1:Wc2Et4UVAP8uFDPN/JYOv/rrVdfjuwR9Gg74tFM8Eow=
+github.com/aws/aws-sdk-go v1.29.15 h1:0ms/213murpsujhsnxnNKNeVouW60aJqSd992Ks3mxs=
+github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
+github.com/aws/aws-sdk-go v1.29.23 h1:wtiGLOzxAP755OfuVTDIy/NbUIYEDxbIbBEDfNhUpeU=
+github.com/aws/aws-sdk-go v1.29.23/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

tests/integration/integration.tf

@@ -0,0 +1,14 @@
+resource "aws_cloudwatch_log_group" "integration_test" {
+  name = "integration_test"
+}
+
+module "integration_test" {
+  source            = "../../"
+  accounts_info     = "self"
+  project_name      = "grace"
+  appenv            = "integration-test"
+  master_account_id = "123456789012"
+  master_role_name  = "role"
+  tenant_role_name  = "tenant-role"
+  source_file       = var.source_file
+}