diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000..bff08b0 --- /dev/null +++ b/.eslintignore @@ -0,0 +1,12 @@ +# don't ever lint node_modules +node_modules +cdk.out +# don't lint build output (make sure it's set to your correct build folder name) +dist +# don't lint nyc coverage output +coverage +*.config.js +*.eslint* + + + diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 0000000..7ad0b0d --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,23 @@ +module.exports = { + root: true, + env: { + node: true, + es2017: true, + mocha: true, + }, + extends: ["eslint:recommended"], + overrides: [ + { + files: ["**/*.ts"], + parser: "@typescript-eslint/parser", + plugins: ["@typescript-eslint"], + extends: [ + "eslint:recommended", + "plugin:@typescript-eslint/eslint-recommended", + "plugin:@typescript-eslint/recommended", + "prettier", + "prettier/@typescript-eslint", + ], + }, + ], +}; diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100755 index 0000000..75ac7bc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,37 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "" +labels: bug +assignees: "" +--- + +**Describe the bug** + + + +**To Reproduce** + + + +**Expected behavior** + + + +**Please complete the following information about the solution:** + +- [ ] Version: [e.g. v1.0.0] + +To get the version of the solution, you can look at the description of the created CloudFormation stack. For example, "_(SO0134) - The AWS CloudFormation template for deployment of the AWS Centralized WAF & SG Management. Version **v1.0.0**_". You can also find the version from [releases](https://github.com/awslabs/aws-centralized-logging/releases) + +- [ ] Region: [e.g. us-east-1] +- [ ] Was the solution modified from the version published on this repository? +- [ ] If the answer to the previous question was yes, are the changes available on GitHub? +- [ ] Have you checked your [service quotas](https://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html) for the sevices this solution uses? +- [ ] Were there any errors in the CloudWatch Logs? [How to enable debug mode?](https://docs.aws.amazon.com/solutions/latest/centralized-logging/appendix-d.html) + +**Screenshots** +If applicable, add screenshots to help explain your problem (please **DO NOT include sensitive information**). + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/documentation-improvements.md b/.github/ISSUE_TEMPLATE/documentation-improvements.md new file mode 100644 index 0000000..23d9133 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation-improvements.md @@ -0,0 +1,17 @@ +--- +name: Documentation improvements +about: Suggest a documentation update +title: '' +labels: documentation +assignees: '' + +--- + +**What were you initially searching for in the docs?** + + +**Is this related to an existing part of the documentation? Please share a link** + +**Describe how we could make it clearer** + +**If you have a proposed update, please share it here** \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100755 index 0000000..1d76207 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this solution +title: '' +labels: feature-request, enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** + + +**Describe the feature you'd like** + + +**Additional context** + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100755 index 0000000..de50e4d --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,5 @@ +*Issue #, if available:* + +*Description of changes:* + +By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e2f4160 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# Test and Compiler files +**test.json +**test.js + +# distribution folders +global-s3-assets +regional-s3-assets +open-source + +# Generated ouputs +dist +coverage +docs +npm-debug.log +*.zip +.scannerwork +*.xml +reports + +# Node dependencies +node_modules +package-lock.json + +# CDK asset staging directory +.cdk.staging +cdk.out +__snapshots__ \ No newline at end of file diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..b512c09 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/.prettierrc.yml b/.prettierrc.yml new file mode 100644 index 0000000..7fefec8 --- /dev/null +++ b/.prettierrc.yml @@ -0,0 +1,12 @@ +# .prettierrc or .prettierrc.yaml +arrowParens: "always" +bracketSpacing: true +endOfLine: "lf" +htmlWhitespaceSensitivity: "css" +proseWrap: "preserve" +trailingComma: "es5" +tabWidth: 2 +semi: true +singleQuote: false +quoteProps: "as-needed" +printWidth: 80 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a99cd9..22a39ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,56 @@ # Change Log - All notable changes to this project will be documented in this file. - - The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.2] - BUGFIX 2020-09-28 -- Changed Cognito user pool to only allow account creation by the Cognito Admin user +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [4.0.0] - 2020-12-15 + +### Added + +- VPC with 2 isolated & 2 public subnets +- Elasticsearch domain in isolated subnets +- Kinesis Data Stream and Kinesis Firehose for data streaming +- CloudWatch Logs Destination for cross account/region data streaming +- Windows jumpbox for accessing kibana +- Security group for jumpbox +- Security group for ES and Kinesis resources + +### Updated + +- Elasticsearch V7.7 +- Lambda log event transformer +- AWS CDK constructs for IaC + +### Removed + +- Spoke templates +- Cross account IAM role for Lambda (cross account streaming now uses CloudWatch Logs Destination) + +## [3.2.1] - 2020-09-14 + +### Added + +- SNS topic is now encrypted using KMS CMK +- Optional MFA support for Cognito users + +### Updated + +- Now uses CDK to create deployment templates +- Leverages AWS Solutions Contruct for Lambda/ElasticSearch/Kibana +- Updated to use Amazon Elasticsearch Service v7.7 + +### Removed + +- Demo Access Logging bucket no longer enables versioning +- Removed global egress access from the VPC security group +- Removed all hard-coded logical resource IDs and names to enable multiple stacks to be deployed, such as for testing or migration ## [3.2] - 2019-12-18 ### Added + - Backward-compatible to v3.0.0 - Includes all v3.0.1 changes - Do NOT upgrade from v3.0.1 to v3.2 @@ -17,12 +58,14 @@ ## [3.0.1] - 2019-11-29 ### Added + - Uses SSM Parameters to retrieve the latest HVM x86_64 AMI - Block public access to 2 buckets created for demo - CLFullAccessUserRole replaces CognitoAuthorizedRole. It is associated with the Admin group. Initial user is placed in this group. - CLReadOnlyAccessRole is added. It provides read-only access to users in UserPoolGroupROAccess. This is the default role for Authenticated users in the pool. ### Updated + - Nodejs8.10 to Nodejs12.x Lambda run time. - Updated license to Apache License version 2.0 - Corrected Master_Role environmental variable in spoke template to MASTER_ROLE @@ -36,16 +79,20 @@ - Tightened security on IAM roles to specific methods and resources ### Removed + - Unreferenced SolutionHelperRole in demo template - Unreferenced S3 bucket mapping in demo template - AMIInfo lookup Lambda - CognitoUnAuthorizedRole / unauthenticated Cognito access - + ## [0.0.1] - 2019-09-09 + ### Added + - CHANGELOG template file to fix new pipeline standards ### Updated + - updated buildspec.yml to meet new pipeline build standards - updated build-s3-dist.sh to meet new pipeline build standards - updated run-unit-tests.sh for correct references to folders diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d99687..2e165b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ information to effectively respond to your bug report or contribution. We welcome you to use the GitHub issue tracker to report bugs or suggest features. -When filing an issue, please check [existing open](https://github.com/awslabs/aws-centralized-logging/issues), or [recently closed](https://github.com/awslabs/aws-centralized-logging/issues?q=is%3Aissue+is%3Aclosed), issues to make sure somebody else hasn't already +When filing an issue, please check [existing open](https://github.com/awslabs/%%SOLUTION_NAME%%/issues), or [recently closed](https://github.com/awslabs/%%SOLUTION_NAME%%/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: * A reproducible test case or series of steps @@ -41,7 +41,7 @@ GitHub provides additional document on [forking a repository](https://help.githu ## Finding contributions to work on -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/aws-centralized-logging/labels/help%20wanted) issues is a great place to start. +Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/%%SOLUTION_NAME%%/labels/help%20wanted) issues is a great place to start. ## Code of Conduct @@ -55,7 +55,7 @@ If you discover a potential security issue in this project we ask that you notif ## Licensing -See the [LICENSE](https://github.com/awslabs/aws-centralized-logging/blob/master/LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. +See the [LICENSE](https://github.com/awslabs/%%SOLUTION_NAME%%/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. diff --git a/NOTICE.txt b/NOTICE.txt index d22051f..9aa381f 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,6 +1,6 @@ -AWS Centralized Logging Solution +AWS Centralized Logging -Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at http://www.apache.org/licenses/ or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, @@ -12,15 +12,9 @@ THIRD PARTY COMPONENTS ********************** This software includes third party software subject to the following copyrights: -AWS SDK under the Apache License Version 2.0 -aws-sdk-mock under the Apache License Version 2. -chai under the Massachusetts Institute of Technology (MIT) license -https under the Massachusetts Institute of Technology (MIT) license -mocha under the Massachusetts Institute of Technology (MIT) license -moment under the Massachusetts Institute of Technology (MIT) license -npm-run-all under the Massachusetts Institute of Technology (MIT) license -nyc under the ISC License -sinon under The 3-Clause BSD License -sinon-chai under The 2-Clause BSD License -stream under the Massachusetts Institute of Technology (MIT) license -uuid under the Massachusetts Institute of Technology (MIT) license +aws-sdk under Apache License 2.0 +aws-cdk under Apache License 2.0 +got under MIT License +moment under MIT License +uuid under MIT License +winston under MIT License \ No newline at end of file diff --git a/README.md b/README.md index 85daae4..0572b5e 100644 --- a/README.md +++ b/README.md @@ -1,81 +1,139 @@ # AWS Centralized Logging Solution -The AWS Centralized Logging Solution is a reference implementation that provides a foundation for logging to a centralized account. Customers can leverage the solution to index CloudTrail Logs, CW Logs, VPC Flow Logs on a ElasticSearch domain. The logs can then be searched on different fields. -## Getting Started -To get started with the AWS Centralized Logging Solution, please review the solution documentation. https://aws.amazon.com/answers/logging/centralized-logging/ +Centralized Logging is a reference implementation that provides a foundation for logging to a centralized account. Customers can leverage the solution to index CloudTrail Logs, CW Logs, VPC Flow Logs on a ElasticSearch domain. The logs can then be searched on different fields. -## Upgrading to v3.2 -Customers using v3.0.0 can upgrade to v3.2 by loading the new primary template. Spoke templates, however, must be removed and installed from the v3.2 spoke template. +The solution supports spoke accounts and regions and gives a single pane to gain actionable insight into the logs using Kibana. + +_Note:_ For any relavant information outside the scope of this readme, please refer to the solution landing page and implementation guide. + +**[🚀Solution Landing Page](https://aws.amazon.com/solutions/implementations/centralized-logging/)** | **[🚧Feature request](https://github.com/awslabs/aws-centralized-logging/issues/new?assignees=&labels=feature-request%2C+enhancement&template=feature_request.md&title=)** | **[🐛Bug Report](https://github.com/awslabs/aws-centralized-logging/issues/new?assignees=&labels=bug%2C+triage&template=bug_report.md&title=)** | **[📜Documentation Improvement](https://github.com/awslabs/aws-centralized-logging/issues/new?assignees=&labels=document-update&template=documentation_improvements.md&title=)** + +## Table of content + +- [Installation](#installing-pre-packaged-solution-template) +- [Customization](#customization) + - [Setup](#setup) + - [Changes](#changes) + - [Unit Test](#unit-test) + - [Build](#build) + - [Deploy](#deploy) +- [Sample Scenario](#sample-scenario) +- [File Structure](#file-structure) +- [License](#license) + +## Installing pre-packaged solution template + +- Primary Template: [aws-centralized-logging.template](https://solutions-reference.s3.amazonaws.com/centralized-logging/latest/aws-centralized-logging.template) + +- Demo Template: [Demo.template](https://solutions-reference.s3.amazonaws.com/centralized-logging/latest/aws-centralized-logging-demo.template) + +## Customization + +- Prerequisite: Node.js>10 + +### Setup + +Clone the repository and run the following commands to install dependencies, format and lint as per the project standards -## Running unit tests for customization -* Clone the repository, then make the desired code changes -* Next, run unit tests to make sure added customization passes the tests ``` -cd ./deployment -chmod +x ./run-unit-tests.sh \n -./run-unit-tests.sh \n +npm i +npm run prettier-format +npm run lint ``` -## Building distributable for customization -* Configure the bucket name of your target Amazon S3 distribution bucket +### Changes + +You may make any needed change as per your requirement. If you want to customize the Centralized Logging opinionated defaults, you can modify the [solution manifest file](./source/resources/lib/manifest.json). You can also control sending solution usage metrics to aws-solutions, from the manifest file. + ``` -export TEMPLATE_OUTPUT_BUCKET=my-bucket-name # bucket where cfn template will reside -export DIST_OUTPUT_BUCKET=my-bucket-name # bucket where customized code will reside -export VERSION=my-version # version number for the customized code +"solutionVersion": "%%VERSION%%", #provide a valid value eg. v1.0 +"sendMetric": "Yes", ``` -_Note:_ You would have to create 2 buckets, one with prefix 'my-bucket-name' and another regional bucket with prefix 'my-bucket-name-'; aws_region is where you are testing the customized solution. Also, the assets in bucket should be publicly accessible -* Now build the distributable: +Addtionally, you can customize the code and add any extension to the solution. Please review our [feature request guidelines](./.github/ISSUE_TEMPLATE/feature_request.md), if you want to submit a PR. + +### Unit Test + +You can run unit tests with the following command from the root of the project + ``` -chmod +x ./build-s3-dist.sh \n -./build-s3-dist.sh $DIST_OUTPUT_BUCKET $TEMPLATE_OUTPUT_BUCKET $VERSION \n + npm run test ``` -* Deploy the distributable to an Amazon S3 bucket in your account. _Note:_ you must have the AWS Command Line Interface installed. +### Build + +You can build lambda binaries with the following command from the root of the project + ``` -aws s3 cp ./dist/ s3://my-bucket-name/centralized-logging// --recursive --exclude "*" --include "*.template" --include "*.json" --acl bucket-owner-full-control --profile aws-cred-profile-name \n -aws s3 cp ./dist/ s3://my-bucket-name-/centralized-logging// --recursive --exclude "*" --include "*.zip" --acl bucket-owner-full-control --profile aws-cred-profile-name \n + npm run build ``` -* Get the link of the centralized-logging-primary.template uploaded to your Amazon S3 bucket. -* Deploy the AWS Centralized Logging Solution to your account by launching a new AWS CloudFormation stack using the link of the centralized-logging-primary.template. +### Deploy -## File Structure -The AWS Centralized Logging Solution project consists of indexing microservices which is deployed to a serverless environment in AWS Lambda. +Run the following command from the root of the project. Deploys all the primary solution components needed for centralized logging. **Deploy in Primary Account** ``` -|-source/ - |-services/ - |-indexing/ [ microservice for indexing logs on ES domain ] - |-lib/ - |-[ service module unit tests ] - |-basic-dashboard.json [ sample dashboard for kibana ] - |-logger.js [ logger class ] - |-metrics-helper.js [ helper module for sending anonymous metrics ] - |-index.js [ injection point for microservice ] - |-package.json - |-auth/ [ microservice for enabling Cognito auth ] - |-index.js [ injection point for microservice ] - |-logger.js [ logger class ] - |-package.json +cd source/resources +npm i ``` -*** -## v3.0.1 changes ``` -* Amazon Cognito integration for user login -* Elasticsearch version update to 6.3 -* Elasticsearch encryption at rest -* T-shirt sizing update +./node_modules/aws-cdk/bin/cdk bootstrap --profile +./node_modules/aws-cdk/bin/cdk synth CL-PrimaryStack +./node_modules/aws-cdk/bin/cdk deploy CL-PrimaryStack --parameters AdminEmail= --parameters SpokeAccounts= --parameters JumpboxKey= --parameters JumpboxDeploy='Yes' --profile ``` -*** +_Note:_ for PROFILE_NAME, substitute the name of an AWS CLI profile that contains appropriate credentials for deploying in your preferred region. + +## Sample Scenario (Enabling CloudWatch logging on Elasticsearch domain) -Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +The default deployment uses opinionated values as setup in [solution manifest file](./source/resources/lib/manifest.json). In this scenario let's say we want to enable CloudWatch logging for ES domain. -Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at +You would need to update the **ESDomain** resource in cl-primary-stack.ts as below: - http://www.apache.org/licenses/ +``` + logging: { + slowSearchLogEnabled: true, + appLogEnabled: true, + slowIndexLogEnabled: true, + }, +``` + +## File structure + +AWS Centralized Logging solution consists of: + +- cdk constructs to generate needed resources +- helper for bootstrapping purposes like creating CloudWatch Logs Destinations +- transformer to translate kinesis data stream records into Elasticsearch documents + +
+|-deployment/
+  |dashboard                      [ sample dashboard for demo ]  
+  |build-scripts/                 [ build scripts ]
+|-source/
+  |-resources
+    |-bin/
+      |-app.ts                    [ entry point for CDK app ]
+    |-__tests__/                  [ unit tests for CDK constructs ] 
+    |-lib/
+      |-cl-demo-ec2-construct.ts  [ CDK construct for demo web server resource ]
+      |-cl-demo-stack.ts          [ CDK construct for demo stack]
+      |-cl-jumpbox-construct.ts   [ CDK construct for windows jumpbox resource ]  
+      |-cl-primary-stack.ts       [ CDK construct for primary stack and related resources ]  
+      |-manifest.json             [ manifest file for CDK resources ]
+    |-config_files                [ tsconfig, jest.config.js, package.json etc. ]
+  |-services/
+    |-helper/                     [ lambda backed helper custom resource to help with solution launch/update/delete ]
+    |-transformer/                [ microservice to translate kinesis records into es documents ]
+      |-__tests/                  [ unit tests for all policy managers ]   
+      |-lib/
+        |-common/                 [ common moduel for logging and metrics collection ]
+      |-index.ts                  [ entry point for lambda function]     
+      |-config_files              [ tsconfig, jest.config.js, package.json etc. ]
+  |-config_files                  [ eslint, prettier, tsconfig, jest.config.js, package.json etc. ]  
+
-or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and limitations under the License. +## License +See license [here](./LICENSE.txt) diff --git a/deployment/build-s3-dist.sh b/deployment/build-s3-dist.sh index be106d9..2cf29c5 100755 --- a/deployment/build-s3-dist.sh +++ b/deployment/build-s3-dist.sh @@ -1,103 +1,205 @@ -#!/bin/bash -# -# This assumes all of the OS-level configuration has been completed and git repo has already been cloned -# -# This script should be run from the repo's deployment directory -# cd deployment -# ./build-s3-dist.sh source-bucket-base-name trademarked-solution-name version-code -# -# Paramenters: -# - source-bucket-base-name: Name for the S3 bucket location where the template will source the Lambda -# code from. The template will append '-[region_name]' to this bucket name. -# For example: ./build-s3-dist.sh solutions my-solution v1.0.0 -# The template will then expect the source code to be located in the solutions-[region_name] bucket -# -# - trademarked-solution-name: name of the solution for consistency -# -# - version-code: version of the package - -# Check to see if input has been provided: -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "Please provide the base source bucket name, trademark approved solution name and version where the lambda code will eventually reside." - echo "For example: ./build-s3-dist.sh solutions trademarked-solution-name v1.0.0" - exit 1 -fi - -bucket=$1 -tmsn=$2 -version=$3 - -do_cmd () { - echo "------ EXEC $*" - $* -} -do_replace() { - replace="s/$2/$3/g" - file=$1 - do_cmd sed -i -e $replace $file -} - -# Get reference for all important folders -template_dir="$PWD" -template_dist_dir="$template_dir/global-s3-assets" -build_dist_dir="$template_dir/regional-s3-assets" -source_dir="$template_dir/../source" - -echo "------------------------------------------------------------------------------" -echo "[Init] Clean old dist, node_modules and bower_components folders" -echo "------------------------------------------------------------------------------" -do_cmd rm -rf $template_dist_dir -do_cmd mkdir -p $template_dist_dir -do_cmd rm -rf $build_dist_dir -do_cmd mkdir -p $build_dist_dir - -echo "------------------------------------------------------------------------------" -echo "[Packing] Templates" -echo "------------------------------------------------------------------------------" -for file in $template_dir/*.template -do - do_cmd cp $file $template_dist_dir/ -done -do_cmd cp basic-dashboard-63.json $template_dist_dir/ - -echo "------------------------------------------------------------------------------" -echo "[Updating Bucket name]" -echo "------------------------------------------------------------------------------" -for file in $template_dist_dir/*.template -do - do_replace $file '%%BUCKET_NAME%%' $bucket -done - -echo "------------------------------------------------------------------------------" -echo "[Updating Solution name]" -echo "------------------------------------------------------------------------------" -for file in $template_dist_dir/*.template -do - do_replace $file '%%SOLUTION_NAME%%' $tmsn +#!/bin/bash +# +# This assumes all of the OS-level configuration has been completed and git repo has already been cloned +# +# This script should be run from the repo's deployment directory +# cd deployment +# ./build-s3-dist.sh source-bucket-base-name trademarked-solution-name version-code +# +# Arguments: +# - source-bucket-base-name: Name for the S3 bucket location where the template will source the Lambda +# code from. The template will append '-[region_name]' to this bucket name. +# For example: ./build-s3-dist.sh solutions my-solution v1.0.0 +# The template will then expect the source code to be located in the solutions-[region_name] bucket +# +# - template-bucket: where templates will be hosted for the solution +# +# - trademarked-solution-name: name of the solution for consistency +# +# - version-code: version of the package + +# Check to see if input has been provided: +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ -z "$4" ]; then + echo "Please provide the base source bucket name, trademark approved solution name and version where the lambda code will eventually reside." + echo "For example: ./build-s3-dist.sh solutions solutions-reference trademarked-solution-name v1.0.0" + exit 1 +fi + +# Get reference for all important folders +template_dir="$PWD" +staging_dist_dir="$template_dir/staging" +template_dist_dir="$template_dir/global-s3-assets" +build_dist_dir="$template_dir/regional-s3-assets" +resource_dir="$template_dir/../source/resources" +source_dir="$template_dir/../source/services" + +echo "------------------------------------------------------------------------------" +echo "[Init] Remove any old dist files from previous runs" +echo "------------------------------------------------------------------------------" +echo "rm -rf $template_dist_dir" +rm -rf $template_dist_dir +echo "mkdir -p $template_dist_dir" +mkdir -p $template_dist_dir +echo "rm -rf $build_dist_dir" +rm -rf $build_dist_dir +echo "mkdir -p $build_dist_dir" +mkdir -p $build_dist_dir +echo "rm -rf $staging_dist_dir" +rm -rf $staging_dist_dir +echo "mkdir -p $staging_dist_dir" +mkdir -p $staging_dist_dir + +echo "------------------------------------------------------------------------------" +echo "[Build] Build typescript microservices" +echo "------------------------------------------------------------------------------" +# build helper function +echo "cd $source_dir/helper" +cd $source_dir/helper +echo "npm run build:all" +npm run build:all + +# build pre-req-manager function +echo "cd $source_dir/transformer" +cd $source_dir/transformer +echo "npm run build:all" +npm run build:all + +echo "------------------------------------------------------------------------------" +echo "[Init] Install dependencies for the cdk-solution-helper" +echo "------------------------------------------------------------------------------" +echo "cd $template_dir/cdk-solution-helper" +cd $template_dir/cdk-solution-helper +echo "npm install" +npm install + +echo "------------------------------------------------------------------------------" +echo "[Synth] CDK Project" +echo "------------------------------------------------------------------------------" +# Install the global aws-cdk package +echo "cd $resource_dir" +cd $resource_dir +echo "npm i" +npm i + +# Run 'cdk synth' to generate raw solution outputs +echo "cdk synth CL-PrimaryStack --output=$staging_dist_dir" +./node_modules/aws-cdk/bin/cdk synth CL-PrimaryStack --output=$staging_dist_dir + +# Remove unnecessary output files +echo "cd $staging_dist_dir" +cd $staging_dist_dir +echo "rm tree.json manifest.json cdk.out" +rm tree.json manifest.json cdk.out + +echo "------------------------------------------------------------------------------" +echo "[Packing] Templates" +echo "------------------------------------------------------------------------------" +# Move outputs from staging to template_dist_dir +echo "Move outputs from staging to template_dist_dir" +echo "cp $staging_dist_dir/CL*.template.json $template_dist_dir/" +cp $staging_dist_dir/CL*.template.json $template_dist_dir/ +rm *.template.json + +# Rename all *.template.json files to *.template +echo "Rename all *.template.json to *.template" +echo "copy templates and rename" +for f in $template_dist_dir/*.template.json; +do + if [[ $f == *"CL-PrimaryStack"* ]] + then + mv "$f" "$template_dist_dir/aws-centralized-logging.template" + else + mv "$f" "$template_dist_dir/aws-centralized-logging-demo.template" + fi done -echo "------------------------------------------------------------------------------" -echo "[Updating version name]" -echo "------------------------------------------------------------------------------" -for file in $template_dist_dir/*.template -do - do_replace $file '%%VERSION%%' $version +# Run the helper to clean-up the templates and remove unnecessary CDK elements +echo "Run the helper to clean-up the templates and remove unnecessary CDK elements" +echo "node $template_dir/cdk-solution-helper/index" +node $template_dir/cdk-solution-helper/index +if [ "$?" = "1" ]; then + echo "(cdk-solution-helper) ERROR: there is likely output above." 1>&2 + exit 1 +fi + +# Find and replace bucket_name, solution_name, and version +if [[ "$OSTYPE" == "darwin"* ]]; then + # Mac OS + echo "Updating variables in template with $1" + replace="s/%%BUCKET_NAME%%/$1/g" + echo "sed -i '' -e $replace $template_dist_dir/*.template" + sed -i '' -e $replace $template_dist_dir/*.template + replace="s/%%TEMPLATE_BUCKET%%/$2/g" + echo "sed -i '' -e $replace $template_dist_dir/*.template" + sed -i '' -e $replace $template_dist_dir/*.template + replace="s/%%SOLUTION_NAME%%/$3/g" + echo "sed -i '' -e $replace $template_dist_dir/*.template" + sed -i '' -e $replace $template_dist_dir/*.template + replace="s/%%VERSION%%/$4/g" + echo "sed -i '' -e $replace $template_dist_dir/*.template" + sed -i '' -e $replace $template_dist_dir/*.template +else + # Other linux + echo "Updating variables in template with $1" + replace="s/%%BUCKET_NAME%%/$1/g" + echo "sed -i -e $replace $template_dist_dir/*.template" + sed -i -e $replace $template_dist_dir/*.template + replace="s/%%TEMPLATE_BUCKET%%/$2/g" + echo "sed -i -e $replace $template_dist_dir/*.template" + sed -i -e $replace $template_dist_dir/*.template + replace="s/%%SOLUTION_NAME%%/$3/g" + echo "sed -i -e $replace $template_dist_dir/*.template" + sed -i -e $replace $template_dist_dir/*.template + replace="s/%%VERSION%%/$4/g" + echo "sed -i -e $replace $template_dist_dir/*.template" + sed -i -e $replace $template_dist_dir/*.template +fi + +echo "------------------------------------------------------------------------------" +echo "[Packing] Sample Dashboard" +echo "------------------------------------------------------------------------------" +echo "cp $template_dir/dashboard.ndjson $template_dist_dir/" +cp $template_dir/dashboard.ndjson $template_dist_dir/ + +echo "------------------------------------------------------------------------------" +echo "[Packing] Lambdas" +echo "------------------------------------------------------------------------------" +# General cleanup of node_modules and package-lock.json files +echo "find $staging_dist_dir -iname "node_modules" -type d -exec rm -rf "{}" \; 2> /dev/null" +find $staging_dist_dir -iname "node_modules" -type d -exec rm -rf "{}" \; 2> /dev/null +echo "find $staging_dist_dir -iname "package-lock.json" -type f -exec rm -f "{}" \; 2> /dev/null" +find $staging_dist_dir -iname "package-lock.json" -type f -exec rm -f "{}" \; 2> /dev/null + +# ... For each asset.* source code artifact in the temporary /staging folder... +cd $staging_dist_dir +for i in `find . -mindepth 1 -maxdepth 1 -type f \( -iname "*.zip" \) -or -type d`; do + + # Rename the artifact, removing the period for handler compatibility + pfname="$(basename -- $i)" + fname="$(echo $pfname | sed -e 's/\.//')" + mv $i $fname + + if [[ $fname != *".zip" ]] + then + # Zip the artifact + echo "zip -r $fname.zip $fname/*" + zip -r $fname.zip $fname + fi + +# ... repeat until all source code artifacts are zipped done -echo "------------------------------------------------------------------------------" -echo "[Rebuild] Indexing Code" +# Copy the zipped artifact from /staging to /regional-s3-assets +echo "cp -R *.zip $build_dist_dir" +cp -R *.zip $build_dist_dir + +# Remove the old, unzipped artifact from /staging +echo "rm -rf *.zip" +rm -rf *.zip + echo "------------------------------------------------------------------------------" -cd $source_dir/services/indexing -npm install -npm run build -npm run zip -cp dist/clog-indexing-service.zip $build_dist_dir/clog-indexing-service.zip - -echo "------------------------------------------------------------------------------" -echo "[Rebuild] Auth code" -echo "------------------------------------------------------------------------------" -cd $source_dir/services/auth -npm install -npm run build -npm run zip -cp dist/clog-auth.zip $build_dist_dir/clog-auth.zip +echo "[Cleanup] Remove temporary files" +echo "------------------------------------------------------------------------------" +# Delete the temporary /staging folder +echo "rm -rf $staging_dist_dir" +rm -rf $staging_dist_dir diff --git a/deployment/centralized-logging-demo.template b/deployment/centralized-logging-demo.template deleted file mode 100644 index 1abffc8..0000000 --- a/deployment/centralized-logging-demo.template +++ /dev/null @@ -1,576 +0,0 @@ -# Centralized Logging Solution -# -# template for centralized-logging-solution -# this template will deploy sample log sources -# -# author: aws-solutions-builder@ -AWSTemplateFormatVersion: 2010-09-09 - -Description: (SO0009d) - AWS Centralized Logging Solution, sample sources template (Version %%VERSION%%) - -Parameters: - # Log Streamer Function Arn - LogStreamerArn: - Description: Lambda Arn for Log Streamer function from primer template - Type: String - - # VPC CIDR for proxy servers - DemoVPCCidr: - Description: CIDR block for VPC - Type: String - MinLength: 9 - MaxLength: 18 - AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})' - Default: 10.250.0.0/16 - ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x - - # Subnet for proxy web server - DemoSubnet: - Description: IP address range for subnet - Type: String - MinLength: 9 - MaxLength: 18 - AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})' - Default: 10.250.250.0/24 - ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x - - LatestAMIId: - Description: >- - Automatically selects the latest Amazon Linux AMI. Do not change this - value - Type: 'AWS::SSM::Parameter::Value' - Default: /aws/service/ami-amazon-linux-latest/amzn-ami-hvm-x86_64-gp2 - -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: Network Configuration - Parameters: - - DemoVPCCidr - - DemoSubnet - ParameterLabels: - DemoVPCCidr: - default: VPC for Sample Log Sources - DemoSubnet: - default: Subnet for Sample Web Server - LogStreamerArn: - default: Lambda Arn for Log Streaming - -Mappings: - - # Instance type for demo box - EC2: - Instance: - Type: 't3.micro' - - # CloudWatch logs pattern mapping - FilterPatternLookup: - Common: - Pattern: '[host, ident, authuser, date, request, status, bytes, referrer, agent]' - CloudTrail: - Pattern: '' - FlowLogs: - Pattern: '[version, account_id, interface_id, srcaddr != "-", dstaddr != "-", srcport != "-", dstport != "-", protocol, packets, bytes, start, end, action, log_status]' - Lambda: - Pattern: '[timestamp=*Z, request_id="*-*", event]' - SpaceDelimited: - Pattern: '[]' - Other: - Pattern: '' - -Resources: - # - # Demo VPC resources - # [DemoVPC, PublicSubnet, InternetGateway, GatewayAttachment, PublicRtb, PublicRoute, SubnetRtbAssoc] - # - DemoVPC: - Type: AWS::EC2::VPC - Properties: - CidrBlock: !Sub ${DemoVPCCidr} - Tags: - - Key: Name - Value: centralized-logging-demo VPC - Metadata: - cfn_nag: - rules_to_suppress: - - id: W60 - reason: "This is a demo VPC with no ingress other than to the demo web server. VPC flow logs are not necessary." - - PublicSubnet: - Type: AWS::EC2::Subnet - Properties: - VpcId: !Sub ${DemoVPC} - AvailabilityZone: !Select [ 0, !GetAZs '' ] - MapPublicIpOnLaunch: true - CidrBlock: !Sub ${DemoSubnet} - Tags: - - Key: Name - Value: centralized-logging-demo subnet - Metadata: - cfn_nag: - rules_to_suppress: - - id: W33 - reason: "This is where the test webserver is deployed for for a static page to demo the functionality of the solution by generating log for demo purposes. Hence requires a public IP" - - InternetGateway: - Type: AWS::EC2::InternetGateway - Properties: {} - - GatewayAttachment: - Type: AWS::EC2::VPCGatewayAttachment - Properties: - VpcId: !Sub ${DemoVPC} - InternetGatewayId: !Sub ${InternetGateway} - - PublicRtb: - Type: AWS::EC2::RouteTable - Properties: - VpcId: !Sub ${DemoVPC} - - PublicRoute: - Type: AWS::EC2::Route - DependsOn: GatewayAttachment - Properties: - RouteTableId: !Sub ${PublicRtb} - DestinationCidrBlock: 0.0.0.0/0 - GatewayId: !Sub ${InternetGateway} - - SubnetRtbAssoc: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: !Sub ${PublicSubnet} - RouteTableId: !Sub ${PublicRtb} - - - # - # VPC flow log resources - # [VPCFlowLogGroup, FlowlogsRole, VPCFlowLog, VPCFlowLogtoLambda] - # - VPCFlowLogGroup: - Type: AWS::Logs::LogGroup - Properties: - RetentionInDays: 1 - - FlowlogsRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - vpc-flow-logs.amazonaws.com - Action: - - sts:AssumeRole - Path: / - Policies: - - PolicyName: LogRolePolicy - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:DescribeLogGroups - - logs:DescribeLogStreams - - logs:PutLogEvents - Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:* - Metadata: - cfn_nag: - rules_to_suppress: - - id: W11 - reason: "Push logs from various resources to cloudwatch logs. Hence open" - - VPCFlowLog: - Type: AWS::EC2::FlowLog - Properties: - DeliverLogsPermissionArn: !Sub ${FlowlogsRole.Arn} - LogGroupName: !Sub ${VPCFlowLogGroup} - ResourceId: !Sub ${DemoVPC} - ResourceType: VPC - TrafficType: ALL - - VPCFlowLogtoLambda: - Type: AWS::Logs::SubscriptionFilter - Properties: - DestinationArn: !Sub ${LogStreamerArn} - FilterPattern: !FindInMap [FilterPatternLookup, FlowLogs, Pattern] - LogGroupName: !Sub ${VPCFlowLogGroup} - - S3LoggingBucket: - DeletionPolicy: Retain - Type: AWS::S3::Bucket - Properties: - BucketName: !Sub aws-trailbucket-s3-access-logs-${AWS::AccountId}-${AWS::Region} - AccessControl: LogDeliveryWrite - VersioningConfiguration: - Status: Enabled - BucketEncryption: - ServerSideEncryptionConfiguration: - - ServerSideEncryptionByDefault: - SSEAlgorithm: AES256 - Tags: - - Key: Name - Value: 'AWS Centralized Logging Demo' - PublicAccessBlockConfiguration: - BlockPublicAcls: True - BlockPublicPolicy: True - IgnorePublicAcls: True - RestrictPublicBuckets: True - Metadata: - cfn_nag: - rules_to_suppress: - - id: W35 - reason: "This S3 bucket is used as the destination for storing access logs" - - id: W51 - reason: "Log delivery controlled by ACL, not bucket policy" - - # - # CloudTrail resources - # [TrailBucket, TrailBucketPolicy, TrailLogGroup, TrailLogGroupRole, TrailLogtoLambda] - # - TrailBucket: - DeletionPolicy: Retain - Type: AWS::S3::Bucket - Properties: - LoggingConfiguration: - DestinationBucketName: !Ref S3LoggingBucket - LogFilePrefix: access-logs - BucketEncryption: - ServerSideEncryptionConfiguration: - - ServerSideEncryptionByDefault: - SSEAlgorithm: AES256 - Tags: - - Key: Name - Value: 'AWS Centralized Logging Demo' - PublicAccessBlockConfiguration: - BlockPublicAcls: True - BlockPublicPolicy: True - IgnorePublicAcls: True - RestrictPublicBuckets: True - - TrailBucketPolicy: - Type: AWS::S3::BucketPolicy - Properties: - Bucket: !Ref TrailBucket - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: cloudtrail.amazonaws.com - Action: s3:GetBucketAcl - Resource: !Sub arn:aws:s3:::${TrailBucket} - - Effect: Allow - Principal: - Service: cloudtrail.amazonaws.com - Action: s3:PutObject - Resource: !Sub arn:aws:s3:::${TrailBucket}/AWSLogs/${AWS::AccountId}/* - Condition: - StringEquals: - s3:x-amz-acl: bucket-owner-full-control - - TrailLogGroup: - Type: AWS::Logs::LogGroup - Properties: - RetentionInDays: 1 - - TrailLogGroupRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: cloudtrail.amazonaws.com - Action: sts:AssumeRole - Policies: - - PolicyName: cloudtrail-policy - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - logs:CreateLogStream - - logs:PutLogEvents - Resource: !Sub ${TrailLogGroup.Arn} - - Trail: - Type: AWS::CloudTrail::Trail - Properties: - IncludeGlobalServiceEvents: true - IsLogging: true - IsMultiRegionTrail: false - S3BucketName: !Sub ${TrailBucket} - CloudWatchLogsLogGroupArn: !Sub ${TrailLogGroup.Arn} - CloudWatchLogsRoleArn: !Sub ${TrailLogGroupRole.Arn} - DependsOn: TrailBucketPolicy - - TrailLogtoLambda: - Type: AWS::Logs::SubscriptionFilter - Properties: - DestinationArn: !Sub ${LogStreamerArn} - FilterPattern: !FindInMap - - FilterPatternLookup - - CloudTrail - - Pattern - LogGroupName: !Sub ${TrailLogGroup} - - # - # WebServer log resources - # [EC2LogRole, InstanceProfile, WebServerSecurityGroup, WebServerSecurityGroupIngress, WebServerLogtoLambda] - # - EC2LogRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - ec2.amazonaws.com - Action: - - sts:AssumeRole - Path: / - Policies: - - PolicyName: LogRolePolicy - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - logs:Create* - - logs:PutLogEvents - - s3:GetObject - Resource: - - !Sub ${WebServerLogGroup.Arn} - - !GetAtt TrailBucket.Arn - - InstanceProfile: - Type: AWS::IAM::InstanceProfile - Properties: - Path: / - Roles: - - !Sub ${EC2LogRole} - - WebServerHost: - Type: AWS::EC2::Instance - Metadata: - Comment: Install a simple PHP application - AWS::CloudFormation::Init: - config: - packages: - yum: - httpd: [] - php: [] - files: - /tmp/cwlogs/apacheaccess.conf: - content: !Join - - '' - - - | - [general] - - | - state_file= /var/awslogs/agent-state - - | - [/var/log/httpd/access_log] - - | - file = /var/log/httpd/access_log - - 'log_group_name = ' - - !Ref WebServerLogGroup - - |+ - - - | - log_stream_name = {instance_id}/apache.log - - 'datetime_format = %d/%b/%Y:%H:%M:%S' - mode: '000400' - owner: apache - group: apache - /var/www/html/index.php: - content: !Join - - '' - - - | - AWS CloudFormation sample PHP application'; - - | - ?> - mode: '000644' - owner: apache - group: apache - /etc/cfn/cfn-hup.conf: - content: !Join - - '' - - - | - [main] - - stack= - - !Ref 'AWS::StackId' - - |+ - - - region= - - !Ref 'AWS::Region' - - |+ - - mode: '000400' - owner: root - group: root - /etc/cfn/hooks.d/cfn-auto-reloader.conf: - content: !Join - - '' - - - | - [cfn-auto-reloader-hook] - - | - triggers=post.update - - > - path=Resources.WebServerHost.Metadata.AWS::CloudFormation::Init - - 'action=/opt/aws/bin/cfn-init -s ' - - !Ref 'AWS::StackId' - - ' -r WebServerHost ' - - ' --region ' - - !Ref 'AWS::Region' - - |+ - - - | - runas=root - services: - sysvinit: - httpd: - enabled: 'true' - ensureRunning: 'true' - sendmail: - enabled: 'false' - ensureRunning: 'false' - CreationPolicy: - ResourceSignal: - Timeout: PT15M - Properties: - ImageId: !Ref LatestAMIId - Tags: - - Key: Name - Value: Web Server centralized-logging-demo - NetworkInterfaces: - - GroupSet: - - !Sub ${WebServerSecurityGroup} - AssociatePublicIpAddress: true - DeviceIndex: 0 - DeleteOnTermination: true - SubnetId: !Sub ${PublicSubnet} - InstanceType: !FindInMap [EC2, Instance, Type] - IamInstanceProfile: !Sub ${InstanceProfile} - UserData: !Base64 - 'Fn::Join': - - '' - - - | - #!/bin/bash -xe - - | - # Get the latest CloudFormation package - - | - yum update -y aws-cfn-bootstrap - - | - # Start cfn-init - - '/opt/aws/bin/cfn-init -s ' - - !Ref 'AWS::StackId' - - ' -r WebServerHost ' - - ' --region ' - - !Ref 'AWS::Region' - - |2 - || error_exit 'Failed to run cfn-init' - - > - # Start up the cfn-hup daemon to listen for changes to the EC2 - instance metadata - - | - /opt/aws/bin/cfn-hup || error_exit 'Failed to start cfn-hup' - - | - # Get the CloudWatch Logs agent - - > - wget - https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py - - | - # Install the CloudWatch Logs agent - - 'python awslogs-agent-setup.py -n -r ' - - !Ref 'AWS::Region' - - |2 - -c /tmp/cwlogs/apacheaccess.conf || error_exit 'Failed to run CloudWatch Logs agent setup' - - | - # pre-warm the apache logs - - | - curl 127.0.0.1 - - | - curl 127.0.0.1/404 - - | - # All done so signal success - - '/opt/aws/bin/cfn-signal -e $? ' - - ' --stack ' - - !Ref 'AWS::StackName' - - ' --resource WebServerHost ' - - ' --region ' - - !Ref 'AWS::Region' - - |+ - - WebServerLogGroup: - Type: AWS::Logs::LogGroup - Properties: - RetentionInDays: 1 - - WebServerSecurityGroup: - Type: AWS::EC2::SecurityGroup - ## - # V56617021 - 10/08/2018 - Suppress cfn_nag - Metadata: - cfn_nag: - rules_to_suppress: - - id: F1000 - reason: Override the SecurityGroup, egress rule defined as well - Properties: - GroupDescription: Enable HTTP access via port 80 - VpcId: !Sub ${DemoVPC} - - WebServerSecurityGroupIngress: - Type: AWS::EC2::SecurityGroupIngress - Properties: - Description: Allow inbound access to the webserver from the internet - GroupId: !Sub ${WebServerSecurityGroup} - IpProtocol: tcp - FromPort: 80 - ToPort: 80 - CidrIp: 0.0.0.0/0 - Metadata: - cfn_nag: - rules_to_suppress: - - id: W9 - reason: "The server is a public webserver and hence CIDR mask is open" - - id: W2 - reason: "To demo the functionality of this solution, there is public facing webserver with which the SecurityGroup will be associated. This webserver instance will be used only as a mechanism to generate logs. Hence no ELB has been added to the infrastructure" - - WebServerSecurityGroupEgress: - Type: AWS::EC2::SecurityGroupEgress - Properties: - Description: Allow outbound access to the internet for all requests coming in - CidrIp: 0.0.0.0/0 - FromPort: 0 - GroupId: !Sub ${WebServerSecurityGroup} - IpProtocol: tcp - ToPort: 65535 - Metadata: - cfn_nag: - rules_to_suppress: - - id: W5 - reason: "The webserver is a public access server and hence CIDR range is open" - - id: W29 - reason: "The TCP/IP protocol requires the ephemeral port range for communication over the internet for a public server" - - WebServerLogtoLambda: - Type: AWS::Logs::SubscriptionFilter - Properties: - DestinationArn: !Sub ${LogStreamerArn} - FilterPattern: !FindInMap [FilterPatternLookup, Common, Pattern] - LogGroupName: !Sub ${WebServerLogGroup} - -Outputs: - PublicIP: - Description: Public IP of sample web server - Value: !Sub http://${WebServerHost.PublicIp} diff --git a/deployment/centralized-logging-primary.template b/deployment/centralized-logging-primary.template deleted file mode 100644 index a90aa01..0000000 --- a/deployment/centralized-logging-primary.template +++ /dev/null @@ -1,863 +0,0 @@ -# Centralized Logging Solution -# -# template for centralized-logging-solution -# **DO NOT DELETE** -# -# author: aws-solutions-builder@ -AWSTemplateFormatVersion: 2010-09-09 - -Description: (SO0009) - AWS Centralized Logging Solution, primary template (Version %%VERSION%%) - -Parameters: - # Name for ES Domain - DOMAINNAME: - Description: Name for the Amazon ES domain that this template will create. Domain names must start with a lowercase letter and must be between 3 and 28 characters. Valid characters are a-z (lowercase only), 0-9. - Type: String - Default: centralizedlogging - - # Email address for the Elasticsearch domain admin - DomainAdminEmail: - Type: String - AllowedPattern: '^[_A-Za-z0-9-\+\.]+(\.[_A-Za-z0-9-]+)*@[A-Za-z0-9-]+(\.[A-Za-z0-9]+)*(\.[A-Za-z]{2,})$' - Description: E-mail address of the Elasticsearch admin - - # Email address for Cognito Admin user - CognitoAdminEmail: - Type: String - AllowedPattern: '^[_A-Za-z0-9-\+\.]+(\.[_A-Za-z0-9-]+)*@[A-Za-z0-9-]+(\.[A-Za-z0-9]+)*(\.[A-Za-z]{2,})$' - Description: E-mail address of the Cognito admin - - # ES cluster size - ClusterSize: - Description: Amazon ES cluster size; small (4 data nodes), medium (6 data nodes), large (6 data nodes) - Type: String - Default: Small - AllowedValues: - - Small - - Medium - - Large - - # Demo template for sample logs - DemoTemplate: - Description: Deploy template for sample data and logs? - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - - # Spoke accounts which would use the same ES - SpokeAccounts: - Description: Account IDs which you want to allow for centralized logging (comma separated list eg. 11111111,22222222) - Type: CommaDelimitedList - - # VPC CIDR for sample sources - DemoVPC: - Description: CIDR for VPC with sample sources (Only required if you chose 'Yes' above) - Type: String - MinLength: 9 - MaxLength: 18 - AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})' - Default: 10.250.0.0/16 - ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x - - # Subnet for sample web server - DemoSubnet: - Description: IP address range for subnet with sample web server (Only required if you chose 'Yes' above) - Type: String - MinLength: 9 - MaxLength: 18 - AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})' - Default: 10.250.250.0/24 - ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x - -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: Elasticsearch Configuration - Parameters: - - DOMAINNAME - - DomainAdminEmail - - ClusterSize - - SpokeAccounts - - Label: - default: Cognito Configuration - Parameters: - - CognitoAdminEmail - - Label: - default: Do you want to deploy sample log sources? - Parameters: - - DemoTemplate - - DemoVPC - - DemoSubnet - ParameterLabels: - CognitoAdminEmail: - default: Cognito Admin email address - DomainAdminEmail: - default: Elasticsearch Domain Admin email address - DOMAINNAME: - default: Elasticsearch Domain name - ClusterSize: - default: Cluster Size - DemoTemplate: - default: Sample Logs - SpokeAccounts: - default: Spoke Accounts - DemoVPC: - default: VPC CIDR for Sample Sources - DemoSubnet: - default: Subnet for Sample Web Server - -Mappings: - InstanceMap: - send-data: {"SendAnonymousData": "Yes"} - - ElasticSearch: - Parameters: - Version: '6.3' - NodeCount: - Small: '4' - Medium: '6' - Large: '6' - MasterSize: - Small: c5.large.elasticsearch - Medium: c5.large.elasticsearch - Large: c5.large.elasticsearch - InstanceSize: - Small: i3.large.elasticsearch - Medium: i3.2xlarge.elasticsearch - Large: i3.4xlarge.elasticsearch - - # Lambda source code mapping - SourceCode: - General: - S3Bucket: "%%BUCKET_NAME%%" - KeyPrefix: "%%SOLUTION_NAME%%/%%VERSION%%" - -Conditions: - DemoData: !Equals [!Ref DemoTemplate, 'Yes'] - SingleAccnt: !Equals [!Select [ 0, !Ref SpokeAccounts ], ''] - -Resources: - # - # Cognito and IAM - # - # Creates a user pool in cognito to auth against - UserPool: - DeletionPolicy: 'Retain' - Type: 'AWS::Cognito::UserPool' - Properties: - UserPoolName: !Sub ${DOMAINNAME}_kibana_access - AutoVerifiedAttributes: - - email - MfaConfiguration: 'OFF' - EmailVerificationSubject: !Ref AWS::StackName - Schema: - - Name: name - AttributeDataType: String - Mutable: true - Required: true - - Name: email - AttributeDataType: String - Mutable: false - Required: true - AdminCreateUserConfig: - AllowAdminCreateUserOnly: True - Metadata: - cfn_nag: - rules_to_suppress: - - id: F78 - reason: "MFAConfiguration can be enabled by the customer upon implementation." - - # Custom resource to configure Cognito and ES - SetupESCognito: - DependsOn: - - 'ElasticsearchAWSLogs' - - 'UserPoolDomain' - Type: 'Custom::SetupESCognito' - Version: 1.0 - Properties: - ServiceToken: !GetAtt LambdaESCognito.Arn - Domain: !Ref DOMAINNAME - CognitoDomain: !Sub ${DOMAINNAME}-${AWS::AccountId} - UserPoolId: !Ref UserPool - IdentityPoolId: !Ref IdentityPool - RoleArn: !GetAtt CognitoESAccessRole.Arn - - UserPoolDomain: - DeletionPolicy: 'Retain' - Type: 'AWS::Cognito::UserPoolDomain' - Properties: - Domain: !Sub ${DOMAINNAME}-${AWS::AccountId} - UserPoolId: !Ref UserPool - - # Creates a group in Cognito for Kibana ADMIN access - UserPoolGroup: - DeletionPolicy: 'Retain' - Type: "AWS::Cognito::UserPoolGroup" - Properties: - Description: 'User pool group for Kibana full access' - GroupName: !Sub ${DOMAINNAME}_kibana_admin - Precedence: 0 - UserPoolId: !Ref UserPool - RoleArn: !GetAtt CognitoAuthorizedRole.Arn - - # Creates a User Pool Client to be used by the identity pool - UserPoolClient: - DeletionPolicy: 'Retain' - Type: 'AWS::Cognito::UserPoolClient' - Properties: - ClientName: !Sub ${DOMAINNAME}-client - GenerateSecret: false - UserPoolId: !Ref UserPool - - # Creates a federated Identity pool - IdentityPool: - DeletionPolicy: 'Retain' - Type: 'AWS::Cognito::IdentityPool' - Properties: - IdentityPoolName: !Sub ${DOMAINNAME}Identity - AllowUnauthenticatedIdentities: false # Prevent anonymous access - CognitoIdentityProviders: - - ClientId: !Ref UserPoolClient - ProviderName: !GetAtt UserPool.ProviderName - ServerSideTokenCheck: true - - # Assigns the roles to the Identity Pool - IdentityPoolRoleMapping: - DeletionPolicy: 'Retain' - Type: 'AWS::Cognito::IdentityPoolRoleAttachment' - Properties: - IdentityPoolId: !Ref IdentityPool - Roles: - authenticated: !GetAtt CognitoAuthorizedRole.Arn - - CognitoAuthorizedRole: - DeletionPolicy: 'Retain' - Type: 'AWS::IAM::Role' - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: 'Allow' - Principal: - Federated: 'cognito-identity.amazonaws.com' - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - 'cognito-identity.amazonaws.com:aud': !Ref IdentityPool - 'ForAnyValue:StringLike': - 'cognito-identity.amazonaws.com:amr': authenticated - Policies: - - PolicyName: 'CognitoAccessPolicy' - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: 'Allow' - Action: - - 'es:ESHttp*' - Resource: !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DOMAINNAME}/*' - - # Policy passed to the ES API to execute updateElasticsearchDomainConfig - CognitoESAccessPolicy: - Type: 'AWS::IAM::Policy' - DeletionPolicy: 'Retain' - Properties: - PolicyName: 'CentralizedLoggingESCognitoAccess' - PolicyDocument: - Version: 2012-10-17 - Statement: - - Sid: ESCognitoPerms - Effect: Allow - Action: - - 'cognito-idp:DescribeUserPool' - - 'cognito-idp:CreateUserPoolClient' - - 'cognito-idp:DeleteUserPoolClient' - - 'cognito-idp:DescribeUserPoolClient' - - 'cognito-idp:AdminInitiateAuth' - - 'cognito-idp:AdminUserGlobalSignOut' - - 'cognito-idp:ListUserPoolClients' - - 'cognito-identity:DescribeIdentityPool' - - 'cognito-identity:UpdateIdentityPool' - - 'cognito-identity:SetIdentityPoolRoles' - - 'cognito-identity:GetIdentityPoolRoles' - - 'es:UpdateElasticsearchDomainConfig' - Resource: - - !Sub 'arn:aws:cognito-idp:${AWS::Region}:${AWS::AccountId}:userpool/${UserPool}' - - !Sub 'arn:aws:cognito-identity:${AWS::Region}:${AWS::AccountId}:identitypool/${IdentityPool}' - - !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DOMAINNAME}' - - Sid: 'PassRole' - Effect: 'Allow' - Action: 'iam:PassRole' - Condition: - StringLike: - 'iam:PassedToService': 'cognito-identity.amazonaws.com' - Resource: !GetAtt CognitoESAccessRole.Arn - Roles: - - !Ref CognitoESAccessRole - - # with CognitoESAcessPolicy above - CognitoESAccessRole: - Type: 'AWS::IAM::Role' - DeletionPolicy: 'Retain' - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: 'Allow' - Principal: - Service: 'es.amazonaws.com' - Action: - - 'sts:AssumeRole' - - # Create initial cognito user - AdminUser: - Type: 'AWS::Cognito::UserPoolUser' - DeletionPolicy: 'Retain' - Properties: - DesiredDeliveryMediums: - - 'EMAIL' - UserAttributes: - - Name: email - Value: !Ref CognitoAdminEmail - Username: !Ref CognitoAdminEmail - UserPoolId: !Ref UserPool - - # Place user in UserPoolGroup - CognitoUserGroupAttachment: - Type: 'AWS::Cognito::UserPoolUserToGroupAttachment' - Properties: - GroupName: !Ref UserPoolGroup - Username: !Ref CognitoAdminEmail - UserPoolId: !Ref UserPool - - LambdaESCognito: - Type: 'AWS::Lambda::Function' - Properties: - Description: Centralized Logging - Lambda solution helper functions - Environment: - Variables: - # V56536055 - 10/08/2018 - better logging capabilities - LOG_LEVEL: 'INFO' #change to WARN, ERROR or DEBUG as needed - Handler: index.handler - Runtime: nodejs12.x - Timeout: 300 - Role: !GetAtt LambdaESCognitoRole.Arn - Code: - S3Bucket: !Join ["-", [!FindInMap ["SourceCode", "General", "S3Bucket"], Ref: "AWS::Region"]] - S3Key: !Join ["/", [!FindInMap ["SourceCode", "General", "KeyPrefix"], "clog-auth.zip"]] - - LambdaESCognitoRole: - Type: AWS::IAM::Role - DependsOn: ElasticsearchAWSLogs - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: "/" - Policies: - - PolicyName: root - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: arn:aws:logs:*:*:* - - Effect: Allow - Action: - - es:UpdateElasticsearchDomainConfig - Resource: !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DOMAINNAME}' - - Effect: Allow - Action: - - cognito-idp:CreateUserPoolDomain - - cognito-idp:DeleteUserPoolDomain - Resource: !GetAtt UserPool.Arn - - Effect: Allow - Action: - - iam:PassRole - Resource: !GetAtt CognitoESAccessRole.Arn - Metadata: - cfn_nag: - rules_to_suppress: - - id: W11 - reason: "CloudWatch Logs captures logs from various resources and hence it is open" - - # - # Primer Elasticsearch resources - # [LoggingMasterRole, LoggingMasterPolicies, ElasticsearchAWSLogs] - # - LoggingMasterRole: - DeletionPolicy: 'Retain' - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - AWS: - Fn::If: - - SingleAccnt - - Ref: AWS::AccountId - - Ref: SpokeAccounts - Service: lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: "/" - - LoggingMasterPolicies: - DeletionPolicy: 'Retain' - Type: AWS::IAM::Policy - Properties: - PolicyName: !Sub logging-master-${AWS::Region} - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - es:ESHttpPost - Resource: !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/* - Roles: - - !Ref LoggingMasterRole - Metadata: - cfn_nag: - rules_to_suppress: - - id: W12 - reason: "Allow resources to hit ES domains" - - ElasticsearchAWSLogs: - Type: AWS::Elasticsearch::Domain - DeletionPolicy: 'Retain' - Properties: - DomainName: !Ref DOMAINNAME - ElasticsearchVersion: !FindInMap [ElasticSearch, Parameters, Version] - # V5804310 - 10/09/2018 - ES Encryption at rest - EncryptionAtRestOptions: - Enabled: true - ElasticsearchClusterConfig: - DedicatedMasterEnabled: true - InstanceCount: !FindInMap [ElasticSearch, NodeCount, !Ref ClusterSize] - ZoneAwarenessEnabled: true - InstanceType: !FindInMap [ElasticSearch, InstanceSize, !Ref ClusterSize] - DedicatedMasterType: !FindInMap [ElasticSearch, MasterSize, !Ref ClusterSize] - DedicatedMasterCount: 3 - SnapshotOptions: - AutomatedSnapshotStartHour: '1' - AccessPolicies: - Version: 2012-10-17 - Statement: - - Action: 'es:ESHttp*' - Principal: - AWS: !Sub ${LoggingMasterRole.Arn} - Effect: Allow - Resource: !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DOMAINNAME}/*' - - Action: 'es:ESHttp*' - Principal: - AWS: !Sub - - arn:aws:sts::${AWS::AccountId}:assumed-role/${AuthRole}/CognitoIdentityCredentials - - { AuthRole: !Ref CognitoAuthorizedRole } - Effect: Allow - Resource: !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DOMAINNAME}/*' - # V57095985 - 10/08/2018 - ES Domain needed configurations - # https://github.com/awslabs/aws-centralized-logging/issues/2 - AdvancedOptions: - rest.action.multi.allow_explicit_index: 'true' - indices.fielddata.cache.size: 40 - Metadata: - cfn_nag: - rules_to_suppress: - - id: W28 - reason: "The ES Domain is passed dynamically as as parameter and explicitly specified to ensure that IAM policies are configured to lockdown access to this specific ES instance only" - - # - # SNS Topic - # - Topic: - Type: 'AWS::SNS::Topic' - Metadata: - cfn_nag: - rules_to_suppress: - - id: W47 - reason: "Suppressing latest warning" - Properties: - DisplayName: 'Centralized Logging CloudWatch alarms notification topic' - - TopicPolicy: - Type: 'AWS::SNS::TopicPolicy' - Properties: - PolicyDocument: - Id: Id1 - Version: '2012-10-17' - Statement: - - Sid: Sid1 - Effect: Allow - Principal: - AWS: !Sub '${AWS::AccountId}' # Allow CloudWatch Alarms - Action: 'sns:Publish' - Resource: '*' - Topics: - - !Ref Topic - - TopicEndpointSubscription: - DependsOn: TopicPolicy - Type: 'AWS::SNS::Subscription' - Properties: - Endpoint: !Ref DomainAdminEmail - Protocol: email - TopicArn: !Ref Topic - - # - # CloudWatch Alarms - # - - StatusYellowAlarm: - DependsOn: TopicEndpointSubscription - Type: 'AWS::CloudWatch::Alarm' - Properties: - AlarmActions: - - !Ref Topic - AlarmDescription: 'Replica shards for at least one index are not allocated to nodes in a cluster.' - ComparisonOperator: GreaterThanOrEqualToThreshold - Dimensions: - - Name: ClientId - Value: !Ref 'AWS::AccountId' - - Name: DomainName - Value: !Ref DOMAINNAME - EvaluationPeriods: 1 - MetricName: 'ClusterStatus.yellow' - Namespace: 'AWS/ES' - OKActions: - - !Ref Topic - Period: 60 - Statistic: Maximum - Threshold: 1 - - StatusRedAlarm: - DependsOn: TopicEndpointSubscription - Type: 'AWS::CloudWatch::Alarm' - Properties: - AlarmActions: - - !Ref Topic - AlarmDescription: 'Primary and replica shards of at least one index are not allocated to nodes in a cluster.' - ComparisonOperator: GreaterThanOrEqualToThreshold - Dimensions: - - Name: ClientId - Value: !Ref 'AWS::AccountId' - - Name: DomainName - Value: !Ref DOMAINNAME - EvaluationPeriods: 1 - MetricName: 'ClusterStatus.red' - Namespace: 'AWS/ES' - OKActions: - - !Ref Topic - Period: 60 - Statistic: Maximum - Threshold: 1 - - CPUUtilizationTooHighAlarm: - DependsOn: TopicEndpointSubscription - Type: 'AWS::CloudWatch::Alarm' - Properties: - AlarmActions: - - !Ref Topic - AlarmDescription: 'Average CPU utilization over last 45 minutes too high.' - ComparisonOperator: GreaterThanOrEqualToThreshold - Dimensions: - - Name: ClientId - Value: !Ref 'AWS::AccountId' - - Name: DomainName - Value: !Ref DOMAINNAME - EvaluationPeriods: 3 - MetricName: 'CPUUtilization' - Namespace: 'AWS/ES' - OKActions: - - !Ref Topic - Period: 900 - Statistic: Average - Threshold: 80 - - MasterCPUUtilizationTooHighAlarm: - DependsOn: TopicEndpointSubscription - Type: 'AWS::CloudWatch::Alarm' - Properties: - AlarmActions: - - !Ref Topic - AlarmDescription: 'Average CPU utilization over last 45 minutes too high.' - ComparisonOperator: GreaterThanOrEqualToThreshold - Dimensions: - - Name: ClientId - Value: !Ref 'AWS::AccountId' - - Name: DomainName - Value: !Ref DOMAINNAME - EvaluationPeriods: 3 - MetricName: 'MasterCPUUtilization' - Namespace: 'AWS/ES' - OKActions: - - !Ref Topic - Period: 900 - Statistic: Average - Threshold: 50 - - FreeStorageSpaceTooLowAlarm: - DependsOn: TopicEndpointSubscription - Type: 'AWS::CloudWatch::Alarm' - Properties: - AlarmActions: - - !Ref Topic - AlarmDescription: 'Cluster has less than 2GB of storage space.' - ComparisonOperator: LessThanOrEqualToThreshold - Dimensions: - - Name: ClientId - Value: !Ref 'AWS::AccountId' - - Name: DomainName - Value: !Ref DOMAINNAME - EvaluationPeriods: 1 - MetricName: 'FreeStorageSpace' - Namespace: 'AWS/ES' - OKActions: - - !Ref Topic - Period: 60 - Statistic: Minimum - Threshold: 2000 - - IndexWritesBlockedTooHighAlarm: - DependsOn: TopicEndpointSubscription - Type: 'AWS::CloudWatch::Alarm' - Properties: - AlarmActions: - - !Ref Topic - AlarmDescription: 'Cluster is blocking incoming write requests.' - ComparisonOperator: GreaterThanOrEqualToThreshold - Dimensions: - - Name: ClientId - Value: !Ref 'AWS::AccountId' - - Name: DomainName - Value: !Ref DOMAINNAME - EvaluationPeriods: 1 - MetricName: 'ClusterIndexWritesBlocked' - Namespace: 'AWS/ES' - OKActions: - - !Ref Topic - Period: 300 - Statistic: Maximum - Threshold: 1 - - JVMMemoryPressureTooHighAlarm: - DependsOn: TopicEndpointSubscription - Type: 'AWS::CloudWatch::Alarm' - Properties: - AlarmActions: - - !Ref Topic - AlarmDescription: 'Average JVM memory pressure over last 15 minutes too high.' - ComparisonOperator: GreaterThanOrEqualToThreshold - Dimensions: - - Name: ClientId - Value: !Ref 'AWS::AccountId' - - Name: DomainName - Value: !Ref DOMAINNAME - EvaluationPeriods: 1 - MetricName: 'JVMMemoryPressure' - Namespace: 'AWS/ES' - OKActions: - - !Ref Topic - Period: 900 - Statistic: Average - Threshold: 80 - - MasterJVMMemoryPressureTooHighAlarm: - DependsOn: TopicEndpointSubscription - Type: 'AWS::CloudWatch::Alarm' - Properties: - AlarmActions: - - !Ref Topic - AlarmDescription: 'Average JVM memory pressure over last 15 minutes too high.' - ComparisonOperator: GreaterThanOrEqualToThreshold - Dimensions: - - Name: ClientId - Value: !Ref 'AWS::AccountId' - - Name: DomainName - Value: !Ref DOMAINNAME - EvaluationPeriods: 1 - MetricName: 'MasterJVMMemoryPressure' - Namespace: 'AWS/ES' - OKActions: - - !Ref Topic - Period: 900 - Statistic: Average - Threshold: 50 - - MasterNotReachableFromNodeAlarm: - DependsOn: TopicEndpointSubscription - Type: 'AWS::CloudWatch::Alarm' - Properties: - AlarmActions: - - !Ref Topic - AlarmDescription: 'Master node stopped or not reachable. Usually the result of a network connectivity issue or AWS dependency problem.' - ComparisonOperator: LessThanThreshold - Dimensions: - - Name: ClientId - Value: !Ref 'AWS::AccountId' - - Name: DomainName - Value: !Ref DOMAINNAME - EvaluationPeriods: 1 - MetricName: 'MasterReachableFromNode' - Namespace: 'AWS/ES' - OKActions: - - !Ref Topic - Period: 60 - Statistic: Minimum - Threshold: 1 - - AutomatedSnapshotFailureTooHighAlarm: - DependsOn: TopicEndpointSubscription - Type: 'AWS::CloudWatch::Alarm' - Properties: - AlarmActions: - - !Ref Topic - AlarmDescription: 'No automated snapshot was taken for the domain in the previous 36 hours (created by marbot).' - ComparisonOperator: GreaterThanOrEqualToThreshold - Dimensions: - - Name: ClientId - Value: !Ref 'AWS::AccountId' - - Name: DomainName - Value: !Ref DOMAINNAME - EvaluationPeriods: 1 - MetricName: 'AutomatedSnapshotFailure' - Namespace: 'AWS/ES' - OKActions: - - !Ref Topic - Period: 60 - Statistic: Maximum - Threshold: 1 - - # - # Log Streamer and Demo resources - # [LogStreamerRole, LogStreamer, LogStreamerInvokePermission, DemoStack] - # - LogStreamerRole: - DeletionPolicy: 'Retain' - Type: 'AWS::IAM::Role' - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: lambda.amazonaws.com - Action: 'sts:AssumeRole' - Path: / - Policies: - - PolicyName: !Sub logstreamer-${AWS::Region} - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/* - - Effect: Allow - Action: - - es:ESHttpPost - Resource: !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/* - Metadata: - cfn_nag: - rules_to_suppress: - - id: W11 - reason: "The resource for log-group is restricted to lambda functions. The name of the fuction is not specified because if specified any change to lambda function would require replacement" - - - LogStreamer: - Type: AWS::Lambda::Function - Properties: - Description: Centralized Logging - Lambda function to stream logs on ES Domain - Environment: - Variables: - # V56536055 - 10/08/2018 - better logging capabilities - LOG_LEVEL: 'INFO' #change to WARN, ERROR or DEBUG as needed - DOMAIN_ENDPOINT: !Sub ${ElasticsearchAWSLogs.DomainEndpoint} - MASTER_ROLE: !Sub ${LoggingMasterRole} - SESSION_ID: !Sub ${AWS::AccountId}-${AWS::Region} - OWNER: Hub - SOLUTION: SO0009 - CLUSTER_SIZE: !Ref ClusterSize - UUID: !Sub ${CreateUniqueID.UUID} - ANONYMOUS_DATA: !FindInMap [InstanceMap, send-data, SendAnonymousData] - Handler: index.handler - Role: !Sub ${LogStreamerRole.Arn} - Code: - S3Bucket: !Join ["-", [!FindInMap ["SourceCode", "General", "S3Bucket"], Ref: "AWS::Region"]] - S3Key: !Join ["/", [!FindInMap ["SourceCode", "General", "KeyPrefix"], "clog-indexing-service.zip"]] - Runtime: nodejs12.x - Timeout: 300 - Metadata: - cfn_nag: - rules_to_suppress: - - id: W58 - reason: "CloudWatch Logs write permissions granted to lambda-specific loggroup via LogStreamerRole" - - LogStreamerInvokePermission: - Type: AWS::Lambda::Permission - Properties: - FunctionName: !Sub ${LogStreamer} - Action: lambda:InvokeFunction - Principal: !Sub logs.${AWS::Region}.amazonaws.com - SourceAccount: !Sub ${AWS::AccountId} - - DemoStack: - Type: AWS::CloudFormation::Stack - Condition: DemoData - Properties: - Parameters: - LogStreamerArn: !Sub ${LogStreamer.Arn} - DemoVPCCidr: !Sub ${DemoVPC} - DemoSubnet: !Sub ${DemoSubnet} - TemplateURL: !Join - - '' - - - 'https://' - - !Join ["-", [!FindInMap ["SourceCode", "General", "S3Bucket"], "reference"]] - - '.s3.amazonaws.com/' - - !Join ["/", [!FindInMap ["SourceCode", "General", "KeyPrefix"], "centralized-logging-demo.template"]] - - CreateUniqueID: - Type: Custom::LoadLambda - Properties: - ServiceToken: !GetAtt LambdaESCognito.Arn - Resource: UUID - -Outputs: - DomainEndpoint: - Description: ES domain endpoint URL - Value: !Sub ${ElasticsearchAWSLogs.DomainEndpoint} - - KibanaLoginURL: - Description: Kibana login URL - Value: !Sub https://${ElasticsearchAWSLogs.DomainEndpoint}/_plugin/kibana/ - - MasterRole: - Description: IAM role for ES cross account access - Value: !Sub ${LoggingMasterRole.Arn} - - SpokeAccountIds: - Description: Accounts that are allowed to index on ES - Value: !Join [ ',', !Ref SpokeAccounts] - - LambdaArn: - Description: Lambda function to index logs on ES Domain - Value: !Sub ${LogStreamer.Arn} - - ClusterSize: - Description: Cluster size for the deployed ES Domain - Value: !Sub ${ClusterSize} diff --git a/deployment/centralized-logging-spoke.template b/deployment/centralized-logging-spoke.template deleted file mode 100644 index dd5306c..0000000 --- a/deployment/centralized-logging-spoke.template +++ /dev/null @@ -1,247 +0,0 @@ -# Centralized Logging Solution -# -# template for centralized-logging-solution -# **DO NOT DELETE** -# -# author: aws-solutions-builder@ -AWSTemplateFormatVersion: 2010-09-09 - -Description: (SO0009s) - AWS Centralized Logging Solution, spoke accounts template (Version %%VERSION%%) - -Parameters: - # Name for ES Domain - ESDomain: - Description: Elasticsearch Domain Endpoint for centralized logging - Type: String - - # ES cluster size - ClusterSize: - Description: Amazon ES cluster size, as deployed in primary account - Type: String - Default: Small - AllowedValues: - - Small - - Medium - - Large - - # Master IAM role for cross account log indexing - MasterRole: - Description: IAM Role Arn for cross account log indexing - Type: String - - # Demo template for sample logs - DemoTemplate: - Description: Deploy demo template for sample logs? - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - - # VPC CIDR for sample sources - DemoVPC: - Description: CIDR for VPC with sample sources (Only required if you chose 'Yes' above) - Type: String - MinLength: 9 - MaxLength: 18 - AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})' - Default: 10.250.0.0/16 - ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x - - # Subnet for sample web server - DemoSubnet: - Description: IP address range for subnet with sample web server (Only required if you chose 'Yes' above) - Type: String - MinLength: 9 - MaxLength: 18 - AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})' - Default: 10.250.250.0/24 - ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x - -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: Elasticsearch Configuration - Parameters: - - ESDomain - - MasterRole - - ClusterSize - - Label: - default: Do you want to deploy sample log sources? - Parameters: - - DemoTemplate - - DemoVPC - - DemoSubnet - ParameterLabels: - ESDomain: - default: Elasticsearch Endpoint - ClusterSize: - default: Cluster Size - DemoTemplate: - default: Sample Logs - DemoVPC: - default: VPC CIDR for Sample Sources - DemoSubnet: - default: Subnet for Sample Web Server - MasterRole: - default: Master Account Role - -Mappings: - InstanceMap: - send-data: {"SendAnonymousData": "Yes"} - - # Lambda source code mapping - SourceCode: - General: - S3Bucket: "%%BUCKET_NAME%%" - KeyPrefix: "%%SOLUTION_NAME%%/%%VERSION%%" - -Conditions: - DemoData: !Equals [!Ref DemoTemplate, 'Yes'] - -Resources: - # - # Log Streamer and Demo resources - # [LogStreamerRole, LogStreamer, LogStreamerInvokePermission, DemoStack] - # - LogStreamerRole: - DeletionPolicy: 'Retain' - Type: 'AWS::IAM::Role' - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: lambda.amazonaws.com - Action: 'sts:AssumeRole' - Path: / - Policies: - - PolicyName: !Sub logstreamer-${AWS::Region} - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/* - - Effect: Allow - Action: - - sts:AssumeRole - Resource: !Sub ${MasterRole} - Metadata: - cfn_nag: - rules_to_suppress: - - id: W11 - reason: "The resource access is restricted to lambda functions. The lambda function is generated dynamically during CF execution to provide update flexibility without replacement. Adding the function name dynamically would add cyclic dependency" - - LogStreamer: - Type: AWS::Lambda::Function - Properties: - Description: Centralized Logging - Lambda function to stream logs cross account on ES Domain - Environment: - Variables: - # V56536055 - 10/08/2018 - better logging capabilities - LOG_LEVEL: 'INFO' #change to WARN, ERROR or DEBUG as needed - DOMAIN_ENDPOINT: !Sub ${ESDomain} - MASTER_ROLE: !Sub ${MasterRole} - SESSION_ID: !Sub ${AWS::AccountId}-${AWS::Region} - OWNER: Spoke - SOLUTION: SO0009s - CLUSTER_SIZE: !Ref ClusterSize - UUID: !Sub ${CreateUniqueID.UUID} - ANONYMOUS_DATA: !FindInMap [InstanceMap, send-data, SendAnonymousData] - Handler: index.handler - Role: !Sub ${LogStreamerRole.Arn} - Code: - S3Bucket: !Join ["-", [!FindInMap ["SourceCode", "General", "S3Bucket"], Ref: "AWS::Region"]] - S3Key: !Join ["/", [!FindInMap ["SourceCode", "General", "KeyPrefix"], "clog-indexing-service.zip"]] - Runtime: nodejs12.x - Timeout: 300 - Metadata: - cfn_nag: - rules_to_suppress: - - id: W58 - reason: "CloudWatch Logs write permissions granted to lambda-specific loggroup via LogStreamerRole" - - LogStreamerInvokePermission: - Type: AWS::Lambda::Permission - Properties: - FunctionName: !Sub ${LogStreamer} - Action: lambda:InvokeFunction - Principal: !Sub logs.${AWS::Region}.amazonaws.com - SourceAccount: !Sub ${AWS::AccountId} - - DemoStack: - Type: AWS::CloudFormation::Stack - Condition: DemoData - Properties: - Parameters: - LogStreamerArn: !Sub ${LogStreamer.Arn} - DemoVPCCidr: !Sub ${DemoVPC} - DemoSubnet: !Sub ${DemoSubnet} - TemplateURL: !Join - - '' - - - 'https://' - - !Join ["-", [!FindInMap ["SourceCode", "General", "S3Bucket"], "reference"]] - - '.s3.amazonaws.com/' - - !Join ["/", [!FindInMap ["SourceCode", "General", "KeyPrefix"], "centralized-logging-demo.template"]] - - # - # Solution Helper resources - # [SolutionHelperRole, SolutionHelper, CreateUniqueID] - # - SolutionHelper: - Type: 'AWS::Lambda::Function' - Properties: - Description: Centralized Logging - Lambda function to generate a unique ID - Environment: - Variables: - # V56536055 - 10/08/2018 - better logging capabilities - LOG_LEVEL: 'INFO' #change to WARN, ERROR or DEBUG as needed - Handler: index.handler - Runtime: nodejs12.x - Timeout: 300 - Role: !GetAtt SolutionHelperRole.Arn - Code: - S3Bucket: !Join ["-", [!FindInMap ["SourceCode", "General", "S3Bucket"], Ref: "AWS::Region"]] - S3Key: !Join ["/", [!FindInMap ["SourceCode", "General", "KeyPrefix"], "clog-auth.zip"]] - - SolutionHelperRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: "/" - Policies: - - PolicyName: root - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${AWS::StackName}-*' - - CreateUniqueID: - Type: Custom::LoadLambda - Properties: - ServiceToken: !GetAtt SolutionHelper.Arn - Resource: UUID - -Outputs: - LambdaArn: - Description: Lambda function to index logs on ES Domain - Value: !Sub ${LogStreamer.Arn} diff --git a/deployment/dashboard.ndjson b/deployment/dashboard.ndjson new file mode 100644 index 0000000..cb9fca8 --- /dev/null +++ b/deployment/dashboard.ndjson @@ -0,0 +1,20 @@ +{"attributes":{"fields":"[{\"name\":\"@log_group\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@log_group.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@log_group\"}}},{\"name\":\"@log_stream\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@log_stream.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@log_stream\"}}},{\"name\":\"@message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@message\"}}},{\"name\":\"@owner\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@owner.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@owner\"}}},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"account_id\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"action\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"action.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"action\"}}},{\"name\":\"additionalEventData.AuthenticationMethod\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.AuthenticationMethod.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.AuthenticationMethod\"}}},{\"name\":\"additionalEventData.CipherSuite\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.CipherSuite.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.CipherSuite\"}}},{\"name\":\"additionalEventData.LoginTo\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.LoginTo.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.LoginTo\"}}},{\"name\":\"additionalEventData.MFAUsed\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.MFAUsed.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.MFAUsed\"}}},{\"name\":\"additionalEventData.MobileVersion\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.MobileVersion.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.MobileVersion\"}}},{\"name\":\"additionalEventData.SignatureVersion\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.SignatureVersion.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.SignatureVersion\"}}},{\"name\":\"additionalEventData.bytesTransferredIn\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"additionalEventData.bytesTransferredOut\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"additionalEventData.configRuleArn\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.configRuleArn.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.configRuleArn\"}}},{\"name\":\"additionalEventData.configRuleInputParameters\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.configRuleInputParameters.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.configRuleInputParameters\"}}},{\"name\":\"additionalEventData.configRuleName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.configRuleName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.configRuleName\"}}},{\"name\":\"additionalEventData.managedRuleIdentifier\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.managedRuleIdentifier.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.managedRuleIdentifier\"}}},{\"name\":\"additionalEventData.notificationJobType\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.notificationJobType.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.notificationJobType\"}}},{\"name\":\"additionalEventData.service\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.service.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.service\"}}},{\"name\":\"additionalEventData.sub\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.sub.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.sub\"}}},{\"name\":\"additionalEventData.x-amz-id-2\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"additionalEventData.x-amz-id-2.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"additionalEventData.x-amz-id-2\"}}},{\"name\":\"agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"agent\"}}},{\"name\":\"apiVersion\",\"type\":\"conflict\",\"esTypes\":[\"date\",\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false,\"conflictDescriptions\":{\"date\":[\"cwl-2020-11-19\"],\"text\":[\"cwl-2020-11-18\"]}},{\"name\":\"apiVersion.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"apiVersion\"}}},{\"name\":\"authuser\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"authuser.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"authuser\"}}},{\"name\":\"awsRegion\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"awsRegion.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"awsRegion\"}}},{\"name\":\"bytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"date\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"date.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"date\"}}},{\"name\":\"dstaddr\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"dstaddr.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"dstaddr\"}}},{\"name\":\"dstport\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"errorCode\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"errorCode.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"errorCode\"}}},{\"name\":\"errorMessage\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"errorMessage.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"errorMessage\"}}},{\"name\":\"eventCategory\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"eventCategory.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"eventCategory\"}}},{\"name\":\"eventID\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"eventID.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"eventID\"}}},{\"name\":\"eventName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"eventName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"eventName\"}}},{\"name\":\"eventSource\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"eventSource.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"eventSource\"}}},{\"name\":\"eventTime\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"eventType\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"eventType.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"eventType\"}}},{\"name\":\"eventVersion\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"eventVersion.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"eventVersion\"}}},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"host\"}}},{\"name\":\"id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"id\"}}},{\"name\":\"ident\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ident.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"ident\"}}},{\"name\":\"interface_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"interface_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"interface_id\"}}},{\"name\":\"log_status\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"log_status.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"log_status\"}}},{\"name\":\"managementEvent\",\"type\":\"boolean\",\"esTypes\":[\"boolean\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"packets\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"readOnly\",\"type\":\"boolean\",\"esTypes\":[\"boolean\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"recipientAccountId\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"recipientAccountId.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"recipientAccountId\"}}},{\"name\":\"referrer\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"referrer.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"referrer\"}}},{\"name\":\"request\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"request\"}}},{\"name\":\"requestID\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"requestID.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"requestID\"}}},{\"name\":\"requestParameters\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"requestParameters.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"requestParameters\"}}},{\"name\":\"resources.ARN\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resources.ARN.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"resources.ARN\"}}},{\"name\":\"resources.accountId\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resources.accountId.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"resources.accountId\"}}},{\"name\":\"resources.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resources.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"resources.type\"}}},{\"name\":\"responseElements\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"responseElements.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"responseElements\"}}},{\"name\":\"serviceEventDetails.snapshotId\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"serviceEventDetails.snapshotId.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"serviceEventDetails.snapshotId\"}}},{\"name\":\"sharedEventID\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sharedEventID.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"sharedEventID\"}}},{\"name\":\"sourceIPAddress\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sourceIPAddress.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"sourceIPAddress\"}}},{\"name\":\"srcaddr\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"srcaddr.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"srcaddr\"}}},{\"name\":\"srcport\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"start\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"type\"}}},{\"name\":\"userAgent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userAgent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userAgent\"}}},{\"name\":\"userIdentity.accessKeyId\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.accessKeyId.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.accessKeyId\"}}},{\"name\":\"userIdentity.accountId\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.accountId.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.accountId\"}}},{\"name\":\"userIdentity.arn\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.arn.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.arn\"}}},{\"name\":\"userIdentity.identityProvider\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.identityProvider.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.identityProvider\"}}},{\"name\":\"userIdentity.invokedBy\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.invokedBy.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.invokedBy\"}}},{\"name\":\"userIdentity.principalId\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.principalId.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.principalId\"}}},{\"name\":\"userIdentity.sessionContext.attributes.creationDate\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userIdentity.sessionContext.attributes.mfaAuthenticated\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.sessionContext.attributes.mfaAuthenticated.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.sessionContext.attributes.mfaAuthenticated\"}}},{\"name\":\"userIdentity.sessionContext.ec2RoleDelivery\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.sessionContext.ec2RoleDelivery.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.sessionContext.ec2RoleDelivery\"}}},{\"name\":\"userIdentity.sessionContext.sessionIssuer.accountId\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.sessionContext.sessionIssuer.accountId.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.sessionContext.sessionIssuer.accountId\"}}},{\"name\":\"userIdentity.sessionContext.sessionIssuer.arn\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.sessionContext.sessionIssuer.arn.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.sessionContext.sessionIssuer.arn\"}}},{\"name\":\"userIdentity.sessionContext.sessionIssuer.principalId\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.sessionContext.sessionIssuer.principalId.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.sessionContext.sessionIssuer.principalId\"}}},{\"name\":\"userIdentity.sessionContext.sessionIssuer.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.sessionContext.sessionIssuer.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.sessionContext.sessionIssuer.type\"}}},{\"name\":\"userIdentity.sessionContext.sessionIssuer.userName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.sessionContext.sessionIssuer.userName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.sessionContext.sessionIssuer.userName\"}}},{\"name\":\"userIdentity.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.type\"}}},{\"name\":\"userIdentity.userName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userIdentity.userName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"userIdentity.userName\"}}},{\"name\":\"version\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"vpcEndpointId\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"vpcEndpointId.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"vpcEndpointId\"}}}]","timeFieldName":"timestamp","title":"cwl-*"},"id":"49e8e120-2a6e-11eb-b505-cb445bbdb0f9","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2020-11-19T13:51:15.529Z","version":"WzE4NiwxXQ=="} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":{\"query_string\":{\"query\":\"status:200\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"HTTP 200 Code","version":1},"id":"HTTP-200-Code","migrationVersion":{"search":"7.4.0"},"references":[{"id":"49e8e120-2a6e-11eb-b505-cb445bbdb0f9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-11-19T18:02:53.536Z","version":"WzIyNSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"HTTP 200 Code Count","uiStateJSON":"{}","version":1,"visState":"{\"type\":\"metric\",\"params\":{\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{},\"title\":\"HTTP 200 Code Count\"}"},"id":"HTTP-200-Code-Count","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"HTTP-200-Code","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-11-19T18:02:54.579Z","version":"WzI0MCwxXQ=="} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":{\"query_string\":{\"query\":\"status:404\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"HTTP 404 Code","version":1},"id":"HTTP-404-Code","migrationVersion":{"search":"7.4.0"},"references":[{"id":"49e8e120-2a6e-11eb-b505-cb445bbdb0f9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-11-19T18:02:53.553Z","version":"WzIyOCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"HTTP 404 Code Count","uiStateJSON":"{}","version":1,"visState":"{\"type\":\"metric\",\"params\":{\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{},\"title\":\"HTTP 404 Code Count\"}"},"id":"HTTP-404-Code-Count","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"HTTP-404-Code","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-11-19T18:02:54.549Z","version":"WzIzMiwxXQ=="} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"query\":{\"query\":{\"query_string\":{\"query\":\"action:accept\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"Accept","version":1},"id":"Accept","migrationVersion":{"search":"7.4.0"},"references":[{"id":"49e8e120-2a6e-11eb-b505-cb445bbdb0f9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-11-19T18:02:53.545Z","version":"WzIyNiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Packets - Accept","uiStateJSON":"{}","version":1,"visState":"{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"interval\":\"30s\",\"min_doc_count\":1,\"extended_bounds\":{},\"scaleMetricValues\":true}}],\"listeners\":{},\"title\":\"Packets - Accept\"}"},"id":"Packets-Accept","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"Accept","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-11-19T18:02:54.555Z","version":"WzIzMywxXQ=="} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"query\":{\"query\":{\"query_string\":{\"query\":\"action:reject\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"Reject","version":1},"id":"Reject","migrationVersion":{"search":"7.4.0"},"references":[{"id":"49e8e120-2a6e-11eb-b505-cb445bbdb0f9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-11-19T18:02:53.555Z","version":"WzIyOSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Packets - Reject","uiStateJSON":"{}","version":1,"visState":"{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"interval\":\"30s\",\"min_doc_count\":1,\"extended_bounds\":{},\"scaleMetricValues\":true}}],\"listeners\":{},\"title\":\"Packets - Reject\"}"},"id":"Packets-Reject","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"Reject","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-11-19T18:02:54.555Z","version":"WzIzNCwxXQ=="} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"All Data","version":1},"id":"All-Data","migrationVersion":{"search":"7.4.0"},"references":[{"id":"49e8e120-2a6e-11eb-b505-cb445bbdb0f9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-11-19T18:02:53.550Z","version":"WzIyNywxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Total Src Addr","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Total Src Addr\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"srcaddr.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"srcaddr\"}}],\"listeners\":{}}"},"id":"Top-10-Src-Address-Packets-Total","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"All-Data","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-11-19T18:02:54.560Z","version":"WzIzNiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Reject Src Addr","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Reject Src Addr\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"srcaddr.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"srcaddr\"}}],\"listeners\":{}}"},"id":"Top-10-Src-Address-Packets-Reject","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"Reject","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-11-19T18:02:54.568Z","version":"WzIzOCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Total Packets Src Address","uiStateJSON":"{\"vis\":{\"legendOpen\":true}}","version":1,"visState":"{\"title\":\"Total Packets Src Address\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":false,\"isDonut\":false,\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"srcaddr.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"\"}}],\"listeners\":{}}"},"id":"Total-Packets-Src-Address","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"All-Data","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-11-19T18:02:54.569Z","version":"WzIzOSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Reject Packets Src Address","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Reject Packets Src Address\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":false,\"isDonut\":false,\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"srcaddr.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}"},"id":"Reject-Packets-Src-Address","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"Reject","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-11-19T18:02:54.589Z","version":"WzI0MSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Top 10 Access Keys","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Top 10 Access Keys\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userIdentity.accessKeyId.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"userIdentity.accessKeyId\"}}],\"listeners\":{}}"},"id":"Top-10-Access-Keys","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"All-Data","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-11-19T18:02:54.557Z","version":"WzIzNSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Top 10 Event Sources","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Top 10 Event Sources\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"eventSource.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"eventSource\"}}],\"listeners\":{}}"},"id":"Top-10-Event-Sources","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"All-Data","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-11-19T18:02:54.568Z","version":"WzIzNywxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":{\"match_all\":{}},\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Top Account Ids","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Top Account Ids\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"account_id\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}"},"id":"f1709820-f182-11e7-8ed1-e93cb1e71cec","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"49e8e120-2a6e-11eb-b505-cb445bbdb0f9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-11-19T18:02:53.557Z","version":"WzIzMCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":{\"match_all\":{}},\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Top Regions","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Top Regions\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"awsRegion.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"awsRegion\"}}],\"listeners\":{}}"},"id":"74a9bbe0-f183-11e7-8ed1-e93cb1e71cec","migrationVersion":{"visualization":"7.7.0"},"references":[{"id":"49e8e120-2a6e-11eb-b505-cb445bbdb0f9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-11-19T18:02:53.559Z","version":"WzIzMSwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"*\",\"language\":\"lucene\"}}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"1\",\"w\":12,\"x\":12,\"y\":24},\"panelIndex\":\"1\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"2\",\"w\":12,\"x\":0,\"y\":24},\"panelIndex\":\"2\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"4\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"4\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"5\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"5\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":16,\"i\":\"6\",\"w\":12,\"x\":0,\"y\":8},\"panelIndex\":\"6\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":16,\"i\":\"7\",\"w\":12,\"x\":24,\"y\":8},\"panelIndex\":\"7\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":16,\"i\":\"8\",\"w\":12,\"x\":12,\"y\":8},\"panelIndex\":\"8\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":16,\"i\":\"9\",\"w\":12,\"x\":36,\"y\":8},\"panelIndex\":\"9\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"10\",\"w\":24,\"x\":0,\"y\":32},\"panelIndex\":\"10\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"11\",\"w\":24,\"x\":24,\"y\":24},\"panelIndex\":\"11\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_9\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"12\",\"w\":12,\"x\":24,\"y\":32},\"panelIndex\":\"12\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_10\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"13\",\"w\":12,\"x\":36,\"y\":32},\"panelIndex\":\"13\",\"version\":\"7.7.0\",\"panelRefName\":\"panel_11\"}]","refreshInterval":{"pause":false,"value":300000},"timeFrom":"now/d","timeRestore":true,"timeTo":"now/d","title":"Basic","version":1},"id":"Basic","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"HTTP-200-Code-Count","name":"panel_0","type":"visualization"},{"id":"HTTP-404-Code-Count","name":"panel_1","type":"visualization"},{"id":"Packets-Accept","name":"panel_2","type":"visualization"},{"id":"Packets-Reject","name":"panel_3","type":"visualization"},{"id":"Top-10-Src-Address-Packets-Total","name":"panel_4","type":"visualization"},{"id":"Top-10-Src-Address-Packets-Reject","name":"panel_5","type":"visualization"},{"id":"Total-Packets-Src-Address","name":"panel_6","type":"visualization"},{"id":"Reject-Packets-Src-Address","name":"panel_7","type":"visualization"},{"id":"Top-10-Access-Keys","name":"panel_8","type":"visualization"},{"id":"Top-10-Event-Sources","name":"panel_9","type":"visualization"},{"id":"f1709820-f182-11e7-8ed1-e93cb1e71cec","name":"panel_10","type":"visualization"},{"id":"74a9bbe0-f183-11e7-8ed1-e93cb1e71cec","name":"panel_11","type":"visualization"}],"type":"dashboard","updated_at":"2020-11-19T18:04:03.877Z","version":"WzI0MiwxXQ=="} +{"exportedCount":19,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/deployment/run-unit-tests.sh b/deployment/run-unit-tests.sh index aca1a45..3d01e3f 100755 --- a/deployment/run-unit-tests.sh +++ b/deployment/run-unit-tests.sh @@ -1,21 +1,32 @@ -#!/bin/bash -# -# This assumes all of the OS-level configuration has been completed and git repo has already been cloned -# -# This script should be run from the repo's deployment directory -# cd deployment -# ./run-unit-tests.sh -# - -# Get reference for all important folders -template_dir="$PWD" -source_dir="$template_dir/../source" - -echo "------------------------------------------------------------------------------" -echo "[Test] Services" -echo "------------------------------------------------------------------------------" -cd $source_dir/services/indexing -npm install -npm run build -npm test - +#!/bin/bash +# +# This assumes all of the OS-level configuration has been completed and git repo has already been cloned +# +# This script should be run from the repo's deployment directory +# cd deployment +# ./run-unit-tests.sh +# + +# Get reference for all important folders +template_dir="$PWD" +resource_dir="$template_dir/../source/resources" +source_dir="$template_dir/../source/services" + +echo "------------------------------------------------------------------------------" +echo "[Test] Resources" +echo "------------------------------------------------------------------------------" +cd $resource_dir +npm run test -- -u + +echo "------------------------------------------------------------------------------" +echo "[Test] helper" +echo "------------------------------------------------------------------------------" +cd $source_dir/helper +npm run test + +echo "------------------------------------------------------------------------------" +echo "[Test] transformer" +echo "------------------------------------------------------------------------------" +cd $source_dir/transformer +npm run test + diff --git a/package.json b/package.json new file mode 100644 index 0000000..46ef567 --- /dev/null +++ b/package.json @@ -0,0 +1,27 @@ +{ + "name": "aws-centralized-logging", + "version": "4.0.0", + "description": "AWS Centralized Logging", + "scripts": { + "docs": "./node_modules/typedoc/bin/typedoc --out docs --name \"AWS Centralized Logging\"", + "lint": "./node_modules/eslint/bin/eslint.js . --ext .ts,.js", + "prettier-format": "./node_modules/prettier/bin-prettier.js --config .prettierrc.yml '**/*.ts' --write", + "build:helper": "cd source/services/helper && npm run build:all", + "build:transformer": "cd source/services/transformer && npm run build:all", + "build": "npm run build:helper && npm run build:transformer", + "test": "cd ./deployment && chmod +x run-unit-tests.sh && ./run-unit-tests.sh" + }, + "author": "aws-solutions", + "license": "Apache-2.0", + "devDependencies": { + "@typescript-eslint/eslint-plugin": "^4.9.0", + "@typescript-eslint/parser": "^4.9.0", + "eslint": "^7.15.0", + "eslint-config-prettier": "^7.0.0", + "eslint-plugin-prettier": "^3.2.0", + "prettier": "^2.2.0", + "typedoc": "^0.19.2", + "typedoc-plugin-no-inherit": "^1.2.0", + "typescript": "^4.1.2" + } +} diff --git a/source/resources/README.md b/source/resources/README.md new file mode 100644 index 0000000..894e39d --- /dev/null +++ b/source/resources/README.md @@ -0,0 +1,12 @@ +# AWS Centralized Logging CDK Project + +The `cdk.json` file tells the CDK Toolkit how to execute your app. + +## Useful commands + +- `npm run build` compile typescript to js +- `npm run watch` watch for changes and compile +- `npm run test` perform the jest unit tests +- `cdk deploy` deploy this stack to your default AWS account/region +- `cdk diff` compare deployed stack with current state +- `cdk synth` emits the synthesized CloudFormation template diff --git a/source/resources/__tests__/cl.test.ts b/source/resources/__tests__/cl.test.ts new file mode 100644 index 0000000..4fef6ed --- /dev/null +++ b/source/resources/__tests__/cl.test.ts @@ -0,0 +1,27 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import "@aws-cdk/assert/jest"; +import { CLPrimary } from "../lib/cl-primary-stack"; +import { App, Stack } from "@aws-cdk/core"; + +describe("==Primary Stack Tests==", () => { + const app = new App(); + const stack: Stack = new CLPrimary(app, "CL-PrimaryStack"); + + describe("Test resources", () => { + test("snapshot test", () => { + expect(stack).toMatchSnapshot(); + }); + }); +}); diff --git a/source/resources/bin/app.ts b/source/resources/bin/app.ts new file mode 100755 index 0000000..4dc72be --- /dev/null +++ b/source/resources/bin/app.ts @@ -0,0 +1,22 @@ +/***************************************************************************** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * + * * + * Licensed under the Apache License, Version 2.0 (the "License"). You may * + * not use this file except in compliance with the License. A copy of the * + * License is located at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * or in the 'license' file accompanying this file. This file is distributed * + * on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, * + * express or implied. See the License for the specific language governing * + * permissions and limitations under the License. * + *****************************************************************************/ + +import { App } from "@aws-cdk/core"; +import { CLPrimary } from "../lib/cl-primary-stack"; + +const app = new App(); + +// Primary Stack +new CLPrimary(app, "CL-PrimaryStack"); diff --git a/source/resources/cdk.json b/source/resources/cdk.json new file mode 100755 index 0000000..97bafc2 --- /dev/null +++ b/source/resources/cdk.json @@ -0,0 +1,7 @@ +{ + "app": "npx ts-node bin/app.ts", + "context": { + "@aws-cdk/core:enableStackNameDuplicates": "true", + "aws-cdk:enableDiffNoFail": "true" + } +} diff --git a/source/resources/jest.config.js b/source/resources/jest.config.js new file mode 100644 index 0000000..39730f7 --- /dev/null +++ b/source/resources/jest.config.js @@ -0,0 +1,43 @@ +module.exports = { + // Automatically clear mock calls and instances between every test + clearMocks: true, + + // The directory where Jest should output its coverage files + coverageDirectory: "coverage", + + // An array of regexp pattern strings used to skip coverage collection + coveragePathIgnorePatterns: ["/node_modules/"], + + // An object that configures minimum threshold enforcement for coverage results + coverageThreshold: { + global: { + // branches: 80, + functions: 80, + lines: 80, + statements: 80, + }, + }, + + // An array of directory names to be searched recursively up from the requiring module's location + moduleDirectories: ["node_modules"], + + // An array of file extensions your modules use + moduleFileExtensions: ["ts", "json", "jsx", "js", "tsx", "node"], + + // Automatically reset mock state between every test + resetMocks: true, + + // The glob patterns Jest uses to detect test files + testMatch: ["**/?(*.)+(spec|test).[t]s?(x)"], + + // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped + testPathIgnorePatterns: ["/node_modules/"], + + // A map from regular expressions to paths to transformers + transform: { + "^.+\\.(t|j)sx?$": "ts-jest", + }, + + // Indicates whether each individual test should be reported during the run + verbose: true, +}; diff --git a/source/resources/lib/cl-demo-ec2-construct.ts b/source/resources/lib/cl-demo-ec2-construct.ts new file mode 100644 index 0000000..1754d80 --- /dev/null +++ b/source/resources/lib/cl-demo-ec2-construct.ts @@ -0,0 +1,210 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +/** + * @description + * This is EC2 construct for WebServer resource + * @author @aws-solutions + */ + +import { Stack, Construct, RemovalPolicy, CfnResource } from "@aws-cdk/core"; +import { + Vpc, + Instance, + InstanceType, + InitFile, + InitService, + InitServiceRestartHandle, + CloudFormationInit, + MachineImage, + AmazonLinuxVirt, + AmazonLinuxGeneration, + AmazonLinuxCpuType, + SecurityGroup, + Peer, + Port, + InitPackage, +} from "@aws-cdk/aws-ec2"; +import { + LogGroup, + RetentionDays, + CfnSubscriptionFilter, +} from "@aws-cdk/aws-logs"; +import { Effect, PolicyStatement } from "@aws-cdk/aws-iam"; +import manifest from "./manifest.json"; + +/** + * @interface + * @description web server interface + */ +interface IEC2Demo { + /** + * @description destination arn for log streaming + * @type {string} + */ + destination: string; + /** + * @description vpc for creating demo resources + * @type {Vpc} + */ + demoVpc: Vpc; +} +/** + * @class + * @description web server resources construct + * @property {string} region of deployment + */ +export class EC2Demo extends Construct { + readonly region: string; + readonly publicIp: string; + constructor(scope: Construct, id: string, props: IEC2Demo) { + super(scope, id); + + const stack = Stack.of(this); + + this.region = stack.region; // Returns the AWS::Region for this stack (or the literal value if known) + + /** + * @description security group for web server + * @type {SecurityGroup} + */ + const demoSg: SecurityGroup = new SecurityGroup(this, "DemoSG", { + vpc: props.demoVpc, + }); + demoSg.addIngressRule(Peer.anyIpv4(), Port.tcp(80), "allow HTTP traffic"); + (demoSg.node.defaultChild as CfnResource).cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W40", + reason: "Demo resource", + }, + { + id: "W5", + reason: "Demo resource", + }, + { + id: "W9", + reason: "Demo resource", + }, + { + id: "W2", + reason: "Demo resource", + }, + ], + }, + }; + + /** + * @description log group for web server + * @type {LogGroup} + */ + const ec2Lg: LogGroup = new LogGroup(this, "EC2LogGroup", { + removalPolicy: RemovalPolicy.DESTROY, + retention: RetentionDays.ONE_WEEK, + }); + + const handle: InitServiceRestartHandle = new InitServiceRestartHandle(); + + /** + * @description cloudformation init configuration for web server + * @type {CloudFormationInit} + */ + const init: CloudFormationInit = CloudFormationInit.fromElements( + InitPackage.yum("httpd", { serviceRestartHandles: [handle] }), + InitPackage.yum("php", { serviceRestartHandles: [handle] }), + InitPackage.yum("amazon-cloudwatch-agent", { + serviceRestartHandles: [handle], + }), + InitFile.fromObject("/tmp/cw-config.json", { + agent: { + run_as_user: "root", + }, + logs: { + logs_collected: { + files: { + collect_list: [ + { + file_path: "/var/log/httpd/access_log", + log_group_name: ec2Lg.logGroupName, + log_stream_name: "{instance_id}/apache.log", + timezone: "UTC", + }, + ], + }, + }, + }, + }), + InitFile.fromString( + "/var/www/html/index.php", + `AWS CloudFormation sample PHP application'; + ?>`, + { + mode: "000644", + owner: "apache", + group: "apache", + serviceRestartHandles: [handle], + } + ), + InitService.enable("httpd", { + enabled: true, + ensureRunning: true, + serviceRestartHandle: handle, + }) + ); + + /** + * @description web server instance + * @type {Instance} + */ + const demoEC2: Instance = new Instance(this, "DemoEC2", { + vpc: props.demoVpc, + instanceType: new InstanceType(manifest.jumpboxInstanceType), + machineImage: MachineImage.latestAmazonLinux({ + virtualization: AmazonLinuxVirt.HVM, + generation: AmazonLinuxGeneration.AMAZON_LINUX_2, + cpuType: AmazonLinuxCpuType.X86_64, + }), + init: init, + allowAllOutbound: true, + securityGroup: demoSg, + }); + + demoEC2.addUserData( + "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop", + "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/tmp/cw-config.json -s", + "curl 127.0.0.1" + ); + demoEC2.addToRolePolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + sid: "LogWrite", + actions: ["logs:Create*", "logs:PutLogEvents"], + resources: [ec2Lg.logGroupArn], + }) + ); + this.publicIp = demoEC2.instancePublicIp; + + /** + * @description subscription filter on web server log group + * @type {SubscriptionFilter} + */ + new CfnSubscriptionFilter(this, "WebServerSubscription", { + destinationArn: props.destination, + filterPattern: + "[host, ident, authuser, date, request, status, bytes, referrer, agent]", + logGroupName: ec2Lg.logGroupName, + }); + } +} diff --git a/source/resources/lib/cl-demo-stack.ts b/source/resources/lib/cl-demo-stack.ts new file mode 100644 index 0000000..4528dc9 --- /dev/null +++ b/source/resources/lib/cl-demo-stack.ts @@ -0,0 +1,325 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +/** + * @description + * This is Demo Stack for AWS Centralized Logging Solution + * @author @aws-solutions + */ + +import { + CfnMapping, + CfnOutput, + CfnParameter, + CfnResource, + Construct, + Fn, + NestedStack, + NestedStackProps, + RemovalPolicy, + Stack, +} from "@aws-cdk/core"; +import { + Vpc, + SubnetType, + FlowLog, + FlowLogResourceType, + FlowLogTrafficType, + FlowLogDestination, +} from "@aws-cdk/aws-ec2"; +import { + Effect, + PolicyStatement, + Role, + ServicePrincipal, +} from "@aws-cdk/aws-iam"; +import { + CfnSubscriptionFilter, + LogGroup, + RetentionDays, +} from "@aws-cdk/aws-logs"; +import { Trail } from "@aws-cdk/aws-cloudtrail"; +import { BlockPublicAccess, Bucket, BucketEncryption } from "@aws-cdk/aws-s3"; +import { EC2Demo } from "./cl-demo-ec2-construct"; +import manifest from "./manifest.json"; + +/** + * @class + * @description demo stack + * @property {string} account id + * @property {string} region of deployment + */ +export class CLDemo extends NestedStack { + readonly account: string; + readonly region: string; + /** + * @constructor + * @param {Construct} scope parent of the construct + * @param {string} id unique identifier for the object + * @param {NestedStackProps} props props for the construct + */ + constructor(scope: Construct, id: string, props?: NestedStackProps) { + super(scope, id, props); + const stack = Stack.of(this); + + this.account = stack.account; // Returns the AWS::AccountId for this stack (or the literal value if known) + this.region = stack.region; // Returns the AWS::Region for this stack (or the literal value if known) + + //============================================================================================= + // Parameters + //============================================================================================= + /** + * @description parameter for CW Logs Destination Arn + * @type {CfnParameter} + */ + const cwLogsDestinationArn: CfnParameter = new CfnParameter( + this, + "CWDestinationParm", + { + type: "String", + } + ); + + //============================================================================================= + // Metadata + //============================================================================================= + this.templateOptions.metadata = { + "AWS::CloudFormation::Interface": { + ParameterGroups: [ + { + Label: { default: "Destination Configuration" }, + Parameters: [cwLogsDestinationArn.logicalId], + }, + ], + ParameterLabels: { + [cwLogsDestinationArn.logicalId]: { + default: "CloudWatch Logs Destination Arn for Log Streaming", + }, + }, + }, + }; + + this.templateOptions.description = `(${manifest.solutionId}D) - The AWS CloudFormation template for deployment of the ${manifest.solutionName}. Version ${manifest.solutionVersion}`; + this.templateOptions.templateFormatVersion = manifest.templateVersion; + + //============================================================================================= + // Map + //============================================================================================= + new CfnMapping(this, "EC2", { + mapping: { Instance: { Type: "t3.micro" } }, + }); + + new CfnMapping(this, "FilterPatternLookup", { + mapping: { + Common: { + Pattern: + "[host, ident, authuser, date, request, status, bytes, referrer, agent]", + }, + CloudTrail: { + Pattern: "", + }, + FlowLogs: { + Pattern: + '[version, account_id, interface_id, srcaddr != "-", dstaddr != "-", srcport != "-", dstport != "-", protocol, packets, bytes, start, end, action, log_status]', + }, + Lambda: { + Pattern: '[timestamp=*Z, request_id="*-*", event]', + }, + SpaceDelimited: { + Pattern: "[]", + }, + Other: { + Pattern: "", + }, + }, + }); + + //============================================================================================= + // Resources + //============================================================================================= + + /** + * @description demo vpc with 1 public subnet + * @type {Vpc} + */ + const demoVPC: Vpc = new Vpc(this, "DemoVPC", { + cidr: "10.0.1.0/26", + natGateways: 0, + vpnGateway: false, + subnetConfiguration: [ + { + cidrMask: 28, + name: "PublicSubnet", + subnetType: SubnetType.PUBLIC, + }, + ], + }); + demoVPC.publicSubnets.forEach((subnet) => { + const hs = subnet.node.defaultChild as CfnResource; + hs.cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W33", + reason: "Need public IP for demo web server ", + }, + ], + }, + }; + }); + + //=================== + // FlowLog resources + //=================== + /** + * @description log group for VPC flow logs + * @type {LogGroup} + */ + const flowLg: LogGroup = new LogGroup(this, "VPCFlowLogGroup", { + removalPolicy: RemovalPolicy.DESTROY, + retention: RetentionDays.ONE_WEEK, + }); + + /** + * @description iam role for flow logs + * @type {Role} + */ + const flowRole: Role = new Role(this, "flowRole", { + assumedBy: new ServicePrincipal("vpc-flow-logs.amazonaws.com"), + }); + + /** + * @description demo flow logs + * @type {FlowLog} + */ + new FlowLog(this, "DemoFlowLog", { + resourceType: FlowLogResourceType.fromVpc(demoVPC), + trafficType: FlowLogTrafficType.ALL, + destination: FlowLogDestination.toCloudWatchLogs(flowLg, flowRole), + }); + + /** + * @description subscription filter for flow logs + * @type {SubscriptionFilter} + */ + new CfnSubscriptionFilter(this, "FlowLogSubscription", { + destinationArn: cwLogsDestinationArn.valueAsString, + filterPattern: Fn.findInMap("FilterPatternLookup", "FlowLogs", "Pattern"), + logGroupName: flowLg.logGroupName, + }); + + //==================== + // WebServer resources + //==================== + /** + * @description ec2 web server resources + * @type {EC2Demo} + */ + const ec2: EC2Demo = new EC2Demo(this, "WebServer", { + destination: cwLogsDestinationArn.valueAsString, + demoVpc: demoVPC, + }); + + //===================== + // CloudTrail resources + //===================== + /** + * @description log group for CloudTrail + * @type {LogGroup} + */ + const cloudtrailLg: LogGroup = new LogGroup(this, "CloudTrailLogGroup", { + removalPolicy: RemovalPolicy.DESTROY, + retention: RetentionDays.ONE_WEEK, + }); + + /** + * @description bucket for CloudTrail + * @type {Bucket} + */ + const trailBucket: Bucket = new Bucket(this, "TrailBucket", { + encryption: BucketEncryption.S3_MANAGED, + blockPublicAccess: BlockPublicAccess.BLOCK_ALL, + }); + trailBucket.addToResourcePolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + principals: [new ServicePrincipal("cloudtrail.amazonaws.com")], + sid: "CloudTrailRead", + actions: ["s3:GetBucketAcl"], + resources: [trailBucket.bucketArn], + }) + ); + trailBucket.addToResourcePolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + principals: [new ServicePrincipal("cloudtrail.amazonaws.com")], + sid: "CloudTrailWrite", + actions: ["s3:PutObject"], + resources: [`${trailBucket.bucketArn}/AWSLogs/${this.account}/*`], + }) + ); + (trailBucket.node.defaultChild as CfnResource).cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W35", + reason: "Demo resource", + }, + { + id: "W41", + reason: "Demo resource, no production data", + }, + ], + }, + }; + + /** + * @description demo trail + * @type {Trail} + */ + new Trail(this, "demoTrail", { + bucket: trailBucket, + cloudWatchLogGroup: cloudtrailLg, + isMultiRegionTrail: false, + sendToCloudWatchLogs: true, + includeGlobalServiceEvents: true, + }); + + /** + * @description subscription filter for cloudtrail logs + * @type {SubscriptionFilter} + */ + new CfnSubscriptionFilter(this, "CloudTrailSubscription", { + destinationArn: cwLogsDestinationArn.valueAsString, + filterPattern: Fn.findInMap( + "FilterPatternLookup", + "CloudTrail", + "Pattern" + ), + logGroupName: cloudtrailLg.logGroupName, + }); + + //============================================================================================= + // Output + //============================================================================================= + new CfnOutput(this, "Destination Arn", { + description: "CloudWatch Logs destination arn", + value: cwLogsDestinationArn.valueAsString, + }); + + new CfnOutput(this, "URL", { + description: "URL for demo web server", + value: `http://${ec2.publicIp}`, + }); + } +} diff --git a/source/resources/lib/cl-jumpbox-construct.ts b/source/resources/lib/cl-jumpbox-construct.ts new file mode 100644 index 0000000..91706f3 --- /dev/null +++ b/source/resources/lib/cl-jumpbox-construct.ts @@ -0,0 +1,112 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +/** + * @description + * This is Jumpbox construct for AWS Centralized Logging Solution + * @author @aws-solutions + */ + +import { Stack, Construct, CfnCondition, CfnResource } from "@aws-cdk/core"; +import { + Instance, + InstanceType, + MachineImage, + WindowsVersion, + Vpc, + SecurityGroup, + ISubnet, + Port, + Peer, +} from "@aws-cdk/aws-ec2"; +import manifest from "./manifest.json"; + +interface IJumpbox { + /** + * @description vpc to launch jumpbox + * @type {Vpc} + */ + vpc: Vpc; + /** + * @description public subnets to launch jumpbox + * @type {ISubnet[]} + */ + subnets: ISubnet[]; + /** + * @description ssh key for jumpbox + * @type {string} + */ + keyname: string; + /** + * @description deploy jumpbox + * @type {CfnCondition} + */ + deploy: CfnCondition; +} +/** + * @class + * @description web server resources construct + * @property {string} region of deployment + */ +export class Jumpbox extends Construct { + readonly region: string; + constructor(scope: Construct, id: string, props: IJumpbox) { + super(scope, id); + + const stack = Stack.of(this); + + this.region = stack.region; // Returns the AWS::Region for this stack (or the literal value if known) + + //========================================================================= + // Resource + //========================================================================= + /** + * @description security group for jumpbox + * @type {SecurityGroup} + */ + const sg: SecurityGroup = new SecurityGroup(this, "JumpboxSG", { + vpc: props.vpc, + allowAllOutbound: false, + }); + sg.addEgressRule(Peer.anyIpv4(), Port.tcp(80), "allow outbound https"); + sg.addEgressRule(Peer.anyIpv4(), Port.tcp(443), "allow outbound https"); + (sg.node.defaultChild as CfnResource).cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W5", + reason: "outbound traffic for http[s]", + }, + ], + }, + }; + (sg.node.defaultChild as CfnResource).cfnOptions.condition = props.deploy; + + /** + * @description jumpbox instance + * @type {Instance} + */ + const jumpbox: Instance = new Instance(this, "JumpboxEC2", { + vpc: props.vpc, + instanceType: new InstanceType(manifest.jumpboxInstanceType), + machineImage: MachineImage.latestWindows( + WindowsVersion.WINDOWS_SERVER_2019_ENGLISH_FULL_BASE + ), + securityGroup: sg, + vpcSubnets: { subnets: props.subnets }, + keyName: props.keyname, + }); + (jumpbox.node.defaultChild as CfnResource).cfnOptions.condition = + props.deploy; + } +} diff --git a/source/resources/lib/cl-primary-stack.ts b/source/resources/lib/cl-primary-stack.ts new file mode 100644 index 0000000..6035a40 --- /dev/null +++ b/source/resources/lib/cl-primary-stack.ts @@ -0,0 +1,1302 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may + * not use this file except in compliance with the License. A copy of the + * License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed + * on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express + * or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +/** + * @description + * This is Primary Stack for AWS Centralized Logging + * @author @aws-solutions + */ + +import { + FlowLogDestination, + FlowLogTrafficType, + Peer, + Port, + SecurityGroup, + SubnetType, + Vpc, +} from "@aws-cdk/aws-ec2"; +import { SnsAction } from "@aws-cdk/aws-cloudwatch-actions"; +import { + Code, + Runtime, + Function, + CfnFunction, + StartingPosition, +} from "@aws-cdk/aws-lambda"; +import { + App, + CfnCondition, + CfnMapping, + CfnOutput, + CfnParameter, + CfnResource, + CustomResource, + Duration, + Fn, + NestedStack, + RemovalPolicy, + Stack, +} from "@aws-cdk/core"; +import { + Domain, + ElasticsearchVersion, + CfnDomain, +} from "@aws-cdk/aws-elasticsearch"; +import { + AccountRecovery, + CfnIdentityPool, + CfnIdentityPoolRoleAttachment, + CfnUserPool, + CfnUserPoolUser, + UserPool, +} from "@aws-cdk/aws-cognito"; +import { + CfnRole, + Effect, + FederatedPrincipal, + Policy, + PolicyDocument, + PolicyStatement, + Role, +} from "@aws-cdk/aws-iam"; +import { Provider } from "@aws-cdk/custom-resources"; +import { ServicePrincipal, ArnPrincipal } from "@aws-cdk/aws-iam"; +import { StreamEncryption, Stream } from "@aws-cdk/aws-kinesis"; +import { + BlockPublicAccess, + Bucket, + BucketAccessControl, + BucketEncryption, +} from "@aws-cdk/aws-s3"; +import { CfnDeliveryStream } from "@aws-cdk/aws-kinesisfirehose"; +import { CLDemo } from "./cl-demo-stack"; +import manifest from "./manifest.json"; +import { LogGroup, LogStream } from "@aws-cdk/aws-logs"; +import { Jumpbox } from "./cl-jumpbox-construct"; +import { KinesisEventSource } from "@aws-cdk/aws-lambda-event-sources"; +import { Queue, QueueEncryption } from "@aws-cdk/aws-sqs"; +import { Topic } from "@aws-cdk/aws-sns"; +import { Alias, IAlias } from "@aws-cdk/aws-kms"; +import { EmailSubscription } from "@aws-cdk/aws-sns-subscriptions"; + +enum LogLevel { + ERROR = "error", + WARN = "warn", + INFO = "info", + DEBUG = "debug", +} + +export class CLPrimary extends Stack { + readonly account: string; + readonly region: string; + readonly partn: string; + + constructor(scope: App, id: string) { + super(scope, id); + + const stack = Stack.of(this); + this.account = stack.account; // Returns the AWS::AccountId for this stack (or the literal value if known) + this.region = stack.region; // Returns the AWS::Region for this stack (or the literal value if known) + this.partn = stack.partition; // Returns the AWS::Partition for this stack + + //========================================================================= + // Parameter + //========================================================================= + /** + * @description ES domain name + * @type {CfnParameter} + */ + const esDomain: CfnParameter = new CfnParameter(this, "DomainName", { + type: "String", + default: "centralizedlogging", + }); + + /** + * @description email address for Cognito admin + * @type {CfnParameter} + */ + const adminEmail: CfnParameter = new CfnParameter(this, "AdminEmail", { + type: "String", + allowedPattern: "^[\\w]+\\@[\\w]+\\.[a-z]+$", + }); + + /** + * @description ES cluster size + * @type {CfnParameter} + */ + const clusterSize: CfnParameter = new CfnParameter(this, "ClusterSize", { + description: + "Elasticsearch cluster size; small (4 data nodes), medium (6 data nodes), large (6 data nodes)", + type: "String", + default: "Small", + allowedValues: ["Small", "Medium", "Large"], + }); + + /** + * @description Option to deploy demo template + * @type {CfnParameter} + */ + const demoTemplate: CfnParameter = new CfnParameter(this, "DemoTemplate", { + description: "Deploy demo template for sample data and logs?", + type: "String", + default: "No", + allowedValues: ["No", "Yes"], + }); + + /** + * @description List of spoke account ids + * @type {CfnParameter} + */ + const spokeAccts: CfnParameter = new CfnParameter(this, "SpokeAccounts", { + description: + "Account IDs which you want to allow for centralized logging (comma separated list eg. 11111111,22222222)", + type: "CommaDelimitedList", + }); + + /** + * @regions List of regions for CW Logs Destination + * @type {CfnParameter} + */ + const spokeRegions: CfnParameter = new CfnParameter(this, "SpokeRegions", { + description: + "Regions which you want to allow for centralized logging (comma separated list eg. us-east-1,us-west-2)", + type: "CommaDelimitedList", + default: "All", + }); + + /** + * @description deploy jumbox + * @type {CfnParameter} + */ + const jumpboxDeploy: CfnParameter = new CfnParameter( + this, + "JumpboxDeploy", + { + description: "Do you want to deploy jumbox?", + type: "String", + default: "No", + allowedValues: ["No", "Yes"], + } + ); + + /** + * @description key pair for jump box + * @type {CfnParameter} + */ + const jumpboxKey: CfnParameter = new CfnParameter(this, "JumpboxKey", { + description: + "Key pair name for jumpbox (You may leave this empty if you chose 'No' above)", + type: "String", + }); + + //============================================================================================= + // Metadata + //============================================================================================= + this.templateOptions.metadata = { + "AWS::CloudFormation::Interface": { + ParameterGroups: [ + { + Label: { + default: "Elasticsearch Configuration", + }, + Parameters: [ + esDomain.logicalId, + clusterSize.logicalId, + adminEmail.logicalId, + ], + }, + { + Label: { + default: "Spoke Configuration", + }, + Parameters: [spokeAccts.logicalId, spokeRegions.logicalId], + }, + { + Label: { + default: "Do you want to deploy sample log sources?", + }, + Parameters: [demoTemplate.logicalId], + }, + { + Label: { + default: "Jumpbox Configuration", + }, + Parameters: [jumpboxDeploy.logicalId, jumpboxKey.logicalId], + }, + ], + ParameterLabels: { + [adminEmail.logicalId]: { + default: "Admin Email Address", + }, + [esDomain.logicalId]: { + default: "Elasticsearch Domain Name", + }, + [jumpboxKey.logicalId]: { + default: "Key pair for jumpbox", + }, + [jumpboxDeploy.logicalId]: { + default: "Deployment", + }, + [clusterSize.logicalId]: { + default: "Cluster Size", + }, + [demoTemplate.logicalId]: { + default: "Sample Logs", + }, + [spokeAccts.logicalId]: { + default: "Spoke Accounts", + }, + [spokeRegions.logicalId]: { + default: "Spoke Regions", + }, + }, + }, + }; + this.templateOptions.description = `(${manifest.solutionId}) - The AWS CloudFormation template for deployment of the ${manifest.solutionName}. Version ${manifest.solutionVersion}`; + this.templateOptions.templateFormatVersion = manifest.templateVersion; + + //========================================================================= + // Mapping + //========================================================================= + const metricsMap = new CfnMapping(this, "CLMap", { + mapping: { + Metric: { + SendAnonymousMetric: manifest.sendMetric, + MetricsEndpoint: manifest.metricsEndpoint, // aws-solutions metrics endpoint + }, + }, + }); + + const esMap = new CfnMapping(this, "ESMap", { + mapping: { + NodeCount: { + Small: 4, + Medium: 6, + Large: 6, + }, + MasterSize: { + Small: "c5.large.elasticsearch", + Medium: "c5.large.elasticsearch", + Large: "c5.large.elasticsearch", + }, + InstanceSize: { + Small: "r5.large.elasticsearch", + Medium: "r5.2xlarge.elasticsearch", + Large: "r5.4xlarge.elasticsearch", + }, + }, + }); + + //============================================================================================= + // Condition + //============================================================================================= + const demoDeploymentCheck = new CfnCondition(this, "demoDeploymentCheck", { + expression: Fn.conditionEquals(demoTemplate.valueAsString, "Yes"), + }); + const jumpboxDeploymentCheck = new CfnCondition( + this, + "JumpboxDeploymentCheck", + { + expression: Fn.conditionEquals(jumpboxDeploy.valueAsString, "Yes"), + } + ); + + //============================================================================================= + // Resource + //============================================================================================= + /** + * @description helper lambda role + * @type {Role} + */ + const helperRole: Role = new Role(this, "HelperRole", { + assumedBy: new ServicePrincipal("lambda.amazonaws.com"), + }); + const helperPolicy1 = new Policy(this, "HelperRolePolicy1", { + roles: [helperRole], + statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:CreateLogGroup", + ], + resources: [ + `arn:${this.partn}:logs:${this.region}:${this.account}:log-group:*`, + `arn:${this.partn}:logs:${this.region}:${this.account}:log-group:*:log-stream:*`, + ], + }), + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "ec2:DescribeRegions", + "logs:PutDestination", + "logs:DeleteDestination", + "logs:PutDestinationPolicy", + ], + resources: ["*"], + }), + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["iam:CreateServiceLinkedRole"], + resources: [ + `arn:${this.partn}:iam::*:role/aws-service-role/es.amazonaws.com/AWSServiceRoleForAmazonElasticsearchService*`, + ], + conditions: { + ["StringLike"]: { + "iam:AWSServiceName": "es.amazonaws.com", + }, + }, + }), + ], + }); + (helperPolicy1.node.defaultChild as CfnResource).cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W12", + reason: + "* needed, actions do no support resource level permissions", + }, + ], + }, + }; + + /** + * @description helper lambda + * @type {Function} + */ + // eslint-disable-next-line @typescript-eslint/ban-types + const helperFunc: Function = new Function(this, "HelperLambda", { + description: manifest.solutionName + " - solution helper functions", + environment: { + LOG_LEVEL: LogLevel.INFO, //change as needed + METRICS_ENDPOINT: metricsMap.findInMap("Metric", "MetricsEndpoint"), + SEND_METRIC: metricsMap.findInMap("Metric", "SendAnonymousMetric"), + }, + handler: "index.handler", + code: Code.fromAsset("../../source/services/helper/dist/cl-helper.zip"), + runtime: Runtime.NODEJS_12_X, + timeout: Duration.seconds(300), + role: helperRole, + }); + const hF = helperFunc.node.findChild("Resource") as CfnFunction; + hF.addDependsOn(helperPolicy1.node.defaultChild as CfnResource); + hF.cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W58", + reason: + "CloudWatch logs write permissions added with managed role AWSLambdaBasicExecutionRole", + }, + ], + }, + }; + + /** + * @description custom resource for helper functions + * @type {Provider} + */ + const helperProvider: Provider = new Provider(this, "HelperProvider", { + onEventHandler: helperFunc, + }); + (helperProvider.node.children[0].node.findChild( + "Resource" + ) as CfnFunction).cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W58", + reason: + "CloudWatch logs write permissions added with managed role AWSLambdaBasicExecutionRole", + }, + ], + }, + }; + + /** + * Get UUID for deployment + */ + const createUniqueId = new CustomResource(this, "CreateUUID", { + resourceType: "Custom::CreateUUID", + serviceToken: helperProvider.serviceToken, + }); + + /** + * Create service linked role for ES + */ + new CustomResource(this, "CreateESServiceRole", { + resourceType: "Custom::CreateESServiceRole", + serviceToken: helperProvider.serviceToken, + }); + + /** + * Send launch data to aws-solutions + */ + new CustomResource(this, "LaunchData", { + resourceType: "Custom::LaunchData", + serviceToken: helperProvider.serviceToken, + properties: { + SolutionId: manifest.solutionId, + SolutionVersion: manifest.solutionVersion, + SolutionUuid: createUniqueId.getAttString("UUID"), + Stack: "PrimaryStack", + }, + }); + + /** + * @description cognito user pool + * @type {UserPool} + */ + const esUserPool: UserPool = new UserPool(this, "ESUserPool", { + standardAttributes: { + email: { + mutable: true, + required: true, + }, + }, + passwordPolicy: { + minLength: 8, + requireLowercase: true, + requireUppercase: true, + requireDigits: true, + requireSymbols: true, + tempPasswordValidity: Duration.days(3), + }, + signInAliases: { email: true }, + accountRecovery: AccountRecovery.EMAIL_ONLY, + selfSignUpEnabled: false, + }); + // enforce advaned security mode + (esUserPool.node.defaultChild as CfnUserPool).addPropertyOverride( + "UserPoolAddOns", + { + AdvancedSecurityMode: "ENFORCED", + } + ); + // add domain to user pool + const upDomain = esUserPool.addDomain("ESCognitoDomain", { + cognitoDomain: { + domainPrefix: `${esDomain.valueAsString}-${createUniqueId.getAttString( + "UUID" + )}`, + }, + }); + + /** + * @description adding admin to user pool + * @type {CfnUserPoolUser} + */ + new CfnUserPoolUser(this, "AdminUser", { + userPoolId: esUserPool.userPoolId, + userAttributes: [{ name: "email", value: adminEmail.valueAsString }], + username: adminEmail.valueAsString, + }); + + /** + * @description cognito user pool + * @type {CfnIdentityPool} + * @remarks higher level constructs for Identity pools are yet not developed + * @see https://docs.aws.amazon.com/cdk/api/latest/docs/aws-cognito-readme.html + */ + const identityPool: CfnIdentityPool = new CfnIdentityPool( + this, + "ESIdentityPool", + { + allowUnauthenticatedIdentities: false, + } + ); + + /** + * @description cognito authenticated role + * @type {Role} + */ + const idpAuthRole: Role = new Role(this, "CognitoAuthRole", { + assumedBy: new FederatedPrincipal( + "cognito-identity.amazonaws.com", + { + ["StringEquals"]: { + "cognito-identity.amazonaws.com:aud": identityPool.ref, + }, + ["ForAnyValue:StringLike"]: { + "cognito-identity.amazonaws.com:amr": "authenticated", + }, + }, + "sts:AssumeRoleWithWebIdentity" + ), + }); + + // identity pool authorized role + new CfnIdentityPoolRoleAttachment(this, "IdentityPoolRoleAttachment", { + identityPoolId: identityPool.ref, + roles: { authenticated: idpAuthRole.roleArn }, + }); + + /** + * @description es role for cognito access + * @type {Role} + * @remark same policy as arn:aws:iam::aws:policy/AmazonESCognitoAccess + */ + const esCognitoRole: Role = new Role(this, "ESCognitoRole", { + assumedBy: new ServicePrincipal("es.amazonaws.com"), + inlinePolicies: { + ["ESCognitoAccess"]: PolicyDocument.fromJson({ + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Action: [ + "cognito-idp:DescribeUserPool", + "cognito-idp:CreateUserPoolClient", + "cognito-idp:DeleteUserPoolClient", + "cognito-idp:DescribeUserPoolClient", + "cognito-idp:AdminInitiateAuth", + "cognito-idp:AdminUserGlobalSignOut", + "cognito-idp:ListUserPoolClients", + "cognito-identity:DescribeIdentityPool", + "cognito-identity:UpdateIdentityPool", + "cognito-identity:SetIdentityPoolRoles", + "cognito-identity:GetIdentityPoolRoles", + ], + Resource: "*", + }, + ], + }), + }, + }); + esCognitoRole.addToPolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["iam:PassRole"], + resources: [esCognitoRole.roleArn], + conditions: { + ["StringLike"]: { + "iam:PassedToService": "cognito-identity.amazonaws.com", + }, + }, + }) + ); + (esCognitoRole.node.defaultChild as CfnResource).cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W11", + reason: "cognito actions do not allow resource level permissions", + }, + ], + }, + }; + + /** + * @description IAM role for kinesis firehose + * @type {Role} + */ + const firehoseRole: Role = new Role(this, "FirehoseRole", { + assumedBy: new ServicePrincipal("firehose.amazonaws.com"), + }); + + /** + * @description log group for VPC flow logs + * @type {LogGroup} + */ + const flowLg: LogGroup = new LogGroup(this, "VPCFlowLogGroup", { + removalPolicy: RemovalPolicy.RETAIN, + }); + + /** + * @description iam role for flow logs + * @type {Role} + */ + const flowRole: Role = new Role(this, "flowRole", { + assumedBy: new ServicePrincipal("vpc-flow-logs.amazonaws.com"), + }); + + /** + * @description es vpc with 2 isolated subnets + * @type {Vpc} + */ + const VPC: Vpc = new Vpc(this, "ESVPC", { + cidr: manifest.esdomain.vpcCIDR, + vpnGateway: false, + flowLogs: { + ["ESVpcFlow"]: { + destination: FlowLogDestination.toCloudWatchLogs(flowLg, flowRole), + trafficType: FlowLogTrafficType.ALL, + }, + }, + subnetConfiguration: [ + { + cidrMask: 24, + subnetType: SubnetType.ISOLATED, + name: "ESIsolatedSubnet", + }, + { + cidrMask: 24, + subnetType: SubnetType.PUBLIC, + name: "ESPublicSubnet", + }, + ], + }); + VPC.publicSubnets.map((subnet) => { + (subnet.node.defaultChild as CfnResource).cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W33", + reason: + "public ip needed for jumpbox, restricted by appropriate security group rule", + }, + ], + }, + }; + }); + + /** + * @description security group for es domain + * @type {SecurityGroup} + */ + const esSg: SecurityGroup = new SecurityGroup(this, "ESSG", { + vpc: VPC, + allowAllOutbound: false, + }); + esSg.addIngressRule( + Peer.ipv4(VPC.vpcCidrBlock), + Port.tcp(443), + "allow inbound https traffic" + ); + esSg.addEgressRule( + Peer.ipv4(VPC.vpcCidrBlock), + Port.tcp(443), + "allow outbound https" + ); + + /** + * @description es domain + * @type {Domain} + */ + const domain: Domain = new Domain(this, "ESDomain", { + version: ElasticsearchVersion.V7_7, + domainName: esDomain.valueAsString, + enforceHttps: true, + vpcOptions: { + subnets: VPC.isolatedSubnets, + securityGroups: [esSg], + }, + encryptionAtRest: { + enabled: true, + }, + zoneAwareness: { + availabilityZoneCount: 2, + }, + nodeToNodeEncryption: true, + automatedSnapshotStartHour: 0, + cognitoKibanaAuth: { + identityPoolId: identityPool.ref, + role: esCognitoRole, + userPoolId: esUserPool.userPoolId, + }, + }); + const _d = domain.node.defaultChild as CfnResource; + _d.cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W28", + reason: "using customer provided domain name", + }, + ], + }, + }; + + // attach policy to idp auth role + idpAuthRole.attachInlinePolicy( + new Policy(this, "authRolePolicy", { + statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "es:ESHttpGet", + "es:ESHttpDelete", + "es:ESHttpPut", + "es:ESHttpPost", + "es:ESHttpHead", + "es:ESHttpPatch", + ], + resources: [domain.domainArn], + }), + ], + }) + ); + + /** + * @description cluster configurations for es domain + * @remark property is not supported on higher level construct + */ + const clusterConfig = { + DedicatedMasterEnabled: true, + InstanceCount: esMap.findInMap("NodeCount", clusterSize.valueAsString), + ZoneAwarenessEnabled: true, + InstanceType: esMap.findInMap("InstanceSize", clusterSize.valueAsString), + DedicatedMasterType: esMap.findInMap( + "MasterSize", + clusterSize.valueAsString + ), + DedicatedMasterCount: 3, + }; + // adding cluster config + const cfnDomain = domain.node.defaultChild as CfnDomain; + cfnDomain.addDependsOn(upDomain.node.defaultChild as CfnResource); + cfnDomain.addPropertyOverride("ElasticsearchClusterConfig", clusterConfig); + + /** + * @description es domain access policy + * @remark domain construct adds access policy using lambda function + */ + const accessPolicies = { + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Action: [ + "es:ESHttpGet", + "es:ESHttpDelete", + "es:ESHttpPut", + "es:ESHttpPost", + "es:ESHttpHead", + "es:ESHttpPatch", + ], + Principal: { AWS: idpAuthRole.roleArn }, + Resource: `arn:${this.partn}:es:${this.region}:${this.account}:domain/${esDomain.valueAsString}/*`, + }, + { + Effect: "Allow", + Action: [ + "es:DescribeElasticsearchDomain", + "es:DescribeElasticsearchDomains", + "es:DescribeElasticsearchDomainConfig", + "es:ESHttpPost", + "es:ESHttpPut", + "es:HttpGet", + ], + Principal: { AWS: firehoseRole.roleArn }, + Resource: `arn:${this.partn}:es:${this.region}:${this.account}:domain/${esDomain.valueAsString}/*`, + }, + ], + }; + // adding access policy + cfnDomain.addPropertyOverride("AccessPolicies", accessPolicies); + + /** + * @description dead letter queue for lambda + * @type {Queue} + */ + const dlq: Queue = new Queue(this, `dlq`, { + encryption: QueueEncryption.KMS_MANAGED, + }); + + /** + * @description Lambda transformer for log events + * @type {Function} + */ + // eslint-disable-next-line @typescript-eslint/ban-types + const logTransformer: Function = new Function(this, "CLTransformer", { + description: `${manifest.solutionName} - Lambda function to transform log events and send to kinesis firehose`, + environment: { + LOG_LEVEL: LogLevel.INFO, //change as needed + SOLUTION_ID: manifest.solutionId, + SOLUTION_VERSION: manifest.solutionVersion, + UUID: createUniqueId.getAttString("UUID"), + CLUSTER_SIZE: clusterSize.valueAsString, + DELIVERY_STREAM: manifest.firehoseName, + METRICS_ENDPOINT: metricsMap.findInMap("Metric", "MetricsEndpoint"), + SEND_METRIC: metricsMap.findInMap("Metric", "SendAnonymousMetric"), + }, + handler: "index.handler", + code: Code.fromAsset( + "../../source/services/transformer/dist/cl-transformer.zip" + ), + runtime: Runtime.NODEJS_12_X, + timeout: Duration.seconds(300), + deadLetterQueue: dlq, + deadLetterQueueEnabled: true, + }); + (logTransformer.node.defaultChild as CfnResource).cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W58", + reason: + "CloudWatch logs write permissions added with managed role AWSLambdaBasicExecutionRole", + }, + ], + }, + }; + + /** + * @description Kms key for SNS topic + * @type {IAlias} + */ + const snsKeyAlias: IAlias = Alias.fromAliasName( + this, + "snsKey", + "alias/aws/sns" + ); + + /** + * @description sns topic for alarms + * @type {Topic} + */ + const topic: Topic = new Topic(this, "Topic", { + displayName: "CL-Lambda-Error", + masterKey: snsKeyAlias, + }); + // add email subscription for admin + topic.addSubscription(new EmailSubscription(adminEmail.valueAsString)); + + // adding cw alarm for lambda error rate + const alarm = logTransformer + .metricErrors() + .createAlarm(this, "CL-LambdaError-Alarm", { + threshold: 0.05, + evaluationPeriods: 1, + }); + alarm.addAlarmAction(new SnsAction(topic)); + + /** + * @description kinesis data stream for centralized logging + * @type {Stream} + */ + const clDataStream: Stream = new Stream(this, "CLDataStream", { + shardCount: manifest.kinesisDataStream.shard, + retentionPeriod: Duration.hours( + manifest.kinesisDataStream.retentionInHrs + ), + encryption: StreamEncryption.MANAGED, + }); + // add event source for kinesis data stream + logTransformer.addEventSource( + new KinesisEventSource(clDataStream, { + batchSize: 100, // default + startingPosition: StartingPosition.TRIM_HORIZON, + }) + ); + + /** + * @description S3 bucket for access logs + * @type {Bucket} + */ + const accessLogsBucket: Bucket = new Bucket(this, "AccessLogsBucket", { + encryption: BucketEncryption.S3_MANAGED, + blockPublicAccess: BlockPublicAccess.BLOCK_ALL, + accessControl: BucketAccessControl.LOG_DELIVERY_WRITE, + }); + // cfn_nag warning suppress rule + const ab = accessLogsBucket.node.defaultChild as CfnResource; + ab.cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W35", + reason: "access logging disabled, its a logging bucket", + }, + { + id: "W51", + reason: "permission given for log delivery", + }, + ], + }, + }; + + /** + * @description S3 bucket for Firehose + * @type {Bucket} + */ + const firehoseBucket: Bucket = new Bucket(this, "CLBucket", { + encryption: BucketEncryption.S3_MANAGED, + blockPublicAccess: BlockPublicAccess.BLOCK_ALL, + serverAccessLogsBucket: accessLogsBucket, + serverAccessLogsPrefix: "cl-access-logs", + }); + // adding bucket policy + firehoseBucket.addToResourcePolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + principals: [new ArnPrincipal(firehoseRole.roleArn)], + actions: ["s3:Put*", "s3:Get*"], + resources: [firehoseBucket.bucketArn, `${firehoseBucket.bucketArn}/*`], + }) + ); + // apply deletion policy + const fb = firehoseBucket.node.defaultChild as CfnResource; + fb.applyRemovalPolicy(RemovalPolicy.RETAIN); + + /** + * @description log group for firehose error events + * @type {LogGroup} + */ + const firehoseLG: LogGroup = new LogGroup(this, "FirehoseLogGroup", { + removalPolicy: RemovalPolicy.RETAIN, + logGroupName: `/aws/kinesisfirehose/${manifest.firehoseName}`, + }); + + /** + * @description log stream for elasticsearch delivery logs + * @type {LogStream} + */ + const firehoseLS: LogStream = new LogStream(this, "FirehoseESLogStream", { + logGroup: firehoseLG, + logStreamName: "ElasticsearchDelivery", + }); + + /** + * @description log stream for s3 delivery logs + * @type {LogStream} + */ + const firehoseLSS3: LogStream = new LogStream(this, "FirehoseS3LogStream", { + logGroup: firehoseLG, + logStreamName: "S3Delivery", + }); + + /** + * @description iam policy for firehose role + * @type {Policy} + */ + const firehosePolicy: Policy = new Policy(this, "FirehosePolicy", { + policyName: manifest.firehosePolicy, + roles: [firehoseRole], + statements: [ + // policy to access S3 bucket + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject", + ], + resources: [ + `arn:${this.partn}:s3:::${firehoseBucket.bucketName}`, + `arn:${this.partn}:s3:::${firehoseBucket.bucketName}/*`, + ], + }), + // policy for kms key + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["kms:GenerateDataKey", "kms:Decrypt"], + resources: [ + `arn:${this.partn}:kms:${this.region}:${this.account}:key/*`, + ], + conditions: { + ["StringEquals"]: { + "kms:ViaService": `s3.${this.region}.amazonaws.com`, + }, + ["StringLike"]: { + "kms:EncryptionContext:aws:s3:arn": [ + `arn:${this.partn}:s3:::${firehoseBucket.bucketName}/*`, + ], + }, + }, + }), + // policy for es vpc + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "ec2:DescribeVpcs", + "ec2:DescribeVpcAttribute", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeNetworkInterfaces", + "ec2:CreateNetworkInterface", + "ec2:CreateNetworkInterfacePermission", + "ec2:DeleteNetworkInterface", + ], + resources: ["*"], + }), + // policy for es + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "es:DescribeElasticsearchDomain", + "es:DescribeElasticsearchDomains", + "es:DescribeElasticsearchDomainConfig", + "es:ESHttpPost", + "es:ESHttpPut", + ], + resources: [ + `arn:${this.partn}:es:${this.region}:${this.account}:domain/${domain.domainName}`, + `arn:${this.partn}:es:${this.region}:${this.account}:domain/${domain.domainName}/*`, + ], + }), + // policy for HTTP Get + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["es:ESHttpGet"], + resources: [ + `arn:${this.partn}:es:${this.region}:${this.account}:domain/${domain.domainName}/_all/_settings`, + `arn:${this.partn}:es:${this.region}:${this.account}:domain/${domain.domainName}/_cluster/stats`, + `arn:${this.partn}:es:${this.region}:${this.account}:domain/${domain.domainName}/cwl-kinesis/_mapping/kinesis`, + `arn:${this.partn}:es:${this.region}:${this.account}:domain/${domain.domainName}/_nodes`, + `arn:${this.partn}:es:${this.region}:${this.account}:domain/${domain.domainName}/_nodes/*/stats`, + `arn:${this.partn}:es:${this.region}:${this.account}:domain/${domain.domainName}/_stats`, + `arn:${this.partn}:es:${this.region}:${this.account}:domain/${domain.domainName}/cwl-kinesis/_stats`, + ], + }), + // policy for CW logs + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["logs:PutLogEvents", "logs:CreateLogStream"], + resources: [`${firehoseLG.logGroupArn}`], + }), + // policy for kms decryption + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["kms:Decrypt"], + resources: [ + `arn:${this.partn}:kms:${this.region}:${this.account}:key/*`, + ], + conditions: { + ["StringEquals"]: { + "kms:ViaService": `kinesis.${this.region}.amazonaws.com`, + }, + ["StringLike"]: { + "kms:EncryptionContext:aws:kinesis:arn": `${clDataStream.streamArn}`, + }, + }, + }), + ], + }); + (firehosePolicy.node.defaultChild as CfnResource).cfnOptions.metadata = { + cfn_nag: { + rules_to_suppress: [ + { + id: "W12", + reason: + "* needed for actions that do no support resource level permissions", + }, + { + id: "W76", + reason: "policy verified", + }, + ], + }, + }; + + /** + * @description CL Firehose + * @type {CfnDeliveryStream} + */ + const clFirehose: CfnDeliveryStream = new CfnDeliveryStream( + this, + "CLFirehose", + { + elasticsearchDestinationConfiguration: { + indexName: "cwl", + domainArn: domain.domainArn, + roleArn: firehoseRole.roleArn, + indexRotationPeriod: "OneDay", + s3Configuration: { + bucketArn: firehoseBucket.bucketArn, + roleArn: firehoseRole.roleArn, + cloudWatchLoggingOptions: { + enabled: true, + logGroupName: `/aws/kinesisfirehose/${manifest.firehoseName}`, + logStreamName: firehoseLSS3.logStreamName, + }, + }, + s3BackupMode: "AllDocuments", + vpcConfiguration: { + roleArn: firehoseRole.roleArn, + subnetIds: VPC.isolatedSubnets.map((subnet) => subnet.subnetId), + securityGroupIds: [esSg.securityGroupId], + }, + cloudWatchLoggingOptions: { + enabled: true, + logGroupName: `/aws/kinesisfirehose/${manifest.firehoseName}`, + logStreamName: firehoseLS.logStreamName, + }, + }, + deliveryStreamType: "DirectPut", + deliveryStreamName: manifest.firehoseName, + deliveryStreamEncryptionConfigurationInput: { + keyType: "AWS_OWNED_CMK", + }, + } + ); + clFirehose.addDependsOn(firehosePolicy.node.defaultChild as CfnResource); + + // allow lambda to put records on firehose + logTransformer.addToRolePolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["firehose:PutRecordBatch"], + resources: [clFirehose.attrArn], + }) + ); + + /** + * @description IAM role for cw logs destination + * @type {Role} + */ + const cwDestinationRole: Role = new Role(this, "CWDestinationRole", { + assumedBy: new ServicePrincipal("logs.amazonaws.com"), + }); + const assumeBy = { + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: { + Service: "logs.amazonaws.com", + }, + Action: "sts:AssumeRole", + }, + ], + }; + (cwDestinationRole.node.defaultChild as CfnRole).addOverride( + "Properties.AssumeRolePolicyDocument", + assumeBy + ); + + /** + * @description iam permissions for putting record on kinesis data stream + * @type {Policy} + */ + const cwDestPolicy: Policy = new Policy(this, "CWDestPolicy", { + roles: [cwDestinationRole], + statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["kinesis:PutRecord"], + resources: [`${clDataStream.streamArn}`], + }), + ], + }); + + /** + * @description iam permission to pass role for creating cw destinations + * @type {Policy} + */ + const helperPolicy2: Policy = new Policy(this, "HelperRolePolicy2", { + roles: [helperRole], + statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["iam:PassRole"], + resources: [cwDestinationRole.roleArn], + }), + ], + }); + (helperPolicy2.node.defaultChild as CfnResource).addDependsOn( + cwDestPolicy.node.defaultChild as CfnResource + ); + + /** + * @description create CW Logs Destination + * @type {CustomResource} + */ + const cwDestination: CustomResource = new CustomResource( + this, + "CWDestination", + { + resourceType: "Custom::CWDestination", + serviceToken: helperProvider.serviceToken, + properties: { + Regions: spokeRegions.valueAsList, + DestinationName: manifest.cwDestinationName, + Role: cwDestinationRole.roleArn, + DataStream: clDataStream.streamArn, + SpokeAccounts: spokeAccts.valueAsList, + }, + } + ); + (cwDestination.node.defaultChild as CfnResource).addDependsOn( + helperPolicy2.node.defaultChild as CfnResource + ); + + /** + * @description Jumpbox resources + * @type {Construct} + */ + new Jumpbox(this, "CL-Jumpbox", { + vpc: VPC, + subnets: VPC.publicSubnets, + keyname: jumpboxKey.valueAsString, + deploy: jumpboxDeploymentCheck, + }); + + /** + * @description Demo stack + * @type {NestedStack} + */ + const demo: NestedStack = new CLDemo(this, "CL-DemoStack", { + parameters: { + ["CWDestinationParm"]: `arn:${this.partn}:logs:${this.region}:${this.account}:destination:${manifest.cwDestinationName}`, + }, + }); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + demo.nestedStackResource!.cfnOptions.condition = demoDeploymentCheck; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + demo.nestedStackResource!.addDependsOn( + domain.node.defaultChild as CfnResource + ); + + //============================================================================================= + // Output + //============================================================================================= + new CfnOutput(this, "Destination Subscription Command", { + description: "Command to run in spoke accounts/regions", + value: `aws logs put-subscription-filter \ + --destination-arn arn:${this.partn}:logs::${this.account}:destination:${manifest.cwDestinationName} \ + --log-group-name \ + --filter-name \ + --filter-pattern \ + --profile `, + }); + + new CfnOutput(this, "Unique ID", { + description: "UUID for Centralized Logging Stack", + value: createUniqueId.getAttString("UUID"), + }); + + new CfnOutput(this, "Admin Email", { + description: "Admin Email address", + value: adminEmail.valueAsString, + }); + + new CfnOutput(this, "Domain Name", { + description: "ES Domain Name", + value: esDomain.valueAsString, + }); + + new CfnOutput(this, "Kibana URL", { + description: "Kibana URL", + value: `https://${domain.domainEndpoint}/_plugin/kibana/`, + }); + + new CfnOutput(this, "Cluster Size", { + description: "ES Cluster Size", + value: clusterSize.valueAsString, + }); + + new CfnOutput(this, "Demo Deployment", { + description: "Demo data deployed?", + value: demoTemplate.valueAsString, + }); + } +} diff --git a/source/resources/lib/manifest.json b/source/resources/lib/manifest.json new file mode 100644 index 0000000..9cfb8be --- /dev/null +++ b/source/resources/lib/manifest.json @@ -0,0 +1,21 @@ +{ + "solutionId": "SO0009", + "metricsEndpoint": "https://metrics.awssolutionsbuilder.com/generic", + "solutionName": "%%SOLUTION_NAME%%", + "templateVersion": "2010-09-09", + "solutionVersion": "%%VERSION%%", + "sendMetric": "Yes", + "streamName": "CL-KinesisStream", + "firehoseName": "CL-Firehose", + "cwDestinationName": "CL-Destination", + "firehosePolicy": "CL-Firehose-Policy", + "jumpboxInstanceType": "t3.micro", + "esdomain": { + "vpcCIDR": "10.0.0.0/16", + "masterNodes": 3 + }, + "kinesisDataStream": { + "shard": 1, + "retentionInHrs": 24 + } +} diff --git a/source/resources/package.json b/source/resources/package.json new file mode 100755 index 0000000..dbac7d3 --- /dev/null +++ b/source/resources/package.json @@ -0,0 +1,43 @@ +{ + "name": "centralized-logging", + "version": "4.0.0", + "license": "Apache-2.0", + "bin": { + "app": "bin/app.js" + }, + "scripts": { + "pretest": "npm i", + "test": "./node_modules/jest/bin/jest.js --coverage ./__tests__" + }, + "devDependencies": { + "@aws-cdk/assert": "1.74.0", + "@types/jest": "^25.2.1", + "@types/node": "14.14.0", + "aws-cdk": "1.74.0", + "jest": "^26.4.1", + "ts-node": "^8.1.0", + "ts-jest": "^26.2.0", + "typescript": "^4.0.2" + }, + "dependencies": { + "@aws-cdk/aws-cloudtrail": "1.74.0", + "@aws-cdk/aws-cognito": "1.74.0", + "@aws-cdk/aws-ec2": "1.74.0", + "@aws-cdk/aws-elasticsearch": "1.74.0", + "@aws-cdk/aws-iam": "1.74.0", + "@aws-cdk/aws-kinesis": "1.74.0", + "@aws-cdk/aws-kms": "1.74.0", + "@aws-cdk/aws-kinesisfirehose": "1.74.0", + "@aws-cdk/aws-lambda": "1.74.0", + "@aws-cdk/aws-lambda-event-sources": "1.74.0", + "@aws-cdk/aws-logs": "1.74.0", + "@aws-cdk/aws-logs-destinations": "1.74.0", + "@aws-cdk/aws-s3": "1.74.0", + "@aws-cdk/aws-sqs": "1.74.0", + "@aws-cdk/aws-sns": "1.74.0", + "@aws-cdk/core": "1.74.0", + "@aws-cdk/custom-resources": "1.74.0", + "@aws-cdk/aws-cloudwatch-actions": "1.74.0", + "@aws-cdk/aws-sns-subscriptions": "1.74.0" + } +} diff --git a/source/resources/tsconfig.json b/source/resources/tsconfig.json new file mode 100644 index 0000000..da7b834 --- /dev/null +++ b/source/resources/tsconfig.json @@ -0,0 +1,41 @@ +{ + "compilerOptions": { + /* Basic Options */ + "target": "ES2018" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, + "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, + "lib": [ + "DOM", + "ES2018" + ] /* Specify library files to be included in the compilation. */, + "declaration": false /* Generates corresponding '.d.ts' file. */, + "noEmit": true, + "outDir": "./dist", + "types": ["jest", "node"], + "removeComments": true /* Do not emit comments to output. */, + "resolveJsonModule": true /* Allows importing modules with a ‘.json’ extension */, + /* Strict Type-Checking Options */ + "strict": true /* Enable all strict type-checking options. */, + "noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */, + "strictNullChecks": true /* Enable strict null checks. */, + "strictFunctionTypes": true /* Enable strict checking of function types. */, + "strictPropertyInitialization": true /* Enable strict checking of property initialization in classes. */, + "noImplicitThis": true /* Raise error on 'this' expressions with an implied 'any' type. */, + "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */, + /* Additional Checks */ + "noUnusedLocals": true /* Report errors on unused locals. */, + "noUnusedParameters": true /* Report errors on unused parameters. */, + "noImplicitReturns": true /* Report error when not all code paths in function return a value. */, + "noFallthroughCasesInSwitch": true /* Report errors for fallthrough cases in switch statement. */, + + /* Module Resolution Options */ + "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, + + /* Experimental Options */ + "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */, + // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + + /* Advanced Options */ + "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ + }, + "exclude": ["cdk.out", "node_modules"] +} diff --git a/source/services/auth/index.js b/source/services/auth/index.js deleted file mode 100644 index 4ed0845..0000000 --- a/source/services/auth/index.js +++ /dev/null @@ -1,115 +0,0 @@ -/******************************************************************************* -* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -* -* Licensed under the Apache License Version 2.0 (the "License"). You may not -* use this file except in compliance with the License. A copy of the License is -* located at -* -* http://www.apache.org/licenses/ -* -* or in the "license" file accompanying this file. This file is distributed on -* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or -* implied. See the License for the specific language governing permissions and -* limitations under the License. -********************************************************************************/ -'use strict'; -/** -* A Lambda function that creates a Cognito User Pool domain -* and updates an Elasticsearch Domain config for Cognito Authentication -**/ - -const AWS = require("aws-sdk"); -const uuid = require("uuid"); -const LOGGER = new(require('./logger'))(); - -exports.handler = function(event, context) { - - LOGGER.log('DEBUG',`REQUEST RECEIVED: ${JSON.stringify(event,null,2)}`); - const cognitoidentityserviceprovider = new AWS.CognitoIdentityServiceProvider(); - const es = new AWS.ES(); - let responseData = {}; - - if (event.RequestType == "Delete" || event.RequestType == "Update" ) { - sendResponse(event, context, "SUCCESS", responseData); - } - else if (event.RequestType == "Create") { - - if (event.ResourceProperties.Resource === "UUID") { - responseData = {UUID: uuid.v4()}; - sendResponse(event, context, "SUCCESS", responseData); - } - else { - - let params = { - DomainName: event.ResourceProperties.Domain, - CognitoOptions: { - Enabled: true, - IdentityPoolId: event.ResourceProperties.IdentityPoolId, - RoleArn: event.ResourceProperties.RoleArn, - UserPoolId: event.ResourceProperties.UserPoolId - } - }; - es.updateElasticsearchDomainConfig(params, function(err, data) { - if (err) { - LOGGER.log('ERROR',`error updating the Elasticsearch domain config: ${err.stack}`); - sendResponse(event, context, "FAILED", responseData); - } - else { - LOGGER.log('INFO',"Elasticsearch domain config update SUCCEEDED"); - sendResponse(event, context, "SUCCESS", responseData); - } - }); - - } - } -}; - -// Send response to the pre-signed S3 URL -function sendResponse(event, context, responseStatus, responseData) { - - let responseBody = JSON.stringify({ - Status: responseStatus, - Reason: "See the details in CloudWatch Log Stream: " + context.logStreamName, - PhysicalResourceId: context.logStreamName, - StackId: event.StackId, - RequestId: event.RequestId, - LogicalResourceId: event.LogicalResourceId, - Data: responseData - }); - - LOGGER.log('DEBUG',`RESPONSE BODY: ${JSON.stringify(responseBody,null,2)}`); - - let https = require("https"); - let url = require("url"); - - let parsedUrl = url.parse(event.ResponseURL); - let options = { - hostname: parsedUrl.hostname, - port: 443, - path: parsedUrl.path, - method: "PUT", - headers: { - "content-type": "", - "content-length": responseBody.length - } - }; - - LOGGER.log('DEBUG',"SENDING RESPONSE...\n"); - - let request = https.request(options, function(response) { - LOGGER.log('DEBUG',`STATUS: ${response.statusCode}`); - LOGGER.log('DEBUG',`headers: ${JSON.stringify(response.headers)}`); - // Tell AWS Lambda that the function execution is done - context.done(); - }); - - request.on("error", function(error) { - LOGGER.log('ERROR',`sendResponse Error: ${error}`); - // Tell AWS Lambda that the function execution is done - context.done(); - }); - - // write data to request body - request.write(responseBody); - request.end(); -} diff --git a/source/services/auth/logger.js b/source/services/auth/logger.js deleted file mode 100644 index c0afa23..0000000 --- a/source/services/auth/logger.js +++ /dev/null @@ -1,40 +0,0 @@ -/******************************************************************************* -* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -* -* Licensed under the Apache License Version 2.0 (the "License"). You may not -* use this file except in compliance with the License. A copy of the License is -* located at -* -* http://www.apache.org/licenses/ -* -* or in the "license" file accompanying this file. This file is distributed on -* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or -* implied. See the License for the specific language governing permissions and -* limitations under the License. -********************************************************************************/ -/** - * [logger module] - * V56536055 - 10/08/2018 - better logging capabilities - */ -'use strict'; - -class Logger { - - constructor() { - this.loglevel = process.env.LOG_LEVEL; - this.LOGLEVELS = { - ERROR: 1, - WARN: 2, - INFO: 3, - DEBUG: 4 - }; - } - - log(level, message) { - if (this.LOGLEVELS[level] <= this.LOGLEVELS[this.loglevel]) - console.log(`[${level}]${message}`); - } - -} - -module.exports = Object.freeze(Logger); diff --git a/source/services/auth/package.json b/source/services/auth/package.json deleted file mode 100644 index dc8d0a9..0000000 --- a/source/services/auth/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "centralized-logging-auth", - "description": "A Lambda function for the centralized logging cognito auth", - "main": "index.js", - "author": { - "name": "aws-solutions-builder" - }, - "version": "0.0.1", - "private": "true", - "dependencies": { - "uuid": "*" - }, - "devDependencies": { - "npm-run-all": "*", - "aws-sdk": "*" - }, - "scripts": { - "pretest": "npm install", - "build-init": "rm -rf dist && mkdir dist", - "build:copy": "cp *.js dist/", - "build:install": "cp package.json dist/ && cd dist && npm install --production", - "build": "npm-run-all -s build-init build:copy build:install", - "zip": "cd dist && rm -f package-lock.json && zip -rq clog-auth.zip ." - } -} diff --git a/source/services/helper/PromiseConstructor.d.ts b/source/services/helper/PromiseConstructor.d.ts new file mode 100644 index 0000000..459f652 --- /dev/null +++ b/source/services/helper/PromiseConstructor.d.ts @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +export interface PromiseResolution { + status: "fulfilled"; + value: T; +} + +export interface PromiseRejection { + status: "rejected"; + reason: E; +} + +export type PromiseResult = + | PromiseResolution + | PromiseRejection; + +export type PromiseList = { + [P in keyof T]: Promise; +}; + +export type PromiseResultList = { + [P in keyof T]: PromiseResult; +}; + +declare global { + interface PromiseConstructor { + allSettled(): Promise<[]>; + allSettled( + list: PromiseList + ): Promise>; + allSettled(iterable: Iterable): Promise>>; + } +} diff --git a/source/services/helper/index.ts b/source/services/helper/index.ts new file mode 100644 index 0000000..9781a71 --- /dev/null +++ b/source/services/helper/index.ts @@ -0,0 +1,432 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +import { v4 as uuidv4 } from "uuid"; +import { logger } from "./lib/common/logger"; +import { Metrics } from "./lib/common/metrics"; +import moment from "moment"; +import { CloudWatchLogs, EC2, IAM } from "aws-sdk"; + +interface IEvent { + RequestType: string; + ResponseURL: string; + StackId: string; + RequestId: string; + ResourceType: string; + LogicalResourceId: string; + ResourceProperties: any; + PhysicalResourceId?: string; +} + +const awsClients = { + ec2: "2016-11-15", + cwLogs: "2014-03-28", + iam: "2010-05-08", +}; + +/** + * @description entry point for helper function + * @param {IEvent} event invoking event + * @param {any} context from the invoking event + */ +exports.handler = async (event: IEvent, context: any) => { + logger.debug({ + label: "helper", + message: `received event: ${JSON.stringify(event)}`, + }); + + let responseData: any = { + Data: "NOV", + }; + let status = "SUCCESS"; + + const properties = event.ResourceProperties; + + if (event.ResourceType === "Custom::CreateUUID") { + // generate uuid + if (event.RequestType === "Create") { + responseData = { + UUID: uuidv4(), + }; + logger.info({ + label: "helper/UUID", + message: `uuid create: ${responseData.UUID}`, + }); + } + } else if (event.ResourceType === "Custom::CreateESServiceRole") { + // create service linked role for es + if (event.RequestType === "Create") { + const iam = new IAM({ apiVersion: awsClients.iam }); + try { + await iam + .createServiceLinkedRole({ AWSServiceName: "es.amazonaws.com" }) + .promise(); + logger.info({ + label: "helper/CreateESServiceRole", + message: `es service linked role created`, + }); + } catch (e) { + logger.error({ + label: "helper/createServiceLinkedRole", + message: `${JSON.stringify(e)}`, + }); + if (e.code === "InvalidInput") { + logger.warn({ + label: "helper/createServiceLinkedRole", + message: `needed ES service linked role already exists ${e.message}`, + }); + } else { + logger.error({ + label: "helper/createServiceLinkedRole", + message: `${e.message}`, + }); + responseData = { + Error: + "failed to create ES service linked role, please see in cw logs for more details", + }; + status = "FAILED"; + } + } + } + } else if (event.ResourceType === "Custom::CWDestination") { + // fetching regions + let allRegions: any; + try { + allRegions = await getRegions(); + } catch (e) { + logger.error({ + label: "helper/CWDestination", + message: `${e.message}`, + }); + responseData = { + Error: "failed to get regions, please see in cw logs for more details", + }; + status = "FAILED"; + return await sendResponse( + event, + context.logStreamName, + status, + responseData + ); + } + + // create regional destinations + if (event.RequestType === "Create" || event.RequestType === "Update") { + // create/update destinations + let spokeRegions = properties.Regions; + logger.debug({ + label: "helper/CWDestination", + message: `Regions to ${event.RequestType} CloudWatch destinations: ${spokeRegions}`, + }); + if (spokeRegions[0] === "All") { + spokeRegions = allRegions; + } + await putDestination( + spokeRegions, + allRegions, + properties.DestinationName, + properties.Role, + properties.DataStream, + properties.SpokeAccounts + ).catch(() => { + responseData = { + Error: + "failed to put cw logs destinations, please see in cw logs for more details", + }; + status = "FAILED"; + }); + } + if (event.RequestType === "Delete") { + // delete destinations + await deleteDestination(properties.DestinationName, allRegions).catch( + (e) => { + logger.warn({ + label: "helper/deleteDestination", + message: `${e.message}`, + }); + } + ); + } + } else if (event.ResourceType === "Custom::LaunchData") { + // send metric for launch + if (process.env.SEND_METRIC === "Yes") { + logger.info({ + label: "helper/LaunchData", + message: `sending launch data`, + }); + let eventType = ""; + if (event.RequestType === "Create") { + eventType = "SolutionLaunched"; + } else if (event.RequestType === "Delete") { + eventType = "SolutionDeleted"; + } + + const metric = { + Solution: properties.SolutionId, + UUID: properties.SolutionUuid, + TimeStamp: moment.utc().format("YYYY-MM-DD HH:mm:ss.S"), + Data: { + Event: eventType, + Stack: properties.Stack, + Version: properties.SolutionVersion, + }, + }; + await Metrics.sendAnonymousMetric( + process.env.METRICS_ENDPOINT, + metric + ); + + responseData = { + Data: metric, + }; + } + } + + // send response to custom resource + return await sendResponse(event, context.logStreamName, status, responseData); +}; + +/** + * @description get list of ec2 regions + */ +async function getRegions() { + logger.info({ + label: "helper/getRegions", + message: `getting ec2 regions`, + }); + try { + const ec2 = new EC2({ + apiVersion: awsClients.ec2, + }); + + const _r = await ec2.describeRegions().promise(); + + if (!_r.Regions) throw new Error("failed to describe regions"); + + const regions = _r.Regions.filter((region) => { + return region.RegionName !== "ap-northeast-3"; + }).map((region) => { + return region.RegionName; + }); + logger.debug({ + label: "helper/getRegions", + message: `${JSON.stringify({ regions: regions })}`, + }); + return regions; + } catch (e) { + logger.error({ + label: "helper/getRegions", + message: e, + }); + throw new Error("error fetching regions"); + } +} + +/** + * @description create cw destinations + * @param {string[]} regions - regions for spokes + * @param {string} destinationName - cw logs destination name + * @param {string} roleArn - ARN of IAM role that grants CloudWatch Logs permissions to call the Amazon Kinesis PutRecord operation on the destination stream + * @param {string} kinesisStreamArn - The ARN of an Amazon Kinesis stream to which to deliver matching log events + * @param {string[]} spokeAccnts - list of spoke account ids + */ +async function putDestination( + regions: string[], + awsRegions: string[], + destinationName: string, + roleArn: string, + kinesisStreamArn: string, + spokeAccnts: string[] +) { + logger.info({ + label: "helper/putDestination", + message: `putting cw logs destinations for spokes`, + }); + try { + // check if provided region list is valid + const regionValid = await areRegionsValid(regions, awsRegions); + if (regionValid) { + await deleteDestination(destinationName, regions); + await Promise.all( + regions.map(async (region) => { + logger.debug({ + label: "helper/putDestination", + message: `creating cw logs destination in ${region}`, + }); + + const cwLogs = new CloudWatchLogs({ + apiVersion: awsClients.cwLogs, + region: region, + }); + + //put destination + const dest: CloudWatchLogs.PutDestinationResponse = await cwLogs + .putDestination({ + destinationName: destinationName, + roleArn: roleArn, + targetArn: kinesisStreamArn, + }) + .promise(); + + // put access policy + const accessPolicy = { + Version: "2012-10-17", + Statement: [ + { + Sid: "AllowSpokesSubscribe", + Effect: "Allow", + Principal: { + AWS: spokeAccnts, + }, + Action: "logs:PutSubscriptionFilter", + Resource: dest.destination?.arn, + }, + ], + }; + await cwLogs + .putDestinationPolicy({ + destinationName: destinationName, + accessPolicy: JSON.stringify(accessPolicy), // for spoke accounts as principals + }) + .promise(); + logger.debug({ + label: "helper/putDestinations", + message: `cw logs destination created in ${region}`, + }); + }) + ); + logger.info({ + label: "helper/putDestinations", + message: `All cw logs destinations created`, + }); + } else { + throw new Error("invalid regions"); + } + } catch (e) { + logger.error({ + label: "helper/putDestination", + message: e, + }); + throw new Error("error in creating cw log destination"); + } +} + +/** + * @description delete cw destinations + * @param {string} destinationName - cw logs destination name + */ +async function deleteDestination(destinationName: string, regions: string[]) { + logger.info({ + label: "helper/deleteDestination", + message: `deleting cw logs destinations `, + }); + try { + await Promise.allSettled( + regions.map(async (region) => { + const cwLogs = new CloudWatchLogs({ + apiVersion: awsClients.cwLogs, + region: region, + }); + await cwLogs + .deleteDestination({ destinationName: destinationName }) + .promise() + .then(() => { + logger.debug({ + label: "helper/deleteDestination", + message: `cw logs destination deleted in ${region}`, + }); + }) + .catch((e) => { + logger.warn({ + label: `helper/deleteDestination`, + message: `${region}: ${e.message}`, + }); + }); + }) + ); + logger.info({ + label: "helper/deleteDestinations", + message: `All cw logs destinations deleted`, + }); + return "cw logs destinations deleted"; + } catch (e) { + logger.warn({ + label: "helper/deleteDestinations", + message: e.message, + }); + throw new Error("error in deleting destinations"); + } +} + +/** + * @description check if region list is valid + * @param {string[]} regions - region list for spokes + */ +async function areRegionsValid(regions: string[], awsRegions: string[]) { + logger.debug({ + label: "helper/areRegionsValid", + message: `checking if region parameter is valid`, + }); + + if (!(awsRegions instanceof Array)) throw new Error("no regions found"); + try { + await Promise.all( + regions.map((region) => { + if (!awsRegions.includes(region)) + throw new Error("invalid region provided"); + }) + ); + return true; + } catch (e) { + logger.error({ + label: "helper/areRegionsValid", + message: `${e.message}`, + }); + return false; + } +} + +/** + * Sends a response to custom resource + * for Create/Update/Delete + * @param {any} event - Custom Resource event + * @param {string} logStreamName - CloudWatch logs stream + * @param {string} responseStatus - response status + * @param {any} responseData - response data + */ +const sendResponse = async ( + event: IEvent, + logStreamName: string, + responseStatus: string, + responseData: any +) => { + const responseBody = { + Status: responseStatus, + Reason: `${JSON.stringify(responseData)}`, + PhysicalResourceId: event.PhysicalResourceId + ? event.PhysicalResourceId + : logStreamName, + StackId: event.StackId, + RequestId: event.RequestId, + LogicalResourceId: event.LogicalResourceId, + Data: responseData, + }; + + logger.debug({ + label: "helper/sendResponse", + message: `Response Body: ${JSON.stringify(responseBody)}`, + }); + + if (responseStatus === "FAILED") { + throw new Error(responseBody.Data.Error); + } else return responseBody; +}; diff --git a/source/services/helper/lib/common/logger/index.ts b/source/services/helper/lib/common/logger/index.ts new file mode 100644 index 0000000..140fcd8 --- /dev/null +++ b/source/services/helper/lib/common/logger/index.ts @@ -0,0 +1,92 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +/** + * { + emerg: 0, + alert: 1, + crit: 2, + error: 3, + warning: 4, + notice: 5, + info: 6, + debug: 7 + } + */ +import { createLogger, transports, format } from "winston"; +import { WinstonSNS } from "./winston-sns"; +const { combine, timestamp, printf } = format; + +/* + * Foramting the output as desired + */ +const myFormat = printf(({ level, label, message }) => { + const _level = level.toUpperCase(); + if (label) return `[${_level}] [${label}] ${message}`; + else return `[${_level}] ${message}`; +}); + +/* + * String mask + */ +const maskCardNumbers = (num: any) => { + const str = num.toString(); + const { length } = str; + + return Array.from(str, (n, i) => { + return i < length - 4 ? "*" : n; + }).join(""); +}; + +// Define the format that mutates the info object. +const maskFormat = format((info: any) => { + // You can CHANGE existing property values + if (info.message.securedNumber) { + info.message.securedNumber = maskCardNumbers(info.message.securedNumber); + } + + // You can also ADD NEW properties if you wish + //info.hasCreditCard = !!info.creditCard; + + return info; +}); + +export const logger = createLogger({ + format: combine( + // + // Order is important here, the formats are called in the + // order they are passed to combine. + // + maskFormat(), + timestamp(), + myFormat + ), + + transports: [ + //cw logs transport channel + new transports.Console({ + level: process.env.LOG_LEVEL, + handleExceptions: true, //handle uncaught exceptions + //format: format.splat() + }), + + //sns transport channel + ...(process.env.SNS_ERROR_NOTIFICATION == "true" + ? [ + new WinstonSNS({ + topic_arn: process.env.SNS_TOPIC_ARN, + level: "error", + }), + ] + : []), + ], +}); diff --git a/source/services/helper/lib/common/logger/winston-sns.ts b/source/services/helper/lib/common/logger/winston-sns.ts new file mode 100644 index 0000000..e99f53f --- /dev/null +++ b/source/services/helper/lib/common/logger/winston-sns.ts @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +import Transport = require("winston-transport"); +import "util"; +import { SNS } from "aws-sdk"; + +// +// Inherit from `winston-transport` so you can take advantage +// of the base functionality and `.exceptions.handle()`. +// +export class WinstonSNS extends Transport { + readonly topic_arn: string; + readonly region: string; + private sns: any; + constructor(opts: any) { + super(opts); + this.topic_arn = opts.topic_arn; + this.region = opts.topic_arn.split(":")[3]; + // + // Consume any custom options here. e.g.: + // - Connection information for databases + // - Authentication information for APIs (e.g. loggly, papertrail, + // logentries, etc.). + // + } + formatter = async (info: any) => { + if (info.label) + return `[${info.level.toUpperCase()}] [${info.label}] ${ + info.timestamp + }: ${JSON.stringify(info.message, null, 2)}`; + else + return `[${info.level.toUpperCase()}] ${info.timestamp}: ${JSON.stringify( + info.message, + null, + 2 + )}`; + }; + + log = async (info: any) => { + try { + this.sns = new SNS({ + apiVersion: "2010-03-31", + region: this.region, + }); + const _txt = await this.formatter(info); + await this.sns + .publish({ + Message: _txt, + TopicArn: this.topic_arn, + }) + .promise(); + return "sns message published successfully"; + } catch (e) { + throw new Error(e.message); + } + }; +} diff --git a/source/services/helper/lib/common/metrics/index.ts b/source/services/helper/lib/common/metrics/index.ts new file mode 100644 index 0000000..2d04367 --- /dev/null +++ b/source/services/helper/lib/common/metrics/index.ts @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +import got from "got"; +import { logger } from "../logger/index"; +/** + * Send metrics to solutions endpoint + * @class Metrics + */ +export class Metrics { + /** + * Sends anonymous metric + * @param {object} metric - metric JSON data + */ + static async sendAnonymousMetric(endpoint: string, metric: any) { + logger.debug({ + label: "metrics/sendAnonymousMetric", + message: `metrics endpoint: ${endpoint}`, + }); + logger.debug({ + label: "metrics/sendAnonymousMetric", + message: `sending metric:${JSON.stringify(metric)}`, + }); + try { + await got(endpoint, { + port: 443, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": "" + JSON.stringify(metric).length, + }, + body: JSON.stringify(metric), + }); + logger.info({ + label: "metrics/sendAnonymousMetric", + message: `metric sent successfully`, + }); + return `Metric sent: ${JSON.stringify(metric)}`; + } catch (error) { + logger.warn({ + label: "metrics/sendAnonymousMetric", + message: `Error occurred while sending metric: ${JSON.stringify( + error + )}`, + }); + return `Error occurred while sending metric`; + } + } +} diff --git a/source/services/helper/package.json b/source/services/helper/package.json new file mode 100644 index 0000000..3b63447 --- /dev/null +++ b/source/services/helper/package.json @@ -0,0 +1,30 @@ +{ + "name": "cl-helper", + "version": "4.0.0", + "description": "helper function for Centralized Logging solution", + "main": "index.js", + "scripts": { + "pretest": "npm i", + "test": "echo \"nothing to do\"", + "build:clean": "rm -rf ./node_modules && rm -rf ./dist && rm -f ./package-lock.json", + "build:copy": "cp -r ./node_modules ./dist/node_modules", + "build:ts": "./node_modules/typescript/bin/tsc --project ./tsconfig.json", + "build:install": "npm i", + "watch": "tsc -w", + "build:zip": "cd ./dist && zip -rq cl-helper.zip .", + "build:all": "npm run build:clean && npm run build:install && npm run build:ts && npm prune --production && npm run build:copy && npm run build:zip" + }, + "author": "aws-solutions", + "license": "Apache-2.0", + "dependencies": { + "got": "^11.5.1", + "moment": "^2.27.0", + "uuid": "^8.2.0", + "winston": "^3.3.3", + "aws-sdk": "^2.714.0" + }, + "devDependencies": { + "typescript": "^4.0.2", + "@types/uuid": "^8.0.0" + } +} diff --git a/source/services/helper/tsconfig.json b/source/services/helper/tsconfig.json new file mode 100644 index 0000000..94645b9 --- /dev/null +++ b/source/services/helper/tsconfig.json @@ -0,0 +1,40 @@ +{ + "compilerOptions": { + /* Basic Options */ + "target": "ES2018" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, + "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, + "lib": [ + "DOM", + "ES2018" + ] /* Specify library files to be included in the compilation. */, + "outDir": "./dist", + "declaration": false /* Generates corresponding '.d.ts' file. */, + "noEmit": false, + "removeComments": true /* Do not emit comments to output. */, + "resolveJsonModule": true /* Allows importing modules with a ‘.json’ extension */, + /* Strict Type-Checking Options */ + "strict": true /* Enable all strict type-checking options. */, + "noImplicitAny": false /* Raise error on expressions and declarations with an implied 'any' type. */, + // "strictNullChecks": true /* Enable strict null checks. */, + "strictFunctionTypes": true /* Enable strict checking of function types. */, + "strictPropertyInitialization": true /* Enable strict checking of property initialization in classes. */, + "noImplicitThis": true /* Raise error on 'this' expressions with an implied 'any' type. */, + "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */, + + /* Additional Checks */ + "noUnusedLocals": true /* Report errors on unused locals. */, + "noUnusedParameters": true /* Report errors on unused parameters. */, + "noImplicitReturns": true /* Report error when not all code paths in function return a value. */, + "noFallthroughCasesInSwitch": true /* Report errors for fallthrough cases in switch statement. */, + + /* Module Resolution Options */ + "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, + + /* Experimental Options */ + "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */, + // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + + /* Advanced Options */ + "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ + } +} diff --git a/source/services/indexing/index.js b/source/services/indexing/index.js deleted file mode 100755 index cdfd6db..0000000 --- a/source/services/indexing/index.js +++ /dev/null @@ -1,467 +0,0 @@ -/******************************************************************************* -* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -* -* Licensed under the Apache License Version 2.0 (the "License"). You may not -* use this file except in compliance with the License. A copy of the License is -* located at -* -* http://www.apache.org/licenses/ -* -* or in the "license" file accompanying this file. This file is distributed on -* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or -* implied. See the License for the specific language governing permissions and -* limitations under the License. -********************************************************************************/ -/** - * @author Solution Builders - */ - -'use strict'; - -let https = require('https'); -let zlib = require('zlib'); -let crypto = require('crypto'); -let AWS = require('aws-sdk'); -let moment = require('moment'); -const MetricsHelper = require('./lib/metrics-helper.js'); -const LOGGER = new(require('./lib/logger'))(); - -const endpoint = process.env.DOMAIN_ENDPOINT; -const masterRole = process.env.MASTER_ROLE; -const sessionId = process.env.SESSION_ID; -const owner = process.env.OWNER; -const solution = process.env.SOLUTION; -const clusterSize = process.env.CLUSTER_SIZE; -const uuid = process.env.UUID; -const anonymousData = process.env.ANONYMOUS_DATA; - -function handler(input, context, callback) { - - let eventText = JSON.stringify(input, null, 2); - - //return callback if environment variable not set - if (!endpoint || !masterRole || !sessionId || !owner) { - LOGGER.log('ERROR', `check environment variables endpoint: ${endpoint}, masterRole: ${masterRole}, sessionId: ${sessionId}, owner: ${owner}`); - return callback('environment variables not defined'); - } - - // Log a message to the console, you can view this text in the Monitoring tab in the Lambda console - // or in the CloudWatch Logs console - LOGGER.log('DEBUG', `Received event: ${eventText}`); - - // decode input from base64 - let zippedInput = new Buffer.from(input.awslogs.data, 'base64'); - - // decompress the input - zlib.gunzip(zippedInput, function(error, buffer) { - if (error) { - return callback(error); - } - - // parse the input from JSON - let awslogsData = JSON.parse(buffer.toString('utf8')); - - // transform the input to Elasticsearch documents - let elasticsearchBulkData = transform(awslogsData); - - // skip control messages - if (!elasticsearchBulkData) { - LOGGER.log('DEBUG', `Received a control message`); - return callback(null, 'success'); - } - LOGGER.log('DEBUG', `elasticsearchBulkData: ${elasticsearchBulkData}`); - - // post documents to the Amazon Elasticsearch Service - post(elasticsearchBulkData, function(error, success, - statusCode, - failedItems) { - LOGGER.log('DEBUG',`Response StatusCode: ${statusCode}`); - - if (error) { - LOGGER.log('ERROR',`postElasticSearchBulkData Error: ${JSON.stringify(error, null, 2)}`); - - if (failedItems && failedItems.length > 0) { - LOGGER.log('ERROR', `Failed Items: ${JSON.stringify(failedItems, null, 2)}`); - } - - return callback(error); - - } else { - LOGGER.log('INFO',`success: ${JSON.stringify(success)}`); - - if (anonymousData === 'Yes') { - - //send anonymous metrics ONLY if chosen 'Yes' - /** - * V38463712 - 08/10/18 - fixing callbacks - */ - sendMetrics({ - 'clusterSize': clusterSize, - 'itemsIndexed': success.successfulItems, - 'totalItemSize': success.totalItemSize - }, function(err, data) { - if (err) LOGGER.log('ERROR',`Metrics Status: ${JSON.stringify(err)}`); - else LOGGER.log('DEBUG',`Metrics Status: ${JSON.stringify(data)}`); - return callback(null, 'Success'); - }); - } else return callback(null, 'Success'); - - - } - - }); - - }); - -} - -/** - * Transform CloudWatch Logs stream data for indexing - * on ES domain - * @param {JSON} payload - cw log stream data - */ -function transform(payload) { - if (payload.messageType === 'CONTROL_MESSAGE') { - return null; - } - - let bulkRequestBody = ''; - - payload.logEvents.forEach(function(logEvent) { - let timestamp = new Date(1 * logEvent.timestamp); - - // index name format: cwl-YYYY.MM.DD - let indexName = [ - 'cwl-' + timestamp.getUTCFullYear(), // year - ('0' + (timestamp.getUTCMonth() + 1)).slice(-2), // month - ('0' + timestamp.getUTCDate()).slice(-2) // day - ].join('.'); - - let source = buildSource(logEvent.message, logEvent.extractedFields); - - - /** - * V38463712 - 10/09/2018 - Convert field to string - * Fix inconsistent CloudTrail attributes & thousands of fields - */ - if ('requestParameters' in source) source['requestParameters'] = JSON.stringify(source['requestParameters']); - if ('responseElements' in source) source['responseElements'] = JSON.stringify(source['responseElements']); - source['@id'] = logEvent.id; - source['@timestamp'] = new Date(1 * logEvent.timestamp).toISOString(); - source['@message'] = logEvent.message; - source['@owner'] = payload.owner; - source['@log_group'] = payload.logGroup; - source['@log_stream'] = payload.logStream; - - let action = { - "index": {} - }; - action.index._index = indexName; - action.index._type = 'CloudWatchLogs'; - action.index._id = logEvent.id; - - bulkRequestBody += [ - JSON.stringify(action), - JSON.stringify(source), - ].join('\n') + '\n'; - }); - LOGGER.log('DEBUG',`bulkRequestBody: ${bulkRequestBody}`); - return bulkRequestBody; -} - -/** - * Building item for ES indexing - * @param {String} message - message field from cw event - * @param {Array} extractedFields - extracted fields from cw event - */ -function buildSource(message, extractedFields) { - if (extractedFields) { - let source = {}; - - for (let key in extractedFields) { - if (extractedFields.hasOwnProperty(key) && extractedFields[key]) { - let value = extractedFields[key]; - - if (isNumeric(value)) { - source[key] = 1 * value; - continue; - } - - let jsonSubString = extractJson(value); - if (jsonSubString !== null) { - source['$' + key] = JSON.parse(jsonSubString); - } - - source[key] = value; - } - } - LOGGER.log('DEBUG',`build source output: ${source}`); - return source; - } - - let jsonSubString = extractJson(message); - if (jsonSubString !== null) { - return JSON.parse(jsonSubString); - } - - return {}; -} - -function extractJson(message) { - let jsonStart = message.indexOf('{'); - if (jsonStart < 0) return null; - let jsonSubString = message.substring(jsonStart); - return isValidJson(jsonSubString) ? jsonSubString : null; -} - -function isValidJson(message) { - try { - JSON.parse(message); - } catch (e) { - return false; - } - return true; -} - -function isNumeric(n) { - return !isNaN(parseFloat(n)) && isFinite(n); -} - -/** - * Post logs on ES domain. - * @param {body} body - ES bulk data to be indexed - * @param {post~callback} callback - The callback that handles the response. - */ -function post(body, callback) { - - LOGGER.log('DEBUG',`endpoint: ${endpoint}`); - - assumeRole(function(err, creds) { - if (err) { - LOGGER.log('ERROR',`error in assuming role: ${err}`); - return callback(err); - } - - buildRequest(endpoint, body, creds, function(err, requestParams) { - if (err) { - LOGGER.log('ERROR',`error in http request: ${err}`); - return callback(err); - } - - LOGGER.log('DEBUG',`requestParams: ${requestParams}`); - let request = https.request(requestParams, function(response) { - let responseBody = ''; - response.on('data', function(chunk) { - responseBody += chunk; - }); - response.on('end', function() { - let info = JSON.parse(responseBody); - let failedItems; - let success; - - LOGGER.log('INFO',`post info: ${JSON.stringify(info,null,2)}`); - - if (response.statusCode >= 200 && response.statusCode < - 299) { - failedItems = info.items.filter(function(x) { - return x.index.status >= 300; - }); - - success = { - "attemptedItems": info.items.length, - "successfulItems": info.items.length - - failedItems.length, - "failedItems": failedItems.length, - "totalItemSize": requestParams.headers[ - 'Content-Length'] - }; - } - - let error = response.statusCode !== 200 || info.errors === - true ? { - "statusCode": response.statusCode, - "responseBody": responseBody - } : null; - - LOGGER.log('INFO',`post error: ${JSON.stringify(error,null,2)}`); - - return callback(error, success, response.statusCode, - failedItems); - - }); - }).on('error', function(e) { - return callback(e); - }); - request.end(requestParams.body); - }); - - }); - -} - -/** - * Assumes role with permissions for ES indexing - * @param {assumeRole~callback} cb - The callback that handles the response with credentials - */ -function assumeRole(cb) { - let creds = { - aws_secret_key: process.env.AWS_SECRET_ACCESS_KEY, - aws_access_key: process.env.AWS_ACCESS_KEY_ID, - aws_session_token: process.env.AWS_SESSION_TOKEN - }; - - //assume role in spoke accounts - if (owner === 'Spoke') { - //assume role for posting documents on ES Domain - let sts = new AWS.STS({ - apiVersion: '2011-06-15' - }); - sts.assumeRole({ - RoleArn: masterRole, - /* required */ - RoleSessionName: sessionId, - /* required */ - }, function(err, data) { - if (err) { - LOGGER.log('ERROR',`assume role err: ${err}`); - return cb(err, null); - } // an error occurred - else { - LOGGER.log('DEBUG',`assume role response: ${data}`); - creds = { - aws_secret_key: data.Credentials.SecretAccessKey, - aws_access_key: data.Credentials.AccessKeyId, - aws_session_token: data.Credentials.SessionToken - }; - return cb(null, creds); - } - }); - } else if (owner === 'Hub') return cb(null, creds); - else return cb('invalid owner', null); -} - -/** - * Build https request for indexing on ES - * @param {String} endpoint - ES endpoint for log indexing - * @param {String} body - - ES bulk data to be indexed - * @param {Dictionary} creds - AWS credentials for https request - * @param {buildRequest~callback} cb - The callback that handles the response - */ -function buildRequest(endpoint, body, creds, cb) { - let endpointParts = endpoint.match( - /^([^\.]+)\.?([^\.]*)\.?([^\.]*)\.amazonaws\.com$/); - let region = endpointParts[2]; - let service = endpointParts[3]; - let datetime = (new Date()).toISOString().replace(/[:\-]|\.\d{3}/g, ''); - let date = datetime.substr(0, 8); - let kDate = hmac('AWS4' + creds.aws_secret_key, date); - let kRegion = hmac(kDate, region); - let kService = hmac(kRegion, service); - let kSigning = hmac(kService, 'aws4_request'); - - let request = { - host: endpoint, - method: 'POST', - path: '/_bulk', - body: body, - headers: { - 'Content-Type': 'application/json', - 'Host': endpoint, - 'Content-Length': Buffer.byteLength(body), - 'X-Amz-Security-Token': creds.aws_session_token, - 'X-Amz-Date': datetime - } - }; - - let canonicalHeaders = Object.keys(request.headers) - .sort(function(a, b) { - return a.toLowerCase() < b.toLowerCase() ? -1 : 1; - }) - .map(function(k) { - return k.toLowerCase() + ':' + request.headers[k]; - }) - .join('\n'); - - let signedHeaders = Object.keys(request.headers) - .map(function(k) { - return k.toLowerCase(); - }) - .sort() - .join(';'); - - let canonicalString = [ - request.method, - request.path, '', - canonicalHeaders, '', - signedHeaders, - hash(request.body, 'hex'), - ].join('\n'); - - let credentialString = [date, region, service, 'aws4_request'].join('/'); - - let stringToSign = [ - 'AWS4-HMAC-SHA256', - datetime, - credentialString, - hash(canonicalString, 'hex') - ].join('\n'); - - request.headers.Authorization = [ - 'AWS4-HMAC-SHA256 Credential=' + creds.aws_access_key + '/' + - credentialString, - 'SignedHeaders=' + signedHeaders, - 'Signature=' + hmac(kSigning, stringToSign, 'hex') - ].join(', '); - - return cb(null, request); -} - -function hmac(key, str, encoding) { - return crypto.createHmac('sha256', key).update(str, 'utf8').digest(encoding); -} - -function hash(str, encoding) { - return crypto.createHash('sha256').update(str, 'utf8').digest(encoding); -} - -function sendMetrics(metricData, cb) { - let _metricsHelper = new MetricsHelper(); - - let _metric = { - Solution: solution, - UUID: uuid, - TimeStamp: moment().utc().format('YYYY-MM-DD HH:mm:ss.S'), - Data: { - ClusterSize: metricData.clusterSize, - ItemsIndexed: metricData.itemsIndexed, - TotalItemSize: metricData.totalItemSize - } - }; - - LOGGER.log('DEBUG',`anonymous metric: ${JSON.stringify(_metric)}`); - - _metricsHelper.sendAnonymousMetric(_metric, function(err, data) { - if (err) { - let responseData = { - Error: 'Sending anonymous metric failed' - }; - LOGGER.log('ERROR',`${[responseData.Error, ':\n', err].join('')}`); - cb(responseData, null); - } else { - let responseStatus = 'SUCCESS'; - let responseData = { - Success: 'Anonymous metrics sent to AWS' - }; - cb(null, responseData); - } - }); - -} - -module.exports = { - handler, - transform, - buildSource, - buildRequest, - assumeRole -}; diff --git a/source/services/indexing/lib/basic-dashboard.json b/source/services/indexing/lib/basic-dashboard.json deleted file mode 100644 index 9061134..0000000 --- a/source/services/indexing/lib/basic-dashboard.json +++ /dev/null @@ -1,286 +0,0 @@ -[{ - "_id": "Basic", - "_type": "dashboard", - "_source": { - "title": "Basic", - "hits": 0, - "description": "", - "panelsJSON": "[{\"col\":4,\"id\":\"HTTP-200-Code-Count\",\"panelIndex\":1,\"row\":7,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"HTTP-404-Code-Count\",\"panelIndex\":2,\"row\":7,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Packets-Accept\",\"panelIndex\":4,\"row\":1,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Packets-Reject\",\"panelIndex\":5,\"row\":1,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-10-Src-Address-Packets-Total\",\"panelIndex\":6,\"row\":3,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Top-10-Src-Address-Packets-Reject\",\"panelIndex\":7,\"row\":3,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Total-Packets-Src-Address\",\"panelIndex\":8,\"row\":3,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Reject-Packets-Src-Address\",\"panelIndex\":9,\"row\":3,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-10-Access-Keys\",\"panelIndex\":10,\"row\":9,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Top-10-Event-Sources\",\"panelIndex\":11,\"row\":7,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"f1709820-f182-11e7-8ed1-e93cb1e71cec\",\"panelIndex\":12,\"row\":9,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"size_x\":3,\"size_y\":2,\"panelIndex\":13,\"type\":\"visualization\",\"id\":\"74a9bbe0-f183-11e7-8ed1-e93cb1e71cec\",\"col\":10,\"row\":9}]", - "optionsJSON": "{\"darkTheme\":true}", - "uiStateJSON": "{\"P-10\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-11\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-6\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-7\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-12\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-1\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-2\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-9\":{\"vis\":{\"legendOpen\":true}},\"P-13\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", - "version": 1, - "timeRestore": true, - "timeTo": "now/d", - "timeFrom": "now/d", - "refreshInterval": { - "display": "5 minutes", - "pause": false, - "section": 2, - "value": 300000 - }, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"}}" - } - } -}, { - "_id": "HTTP-200-Code", - "_type": "search", - "_source": { - "title": "HTTP 200 Code", - "description": "", - "hits": 0, - "columns": [ - "_source" - ], - "sort": [ - "@timestamp", - "desc" - ], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"cwl-*\",\"query\":{\"query_string\":{\"query\":\"status:200\",\"analyze_wildcard\":true}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[]}" - } - } -}, { - "_id": "Accept", - "_type": "search", - "_source": { - "title": "Accept", - "description": "", - "hits": 0, - "columns": [ - "_source" - ], - "sort": [ - "@timestamp", - "desc" - ], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"cwl-*\",\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"query\":{\"query_string\":{\"query\":\"action:accept\",\"analyze_wildcard\":true}}}" - } - } -}, { - "_id": "All-Data", - "_type": "search", - "_source": { - "title": "All Data", - "description": "", - "hits": 0, - "columns": [ - "_source" - ], - "sort": [ - "@timestamp", - "desc" - ], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"cwl-*\",\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}" - } - } -}, { - "_id": "Reject", - "_type": "search", - "_source": { - "title": "Reject", - "description": "", - "hits": 0, - "columns": [ - "_source" - ], - "sort": [ - "@timestamp", - "desc" - ], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"cwl-*\",\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"query\":{\"query_string\":{\"query\":\"action:reject\",\"analyze_wildcard\":true}}}" - } - } -}, { - "_id": "HTTP-404-Code", - "_type": "search", - "_source": { - "title": "HTTP 404 Code", - "description": "", - "hits": 0, - "columns": [ - "_source" - ], - "sort": [ - "@timestamp", - "desc" - ], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"cwl-*\",\"query\":{\"query_string\":{\"query\":\"status:404\",\"analyze_wildcard\":true}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[]}" - } - } -}, { - "_id": "HTTP-404-Code-Count", - "_type": "visualization", - "_source": { - "title": "HTTP 404 Code Count", - "visState": "{\"type\":\"metric\",\"params\":{\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{},\"title\":\"HTTP 404 Code Count\"}", - "uiStateJSON": "{}", - "description": "", - "savedSearchId": "HTTP-404-Code", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - } - } -}, { - "_id": "Packets-Accept", - "_type": "visualization", - "_source": { - "title": "Packets - Accept", - "visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"custom\",\"customInterval\":\"30s\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{},\"title\":\"Packets - Accept\"}", - "uiStateJSON": "{}", - "description": "", - "savedSearchId": "Accept", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - } - } -}, { - "_id": "Packets-Reject", - "_type": "visualization", - "_source": { - "title": "Packets - Reject", - "visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"custom\",\"customInterval\":\"30s\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{},\"title\":\"Packets - Reject\"}", - "uiStateJSON": "{}", - "description": "", - "savedSearchId": "Reject", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - } - } -}, { - "_id": "Top-10-Access-Keys", - "_type": "visualization", - "_source": { - "title": "Top 10 Access Keys", - "visState": "{\"title\":\"Top 10 Access Keys\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userIdentity.accessKeyId.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"userIdentity.accessKeyId\"}}],\"listeners\":{}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "savedSearchId": "All-Data", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - } - } -}, { - "_id": "Top-10-Src-Address-Packets-Total", - "_type": "visualization", - "_source": { - "title": "Total Src Addr", - "visState": "{\"title\":\"Total Src Addr\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"srcaddr.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"srcaddr\"}}],\"listeners\":{}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "savedSearchId": "All-Data", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - } - } -}, { - "_id": "Top-10-Event-Sources", - "_type": "visualization", - "_source": { - "title": "Top 10 Event Sources", - "visState": "{\"title\":\"Top 10 Event Sources\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"eventSource.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"eventSource\"}}],\"listeners\":{}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "savedSearchId": "All-Data", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - } - } -}, { - "_id": "HTTP-200-Code-Count", - "_type": "visualization", - "_source": { - "title": "HTTP 200 Code Count", - "visState": "{\"type\":\"metric\",\"params\":{\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{},\"title\":\"HTTP 200 Code Count\"}", - "uiStateJSON": "{}", - "description": "", - "savedSearchId": "HTTP-200-Code", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - } - } -}, { - "_id": "f1709820-f182-11e7-8ed1-e93cb1e71cec", - "_type": "visualization", - "_source": { - "title": "Top Account Ids", - "visState": "{\"title\":\"Top Account Ids\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"account_id\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"cwl-*\",\"query\":{\"match_all\":{}},\"filter\":[]}" - } - } -}, { - "_id": "Top-10-Src-Address-Packets-Reject", - "_type": "visualization", - "_source": { - "title": "Reject Src Addr", - "visState": "{\"title\":\"Reject Src Addr\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"srcaddr.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"srcaddr\"}}],\"listeners\":{}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "savedSearchId": "Reject", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - } - } -}, { - "_id": "74a9bbe0-f183-11e7-8ed1-e93cb1e71cec", - "_type": "visualization", - "_source": { - "title": "Top Regions", - "visState": "{\"title\":\"Top Regions\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"awsRegion.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"awsRegion\"}}],\"listeners\":{}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"cwl-*\",\"query\":{\"match_all\":{}},\"filter\":[]}" - } - } -}, { - "_id": "Total-Packets-Src-Address", - "_type": "visualization", - "_source": { - "title": "Total Packets Src Address", - "visState": "{\"title\":\"Total Packets Src Address\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":false,\"isDonut\":false,\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"srcaddr.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"\"}}],\"listeners\":{}}", - "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", - "description": "", - "savedSearchId": "All-Data", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - } - } -}, { - "_id": "Reject-Packets-Src-Address", - "_type": "visualization", - "_source": { - "title": "Reject Packets Src Address", - "visState": "{\"title\":\"Reject Packets Src Address\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":false,\"isDonut\":false,\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"packets\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"srcaddr.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "", - "savedSearchId": "Reject", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - } - } -}] diff --git a/source/services/indexing/lib/index.spec.js b/source/services/indexing/lib/index.spec.js deleted file mode 100644 index 6e54474..0000000 --- a/source/services/indexing/lib/index.spec.js +++ /dev/null @@ -1,277 +0,0 @@ -/******************************************************************************* -* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -* -* Licensed under the Apache License Version 2.0 (the "License"). You may not -* use this file except in compliance with the License. A copy of the License is -* located at -* -* http://www.apache.org/licenses/ -* -* or in the "license" file accompanying this file. This file is distributed on -* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or -* implied. See the License for the specific language governing permissions and -* limitations under the License. -********************************************************************************/ -'use strict'; - -let assert = require('chai').assert; -let expect = require('chai').expect; -let sinon = require('sinon'); -let AWS = require('aws-sdk-mock'); -let https = require('https'); -let PassThrough = require('stream').PassThrough; - -const Logging = require('../index.js'); -const MetricsHelper = require('./metrics-helper.js'); - -describe('ES_Logging_TestSuite', function() { - //test suite for transform function - describe('#transform', function() { - - let _awslogs_data = { - messageType: 'DATA_MESSAGE', - owner: 'xxxxx', - logGroup: 'testLogGroup', - logStream: 'testLogStream', - subscriptionFilters: ['testFilter'], - logEvents: [{ - id: 'eventId1', - timestamp: 1440442987000, - message: '[ERROR] First test message' - }, { - id: 'eventId2', - timestamp: 1440442987001, - message: '[ERROR] Second test message' - }] - }; - - let _es_bulkdata = - '{\"index\":{\"_index\":\"cwl-2015.08.24\",\"_type\":\"CloudWatchLogs\",\"_id\":\"eventId1\"}}\n{\"@id\":\"eventId1\",\"@timestamp\":\"2015-08-24T19:03:07.000Z\",\"@message\":\"[ERROR] First test message\",\"@owner\":\"xxxxx\",\"@log_group\":\"testLogGroup\",\"@log_stream\":\"testLogStream\"}\n{\"index\":{\"_index\":\"cwl-2015.08.24\",\"_type\":\"CloudWatchLogs\",\"_id\":\"eventId2\"}}\n{\"@id\":\"eventId2\",\"@timestamp\":\"2015-08-24T19:03:07.001Z\",\"@message\":\"[ERROR] Second test message\",\"@owner\":\"xxxxx\",\"@log_group\":\"testLogGroup\",\"@log_stream\":\"testLogStream\"}\n'; - - let _awslogs_data2 = { - message: "2 1234 eni-xxxx 10.x.x.x 10.x.x.x 123 123 17 1 76 1496689911 1496689967 ACCEPT OK", - extractedFields: { - srcaddr: "10.x.x.x", - dstport: "123", - start: "1496689911", - dstaddr: "10.x.x.x", - version: "2", - packets: "1", - protocol: "17", - account_id: "1234", - interface_id: "eni-xxxx", - log_status: "OK", - bytes: "76", - srcport: "123", - action: "ACCEPT", - end: "1496689967" - } - }; - - let _source = { - srcaddr: '10.x.x.x', - dstport: 123, - start: 1496689911, - dstaddr: '10.x.x.x', - version: 2, - packets: 1, - protocol: 17, - account_id: 1234, - interface_id: 'eni-xxxx', - log_status: 'OK', - bytes: 76, - srcport: 123, - action: 'ACCEPT', - end: 1496689967 - }; - - //hooks - beforeEach(function() {}); - afterEach(function() {}); - - /** - * Test cases - * @param {JSON} _awslogs_data - sample CW log event - * @param {JSON} _awslogs_data2 - sample CW log event - * @param {String} _es_bulkdata - data to be indexed on es - * @param {JSON} _source - data after transformation - */ - it('should return success when buildSource successfully', - function( - done) { - - let sourcedata = Logging.buildSource(_awslogs_data2.message, - _awslogs_data2.extractedFields); - assert.deepEqual(sourcedata, _source); - done(); - - }); - - it('should return success when log data transformed correctly', - function(done) { - - let data = Logging.transform(_awslogs_data); - assert.equal(data, _es_bulkdata); - done(); - - }); - - }); - - //test suite for buildRequest function - describe('#buildRequest', function() { - - let _es_bulkdata = - '{\"index\":{\"_index\":\"cwl-2015.08.24\",\"_type\":\"CloudWatchLogs\",\"_id\":\"eventId1\"}}\n{\"@id\":\"eventId1\",\"@timestamp\":\"2015-08-24T19:03:07.000Z\",\"@message\":\"[ERROR] First test message\",\"@owner\":\"123456789123\",\"@log_group\":\"testLogGroup\",\"@log_stream\":\"testLogStream\"}\n{\"index\":{\"_index\":\"cwl-2015.08.24\",\"_type\":\"CloudWatchLogs\",\"_id\":\"eventId2\"}}\n{\"@id\":\"eventId2\",\"@timestamp\":\"2015-08-24T19:03:07.001Z\",\"@message\":\"[ERROR] Second test message\",\"@owner\":\"123456789123\",\"@log_group\":\"testLogGroup\",\"@log_stream\":\"testLogStream\"}\n'; - - let _endpoint = 'centralized-logging.aws_region.es.amazonaws.com'; - - let _creds = { - aws_secret_key: 'xxxxxx', - aws_access_key: 'xxxxxx', - aws_session_token: 'xxxxxx' - }; - - //hooks - before(function() {}); - after(function() {}); - - /** - * Test cases - * @param {String} _endpoint - es endpoint - * @param {String} _es_bulkdata - data to be indexed on es - * @param {String} _creds - aws credentials - */ - it('should return success when post request built successfully', - function( - done) { - - Logging.buildRequest(_endpoint, _es_bulkdata, _creds, - function(err, request) { - if (err) done(err); - else { - done(); - } - }); - - }); - - }); - - //test suite for assume role function - describe('#assumeRole', function() { - - let _assumedRole = { - AssumedRoleUser: { - AssumedRoleId: 'a' - }, - Credentials: { - SecretAccessKey: 'x', - AccessKeyId: 'y', - SessionToken: 'z' - } - }; - - //hooks - beforeEach(function() {}); - afterEach(function() { - AWS.restore('STS'); - }); - - /** - * Test cases - * @param {Service} STS - aws service STS - * @param {Method} assumeRole - STS method assumeRole - - it('should return credentials when sts assume role succeeds', - function( - done) { - - AWS.mock('STS', 'assumeRole', function(params, callback) { - callback(null, _assumedRole); - }); - - Logging.assumeRole(function(err, creds) { - if (err) { - done(err); - } else { - expect(creds.aws_secret_key).to.equal('x'); - done(); - } - }); - - }); - */ - it('should return error when sts assume role fails', function(done) { - - AWS.mock('STS', 'assumeRole', function(params, callback) { - callback('invalid owner', null); - }); - - Logging.assumeRole(function(err, creds) { - if (err) { - expect(err).to.equal('invalid owner'); - done(); - } else { - done('invalid failure for negative test'); - } - }); - }); - - }); - - //test suite for sendMetrics function - describe('#sendMetrics', function() { - - //hooks - beforeEach(function() { - this.request = sinon.stub(https, 'request'); - }); - - afterEach(function() { - https.request.restore(); - }); - - /** - * Test cases - * @param {JSON} {} - sample parameter - * @param {function~callback} - empty callback - */ - it('should return success when metrics successfully sent', function() { - - let request = new PassThrough(); - let write = sinon.spy(request, 'write'); - - this.request.returns(request); - - let _metricsHelper = new MetricsHelper(); - _metricsHelper.sendAnonymousMetric({}, function() {}); - - assert(write.withArgs('{}').calledOnce); - - }); - - it('should return error when failed to send metrics', function(done) { - - let expected = - 'Error occurred when sending metric request {\"response\":\"ERROR\"}'; - let request = new PassThrough(); - - this.request.returns(request); - - let _metricsHelper = new MetricsHelper(); - _metricsHelper.sendAnonymousMetric({}, - function(err) { - assert.equal(err, expected); - done(); - }); - - request.emit('error', { - response: 'ERROR' - }); - - }); - - }); - -}); diff --git a/source/services/indexing/lib/logger.js b/source/services/indexing/lib/logger.js deleted file mode 100644 index c0afa23..0000000 --- a/source/services/indexing/lib/logger.js +++ /dev/null @@ -1,40 +0,0 @@ -/******************************************************************************* -* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -* -* Licensed under the Apache License Version 2.0 (the "License"). You may not -* use this file except in compliance with the License. A copy of the License is -* located at -* -* http://www.apache.org/licenses/ -* -* or in the "license" file accompanying this file. This file is distributed on -* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or -* implied. See the License for the specific language governing permissions and -* limitations under the License. -********************************************************************************/ -/** - * [logger module] - * V56536055 - 10/08/2018 - better logging capabilities - */ -'use strict'; - -class Logger { - - constructor() { - this.loglevel = process.env.LOG_LEVEL; - this.LOGLEVELS = { - ERROR: 1, - WARN: 2, - INFO: 3, - DEBUG: 4 - }; - } - - log(level, message) { - if (this.LOGLEVELS[level] <= this.LOGLEVELS[this.loglevel]) - console.log(`[${level}]${message}`); - } - -} - -module.exports = Object.freeze(Logger); diff --git a/source/services/indexing/lib/logger.spec.js b/source/services/indexing/lib/logger.spec.js deleted file mode 100644 index ab8a303..0000000 --- a/source/services/indexing/lib/logger.spec.js +++ /dev/null @@ -1,59 +0,0 @@ -/******************************************************************************* -* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -* -* Licensed under the Apache License Version 2.0 (the "License"). You may not -* use this file except in compliance with the License. A copy of the License is -* located at -* -* http://www.apache.org/licenses/ -* -* or in the "license" file accompanying this file. This file is distributed on -* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or -* implied. See the License for the specific language governing permissions and -* limitations under the License. -********************************************************************************/ -'use strict'; - -let assert = require('chai').assert; -let expect = require('chai').expect; - -let Logger = new(require('./logger'))(); - -describe("#Logger", function() { - - describe('#logger', function() { - - it('check with LOG_LEVEL=INFO', function() { - Logger.loglevel = 'INFO'; - Logger.log('INFO', 'INFO_MESSAGE'); - Logger.log('WARN', 'WARN_MESSAGE'); - Logger.log('ERROR', 'ERROR_MESSAGE'); - Logger.log('DEBUG', 'DEBUG_MESSAGE'); - }); - - it('check with LOG_LEVEL=WARN', function() { - Logger.loglevel = 'WARN'; - Logger.log('INFO', 'INFO_MESSAGE'); - Logger.log('WARN', 'WARN_MESSAGE'); - Logger.log('ERROR', 'ERROR_MESSAGE'); - Logger.log('DEBUG', 'DEBUG_MESSAGE'); - }); - - it('check with LOG_LEVEL=ERROR', function() { - Logger.loglevel = 'ERROR'; - Logger.log('INFO', 'INFO_MESSAGE'); - Logger.log('WARN', 'WARN_MESSAGE'); - Logger.log('ERROR', 'ERROR_MESSAGE'); - Logger.log('DEBUG', 'DEBUG_MESSAGE'); - }); - - it('check with LOG_LEVEL=DEBUG', function() { - Logger.loglevel = 'DEBUG'; - Logger.log('INFO', 'INFO_MESSAGE'); - Logger.log('WARN', 'WARN_MESSAGE'); - Logger.log('ERROR', 'ERROR_MESSAGE'); - Logger.log('DEBUG', 'DEBUG_MESSAGE'); - }); - - }) -}) diff --git a/source/services/indexing/lib/metrics-helper.js b/source/services/indexing/lib/metrics-helper.js deleted file mode 100644 index 13258c9..0000000 --- a/source/services/indexing/lib/metrics-helper.js +++ /dev/null @@ -1,90 +0,0 @@ -/******************************************************************************* -* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -* -* Licensed under the Apache License Version 2.0 (the "License"). You may not -* use this file except in compliance with the License. A copy of the License is -* located at -* -* http://www.apache.org/licenses/ -* -* or in the "license" file accompanying this file. This file is distributed on -* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or -* implied. See the License for the specific language governing permissions and -* limitations under the License. -********************************************************************************/ -/** - * @author Solution Builders - */ - -'use strict'; - -let https = require('https'); -const LOGGER = new(require('./logger'))(); - - -/** - * Helper function for sending anonymous metrics to Solutions Builder. - * - * @class metricsHelper - */ -let metricsHelper = (function() { - - /** - * @class metricsHelper - * @constructor - */ - let metricsHelper = function() {}; - - /** - * Sends opt-in, anonymous metric. - * @param {JSON} metric - metric to send to opt-in, anonymous collection. - * @param {sendAnonymousMetric~requestCallback} cb - The callback that handles the response. - */ - metricsHelper.prototype.sendAnonymousMetric = function(metric, cb) { - - let _options = { - hostname: 'metrics.awssolutionsbuilder.com', - port: 443, - path: '/generic', - method: 'POST', - headers: { - 'Content-Type': 'application/json' - } - }; - - let request = https.request(_options, function(response) { - // data is streamed in chunks from the server - // so we have to handle the "data" event - let buffer; - let data; - let route; - - response.on('data', function(chunk) { - buffer += chunk; - }); - - response.on('end', function(err) { - data = buffer; - cb(null, data); - }); - }); - - if (metric) { - LOGGER.log('DEBUG','sending https post for metrics'); - request.write(JSON.stringify(metric)); - } - - request.end(); - - request.on('error', (e) => { - LOGGER.log('ERROR', e); - cb(['Error occurred when sending metric request', JSON.stringify( - e)].join(' '), null); - }); - }; - - return metricsHelper; - -})(); - -module.exports = metricsHelper; diff --git a/source/services/indexing/lib/test-setup.spec.js b/source/services/indexing/lib/test-setup.spec.js deleted file mode 100644 index 1905552..0000000 --- a/source/services/indexing/lib/test-setup.spec.js +++ /dev/null @@ -1,29 +0,0 @@ -/******************************************************************************* -* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -* -* Licensed under the Apache License Version 2.0 (the "License"). You may not -* use this file except in compliance with the License. A copy of the License is -* located at -* -* http://www.apache.org/licenses/ -* -* or in the "license" file accompanying this file. This file is distributed on -* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or -* implied. See the License for the specific language governing permissions and -* limitations under the License. -********************************************************************************/ -const sinon = require('sinon'); -const chai = require('chai'); -const sinonChai = require('sinon-chai'); - -before(function() { - chai.use(sinonChai); -}); - -beforeEach(function() { - this.sandbox = sinon.sandbox.create(); -}); - -afterEach(function() { - this.sandbox.restore(); -}); diff --git a/source/services/indexing/package.json b/source/services/indexing/package.json deleted file mode 100644 index ea56255..0000000 --- a/source/services/indexing/package.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "centralized-logging-indexing", - "description": "A Lambda function for the centralized logging indexing microservice", - "main": "index.js", - "author": { - "name": "aws-solutions-builder" - }, - "version": "0.0.1", - "private": "true", - "engines": { - "node": ">=6.10" - }, - "dependencies": { - "https": "*", - "moment": "*" - }, - "devDependencies": { - "aws-sdk": "*", - "chai": "*", - "sinon": ">=4.0.0 <5.0.0", - "sinon-chai": "*", - "mocha": "*", - "aws-sdk-mock": "*", - "https": "*", - "npm-run-all": "*", - "stream": "*", - "nyc": "*" - }, - "scripts": { - "pretest": "npm install", - "test": "env DomainEndpoint='es-endpoint.us-east-1.es.amazonaws.com' MasterRole='master_role_needs_to_be_longer' SessionId='session_id' Owner='Spoke' LOG_LEVEL=DEBUG mocha lib/*.spec.js", - "build-init": "rm -rf dist && rm -f archive.zip && mkdir dist && mkdir dist/lib", - "build:copy": "cp index.js dist/ && cp -r lib/*.js dist/lib && cp lib/basic-dashboard.json dist/lib", - "build:install": "cp package.json dist/ && cd dist && npm install --production", - "build": "npm-run-all -s build-init build:copy build:install", - "zip": "cd dist && rm -f package-lock.json && zip -rq clog-indexing-service.zip .", - "coverage": "nyc cover _mocha $(find ./lib -name \"*.spec.js\" -not -path \"./node_modules/*\")" - } -} diff --git a/source/services/transformer/PromiseConstructor.d.ts b/source/services/transformer/PromiseConstructor.d.ts new file mode 100644 index 0000000..459f652 --- /dev/null +++ b/source/services/transformer/PromiseConstructor.d.ts @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +export interface PromiseResolution { + status: "fulfilled"; + value: T; +} + +export interface PromiseRejection { + status: "rejected"; + reason: E; +} + +export type PromiseResult = + | PromiseResolution + | PromiseRejection; + +export type PromiseList = { + [P in keyof T]: Promise; +}; + +export type PromiseResultList = { + [P in keyof T]: PromiseResult; +}; + +declare global { + interface PromiseConstructor { + allSettled(): Promise<[]>; + allSettled( + list: PromiseList + ): Promise>; + allSettled(iterable: Iterable): Promise>>; + } +} diff --git a/source/services/transformer/index.ts b/source/services/transformer/index.ts new file mode 100644 index 0000000..2d1e018 --- /dev/null +++ b/source/services/transformer/index.ts @@ -0,0 +1,287 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import zlib from "zlib"; +import { Firehose } from "aws-sdk"; +import { Record } from "aws-sdk/clients/firehose"; +import { logger } from "./lib/common/logger"; +import { Metrics } from "./lib/common/metrics"; +import moment from "moment"; +/** + * @description interface for log event + * @property {string} id for the log event + * @property {number} timestamp for the log event + * @property {string} message stringified log evenrt + * @property {any} extractedFields inferred fields from the event + */ +interface ILogEvent { + id: string; + timestamp: number; + message: string; + extractedFields: any; +} + +/** + * @description interface for log event + * @property {any[]} Records kinesis records from the data stream + */ +interface IEvent { + Records: [ + { + kinesis: { + kinesisSchemaVersion: string; + partitionKey: string; + sequenceNumber: string; + data: string; + approximateArrivalTimestamp: number; + }; + eventSource: string; + eventVersion: string; + eventID: string; + eventName: string; + invokeIdentityArn: string; + awsRegion: string; + eventSourceARN: string; + } + ]; +} + +/** + * @description transform log events into es documents + * @param {ILogEvent} logEvent - log event to transform into es document + * @param {string} owner - account id of the owner + * @param {string} logGroup - log group originating the event + * @param {string} logStream - log stream originating the event + */ +function transform( + logEvent: ILogEvent, + owner: string, + logGroup: string, + logStream: string +) { + const source = buildSource(logEvent.message, logEvent.extractedFields); + if ("requestParameters" in source) + source["requestParameters"] = JSON.stringify(source["requestParameters"]); + if ("responseElements" in source) + source["responseElements"] = JSON.stringify(source["responseElements"]); + if ("apiVersion" in source) source["apiVersion"] = "" + source["apiVersion"]; + source["timestamp"] = new Date(1 * logEvent.timestamp).toISOString(); + source["id"] = logEvent.id; + source["type"] = "CloudWatchLogs"; + source["@message"] = logEvent.message; + source["@owner"] = owner; + source["@log_group"] = logGroup; + source["@log_stream"] = logStream; + + return source; +} + +/** + * @description building source for log events + * @param message - log event + * @param extractedFields - fields in the log event + */ +function buildSource(message: string, extractedFields: any) { + if (extractedFields) { + const source = {}; + + for (const key in extractedFields) { + if ( + Object.prototype.hasOwnProperty.call(extractedFields, key) && + extractedFields[key] + ) { + const value = extractedFields[key]; + + if (isNumeric(value)) { + source[key] = 1 * value; + continue; + } + + const jsonSubString = extractJson(value); + if (jsonSubString !== null) { + source["$" + key] = JSON.parse(jsonSubString); + } + + source[key] = value; + } + } + + return source; + } + + const jsonSubString = extractJson(message); + if (jsonSubString !== null) { + return JSON.parse(jsonSubString); + } + + return {}; +} + +/** + * @description extracting json from log event + * @param {string} message - log event + */ +function extractJson(message: string) { + const jsonStart = message.indexOf("{"); + if (jsonStart < 0) return null; + const jsonSubString = message.substring(jsonStart); + return isValidJson(jsonSubString) ? jsonSubString : null; +} + +/** + * @description checking if extracted field has valid JSON + * @param {string} message - log event + */ +function isValidJson(message: string) { + try { + JSON.parse(message); + } catch (e) { + return false; + } + return true; +} + +/** + * @description checking if extracted field has numeric value + * @param n - extracted field to test for numeric value + */ +function isNumeric(n: any) { + return !isNaN(parseFloat(n)) && isFinite(n); +} + +/** + * @description creates records for firehose from the log events + * @param {ILogEvent} logEvent - log event to transform into es document + * @param {string} owner - account id of the owner + * @param {string} logGroup - log group originating the event + * @param {string} logStream - log stream originating the event + */ +function createRecordsFromEvents( + logEvents: ILogEvent[], + owner: string, + logGroup: string, + logStream: string +) { + const records: Record[] = []; + logEvents.forEach((event: ILogEvent) => { + const transformedEvent = transform(event, owner, logGroup, logStream); + logger.debug({ + label: "createRecordsFromEvents", + message: `transformed event: ${JSON.stringify(transformedEvent)}`, + }); + records.push({ + Data: Buffer.from(JSON.stringify(transformedEvent)), + }); + }); + logger.info({ + label: "createRecordsFromEvents", + message: "records created from log events", + }); + return records; +} + +async function putRecords(records: Record[]) { + logger.debug({ + label: "putRecords", + message: "records put on firehose", + }); + const params = { + DeliveryStreamName: "" + process.env.DELIVERY_STREAM /* required */, + Records: records, + }; + const firehose = new Firehose(); + await firehose.putRecordBatch(params).promise(); + + // send usage metric to aws-solutions + if (process.env.SEND_METRIC === "Yes") { + logger.info({ + label: "putRecords", + message: `sending metrics for indexed data`, + }); + + let totalItemSize = 0; + records.forEach((r) => { + totalItemSize += (r.Data as Buffer).byteLength; + }); + logger.debug({ + label: "putRecords/sendMetric", + message: `totalItemSize: ${totalItemSize}`, + }); + + const metric = { + Solution: process.env.SOLUTION_ID, + UUID: process.env.UUID, + TimeStamp: moment.utc().format("YYYY-MM-DD HH:mm:ss.S"), + Data: { + TotalItemSize: totalItemSize, + Version: process.env.SOLUTION_VERSION, + Region: process.env.AWS_REGION, + }, + }; + await Metrics.sendAnonymousMetric( + process.env.METRICS_ENDPOINT, + metric + ); + } +} + +exports.handler = async (event: IEvent) => { + logger.debug({ + label: "handler", + message: `event: ${JSON.stringify(event)}`, + }); + await Promise.allSettled( + event.Records.map(async (r) => { + try { + const buffer = Buffer.from(r.kinesis.data, "base64"); + let decompressed; + try { + decompressed = zlib.gunzipSync(buffer); + } catch (e) { + logger.error({ + label: "handler", + message: `error in reading data: ${JSON.stringify(e)} `, + }); + throw new Error("error in decompressing data"); + } + const payload = JSON.parse(decompressed); + logger.debug({ label: "handler", message: JSON.stringify(payload) }); + + // CONTROL_MESSAGE are sent by CWL to check if the subscription is reachable. + // They do not contain actual data. + if (payload.messageType === "CONTROL_MESSAGE") { + return; + } else if (payload.messageType === "DATA_MESSAGE") { + const records = createRecordsFromEvents( + payload.logEvents, + payload.owner, + payload.logGroup, + payload.logStream + ); + await putRecords(records); + logger.info({ + label: "handler", + message: "records put success", + }); + } else { + return; + } + } catch (e) { + logger.error({ + label: "handler", + message: e, + }); + } + }) + ); +}; diff --git a/source/services/transformer/lib/common/logger/index.ts b/source/services/transformer/lib/common/logger/index.ts new file mode 100644 index 0000000..140fcd8 --- /dev/null +++ b/source/services/transformer/lib/common/logger/index.ts @@ -0,0 +1,92 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +/** + * { + emerg: 0, + alert: 1, + crit: 2, + error: 3, + warning: 4, + notice: 5, + info: 6, + debug: 7 + } + */ +import { createLogger, transports, format } from "winston"; +import { WinstonSNS } from "./winston-sns"; +const { combine, timestamp, printf } = format; + +/* + * Foramting the output as desired + */ +const myFormat = printf(({ level, label, message }) => { + const _level = level.toUpperCase(); + if (label) return `[${_level}] [${label}] ${message}`; + else return `[${_level}] ${message}`; +}); + +/* + * String mask + */ +const maskCardNumbers = (num: any) => { + const str = num.toString(); + const { length } = str; + + return Array.from(str, (n, i) => { + return i < length - 4 ? "*" : n; + }).join(""); +}; + +// Define the format that mutates the info object. +const maskFormat = format((info: any) => { + // You can CHANGE existing property values + if (info.message.securedNumber) { + info.message.securedNumber = maskCardNumbers(info.message.securedNumber); + } + + // You can also ADD NEW properties if you wish + //info.hasCreditCard = !!info.creditCard; + + return info; +}); + +export const logger = createLogger({ + format: combine( + // + // Order is important here, the formats are called in the + // order they are passed to combine. + // + maskFormat(), + timestamp(), + myFormat + ), + + transports: [ + //cw logs transport channel + new transports.Console({ + level: process.env.LOG_LEVEL, + handleExceptions: true, //handle uncaught exceptions + //format: format.splat() + }), + + //sns transport channel + ...(process.env.SNS_ERROR_NOTIFICATION == "true" + ? [ + new WinstonSNS({ + topic_arn: process.env.SNS_TOPIC_ARN, + level: "error", + }), + ] + : []), + ], +}); diff --git a/source/services/transformer/lib/common/logger/winston-sns.ts b/source/services/transformer/lib/common/logger/winston-sns.ts new file mode 100644 index 0000000..e99f53f --- /dev/null +++ b/source/services/transformer/lib/common/logger/winston-sns.ts @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +import Transport = require("winston-transport"); +import "util"; +import { SNS } from "aws-sdk"; + +// +// Inherit from `winston-transport` so you can take advantage +// of the base functionality and `.exceptions.handle()`. +// +export class WinstonSNS extends Transport { + readonly topic_arn: string; + readonly region: string; + private sns: any; + constructor(opts: any) { + super(opts); + this.topic_arn = opts.topic_arn; + this.region = opts.topic_arn.split(":")[3]; + // + // Consume any custom options here. e.g.: + // - Connection information for databases + // - Authentication information for APIs (e.g. loggly, papertrail, + // logentries, etc.). + // + } + formatter = async (info: any) => { + if (info.label) + return `[${info.level.toUpperCase()}] [${info.label}] ${ + info.timestamp + }: ${JSON.stringify(info.message, null, 2)}`; + else + return `[${info.level.toUpperCase()}] ${info.timestamp}: ${JSON.stringify( + info.message, + null, + 2 + )}`; + }; + + log = async (info: any) => { + try { + this.sns = new SNS({ + apiVersion: "2010-03-31", + region: this.region, + }); + const _txt = await this.formatter(info); + await this.sns + .publish({ + Message: _txt, + TopicArn: this.topic_arn, + }) + .promise(); + return "sns message published successfully"; + } catch (e) { + throw new Error(e.message); + } + }; +} diff --git a/source/services/transformer/lib/common/metrics/index.ts b/source/services/transformer/lib/common/metrics/index.ts new file mode 100644 index 0000000..2d04367 --- /dev/null +++ b/source/services/transformer/lib/common/metrics/index.ts @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +import got from "got"; +import { logger } from "../logger/index"; +/** + * Send metrics to solutions endpoint + * @class Metrics + */ +export class Metrics { + /** + * Sends anonymous metric + * @param {object} metric - metric JSON data + */ + static async sendAnonymousMetric(endpoint: string, metric: any) { + logger.debug({ + label: "metrics/sendAnonymousMetric", + message: `metrics endpoint: ${endpoint}`, + }); + logger.debug({ + label: "metrics/sendAnonymousMetric", + message: `sending metric:${JSON.stringify(metric)}`, + }); + try { + await got(endpoint, { + port: 443, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": "" + JSON.stringify(metric).length, + }, + body: JSON.stringify(metric), + }); + logger.info({ + label: "metrics/sendAnonymousMetric", + message: `metric sent successfully`, + }); + return `Metric sent: ${JSON.stringify(metric)}`; + } catch (error) { + logger.warn({ + label: "metrics/sendAnonymousMetric", + message: `Error occurred while sending metric: ${JSON.stringify( + error + )}`, + }); + return `Error occurred while sending metric`; + } + } +} diff --git a/source/services/transformer/package.json b/source/services/transformer/package.json new file mode 100644 index 0000000..a312a1a --- /dev/null +++ b/source/services/transformer/package.json @@ -0,0 +1,32 @@ +{ + "name": "cl-transformer", + "version": "4.0.0", + "description": "transformer lambda for Centralized Logging solution", + "main": "index.js", + "scripts": { + "pretest": "npm i", + "test": "echo \"nothing to do\"", + "build:clean": "rm -rf ./node_modules && rm -rf ./dist && rm -f ./package-lock.json", + "build:copy": "cp -r ./node_modules ./dist/node_modules", + "build:ts": "./node_modules/typescript/bin/tsc --project ./tsconfig.json", + "build:install": "npm i", + "watch": "tsc -w", + "build:zip": "cd ./dist && zip -rq cl-transformer.zip .", + "build:all": "npm run build:clean && npm run build:install && npm run build:ts && npm prune --production && npm run build:copy && npm run build:zip" + }, + "author": "aws-solutions", + "license": "Apache-2.0", + "dependencies": { + "got": "^11.5.1", + "moment": "^2.27.0", + "uuid": "^8.2.0", + "winston": "^3.3.3", + "aws-sdk": "^2.714.0" + }, + "devDependencies": { + "typescript": "^4.0.2", + "@types/uuid": "^8.0.0", + "@types/node": "^14.0.23", + "@types/moment": "^2.13.0" + } +} diff --git a/source/services/transformer/tsconfig.json b/source/services/transformer/tsconfig.json new file mode 100644 index 0000000..94645b9 --- /dev/null +++ b/source/services/transformer/tsconfig.json @@ -0,0 +1,40 @@ +{ + "compilerOptions": { + /* Basic Options */ + "target": "ES2018" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, + "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, + "lib": [ + "DOM", + "ES2018" + ] /* Specify library files to be included in the compilation. */, + "outDir": "./dist", + "declaration": false /* Generates corresponding '.d.ts' file. */, + "noEmit": false, + "removeComments": true /* Do not emit comments to output. */, + "resolveJsonModule": true /* Allows importing modules with a ‘.json’ extension */, + /* Strict Type-Checking Options */ + "strict": true /* Enable all strict type-checking options. */, + "noImplicitAny": false /* Raise error on expressions and declarations with an implied 'any' type. */, + // "strictNullChecks": true /* Enable strict null checks. */, + "strictFunctionTypes": true /* Enable strict checking of function types. */, + "strictPropertyInitialization": true /* Enable strict checking of property initialization in classes. */, + "noImplicitThis": true /* Raise error on 'this' expressions with an implied 'any' type. */, + "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */, + + /* Additional Checks */ + "noUnusedLocals": true /* Report errors on unused locals. */, + "noUnusedParameters": true /* Report errors on unused parameters. */, + "noImplicitReturns": true /* Report error when not all code paths in function return a value. */, + "noFallthroughCasesInSwitch": true /* Report errors for fallthrough cases in switch statement. */, + + /* Module Resolution Options */ + "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, + + /* Experimental Options */ + "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */, + // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + + /* Advanced Options */ + "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ + } +}