From ac7d90034b93e8f7471bbd32ef1f80d89803e43b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 3 Jan 2019 15:34:18 -0800 Subject: [PATCH] Add templates (#4) * Add templates * Render additional templates. Fmt code. * fix typo * Add boilerplate kops documentation * Update README and add missing templates * address CR * Update geodesic and root modules * upgrade geodesic * Add email address to readme * Update placeholders * Update configs/root.tfvars Co-Authored-By: osterman * subnet calcs --- README.md | 7 +- configs/audit.tfvars | 19 +- configs/corp.tfvars | 18 +- configs/data.tfvars | 18 +- configs/dev.tfvars | 17 +- configs/prod.tfvars | 18 +- configs/root.tfvars | 21 +- configs/staging.tfvars | 18 +- configs/testing.tfvars | 18 +- modules/child/main.tf | 11 +- modules/child/variables.tf | 4 +- modules/docker-build/variables.tf | 4 +- modules/export-env/main.tf | 5 +- modules/init-dirs/variables.tf | 4 +- modules/render/variables.tf | 4 +- modules/root/main.tf | 9 +- modules/root/variables.tf | 2 + tasks/Makefile.child | 4 + templates/.editorconfig | 4 + templates/.github/CODEOWNERS | 2 + templates/.github/ISSUE_TEMPLATE/bug.md | 27 + .../.github/ISSUE_TEMPLATE/feature-request.md | 13 + templates/.github/PULL_REQUEST.md | 12 + templates/.gitignore | 1 - templates/README.md | 22 +- templates/conf/account-dns/terraform.tfvars | 1 + templates/conf/bootstrap/terraform.tfvars | 2 + templates/conf/users/terraform.tfvars | 2 + templates/docs/kops.md | 724 ++++++++++++++++++ 29 files changed, 942 insertions(+), 69 deletions(-) create mode 100644 templates/.github/CODEOWNERS create mode 100644 templates/.github/ISSUE_TEMPLATE/bug.md create mode 100644 templates/.github/ISSUE_TEMPLATE/feature-request.md create mode 100644 templates/.github/PULL_REQUEST.md create mode 100644 templates/conf/account-dns/terraform.tfvars create mode 100644 templates/conf/bootstrap/terraform.tfvars create mode 100644 templates/conf/users/terraform.tfvars create mode 100644 templates/docs/kops.md diff --git a/README.md b/README.md index a152d12..c05a849 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,10 @@ Get up and running quickly with one of our reference architectures using our ful - [Table of Contents](#table-of-contents) - [Known Limitations](#known-limitations) - [High Level Overview](#high-level-overview) - - [Architecture](#architecture) + - [Architecture](#architecture) - [Assumptions](#assumptions) - [Checklist](#checklist) + - [Get Started](#get-started) - [1. Provision Root Account](#1-provision-root-account) - [2. Provision Subaccounts](#2-provision-subaccounts) - [3. Delegate DNS](#3-delegate-dns) @@ -39,7 +40,7 @@ When you're done, in the `repos/` directory you'll have one Git repo for each AW See the [Next Steps](#next-steps) section for where to go after this process completes. -## Architecture +### Architecture Our "reference architecture" is an opinionated approach to architecting accounts for AWS. @@ -83,6 +84,8 @@ Before we get started, make sure you have the following - [ ] An available domain we can use for DNS-base service discovery (E.g. `ourcompany.co`). This domain must not be in use elsewhere as the root account will need to be the authoritative name server (`SOA`). - [ ] Ensure that any users who will be added during this bootstrap process have setup their [keybase](http://keybase.io) profile. You'll need this if setting them up in the `users` section of the [`config/root.tfvars`](https://github.com/cloudposse/reference-architectures/blob/master/configs/root.tfvars). For example you should be able to [verify their public key](https://keybase.io/osterman/key.asc) on `keybase.io` by running `curl https://keybase.io/$username/key.asc`. +## Get Started + ### 1. Provision Root Account The "root" account is the top-most AWS account from which all other AWS accounts are programmatically created. diff --git a/configs/audit.tfvars b/configs/audit.tfvars index 46afb27..20e0604 100644 --- a/configs/audit.tfvars +++ b/configs/audit.tfvars @@ -5,17 +5,22 @@ stage = "audit" # List of templates to install templates = [ "README.md", - "Dockerfile.child", - ".gitignore", - ".dockerignore", - "Makefile", - "conf/Makefile" - ] + "Dockerfile.child", + ".github/CODEOWNERS", + ".github/ISSUE_TEMPLATE/feature-request.md", + ".github/ISSUE_TEMPLATE/bug.md", + ".github/PULL_REQUEST.md", + ".editorconfig", + ".gitignore", + ".dockerignore", + "Makefile", + "conf/Makefile", +] # List of terraform root modules to enable terraform_root_modules = [ "aws/tfstate-backend", "aws/account-dns", "aws/chamber", - "aws/audit-cloudtrail" + "aws/audit-cloudtrail", ] diff --git a/configs/corp.tfvars b/configs/corp.tfvars index 20315cb..777dd73 100644 --- a/configs/corp.tfvars +++ b/configs/corp.tfvars @@ -5,12 +5,18 @@ stage = "corp" # List of templates to install templates = [ "README.md", - "Dockerfile.child", - ".gitignore", - ".dockerignore", - "Makefile", + "Dockerfile.child", + ".github/CODEOWNERS", + ".github/ISSUE_TEMPLATE/feature-request.md", + ".github/ISSUE_TEMPLATE/bug.md", + ".github/PULL_REQUEST.md", + ".editorconfig", + ".gitignore", + ".dockerignore", + "Makefile", "conf/Makefile", - "conf/account-dns/terraform.tfvars" + "conf/account-dns/terraform.tfvars", + "docs/kops.md", ] # List of terraform root modules to enable @@ -18,5 +24,5 @@ terraform_root_modules = [ "aws/tfstate-backend", "aws/account-dns", "aws/chamber", - "aws/cloudtrail" + "aws/cloudtrail", ] diff --git a/configs/data.tfvars b/configs/data.tfvars index 9e279f5..4fa4e34 100644 --- a/configs/data.tfvars +++ b/configs/data.tfvars @@ -5,12 +5,18 @@ stage = "data" # List of templates to install templates = [ "README.md", - "Dockerfile.child", - ".gitignore", - ".dockerignore", - "Makefile", + "Dockerfile.child", + ".github/CODEOWNERS", + ".github/ISSUE_TEMPLATE/feature-request.md", + ".github/ISSUE_TEMPLATE/bug.md", + ".github/PULL_REQUEST.md", + ".editorconfig", + ".gitignore", + ".dockerignore", + "Makefile", "conf/Makefile", - "conf/account-dns/terraform.tfvars" + "conf/account-dns/terraform.tfvars", + "docs/kops.md", ] # List of terraform root modules to enable @@ -18,5 +24,5 @@ terraform_root_modules = [ "aws/tfstate-backend", "aws/account-dns", "aws/chamber", - "aws/cloudtrail" + "aws/cloudtrail", ] diff --git a/configs/dev.tfvars b/configs/dev.tfvars index 628a960..82f6c1a 100644 --- a/configs/dev.tfvars +++ b/configs/dev.tfvars @@ -5,12 +5,17 @@ stage = "dev" # List of templates to install templates = [ "README.md", - "Dockerfile.child", - ".gitignore", - ".dockerignore", - "Makefile", + "Dockerfile.child", + ".github/CODEOWNERS", + ".github/ISSUE_TEMPLATE/feature-request.md", + ".github/ISSUE_TEMPLATE/bug.md", + ".github/PULL_REQUEST.md", + ".editorconfig", + ".gitignore", + ".dockerignore", + "Makefile", "conf/Makefile", - "conf/account-dns/terraform.tfvars" + "conf/account-dns/terraform.tfvars", ] # List of terraform root modules to enable @@ -18,5 +23,5 @@ terraform_root_modules = [ "aws/tfstate-backend", "aws/account-dns", "aws/chamber", - "aws/cloudtrail" + "aws/cloudtrail", ] diff --git a/configs/prod.tfvars b/configs/prod.tfvars index c027a97..ce39e93 100644 --- a/configs/prod.tfvars +++ b/configs/prod.tfvars @@ -5,12 +5,18 @@ stage = "prod" # List of templates to install templates = [ "README.md", - "Dockerfile.child", - ".gitignore", - ".dockerignore", - "Makefile", + "Dockerfile.child", + ".github/CODEOWNERS", + ".github/ISSUE_TEMPLATE/feature-request.md", + ".github/ISSUE_TEMPLATE/bug.md", + ".github/PULL_REQUEST.md", + ".editorconfig", + ".gitignore", + ".dockerignore", + "Makefile", "conf/Makefile", - "conf/account-dns/terraform.tfvars" + "conf/account-dns/terraform.tfvars", + "docs/kops.md", ] # List of terraform root modules to enable @@ -18,5 +24,5 @@ terraform_root_modules = [ "aws/tfstate-backend", "aws/account-dns", "aws/chamber", - "aws/cloudtrail" + "aws/cloudtrail", ] diff --git a/configs/root.tfvars b/configs/root.tfvars index e26c9ab..40448b9 100644 --- a/configs/root.tfvars +++ b/configs/root.tfvars @@ -9,6 +9,11 @@ namespace = "test" # The default region for this account aws_region = "us-west-2" +# Network CIDR of Organization +org_network_cidr = "10.0.0.0/8" +org_network_offset = 100 +org_network_newbits = 8 # = /16 + # The docker registry that will be used for the images built (nothing will get pushed) docker_registry = "cloudposse" @@ -16,6 +21,11 @@ docker_registry = "cloudposse" templates = [ "README.md", "Dockerfile.root", + ".github/CODEOWNERS", + ".github/ISSUE_TEMPLATE/feature-request.md", + ".github/ISSUE_TEMPLATE/bug.md", + ".github/PULL_REQUEST.md", + ".editorconfig", ".gitignore", ".dockerignore", "Makefile", @@ -24,6 +34,7 @@ templates = [ "conf/bootstrap/terraform.tfvars", "conf/iam/terraform.tfvars", "conf/root-dns/terraform.tfvars", + "conf/users/terraform.tfvars" ] # Account email address format (e.g. `ops+%s@example.co`). This is not easily changed later. @@ -37,21 +48,21 @@ accounts_enabled = [ "testing", "data", "corp", - "audit" + "audit", ] # Administrator IAM usernames mapped to their keybase usernames for password encryption users = { - "erik@cloudposse.com" = "osterman" +# "erik@cloudposse.com" = "osterman" } # Terraform Root Modules Image (don't change this unless you know what you're doing) # Project: https://github.com/cloudposse/terraform-root-modules -terraform_root_modules_image = "cloudposse/terraform-root-modules:0.14.3" +terraform_root_modules_image = "cloudposse/terraform-root-modules:0.18.1" # Geodesic Base Image (don't change this unless you know what you're doing) # Project: https://github.com/cloudposse/geodesic -geodesic_base_image = "cloudposse/geodesic:0.49.0" +geodesic_base_image = "cloudposse/geodesic:0.56.0" # List of terraform root modules to enable terraform_root_modules = [ @@ -63,7 +74,7 @@ terraform_root_modules = [ "aws/root-iam", "aws/iam", "aws/users", - "aws/cloudtrail" + "aws/cloudtrail", ] # Message of the Day diff --git a/configs/staging.tfvars b/configs/staging.tfvars index c2b2f65..831baba 100644 --- a/configs/staging.tfvars +++ b/configs/staging.tfvars @@ -5,12 +5,18 @@ stage = "staging" # List of templates to install templates = [ "README.md", - "Dockerfile.child", - ".gitignore", - ".dockerignore", - "Makefile", + "Dockerfile.child", + ".github/CODEOWNERS", + ".github/ISSUE_TEMPLATE/feature-request.md", + ".github/ISSUE_TEMPLATE/bug.md", + ".github/PULL_REQUEST.md", + ".editorconfig", + ".gitignore", + ".dockerignore", + "Makefile", "conf/Makefile", - "conf/account-dns/terraform.tfvars" + "conf/account-dns/terraform.tfvars", + "docs/kops.md", ] # List of terraform root modules to enable @@ -18,5 +24,5 @@ terraform_root_modules = [ "aws/tfstate-backend", "aws/account-dns", "aws/chamber", - "aws/cloudtrail" + "aws/cloudtrail", ] diff --git a/configs/testing.tfvars b/configs/testing.tfvars index 456120f..b9a9427 100644 --- a/configs/testing.tfvars +++ b/configs/testing.tfvars @@ -5,12 +5,18 @@ stage = "testing" # List of templates to install templates = [ "README.md", - "Dockerfile.child", - ".gitignore", - ".dockerignore", - "Makefile", + "Dockerfile.child", + ".github/CODEOWNERS", + ".github/ISSUE_TEMPLATE/feature-request.md", + ".github/ISSUE_TEMPLATE/bug.md", + ".github/PULL_REQUEST.md", + ".editorconfig", + ".gitignore", + ".dockerignore", + "Makefile", "conf/Makefile", - "conf/account-dns/terraform.tfvars" + "conf/account-dns/terraform.tfvars", + "docs/kops.md", ] # List of terraform root modules to enable @@ -18,5 +24,5 @@ terraform_root_modules = [ "aws/tfstate-backend", "aws/account-dns", "aws/chamber", - "aws/cloudtrail" + "aws/cloudtrail", ] diff --git a/modules/child/main.tf b/modules/child/main.tf index 652c2e7..af6c9cd 100644 --- a/modules/child/main.tf +++ b/modules/child/main.tf @@ -1,3 +1,12 @@ +locals { + context = { + # Used by `README.md` + account_email_address = "${format(var.account_email, var.stage)}" + } + + vars = "${merge(var.vars, local.context)}" +} + module "account" { source = "../../modules/account" dirs = "${var.dirs}" @@ -11,7 +20,7 @@ module "account" { image_tag = "${var.image_tag}" templates = "${var.templates}" dirs = "${var.dirs}" - vars = "${var.vars}" + vars = "${local.vars}" strip = "${var.strip}" artifacts_dir = "${var.artifacts_dir}" repos_dir = "${var.repos_dir}" diff --git a/modules/child/variables.tf b/modules/child/variables.tf index a77887d..e68d5bc 100644 --- a/modules/child/variables.tf +++ b/modules/child/variables.tf @@ -18,6 +18,8 @@ variable "image_tag" { default = "latest" } +variable "account_email" {} + variable "templates" { type = "list" } @@ -37,7 +39,7 @@ variable "strip" { } variable "networks" { - type = "map" + type = "map" default = {} } diff --git a/modules/docker-build/variables.tf b/modules/docker-build/variables.tf index 10e96f8..173e571 100644 --- a/modules/docker-build/variables.tf +++ b/modules/docker-build/variables.tf @@ -15,7 +15,7 @@ variable "dockerfile" { # NOTE: this variable won't actually be used for anything and the actual `depends_on` keyword # in terraform does not support interpolation. variable "depends_on" { - type = "list" + type = "list" description = "Define a list of variables that this module depends on in order to force serialized execution." - default = [] + default = [] } diff --git a/modules/export-env/main.tf b/modules/export-env/main.tf index cc9b846..ea83356 100644 --- a/modules/export-env/main.tf +++ b/modules/export-env/main.tf @@ -1,6 +1,5 @@ - locals { - keys = "${keys(var.env)}" + keys = "${keys(var.env)}" values = "${values(var.env)}" } @@ -16,7 +15,7 @@ data "null_data_source" "envs" { locals { export = { encoded = "${format(var.template, join("\n", data.null_data_source.envs.*.outputs.encoded))}" - raw = "${format(var.template, join("\n", data.null_data_source.envs.*.outputs.raw))}" + raw = "${format(var.template, join("\n", data.null_data_source.envs.*.outputs.raw))}" } } diff --git a/modules/init-dirs/variables.tf b/modules/init-dirs/variables.tf index 5fbbe19..cfe1dce 100644 --- a/modules/init-dirs/variables.tf +++ b/modules/init-dirs/variables.tf @@ -8,7 +8,7 @@ variable "dirs" { # NOTE: this variable won't actually be used for anything and the actual `depends_on` keyword # in terraform does not support interpolation. variable "depends_on" { - type = "list" + type = "list" description = "Define a list of variables that this module depends on in order to force serialized execution." - default = [] + default = [] } diff --git a/modules/render/variables.tf b/modules/render/variables.tf index 6a5fb6e..6428030 100644 --- a/modules/render/variables.tf +++ b/modules/render/variables.tf @@ -17,7 +17,7 @@ variable "strip" { # NOTE: this variable won't actually be used for anything and the actual `depends_on` keyword # in terraform does not support interpolation. variable "depends_on" { - type = "list" + type = "list" description = "Define a list of variables that this module depends on in order to force serialized execution." - default = [] + default = [] } diff --git a/modules/root/main.tf b/modules/root/main.tf index e6a9ad8..e488325 100644 --- a/modules/root/main.tf +++ b/modules/root/main.tf @@ -1,8 +1,9 @@ locals { context = { # Used by `accounts` - accounts_enabled = "${jsonencode(var.accounts_enabled)}" - account_email = "${var.account_email}" + accounts_enabled = "${jsonencode(var.accounts_enabled)}" + account_email = "${var.account_email}" + account_email_address = "${format(var.account_email, var.stage)}" # Used by `root-dns` root_domain_name = "${var.stage}.${var.domain}" @@ -11,7 +12,6 @@ locals { vars = "${merge(var.vars, local.context)}" } - locals { all_accounts = "${concat(list("root"), var.accounts_enabled)}" } @@ -20,7 +20,7 @@ data "null_data_source" "networks" { count = "${length(local.all_accounts)}" inputs = { - cidr = "${cidrsubnet(var.org_network_cidr, 8, count.index)}" + cidr = "${cidrsubnet(var.org_network_cidr, var.org_network_newbits, var.org_network_offset + count.index)}" } } @@ -28,7 +28,6 @@ locals { networks = "${zipmap(local.all_accounts, data.null_data_source.networks.*.outputs.cidr)}" } - module "account" { source = "../../modules/account/" diff --git a/modules/root/variables.tf b/modules/root/variables.tf index f5a8fa3..af1de85 100644 --- a/modules/root/variables.tf +++ b/modules/root/variables.tf @@ -46,6 +46,8 @@ variable "strip" { } variable "org_network_cidr" {} +variable "org_network_offset" {} +variable "org_network_newbits" {} variable "account_network_cidr" { default = "" diff --git a/tasks/Makefile.child b/tasks/Makefile.child index 7ab9389..3a1286f 100644 --- a/tasks/Makefile.child +++ b/tasks/Makefile.child @@ -60,6 +60,10 @@ children: children/validate $(ACCOUNTS_ENABLED) children/validate: $(addsuffix /validate,$(ACCOUNTS_ENABLED)) @exit 0 +## Initialize all child account configuration +children/init: $(addsuffix /init,$(ACCOUNTS_ENABLED)) + @exit 0 + ## Finalize provisioning of all child accounts children/finalize: $(addsuffix /finalize,$(ACCOUNTS_ENABLED)) @exit 0 diff --git a/templates/.editorconfig b/templates/.editorconfig index b37aa4c..b471316 100644 --- a/templates/.editorconfig +++ b/templates/.editorconfig @@ -22,3 +22,7 @@ indent_size = 4 [*.sh] indent_style = tab indent_size = 4 + +[*.{tf,tfvars,tpl}] +indent_size = 2 +indent_style = space diff --git a/templates/.github/CODEOWNERS b/templates/.github/CODEOWNERS new file mode 100644 index 0000000..fbf88eb --- /dev/null +++ b/templates/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Use this file to define individuals or teams that are responsible for code in a repository. +# Read more: diff --git a/templates/.github/ISSUE_TEMPLATE/bug.md b/templates/.github/ISSUE_TEMPLATE/bug.md new file mode 100644 index 0000000..67c6852 --- /dev/null +++ b/templates/.github/ISSUE_TEMPLATE/bug.md @@ -0,0 +1,27 @@ +--- +title: "Bug report" +labels: bug +assignees: +--- + +## what +* Describe *what* the problem (bug) is you encountered and the steps to reproduce it. +* Use bullet points to be concise and to the point. + +## why +* Explain *why* this is a problem by describing the expected behavior. +* Use bullet points to be concise and to the point. +* If you have a hunch why this happens, add that here. + +## details + +Here is a log of what happens. + +
+Output + +``` +# Paste screen capture in a code block +``` + +
diff --git a/templates/.github/ISSUE_TEMPLATE/feature-request.md b/templates/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000..5da8ffb --- /dev/null +++ b/templates/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,13 @@ +--- +title: "Feature Request: ..." +labels: enhancement +assignees: +--- + +## what +* Describe *what* changes you would like to be made to this project. +* Use bullet points to be concise and to the point. + +## why +* Explain *why* these changes amount to an enhancement which addresses some particular business use-case. +* Use bullet points to be concise and to the point. diff --git a/templates/.github/PULL_REQUEST.md b/templates/.github/PULL_REQUEST.md new file mode 100644 index 0000000..03f1e2c --- /dev/null +++ b/templates/.github/PULL_REQUEST.md @@ -0,0 +1,12 @@ +## what +* Describe high-level what changed as a result of these commits (i.e. in plain-english, what do these changes mean?) +* Use bullet points to be concise and to the point. + +## why +* Provide the justifications for the changes (e.g. business case). +* Describe why these changes were made (e.g. why do these commits fix the problem?) +* Use bullet points to be concise and to the point. + +## references +* Link to any supporting github issues or helpful documentation to add some context (e.g. stackoverflow). +* Use `closes #123`, if this PR closes a GitHub issue `#123` diff --git a/templates/.gitignore b/templates/.gitignore index 55ee851..c52e5bb 100644 --- a/templates/.gitignore +++ b/templates/.gitignore @@ -2,7 +2,6 @@ **/.terraform/* *.tfstate *.tfstate.* -*.tfvars # Module directory .terraform diff --git a/templates/README.md b/templates/README.md index e83102e..4fc2fd2 100644 --- a/templates/README.md +++ b/templates/README.md @@ -1,7 +1,18 @@ -# \${image_name} +# ${image_name} This repository provides all the tooling to manage the `${stage}` account infrastructure on AWS. It distributes a single docker container which bundles the entire tool-chain and infrastructure as code necessary to administer the account. +## Acount Details + +| Property | Value | +| -------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| AWS Account ID | ${aws_account_id} | +| Account Email | ${account_email_address} | +| Login URL | | +| Namespace | ${namespace} | +| Stage | ${stage} | +| Default Region | ${aws_region} | + ## Goals & Principles This project aims to be lightweight. It follows these principles: @@ -32,6 +43,8 @@ Here is the list of some of the tools we use to provision `${image_name}` infras - [helm](https://helm.sh/) - [helmfile](https://github.com/roboll/helmfile) +**NOTE:** Additional documentation can be found in the [`docs/`](docs/) directory. + ## Layout This repo is organized in the following way. @@ -47,6 +60,7 @@ ${image_name}/ │   └── module3/ # Another terraform "root" module │   ├── file1.tf # Overlay additional files │   └── file2.tf # +├── docs/ # Additional documentation ├── Dockerfile # Dockerfile that describes how to build this image ├── Makefile # Makefile that uses the `build-harness` to facilitate building the image └── rootfs/ # "Root" (`/`) filesystem which is overlayed inside of the docker image @@ -57,7 +71,7 @@ ${image_name}/ Most configuration settings are defined as environment variables. These can be set using the `ENV` declaration in the [`Dockerfile`](Dockerfile). These have been set to _sane defaults_ and shouldn't need to be touched. All these settings are required.
-List of Environment Variables +List of Supported Environment Variables | Environment Variable | Description of the setting | | --------------------- | ----------------------------------------------------------------------------- | @@ -179,3 +193,7 @@ For more info, see [Using Geodesic with Terraform](https://docs.cloudposse.com/g - https://github.com/cloudposse/geodesic - https://github.com/cloudposse/packages - https://github.com/cloudposse/build-harness + +## Getting Help + +Did you get stuck? Find us on [slack](https://slack.cloudposse.com) in the `#geodesic` channel. diff --git a/templates/conf/account-dns/terraform.tfvars b/templates/conf/account-dns/terraform.tfvars new file mode 100644 index 0000000..3ce3599 --- /dev/null +++ b/templates/conf/account-dns/terraform.tfvars @@ -0,0 +1 @@ +domain_name = "${domain_name}" diff --git a/templates/conf/bootstrap/terraform.tfvars b/templates/conf/bootstrap/terraform.tfvars new file mode 100644 index 0000000..6708182 --- /dev/null +++ b/templates/conf/bootstrap/terraform.tfvars @@ -0,0 +1,2 @@ +output_path = "/artifacts" +accounts_enabled = ${accounts_enabled} diff --git a/templates/conf/users/terraform.tfvars b/templates/conf/users/terraform.tfvars new file mode 100644 index 0000000..79f3e55 --- /dev/null +++ b/templates/conf/users/terraform.tfvars @@ -0,0 +1,2 @@ +# List of accounts to enable +accounts_enabled = ${accounts_enabled} diff --git a/templates/docs/kops.md b/templates/docs/kops.md new file mode 100644 index 0000000..60d614d --- /dev/null +++ b/templates/docs/kops.md @@ -0,0 +1,724 @@ +# Kops (Kubernetes Operations) + +## Table of Contents +- [Kops (Kubernetes Operations)](#kops-kubernetes-operations) + - [Table of Contents](#table-of-contents) + - [Configuration Settings](#configuration-settings) + - [Provision a Kops Cluster](#provision-a-kops-cluster) + - [Populate `chamber` Secrets](#populate-chamber-secrets) + - [Provision `vpc` from `backing-services` with Terraform](#provision-vpc-from-backing-services-with-terraform) + - [Provision `vpc-peering` from `kops-aws-platform` with Terraform](#provision-vpc-peering-from-kops-aws-platform-with-terraform) + - [Provision the rest of `kops-aws-platform` with Terraform](#provision-the-rest-of-kops-aws-platform-with-terraform) + - [Provision the rest of `backing-services` with Terraform](#provision-the-rest-of-backing-services-with-terraform) + - [Provision Kubernetes Resources](#provision-kubernetes-resources) + - [Deploy heapster](#deploy-heapster) + - [Deploy kubernetes-dashboard](#deploy-kubernetes-dashboard) + - [Deploy kiam](#deploy-kiam) + - [Deploy external-dns](#deploy-external-dns) + - [Deploy kube-lego](#deploy-kube-lego) + - [Deploy prometheus-operator](#deploy-prometheus-operator) + - [Deploy kube-prometheus](#deploy-kube-prometheus) + - [Deploy nginx-ingress](#deploy-nginx-ingress) + - [Deploy fluentd-elasticsearch-logs](#deploy-fluentd-elasticsearch-logs) + - [Deploy portal](#deploy-portal) + - [Check Cluster Health](#check-cluster-health) + - [References](#references) + - [Getting Help](#getting-help) + +## Configuration Settings + +Most configuration settings are defined as environment variables. These can be set using the `ENV` declaration in the [`Dockerfile`](Dockerfile). These have been set to _sane defaults_ and shouldn't need to be touched. All these settings are required. + +
+List of Supported Environment Variables + +| Environment Variable | Description of the setting | +| -------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| BASTION_MACHINE_TYPE | AWS EC2 instance type of bastion host | +| KOPS_ADMISSION_CONTROL_ENABLED | Toggle if adminission controller should be enabled | +| KOPS_API_LOAD_BALANCER_IDLE_TIMEOUT_SECONDS | AWS ELB idle connection timeout for the API load balancer | +| KOPS_AUTHORIZATION_RBAC_ENABLED | Toggle Kubernetes RBAC support | +| KOPS_AVAILABILITY_ZONES | AWS Availability Zones (AZs) to use. Must all reside in the same region. Use an _odd_ number. | +| KOPS_AWS_IAM_AUTHENTICATOR_ENABLED | Toggle IAM Authenticator support | +| KOPS_BASE_IMAGE | AWS AMI base image for all EC2 instances | +| KOPS_BASTION_PUBLIC_NAME | Hostname that will be used for the bastion instance | +| KOPS_CLOUDWATCH_DETAILED_MONITORING | Toggle detailed CloudWatch monitoring (increases operating costs) | +| KOPS_CLUSTER_AUTOSCALER_ENABLED | Toggle the Kubernetes node autoscaler capability | +| KOPS_CLUSTER_NAME | Cluster base hostname (E.g. `${aws_region}.${image_name}`) | +| KOPS_DNS_ZONE | Authoritative DNS Zone that will be populated automatic with hostnames | +| KOPS_KUBE_API_SERVER_AUTHORIZATION_MODE | Ordered list of plug-ins to do authorization on secure port | +| KOPS_KUBE_API_SERVER_AUTHORIZATION_RBAC_SUPER_USER | Username of the Kubernetes Super User | +| KOPS_NETWORK_CIDR | The network used by kubernetes for `Pods` and `Services` in the cluster | +| KOPS_NON_MASQUERADE_CIDR | A list of strings in CIDR notation that specify the non-masquerade ranges. | +| KOPS_PRIVATE_SUBNETS | Subnet CIDRs for all EC2 instances | +| KOPS_STATE_STORE | S3 Bucket that will be used to store the cluster state (E.g. `${aws_region}.${image_name}`) | +| KOPS_UTILITY_SUBNETS | Subnet CIDRs for the publically facing services (e.g. ingress ELBs) | +| KUBERNETES_VERSION | Version of Kubernetes to deploy. Must be compatible with the `kops` release. | +| NODE_MACHINE_TYPE | AWS EC2 instance type for the _default_ node pool | +| NODE_MAX_SIZE | Maximum number of EC2 instances in the _default_ node pool | +| NODE_MIN_SIZE | Minimum number of EC2 instances in the _default_ node pool | + +**IMPORTANT:** + +1. `KOPS_NETWORK_CIDR` and `KOPS_NON_MASQUERADE_CIDR` **MUST NOT** overlap +2. `KOPS_KUBE_API_SERVER_AUTHORIZATION_MODE` is a comma-separated list (e.g.`AlwaysAllow`,`AlwaysDeny`,`ABAC`,`Webhook`,`RBAC`,`Node`) + +
+ +## Provision a Kops Cluster + +We create a [`kops`](https://github.com/kubernetes/kops) cluster from a manifest. + +The manifest template is located in [`/templates/kops/default.yaml`](https://github.com/cloudposse/geodesic/blob/master/rootfs/templates/kops/default.yaml) +and is compiled by running `build-kops-manifest` in the [`Dockerfile`](Dockerfile). + +Provisioning a `kops` cluster takes three steps: + +1. Provision the `kops` backend (config S3 bucket, cluster DNS zone, and SSH keypair to access the k8s masters and nodes) in Terraform +2. Update the [`Dockerfile`](Dockerfile) and rebuild/restart the `geodesic` shell to generate a `kops` manifest file +3. Execute the `kops` manifest file to create the `kops` cluster + +Run Terraform to provision the `kops` backend (S3 bucket, DNS zone, and SSH keypair): + +```bash +make -C /conf/kops init apply +``` + +The `.envrc` for the `kops` config should look like this: + +```docker +# kops config +ENV KOPS_CLUSTER_NAME="${aws_region}.${image_name}" +ENV KOPS_DNS_ZONE=$${KOPS_CLUSTER_NAME} +ENV KOPS_STATE_STORE="s3://${namespace}-${stage}-kops-state" +ENV KOPS_STATE_STORE_REGION="${aws_region}" +ENV KOPS_AVAILABILITY_ZONES="${aws_region}a,${aws_region}d,${aws_region}c" +ENV KOPS_BASTION_PUBLIC_NAME="bastion" +ENV BASTION_MACHINE_TYPE="t2.medium" +ENV MASTER_MACHINE_TYPE="t2.medium" +ENV NODE_MACHINE_TYPE="t2.medium" +ENV NODE_MAX_SIZE="2" +ENV NODE_MIN_SIZE="2" +``` + +Type `exit` (or hit ^D) to leave the shell. + +Note, if you've assumed a role, you'll first need to leave that also by typing `exit` (or hit ^D). + +Rebuild the Docker image: + +```bash +make docker/build +``` + +Run the `geodesic` shell again and assume role to login to AWS: + +```bash +${image_name} +assume-role +``` + +Change directory to `kops` folder: + +```bash +cd /conf/kops +``` + +You will see the `kops` manifest file `manifest.yaml` generated. + +Run the following command to create the cluster. This will just initialize the cluster state and store it in the S3 bucket, but not actually provision any AWS resources for the cluster. + +```bash +kops create -f manifest.yaml +``` + +Run the following command to add the SSH public key to the cluster: + +```bash +kops create secret sshpublickey admin -i /secrets/tf/ssh/${namespace}-${stage}-kops-${aws_region}.pub --name $KOPS_CLUSTER_NAME +``` + +Run the following command to provision the AWS resources for the cluster: + +```bash +kops update cluster --yes +``` + +All done. The `kops` cluster is now up and running. + +To use the `kubectl` command (_e.g._ `kubectl get nodes`, `kubectl get pods`), you need to export the `kubecfg` configuration settings from the cluster. + +Run the following command to export `kubecfg` settings needed to connect to the cluster: + +```bash +kops export kubecfg +``` + +**IMPORTANT:** You need to run this command every time you start a new shell and before you work with the cluster (e.g. before running `kubectl`). + +See the documentation for [`kubecfg` settings for `kubectl`](https://github.com/kubernetes/kops/blob/master/docs/kubectl.md) for more details. +
+ +Run the following command to validate the cluster: + +```bash +kops validate cluster +``` + +
Show Output + +Below is an example of what it should _roughly_ look like (IPs and Availability Zones may differ). + +``` +✓ (${namespace}-${stage}-admin) kops ⨠ kops validate cluster +Validating cluster ${aws_region}.${image_name} + +INSTANCE GROUPS +NAME ROLE MACHINETYPE MIN MAX SUBNETS +bastions Bastion t2.medium 1 1 utility-${aws_region}a,utility-${aws_region}d,utility-${aws_region}c +master-${aws_region}a Master t2.medium 1 1 ${aws_region}a +master-${aws_region}c Master t2.medium 1 1 ${aws_region}c +master-${aws_region}d Master t2.medium 1 1 ${aws_region}d +nodes Node t2.medium 2 2 ${aws_region}a,${aws_region}d,${aws_region}c + +NODE STATUS +NAME ROLE READY +ip-172-20-108-58.${aws_region}.compute.internal node True +ip-172-20-125-166.${aws_region}.compute.internal master True +ip-172-20-62-206.${aws_region}.compute.internal master True +ip-172-20-74-158.${aws_region}.compute.internal master True +ip-172-20-88-143.${aws_region}.compute.internal node True + +Your cluster ${aws_region}.${image_name} is ready +``` + +
+
+ +Run the following command to list all nodes: + +```bash +kubectl get nodes +``` + +
Show Output + +Below is an example of what it should _roughly_ look like (IPs and Availability Zones may differ). + +``` +✓ (${namespace}-${stage}-admin) kops ⨠ kubectl get nodes +NAME STATUS ROLES AGE VERSION +ip-172-20-108-58.${aws_region}.compute.internal Ready node 15m v1.10.8 +ip-172-20-125-166.${aws_region}.compute.internal Ready master 17m v1.10.8 +ip-172-20-62-206.${aws_region}.compute.internal Ready master 18m v1.10.8 +ip-172-20-74-158.${aws_region}.compute.internal Ready master 17m v1.10.8 +ip-172-20-88-143.${aws_region}.compute.internal Ready node 16m v1.10.8 +``` + +
+
+ +Run the following command to list all pods: + +```bash +kubectl get pods --all-namespaces +``` + +
Show Output + +Below is an example of what it should _roughly_ look like (IPs and Availability Zones may differ). + +``` +✓ (${namespace}-${stage}-admin) backing-services ⨠ kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system calico-kube-controllers-69c6bdf999-7sfdg 1/1 Running 0 1h +kube-system calico-node-4qlj2 2/2 Running 0 1h +kube-system calico-node-668x9 2/2 Running 0 1h +kube-system calico-node-jddc9 2/2 Running 0 1h +kube-system calico-node-pszd8 2/2 Running 0 1h +kube-system calico-node-rqfbk 2/2 Running 0 1h +kube-system dns-controller-75b75f6f5d-tdg9s 1/1 Running 0 1h +kube-system etcd-server-events-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system etcd-server-events-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 2 1h +kube-system etcd-server-events-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system etcd-server-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system etcd-server-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 2 1h +kube-system etcd-server-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-apiserver-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-apiserver-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 3 1h +kube-system kube-apiserver-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-controller-manager-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-controller-manager-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-controller-manager-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-dns-5fbcb4d67b-kp2pp 3/3 Running 0 1h +kube-system kube-dns-5fbcb4d67b-wg6gv 3/3 Running 0 1h +kube-system kube-dns-autoscaler-6874c546dd-tvbhq 1/1 Running 0 1h +kube-system kube-proxy-ip-172-20-108-58.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-proxy-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-proxy-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-proxy-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-proxy-ip-172-20-88-143.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-scheduler-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-scheduler-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 0 1h +kube-system kube-scheduler-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 1h +``` + +
+
+
+ +To upgrade the cluster or change settings (_e.g_. number of nodes, instance types, Kubernetes version, etc.): + +1. Modify the `kops` settings in the [`Dockerfile`](Dockerfile) +2. Rebuild Docker image (`make docker/build`) +3. Run `geodesic` shell (`${image_name}`), assume role (`assume-role`) and change directory to `/conf/kops` folder +4. Run `kops export kubecfg` +5. Run `kops replace -f manifest.yaml` to replace the cluster resources (update state) +6. Run `kops update cluster` +7. Run `kops update cluster --yes` +8. Run `kops rolling-update cluster` +9. Run `kops rolling-update cluster --yes --force` to force a rolling update (replace EC2 instances) +
+ +## Populate `chamber` Secrets + +**NOTE:** We use `chamber` to first populate the environment with the secrets from the specified service (`backing-services`, `kops`) +and then execute the given commands (`terraform plan` or `terraform apply`). +You need to do it only once for a given set of secrets. Repeat this step if you want to add new secrets. + +Populate `chamber` secrets for `kops` project (make sure to change the keys and values to reflect your environment; add new secrets as needed) + +```bash +chamber write kops +chamber write kops +... +``` + +**NOTE:** Run `chamber list -e kops` to list the secrets stored for `kops` project + +Populate `chamber` secrets for `backing-services` project (make sure to change the values to reflect your environment; add new secrets as needed) + +```bash +chamber write backing-services TF_VAR_POSTGRES_DB_NAME ${namespace}_${stage} +chamber write backing-services TF_VAR_POSTGRES_ADMIN_NAME ${namespace}_admin +chamber write backing-services TF_VAR_POSTGRES_ADMIN_PASSWORD XXXXXXXXXXXX +``` + +**NOTE:** Run `chamber list -e backing-services` to list the secrets stored for `backing-services` project +
+ +## Provision `vpc` from `backing-services` with Terraform + +**NOTE:** We provision `backing-services` in two phases because: + +- `aurora-postgres` and other backing services depend on `vpc-peering` (they use `kops` Security Group to allow `kops` applications to connect) +- `vpc-peering` depends on `vpc` and `kops` (it creates a peering connection between the two networks) + +To break the circular dependencies, we provision `kops`, then `vpc` (from `backing-services`), then `vpc-peering`, +and finally the rest of `backing-services` (`aurora-postgres` and other services). + +Provision `vpc` for `backing-services`: + +```bash +cd /conf/backing-services +init-terraform +terraform plan -target=data.aws_availability_zones.available -target=module.vpc -target=module.subnets +terraform apply -target=data.aws_availability_zones.available -target=module.vpc -target=module.subnets +``` + +## Provision `vpc-peering` from `kops-aws-platform` with Terraform + +```bash +cd /conf/kops-aws-platform +init-terraform +terraform plan -target=data.aws_vpc.backing_services_vpc -target=module.kops_vpc_peering +terraform apply -target=data.aws_vpc.backing_services_vpc -target=module.kops_vpc_peering +``` + +You should see the following output: + +
Show Output + +``` +Apply complete! Resources: 10 added, 0 changed, 0 destroyed. +Releasing state lock. This may take a few moments... + +Outputs: + +kops_vpc_peering_accept_status = active +kops_vpc_peering_connection_id = pcx-014d91a03f56e3170 +``` + +
+
+ +## Provision the rest of `kops-aws-platform` with Terraform + +```bash +cd /conf/kops-aws-platform +terraform plan +terraform apply +``` + +You should see the following output: + +
Show Output + +``` +Apply complete! Resources: 4 added, 0 changed, 0 destroyed. +Releasing state lock. This may take a few moments... + +Outputs: + +kops_external_dns_policy_arn = arn:aws:iam::xxxxxxxx:policy/${namespace}-${stage}-external-dns +kops_external_dns_policy_id = arn:aws:iam::xxxxxxxx:policy/${namespace}-${stage}-external-dns +kops_external_dns_policy_name = ${namespace}-${stage}-external-dns +kops_external_dns_role_arn = arn:aws:iam::xxxxxxxx:role/${namespace}-${stage}-external-dns +kops_external_dns_role_name = ${namespace}-${stage}-external-dns +kops_external_dns_role_unique_id = XXXXXXXXXXXXXXXX +``` + +
+
+ +## Provision the rest of `backing-services` with Terraform + +**NOTE:** Make sure you have populated the `chamber` secrets for `backing-services` (see Populate `chamber` secrets above) : + +```bash +chamber write backing-services TF_VAR_POSTGRES_DB_NAME ${namespace}_${stage} +chamber write backing-services TF_VAR_POSTGRES_ADMIN_NAME ${namespace}_admin +chamber write backing-services TF_VAR_POSTGRES_ADMIN_PASSWORD XXXXXXXXXXXXXXXX +``` + +Provision `aurora-postgres` and `elasticsearch`: + +```bash +cd /conf/backing-services +chamber exec backing-services -- terraform plan +chamber exec backing-services -- terraform apply +``` + +You should see the following output: + +
Show Output + +``` +aurora_postgres_cluster_name = ${namespace}-${stage}-postgres +aurora_postgres_database_name = ${namespace}_${stage} +aurora_postgres_master_hostname = master.postgres.${image_name} +aurora_postgres_master_username = ${namespace}_admin +aurora_postgres_replicas_hostname = replicas.postgres.${image_name} +elasticsearch_domain_arn = arn:aws:es:${aws_region}:091456519406:domain/${namespace}-${stage}-elasticsearch +elasticsearch_domain_endpoint = vpc-${namespace}-${stage}-elasticsearch-35lgg7m52qybtqf3cftblowblm.${aws_region}.es.amazonaws.com +elasticsearch_domain_hostname = elasticsearch.${image_name} +elasticsearch_domain_id = 091456519406/${namespace}-${stage}-elasticsearch +elasticsearch_kibana_endpoint = vpc-${namespace}-${stage}-elasticsearch-35lgg7m52qybtqf3cftblowblm.${aws_region}.es.amazonaws.com/_plugin/kibana/ +elasticsearch_kibana_hostname = kibana-elasticsearch.${image_name} +elasticsearch_security_group_id = sg-0cc1155c8bd45a6c2 +``` + +
+
+ +## Provision Kubernetes Resources + +**NOTE:** We use [helmfile](https://github.com/roboll/helmfile) to deploy [Helm](https://helm.sh/) [charts](https://github.com/kubernetes/charts) to provision Kubernetes resources. +We use `chamber` to first populate the environment with the secrets from the `kops` chamber service and then execute commands (_e.g._ `helmfile sync`) + +**NOTE:** Make sure to export Kubernetes config by executing `kops export kubecfg` + +Run `helm init` to initialize `helm` and `tiller`: + +
Show Output + +``` +✓ (${namespace}-${stage}-admin) ⨠ helm init +$HELM_HOME has been configured at /var/lib/helm. + +Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster. + +Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy. +To prevent this, run `helm init` with the --tiller-tls-verify flag. +For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation +Happy Helming! +``` + +
+
+ +### Deploy heapster + +```bash +cd /conf +chamber exec kops -- helmfile --selector namespace=kube-system,chart=heapster sync +``` + +
Show Output + +``` +✓ (${namespace}-${stage}-admin) ⨠ chamber exec kops -- helmfile --selector namespace=kube-system,chart=heapster sync +Adding repo stable https://kubernetes-charts.storage.googleapis.com +"stable" has been added to your repositories + +Updating repo +Hang tight while we grab the latest from your chart repositories... +...Skip local chart repository +...Successfully got an update from the "incubator" chart repository +...Successfully got an update from the "cloudposse-incubator" chart repository +...Successfully got an update from the "coreos-stable" chart repository +...Successfully got an update from the "stable" chart repository +Update Complete. ⎈ Happy Helming!⎈ + +Upgrading stable/heapster +Release "heapster" does not exist. Installing it now. +NAME: heapster +LAST DEPLOYED: Fri Nov 30 17:58:10 2018 +NAMESPACE: kube-system +STATUS: DEPLOYED + +RESOURCES: +==> v1/Service +NAME AGE +heapster 3s + +==> v1beta1/Deployment +heapster-heapster 3s + +==> v1/Pod(related) + +NAME READY STATUS RESTARTS AGE +heapster-heapster-68d4c66f4c-b6j85 0/2 ContainerCreating 0 3s +``` + +
+
+ +### Deploy kubernetes-dashboard + +```bash +chamber exec kops -- helmfile --selector namespace=kube-system,chart=kubernetes-dashboard sync +``` + +### Deploy kiam + +```bash +cd /conf/scripts/kiam +make all +make chamber/write/agent +make chamber/write/server +make annotate +cd /conf +chamber exec kops -- helmfile --selector namespace=kube-system,chart=kiam sync +``` + +### Deploy external-dns + +```bash +chamber write kops EXTERNAL_DNS_TXT_OWNER_ID ${aws_region}.${image_name} +chamber write kops EXTERNAL_DNS_TXT_PREFIX 27ba410b-1809-491b-bc06-8f2b7f703209- +chamber write kops EXTERNAL_DNS_IAM_ROLE ${namespace}-${stage}-external-dns +chamber exec kops -- helmfile --selector namespace=kube-system,chart=external-dns sync +``` + +### Deploy kube-lego + +```bash +chamber write kops KUBE_LEGO_EMAIL ops@${domain_name} +chamber write kops KUBE_LEGO_PROD true +chamber exec kops -- helmfile --selector namespace=kube-system,chart=kube-lego sync +``` + +### Deploy prometheus-operator + +```bash +chamber exec kops -- helmfile --selector namespace=kube-system,chart=prometheus-operator sync +``` + +### Deploy kube-prometheus + +**NOTE:** Update `KUBE_PROMETHEUS_ALERT_MANAGER_SLACK_WEBHOOK_URL` and `KUBE_PROMETHEUS_ALERT_MANAGER_SLACK_CHANNEL` to the values specific to the current project. + +```bash +chamber write kops KUBE_PROMETHEUS_ALERT_MANAGER_SLACK_WEBHOOK_URL https://xxxxxxxxxx.xxx +chamber write kops KUBE_PROMETHEUS_ALERT_MANAGER_SLACK_CHANNEL test +chamber write kops KUBE_PROMETHEUS_ALERT_MANAGER_HOSTNAME alerts.${aws_region}.${image_name} +chamber write kops KUBE_PROMETHEUS_ALERT_MANAGER_INGRESS ingress.${aws_region}.${image_name} +chamber write kops KUBE_PROMETHEUS_ALERT_MANAGER_SECRET_NAME alertmanager-general-tls +chamber write kops KUBE_PROMETHEUS_HOSTNAME prometheus.${aws_region}.${image_name} +chamber write kops KUBE_PROMETHEUS_INGRESS ingress.${aws_region}.${image_name} +chamber write kops KUBE_PROMETHEUS_SECRET_NAME prometheus-general-tls +chamber write kops KUBE_PROMETHEUS_EXTERNAL_VALUES_FILE "./values/kube-prometheus.grafana.dashboards.yaml" +chamber exec kops -- helmfile --selector namespace=monitoring,chart=kube-prometheus --selector namespace=monitoring,chart=kube-prometheus-grafana sync +``` + +### Deploy nginx-ingress + +```bash +chamber write kops NGINX_INGRESS_HOSTNAME ingress.${aws_region}.${image_name} +chamber exec kops -- helmfile --selector namespace=kube-system,chart=nginx-ingress sync +``` + +### Deploy fluentd-elasticsearch-logs + +```bash +chamber write kops ELASTICSEARCH_HOST vpc-${namespace}-${stage}-elasticsearch-45lgg4m52qybkqz3cftbxowbxm.${aws_region}.es.amazonaws.com +chamber write kops ELASTICSEARCH_PORT 443 +chamber write kops ELASTICSEARCH_SCHEME "https" +chamber exec kops -- helmfile --selector namespace=kube-system,name=fluentd-elasticsearch-logs sync +``` + +### Deploy portal + +```bash +chamber write kops PORTAL_TITLE "($stage)" +chamber write kops PORTAL_BRAND "NOC" +chamber write kops PORTAL_HOSTNAME portal.${aws_region}.${image_name} +chamber write kops PORTAL_BRAND_IMAGE_FAVICON_URL "https://avatars2.githubusercontent.com/u/16398900?s=200&v=4" +chamber write kops PORTAL_BRAND_IMAGE_URL "https://avatars2.githubusercontent.com/u/16398900?s=200&v=4" +chamber write kops PORTAL_BRAND_IMAGE_WIDTH 35 +chamber write kops PORTAL_INGRESS_TLS_ENABLED true +chamber write kops PORTAL_INGRESS ingress.${aws_region}.${image_name} +chamber write kops PORTAL_COOKIE_DOMAIN .${aws_region}.${image_name} +chamber write kops PORTAL_OAUTH2_PROXY_COOKIE_NAME xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +chamber write kops PORTAL_OAUTH2_PROXY_COOKIE_SECRET xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +chamber write kops PORTAL_OAUTH2_PROXY_GITHUB_ORGANIZATION xxxxxxxx +chamber write kops PORTAL_OAUTH2_PROXY_GITHUB_TEAM li +chamber write kops PORTAL_OAUTH2_PROXY_CLIENT_ID xxxxxxxxxxxxxxxxxx +chamber write kops PORTAL_OAUTH2_PROXY_CLIENT_SECRET xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +chamber write kops PORTAL_OAUTH2_PROXY_REDIRECT_URL https://portal.${aws_region}.${image_name}/oauth2/callback +chamber write kops PORTAL_BACKEND_KIBANA_EXTERNAL_NAME vpc-${namespace}-${stage}-elasticsearch-35lgg7m52qybtqf3cftblowblm.${aws_region}.es.amazonaws.com +chamber write kops PORTAL_BACKEND_KIBANA_ENABLED true +chamber exec kops -- helmfile --selector namespace=monitoring,chart=portal sync +``` + +## Check Cluster Health + +To confirm that all `helm` releases are deployed and running, run the following command: + +```bash +helm list -a +``` + +
Show Output + +Below is an example of what it should _roughly_ look like (IPs and Availability Zones may differ). + +``` +✓ (${namespace}-${stage}-admin) ⨠ helm list -a +NAME REVISION UPDATED STATUS CHART APP VERSION NAMESPACE +dns 1 Fri Nov 30 19:15:23 2018 DEPLOYED external-dns-0.5.4 0.4.8 kube-system +fluentd-elasticsearch-logs 1 Mon Dec 3 19:37:57 2018 DEPLOYED fluentd-kubernetes-0.3.0 0.12 kube-system +heapster 2 Mon Dec 3 15:38:29 2018 DEPLOYED heapster-0.2.10 1.3.0 kube-system +ingress 2 Sat Dec 1 06:05:15 2018 DEPLOYED nginx-ingress-0.25.1 0.17.1 kube-system +ingress-backend 2 Sat Dec 1 06:05:14 2018 DEPLOYED nginx-default-backend-0.2.2 kube-system +ingress-monitoring 2 Sat Dec 1 06:05:15 2018 DEPLOYED monochart-0.4.0 0.4.0 monitoring +kiam 2 Sat Dec 1 04:04:31 2018 DEPLOYED kiam-2.0.0-rc1 3.0-rc1 kube-system +kube-prometheus 1 Mon Dec 3 19:06:53 2018 DEPLOYED kube-prometheus-0.0.105 monitoring +kube-prometheus-grafana 1 Mon Dec 3 18:49:39 2018 DEPLOYED monochart-0.4.0 0.4.0 monitoring +kubernetes-dashboard 1 Fri Nov 30 18:27:09 2018 DEPLOYED kubernetes-dashboard-0.6.7 1.8.3 kube-system +portal 3 Mon Dec 3 17:57:49 2018 DEPLOYED portal-0.2.4 monitoring +prometheus-operator 1 Fri Nov 30 18:43:55 2018 DEPLOYED prometheus-operator-0.0.29 0.20.0 kube-system +tls 2 Sat Dec 1 05:10:12 2018 DEPLOYED kube-lego-0.1.2 kube-system +``` + +
+
+ +To check the status of all Kubernetes pods, run the following command: + +```bash +kubectl get pods --all-namespaces +``` + +
Show Output + +Below is an example of what it should _roughly_ look like (IPs and Availability Zones may differ). + +``` +✓ (${namespace}-${stage}-admin) ⨠ kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system calico-kube-controllers-69c6bdf999-7sfdg 1/1 Running 0 3d +kube-system calico-node-4qlj2 2/2 Running 0 3d +kube-system calico-node-668x9 2/2 Running 0 3d +kube-system calico-node-jddc9 2/2 Running 0 3d +kube-system calico-node-pszd8 2/2 Running 0 3d +kube-system calico-node-rqfbk 2/2 Running 0 3d +kube-system dns-controller-75b75f6f5d-tdg9s 1/1 Running 0 3d +kube-system dns-external-dns-67b99686c4-chl8w 1/1 Running 0 3d +kube-system etcd-server-events-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system etcd-server-events-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 2 3d +kube-system etcd-server-events-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system etcd-server-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system etcd-server-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 2 3d +kube-system etcd-server-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system fluentd-elasticsearch-logs-fluentd-kubernetes-24zcq 1/1 Running 0 21m +kube-system fluentd-elasticsearch-logs-fluentd-kubernetes-5mjbp 1/1 Running 0 21m +kube-system fluentd-elasticsearch-logs-fluentd-kubernetes-7d682 1/1 Running 0 21m +kube-system fluentd-elasticsearch-logs-fluentd-kubernetes-lskkt 1/1 Running 0 21m +kube-system fluentd-elasticsearch-logs-fluentd-kubernetes-n9ddf 1/1 Running 0 21m +kube-system heapster-heapster-59b674774c-8c689 2/2 Running 0 3d +kube-system ingress-backend-default-6f987d4648-bk4kh 1/1 Running 0 2d +kube-system ingress-backend-default-6f987d4648-j8frw 1/1 Running 0 2d +kube-system ingress-nginx-ingress-controller-659759dc98-c8w7z 1/1 Running 0 2d +kube-system ingress-nginx-ingress-controller-659759dc98-f9xsr 1/1 Running 0 2d +kube-system ingress-nginx-ingress-controller-659759dc98-rp6jv 1/1 Running 0 2d +kube-system ingress-nginx-ingress-controller-659759dc98-zl872 1/1 Running 0 2d +kube-system kiam-agent-hh4f8 1/1 Running 0 3d +kube-system kiam-agent-wqqbw 1/1 Running 0 3d +kube-system kiam-server-ft956 1/1 Running 0 3d +kube-system kiam-server-mdfds 1/1 Running 0 3d +kube-system kiam-server-rqp76 1/1 Running 0 3d +kube-system kube-apiserver-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-apiserver-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 3 3d +kube-system kube-apiserver-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-controller-manager-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-controller-manager-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-controller-manager-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-dns-5fbcb4d67b-kp2pp 3/3 Running 0 3d +kube-system kube-dns-5fbcb4d67b-wg6gv 3/3 Running 0 3d +kube-system kube-dns-autoscaler-6874c546dd-tvbhq 1/1 Running 0 3d +kube-system kube-proxy-ip-172-20-108-58.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-proxy-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-proxy-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-proxy-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-proxy-ip-172-20-88-143.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-scheduler-ip-172-20-125-166.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-scheduler-ip-172-20-62-206.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kube-scheduler-ip-172-20-74-158.${aws_region}.compute.internal 1/1 Running 0 3d +kube-system kubernetes-dashboard-76ddc7694c-twxjn 1/1 Running 0 3d +kube-system prometheus-operator-5f4669c455-dt588 1/1 Running 0 1h +kube-system tiller-deploy-6fd8d857bc-v2q5l 1/1 Running 0 3d +kube-system tls-kube-lego-679f49f8ff-pmxhf 1/1 Running 0 2d +monitoring alertmanager-kube-prometheus-0 2/2 Running 0 52m +monitoring alertmanager-kube-prometheus-1 2/2 Running 0 51m +monitoring alertmanager-kube-prometheus-2 2/2 Running 0 51m +monitoring alertmanager-kube-prometheus-3 2/2 Running 0 51m +monitoring kube-prometheus-exporter-kube-state-7cb68c5b6c-wj4z9 2/2 Running 0 52m +monitoring kube-prometheus-exporter-node-4fj5s 1/1 Running 0 52m +monitoring kube-prometheus-exporter-node-fb5p5 1/1 Running 0 52m +monitoring kube-prometheus-exporter-node-gv5jn 1/1 Running 0 52m +monitoring kube-prometheus-exporter-node-xhvqt 1/1 Running 0 52m +monitoring kube-prometheus-exporter-node-xvrsv 1/1 Running 0 52m +monitoring kube-prometheus-grafana-79454f96dc-gmmrd 2/2 Running 0 52m +monitoring portal-oauth2-proxy-8558678f5d-q8wll 1/1 Running 0 4h +monitoring portal-portal-6f8b759d64-v898p 1/1 Running 0 2h +monitoring portal-portal-6f8b759d64-zwp2g 1/1 Running 0 2h +monitoring prometheus-kube-prometheus-0 3/3 Running 1 52m +monitoring prometheus-kube-prometheus-1 3/3 Running 1 51m +monitoring prometheus-kube-prometheus-2 0/3 Pending 0 51m +``` + +
+ +## References + +- https://docs.cloudposse.com +- https://github.com/segmentio/chamber +- https://github.com/kubernetes/kops +- https://github.com/kubernetes-incubator/external-dns/blob/master/docs/faq.md +- https://github.com/gravitational/workshop/blob/master/k8sprod.md + +## Getting Help + +Did you get stuck? Find us on [slack](https://slack.cloudposse.com) in the `#geodesic` channel.