diff --git a/modules/terraform-aws-proxy/.terraform-docs.yaml b/modules/terraform-aws-proxy/.terraform-docs.yaml new file mode 100644 index 0000000..0936036 --- /dev/null +++ b/modules/terraform-aws-proxy/.terraform-docs.yaml @@ -0,0 +1,21 @@ +formatter: markdown +header-from: doc_fragments/header.md +settings: + anchor: true + color: true + default: true + escape: true + html: true + indent: 2 + required: true + sensitive: true + type: true + + +sort: + enabled: true + by: required + +output: + file: README.md + mode: replace \ No newline at end of file diff --git a/modules/terraform-aws-proxy/README.md b/modules/terraform-aws-proxy/README.md new file mode 100644 index 0000000..d805d14 --- /dev/null +++ b/modules/terraform-aws-proxy/README.md @@ -0,0 +1,88 @@ + +# Terraform Module for AWS Transit Gateway + +This module contains resource files and example variable definition files for creation of AWS Transity Gateway (TGW) and attaching a specified list of VPCs via the TGW. This module also updates both the Transit Gateway and VPC route tables. This module can be used to assist in deploying Cloudera Data Platform (CDP) Public Cloud in a fully private networking configuration where a CDP VPC and Networking VPC are connected using the Transit Gateway. + +## Usage + +The [examples](./examples) directory has example of using this module: + +* `ex01-vpc-tgw-attach` demonstrates how this module can be used to use a Transit Gateway to attach a private CDP VPC with a dedicated networking VPC. The [terraform-aws-vpc](../../../terraform-aws-vpc/README.md) module is also used as part of this example. + +The README and sample `terraform.tfvars.sample` describe how to use the example. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | > 1.3.0 | +| [aws](#requirement\_aws) | ~> 4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | ~> 4.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_autoscaling_attachment.proxy_asg_tg_attach](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_attachment) | resource | +| [aws_autoscaling_group.proxy_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | +| [aws_launch_template.proxy_lt](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_lb.proxy_lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb) | resource | +| [aws_lb_listener.proxy_lb_listener](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_listener) | resource | +| [aws_lb_target_group.proxy_tg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_target_group) | resource | +| [aws_route.vpc_tgw_route](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route) | resource | +| [aws_security_group.proxy_sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.proxy_egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.proxy_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.proxy_lb_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_ami.proxy_default_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_network_interface.proxy_lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/network_interface) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | +| [aws_route_table.proxy_rt](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/route_table) | data source | +| [aws_vpc.proxy_vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [lb\_subnet\_ids](#input\_lb\_subnet\_ids) | The IDs of the subnet for the Network Load Balancer | `list(any)` | n/a | yes | +| [network\_load\_balancer\_name](#input\_network\_load\_balancer\_name) | Name of Network Load Balancer for the Proxy. | `string` | n/a | yes | +| [proxy\_autoscaling\_group\_name](#input\_proxy\_autoscaling\_group\_name) | Name of Autoscaling Group for the Proxy VMs. | `string` | n/a | yes | +| [proxy\_aws\_keypair\_name](#input\_proxy\_aws\_keypair\_name) | SSH Keypair name for the proxy VM | `string` | n/a | yes | +| [proxy\_launch\_template\_name](#input\_proxy\_launch\_template\_name) | Name of Launch Template for the Proxy VMs. | `string` | n/a | yes | +| [proxy\_subnet\_ids](#input\_proxy\_subnet\_ids) | The IDs of the subnet where the proxy VMs will run | `list(any)` | n/a | yes | +| [target\_group\_proxy\_name](#input\_target\_group\_proxy\_name) | Name of Target Group for the Proxy. | `string` | n/a | yes | +| [vpc\_id](#input\_vpc\_id) | VPC ID for where the proxy VM will run | `string` | n/a | yes | +| [autoscaling\_group\_scaling](#input\_autoscaling\_group\_scaling) | Minimum, maximum and desired size of EC2 instance in the Auto Scaling Group. | 
object({
    min_size         = number
    max_size         = number
    desired_capacity = number
  })
| 
{
  "desired_capacity": 3,
  "max_size": 6,
  "min_size": 3
}
| no | +| [aws\_region](#input\_aws\_region) | AWS region, used in Proxy Whitelist configuration files. If not provided will perform lookup of aws\_region data source. | `string` | `null` | no | +| [cdp\_region](#input\_cdp\_region) | CDP Control Plane region, used in Proxy Whitelist configuration files. | `string` | `"us-west-1"` | no | +| [create\_proxy\_sg](#input\_create\_proxy\_sg) | Flag to specify if the Security Group for the proxy should be created. | `bool` | `true` | no | +| [egress\_rules](#input\_egress\_rules) | List of egress rules to create. Used only if create\_proxy\_sg is true | 
list(object({
    cidrs     = list(string)
    from_port = number
    to_port   = optional(number)
    protocol  = string
  }))
| 
[
  {
    "cidrs": [
      "0.0.0.0/0"
    ],
    "from_port": 0,
    "protocol": "all",
    "to_port": 0
  }
]
| no | +| [enable\_proxy\_public\_ip](#input\_enable\_proxy\_public\_ip) | Assign a public IP address to the Proxy VM | `bool` | `true` | no | +| [env\_tags](#input\_env\_tags) | Tags applied to provisioned resources | `map(any)` | `{}` | no | +| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules to create. Used only if create\_proxy\_sg is true | 
list(object({
    cidrs     = list(string)
    from_port = number
    to_port   = optional(number)
    protocol  = string
  }))
| `[]` | no | +| [proxy\_aws\_ami](#input\_proxy\_aws\_ami) | The AWS AMI to use for the proxy VM | `string` | `null` | no | +| [proxy\_aws\_instance\_type](#input\_proxy\_aws\_instance\_type) | The EC2 instance type to use for the proxy VM | `string` | `"t3.medium"` | no | +| [proxy\_launch\_template\_user\_data\_file](#input\_proxy\_launch\_template\_user\_data\_file) | Location of the AWS Launch Template user data script. If not specified the files/user-data-proxy.sh.tpl file accompanying the module is used. | `string` | `null` | no | +| [proxy\_port](#input\_proxy\_port) | Port number which the proxy and NLB listens | `number` | `3129` | no | +| [proxy\_security\_group\_id](#input\_proxy\_security\_group\_id) | ID for existing Security Group to be used for the proxy VM. Required when create\_proxy\_sg is false | `string` | `null` | no | +| [proxy\_security\_group\_name](#input\_proxy\_security\_group\_name) | Name of Proxy Security Group for CDP environment. Used only if create\_proxy\_sg is true. | `string` | `null` | no | +| [proxy\_whitelist\_file](#input\_proxy\_whitelist\_file) | Location of the Proxy Whitelist file. If not specified the files/squid-http-whitelist.txt.tpl file accompanying the module is used. | `string` | `null` | no | +| [route\_tables\_to\_update](#input\_route\_tables\_to\_update) | List of any route tables to update to point to the Network interface of the Proxy VM | 
list(object({
    route_tables           = list(string)
    destination_cidr_block = string
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [proxy\_lb\_arn](#output\_proxy\_lb\_arn) | ARN of the Proxy Load Balancer | +| [proxy\_lb\_dns\_name](#output\_proxy\_lb\_dns\_name) | DNS Name of the Proxy Load Balancer | +| [proxy\_port](#output\_proxy\_port) | Port where Proxy is running | + \ No newline at end of file diff --git a/modules/terraform-aws-proxy/data.tf b/modules/terraform-aws-proxy/data.tf new file mode 100644 index 0000000..8fde7ba --- /dev/null +++ b/modules/terraform-aws-proxy/data.tf @@ -0,0 +1,61 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Find AWS region +data "aws_region" "current" {} + +# Find AMI for default proxy VMs +data "aws_ami" "proxy_default_ami" { + + most_recent = true + + filter { + name = "name" + values = ["al2023-ami-2023*-x86_64"] + } + + owners = ["amazon"] +} + +# Find details of the VPC +data "aws_vpc" "proxy_vpc" { + id = var.vpc_id +} + +# Find the network interface for the load balancer +data "aws_network_interface" "proxy_lb" { + + for_each = { for k, v in var.lb_subnet_ids : k => v } + + filter { + name = "description" + values = ["ELB ${aws_lb.proxy_lb.arn_suffix}"] + } + + filter { + name = "subnet-id" + values = [each.value] + } +} + +# Find route table details +data "aws_route_table" "proxy_rt" { + + for_each = { + for k, v in local.route_tables_to_update : k => v + } + + route_table_id = each.value.route_table + +} \ No newline at end of file diff --git a/modules/terraform-aws-proxy/defaults.tf b/modules/terraform-aws-proxy/defaults.tf new file mode 100644 index 0000000..98521c7 --- /dev/null +++ b/modules/terraform-aws-proxy/defaults.tf @@ -0,0 +1,71 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + + # AWS region (derived from provider lookup unless overridden) + aws_region = coalesce(var.aws_region, data.aws_region.current.name) + + # Security Groups + proxy_security_group_id = (var.create_proxy_sg ? + aws_security_group.proxy_sg[0].id : var.proxy_security_group_id) + + # Proxy VM + proxy_aws_ami = coalesce(var.proxy_aws_ami, data.aws_ami.proxy_default_ami.id) + + # User data for Proxy VM (for squid proxy) + proxy_launch_template_user_data_file = coalesce(var.proxy_launch_template_user_data_file, "${path.module}/files/squid-user-data.sh.tpl") + + # Squid whitelist file + proxy_whitelist_file = coalesce(var.proxy_whitelist_file, "${path.module}/files/squid-whitelist.txt.tpl") + + # Local variables to determine route table to Internal NLB eni mapping + route_tables_to_update = flatten([ + for route in var.route_tables_to_update : + [ + for rt in route.route_tables : + { + route_table = rt + destination_cidr_block = route.destination_cidr_block + } + ] + ]) + + lb_eni_details = [ + for eni in data.aws_network_interface.proxy_lb : + { + eni_id = eni.id + az = eni.availability_zone + subnet_id = eni.subnet_id + } + ] + + # TODO: Explore better rt to eni mapping with the below + # route_table_details = [ + # for rt in data.aws_route_table.proxy_rt : + # { + # rt_id = rt.id + # subnet_ids = rt.associations[*].subnet_id + # } + # ] + + route_table_to_lb_eni_assoc = { + for k, v in data.aws_route_table.proxy_rt : v.id => { + # TODO: eni of same subnet assoc if possible otherwise the first eni_id in lb_eni_details + eni = local.lb_eni_details[0].eni_id + } + } + + +} \ No newline at end of file diff --git a/modules/terraform-aws-proxy/doc_fragments/header.md b/modules/terraform-aws-proxy/doc_fragments/header.md new file mode 100644 index 0000000..e1fba47 --- /dev/null +++ b/modules/terraform-aws-proxy/doc_fragments/header.md @@ -0,0 +1,11 @@ +# Terraform Module for AWS Transit Gateway + +This module contains resource files and example variable definition files for creation of AWS Transity Gateway (TGW) and attaching a specified list of VPCs via the TGW. This module also updates both the Transit Gateway and VPC route tables. This module can be used to assist in deploying Cloudera Data Platform (CDP) Public Cloud in a fully private networking configuration where a CDP VPC and Networking VPC are connected using the Transit Gateway. + +## Usage + +The [examples](./examples) directory has example of using this module: + +* `ex01-vpc-tgw-attach` demonstrates how this module can be used to use a Transit Gateway to attach a private CDP VPC with a dedicated networking VPC. The [terraform-aws-vpc](../../../terraform-aws-vpc/README.md) module is also used as part of this example. + +The README and sample `terraform.tfvars.sample` describe how to use the example. diff --git a/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/main.tf b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/main.tf new file mode 100644 index 0000000..838f407 --- /dev/null +++ b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/main.tf @@ -0,0 +1,72 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +provider "aws" { + profile = var.aws_profile + region = var.aws_region +} + +module "ex01_network_vpc" { + source = "../../../terraform-aws-vpc" + + cdp_vpc = false + vpc_name = "${var.name_prefix}-network-vpc" + vpc_cidr = "10.11.0.0/16" + enable_nat_gateway = false + + private_cidr_range = var.network_vpc_private_cidr_range + public_cidr_range = var.network_vpc_public_cidr_range + +} + +module "ex01_proxy" { + source = "../.." + + vpc_id = module.ex01_network_vpc.vpc_id + + proxy_security_group_name = "${var.name_prefix}-sg" + proxy_aws_keypair_name = var.aws_key_pair + + proxy_launch_template_name = "${var.name_prefix}-lt" + proxy_autoscaling_group_name = "${var.name_prefix}-asg" + proxy_subnet_ids = module.ex01_network_vpc.public_subnets + + network_load_balancer_name = "${var.name_prefix}-lb" + target_group_proxy_name = "${var.name_prefix}-tg" + lb_subnet_ids = module.ex01_network_vpc.private_subnets + + ingress_rules = [ + { + cidrs = module.ex01_network_vpc.vpc_cidr_blocks + from_port = 0 + to_port = 65535 + protocol = "tcp" + }, + { + cidrs = var.ingress_extra_cidrs + from_port = 22 + # to_port = + protocol = "tcp" + } + ] + + route_tables_to_update = [ + # Route all Internet traffic in Networking VPC to the proxy instance(s) + { + route_tables = module.ex01_network_vpc.private_route_tables + destination_cidr_block = "0.0.0.0/0" + } + ] + +} diff --git a/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/terraform.tfvars.sample b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/terraform.tfvars.sample new file mode 100644 index 0000000..1312280 --- /dev/null +++ b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/terraform.tfvars.sample @@ -0,0 +1,23 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global Settings ------- +name_prefix = "" + +# ------- Cloud Settings ------- +aws_region = "" # Change this to specify Cloud Provider region, e.g. eu-west-1 +aws_key_pair = "" # Change this with the name of a pre-existing AWS keypair, e.g. my-keypair + +# ------- Proxy settings ------- +ingress_extra_cidrs = "" # Any additional CIDRs the Proxy Security Groups for SSH access diff --git a/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/variables.tf b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/variables.tf new file mode 100644 index 0000000..94c333e --- /dev/null +++ b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/variables.tf @@ -0,0 +1,84 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "aws_profile" { + type = string + description = "Profile for AWS cloud credentials" + + # Profile is default unless explicitly specified + default = "default" +} + +variable "aws_region" { + type = string + description = "Region which Cloud resources will be created" +} + +variable "env_tags" { + type = map(any) + description = "Tags applied to provised resources" + + default = null +} + +variable "name_prefix" { + type = string + description = "Shorthand name to use when naming resources." +} + +# ------- VPC settings ------- +variable "cdp_vpc_private_cidr_range" { + type = number + description = "Size of each private subnet for CDP VPC." + + default = 19 +} + +variable "cdp_vpc_public_cidr_range" { + type = number + description = "Size of each public subnet for CDP VPC." + + default = 24 +} + +variable "network_vpc_private_cidr_range" { + type = number + description = "Size of each private subnet for Network VPC." + + default = 19 +} + +variable "network_vpc_public_cidr_range" { + type = number + description = "Size of each public subnet for Network VPC." + + default = 24 +} + +# ------- Proxy settings ------- +variable "ingress_extra_cidrs" { + type = list(string) + description = "List of extra ingress rules to create." + + default = [] +} + + +variable "aws_key_pair" { + type = string + + description = "Name of the Public SSH key for the CDP environment" + +} \ No newline at end of file diff --git a/modules/terraform-aws-proxy/files/squid-user-data.sh.tpl b/modules/terraform-aws-proxy/files/squid-user-data.sh.tpl new file mode 100644 index 0000000..e3e22fe --- /dev/null +++ b/modules/terraform-aws-proxy/files/squid-user-data.sh.tpl @@ -0,0 +1,145 @@ +#!/bin/bash + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##################################################### +# User Data Bash script to setup suid proxy on a +# Amazon Linux 2 EC2 instance. +##################################################### + +# Install +yum update -y +yum install -y squid + +# Create Squid whitelist file +echo "${whitelist_txt}" > /etc/squid/whitelist.txt + +# Generate self-signed cert +mkdir -p /etc/squid/ssl +openssl req \ + -x509 -new -sha256 -nodes \ + -newkey rsa:2048 -days 365 \ + -keyout /etc/squid/ssl/private.key \ + -out /etc/squid/ssl/cert.pem \ + -subj "/C=XX/ST=XX/L=squid/O=squid/CN=squid" + + +# Create Squid configuration file +cat > /etc/squid/squid.conf << EOF +# Working Config File for non-transparent proxy +# Recommended minimum configuration: +# +# Example rule allowing access from your local networks. +# Adapt to list your (internal) IP networks from where browsing +# should be allowed +acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) +acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) +acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) +acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) +acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines +acl SSL_ports port 443 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http + +# ACL for the whitelist +acl http-whitelist dstdomain "/etc/squid/whitelist.txt" + +# Deny access to URLs not in the whitelist +http_access allow http-whitelist + +http_port ${proxy_port} cert=/etc/squid/ssl/cert.pem key=/etc/squid/ssl/private.key +https_port ${proxy_port} cert=/etc/squid/ssl/cert.pem key=/etc/squid/ssl/private.key +ssl_bump bump all +sslcrtd_program /usr/lib/squid/ssl_crtd -s /var/lib/ssl_db -M 4MB +sslcrtd_children 8 startup=1 idle=1 + +# Deny access to all other URLs +http_access deny all + +# Recommended minimum Access Permission configuration: +# Deny requests to certain unsafe ports +http_access deny !Safe_ports + + +# Deny CONNECT to other than secure SSL ports +http_access deny CONNECT !SSL_ports + + +# Only allow cachemgr access from localhost +http_access allow localhost manager +http_access deny manager + + +# This default configuration only allows localhost requests because a more +# permissive Squid installation could introduce new attack vectors into the +# network by proxying external TCP connections to unprotected services. + + +http_access allow localhost + + +# The two deny rules below are unnecessary in this default configuration +# because they are followed by a "deny all" rule. However, they may become +# critically important when you start allowing external requests below them. +# Protect web applications running on the same server as Squid. They often +# assume that only local users can access them at "localhost" ports. +http_access deny to_localhost +# Protect cloud servers that provide local users with sensitive info about +# their server via certain well-known link-local (a.k.a. APIPA) addresses. +http_access deny to_linklocal + +# +# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS +# +#http_access allow whitelist_url +# For example, to allow access from your local networks, you may uncomment the +# following rule (and/or add rules that match your definition of "local"): +# http_access allow localnet +# And finally deny all other access to this proxy +#http_access deny all + + +http_access allow all +# Squid normally listens to port 3128 +# Squid normally listens to port 3128 +http_port 3128 +# Uncomment and adjust the following to add a disk cache directory. +#cache_dir ufs /var/spool/squid 100 16 256 +# Leave coredumps in the first cache dir +coredump_dir /var/spool/squid +# +# Add any of your own refresh_pattern entries above these. +# +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern . 0 20% 4320 +EOF + + +# Start and enable squid +systemctl enable squid +systemctl start squid \ No newline at end of file diff --git a/modules/terraform-aws-proxy/files/squid-whitelist.txt.tpl b/modules/terraform-aws-proxy/files/squid-whitelist.txt.tpl new file mode 100644 index 0000000..a736fee --- /dev/null +++ b/modules/terraform-aws-proxy/files/squid-whitelist.txt.tpl @@ -0,0 +1,37 @@ +# CDP Region specific endpoints +.v2.${cdp_region}.ccm.cdp.cloudera.com +dbusapi.${cdp_region}.sigma.altus.cloudera.com +api.${cdp_region}.cdp.cloudera.com +.s3.${cdp_region}.amazonaws.com +console.${cdp_region}.cdp.cloudera.com + +# AWS Region specific endpoints +.eks.${aws_region}.amazonaws.com +.autoscaling.${aws_region}.amazonaws.com +.cloudformation.${aws_region}.amazonaws.com + +# CDE deployment +cloudera.com +www.cloudera.com +truststore.pki.rds.amazonaws.com + +# CDW deployment +container.repo.cloudera.com +amazonlinux-2-repos-${aws_region}.s3.dualstack.${aws_region}.amazonaws.com + +# CDF Deployment +consoleauth.altus.cloudera.com +iamapi.${cdp_region}.altus.cloudera.com + +# Other endpoints used +prod-us-west-2-starport-layer-bucket.s3.us-west-2.amazonaws.com +s3-r-w.us-west-2.amazonaws.com +.execute-api.us-west-2.amazonaws.com +cloudera-dbus-prod.s3.amazonaws.com +archive.cloudera.com +cloudera-service-delivery-cache.s3.amazonaws.com +container.repository.cloudera.com +docker.repository.cloudera.com +pypi.org +raw.githubusercontent.com +github.com diff --git a/modules/terraform-aws-proxy/main.tf b/modules/terraform-aws-proxy/main.tf new file mode 100644 index 0000000..ea67d91 --- /dev/null +++ b/modules/terraform-aws-proxy/main.tf @@ -0,0 +1,187 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Networking - Security Group and Security Group rules ------- +# Security Group for proxy +resource "aws_security_group" "proxy_sg" { + + count = var.create_proxy_sg ? 1 : 0 + + vpc_id = var.vpc_id + name = var.proxy_security_group_name + description = var.proxy_security_group_name + tags = merge(var.env_tags, { Name = var.proxy_security_group_name }) +} + +# Create ingress SG rule +resource "aws_security_group_rule" "proxy_ingress" { + + for_each = { for k, v in var.ingress_rules : k => v + if var.create_proxy_sg + } + + security_group_id = aws_security_group.proxy_sg[0].id + type = "ingress" + description = "Ingress rules for ${var.proxy_security_group_name} Proxy Security Group" + + cidr_blocks = each.value.cidrs + from_port = each.value.from_port + to_port = coalesce(each.value.to_port, each.value.from_port) + protocol = each.value.protocol + +} + +# Access from NLB to Proxy VMs within the VPC +resource "aws_security_group_rule" "proxy_lb_ingress" { + + count = var.create_proxy_sg ? 1 : 0 + + security_group_id = aws_security_group.proxy_sg[0].id + type = "ingress" + description = "Allow traffic from NLB to Proxy VMs in ${var.proxy_security_group_name} Proxy Security Group" + + cidr_blocks = data.aws_vpc.proxy_vpc.cidr_block_associations[*].cidr_block + from_port = var.proxy_port + to_port = var.proxy_port + protocol = "TCP" +} + +# Create egress SG rule +resource "aws_security_group_rule" "proxy_egress" { + + for_each = { for k, v in var.egress_rules : k => v + if var.create_proxy_sg + } + + security_group_id = aws_security_group.proxy_sg[0].id + type = "egress" + description = "Egress rules for ${var.proxy_security_group_name} Proxy Security Group" + + cidr_blocks = each.value.cidrs #tfsec:ignore:aws-ec2-no-public-egress-sgr #tfsec:ignore:aws-vpc-no-public-egress-sgr + from_port = each.value.from_port + to_port = coalesce(each.value.to_port, each.value.from_port) + protocol = each.value.protocol + +} + +# ------- Proxy launch template and Auto-Scaling Groups ------- +resource "aws_launch_template" "proxy_lt" { + + name = var.proxy_launch_template_name + + image_id = local.proxy_aws_ami + instance_type = var.proxy_aws_instance_type + key_name = var.proxy_aws_keypair_name + + user_data = base64encode(templatefile(local.proxy_launch_template_user_data_file, { + proxy_port = var.proxy_port + whitelist_txt = templatefile(local.proxy_whitelist_file, { + aws_region = local.aws_region + cdp_region = var.cdp_region }) + })) + + network_interfaces { + associate_public_ip_address = var.enable_proxy_public_ip + security_groups = [local.proxy_security_group_id] + } + + metadata_options { + http_tokens = "required" + } + + tags = var.env_tags + +} + +resource "aws_autoscaling_group" "proxy_asg" { + name = var.proxy_autoscaling_group_name + min_size = var.autoscaling_group_scaling.min_size + max_size = var.autoscaling_group_scaling.max_size + desired_capacity = var.autoscaling_group_scaling.desired_capacity + + target_group_arns = [aws_lb_target_group.proxy_tg.arn] + + vpc_zone_identifier = var.proxy_subnet_ids + + launch_template { + id = aws_launch_template.proxy_lt.id + version = "$Latest" + } + + dynamic "tag" { + for_each = merge(var.env_tags, { Name = "${var.proxy_autoscaling_group_name}-proxy" }) + content { + key = tag.key + value = tag.value + propagate_at_launch = true + } + } + +} + +# ------- Internal Network Load Balancer ------- + +resource "aws_lb" "proxy_lb" { + name = var.network_load_balancer_name + internal = true + load_balancer_type = "network" + subnets = var.lb_subnet_ids + + tags = var.env_tags +} + +resource "aws_lb_listener" "proxy_lb_listener" { + load_balancer_arn = aws_lb.proxy_lb.arn + port = var.proxy_port + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.proxy_tg.arn + } + + tags = var.env_tags +} + +# Target groups are essentially the end point of the LB +resource "aws_lb_target_group" "proxy_tg" { + name = var.target_group_proxy_name + target_type = "instance" + port = var.proxy_port + protocol = "TCP" + vpc_id = var.vpc_id + + # TODO: Review health checks on the TG + + tags = var.env_tags +} + +# Now we have a target group we need to assign something to it. This is done through target group attachments +resource "aws_autoscaling_attachment" "proxy_asg_tg_attach" { + autoscaling_group_name = aws_autoscaling_group.proxy_asg.id + lb_target_group_arn = aws_lb_target_group.proxy_tg.arn +} + + +# ------- Route Table update ------- +# Update the route tables to point to eni of NLB +resource "aws_route" "vpc_tgw_route" { + for_each = { + for k, v in local.route_tables_to_update : k => v + } + + route_table_id = each.value.route_table + destination_cidr_block = each.value.destination_cidr_block + network_interface_id = local.route_table_to_lb_eni_assoc[each.value.route_table].eni +} diff --git a/modules/terraform-aws-proxy/outputs.tf b/modules/terraform-aws-proxy/outputs.tf new file mode 100644 index 0000000..9df92b9 --- /dev/null +++ b/modules/terraform-aws-proxy/outputs.tf @@ -0,0 +1,46 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DNS name of the NLB +output "proxy_lb_dns_name" { + value = aws_lb.proxy_lb.dns_name + + description = "DNS Name of the Proxy Load Balancer" +} + +output "proxy_lb_arn" { + value = aws_lb.proxy_lb.arn + + description = "ARN of the Proxy Load Balancer" +} + +# Proxy port +output "proxy_port" { + value = var.proxy_port + + description = "Port where Proxy is running" +} \ No newline at end of file diff --git a/modules/terraform-aws-proxy/provider.tf b/modules/terraform-aws-proxy/provider.tf new file mode 100644 index 0000000..d45eadd --- /dev/null +++ b/modules/terraform-aws-proxy/provider.tf @@ -0,0 +1,24 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + } + + required_version = "> 1.3.0" +} diff --git a/modules/terraform-aws-proxy/variables.tf b/modules/terraform-aws-proxy/variables.tf new file mode 100644 index 0000000..cd489f0 --- /dev/null +++ b/modules/terraform-aws-proxy/variables.tf @@ -0,0 +1,217 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "env_tags" { + type = map(any) + description = "Tags applied to provisioned resources" + + default = {} +} + +variable "vpc_id" { + type = string + description = "VPC ID for where the proxy VM will run" + +} + +variable "cdp_region" { + type = string + description = "CDP Control Plane region, used in Proxy Whitelist configuration files." + + default = "us-west-1" +} + +variable "aws_region" { + type = string + description = "AWS region, used in Proxy Whitelist configuration files. If not provided will perform lookup of aws_region data source." + + default = null +} + +# ------- Proxy SG ------- +variable "create_proxy_sg" { + type = bool + + description = "Flag to specify if the Security Group for the proxy should be created." + + default = true +} + +variable "proxy_security_group_name" { + type = string + + description = "Name of Proxy Security Group for CDP environment. Used only if create_proxy_sg is true." + + default = null +} + +variable "proxy_security_group_id" { + type = string + + description = "ID for existing Security Group to be used for the proxy VM. Required when create_proxy_sg is false" + + default = null +} + +variable "ingress_rules" { + description = "List of ingress rules to create. Used only if create_proxy_sg is true" + type = list(object({ + cidrs = list(string) + from_port = number + to_port = optional(number) + protocol = string + })) + default = [] +} + +variable "egress_rules" { + description = "List of egress rules to create. Used only if create_proxy_sg is true" + type = list(object({ + cidrs = list(string) + from_port = number + to_port = optional(number) + protocol = string + })) + default = [{ + cidrs = ["0.0.0.0/0"] + from_port = 0 + to_port = 0 + protocol = "all" + }] +} + +# ------- Proxy Settings ------- +variable "proxy_port" { + type = number + description = "Port number which the proxy and NLB listens" + + default = 3129 +} + +variable "proxy_launch_template_name" { + type = string + + description = "Name of Launch Template for the Proxy VMs." + +} + +variable "enable_proxy_public_ip" { + type = bool + + description = "Assign a public IP address to the Proxy VM" + + default = true +} + +variable "proxy_aws_ami" { + type = string + description = "The AWS AMI to use for the proxy VM" + + default = null +} + +variable "proxy_aws_instance_type" { + type = string + description = "The EC2 instance type to use for the proxy VM" + + default = "t3.medium" + +} + +variable "proxy_aws_keypair_name" { + type = string + + description = "SSH Keypair name for the proxy VM" + +} + +variable "proxy_launch_template_user_data_file" { + type = string + + description = "Location of the AWS Launch Template user data script. If not specified the files/user-data-proxy.sh.tpl file accompanying the module is used." + + default = null +} + +variable "proxy_whitelist_file" { + type = string + + description = "Location of the Proxy Whitelist file. If not specified the files/squid-http-whitelist.txt.tpl file accompanying the module is used." + + default = null +} + +variable "proxy_autoscaling_group_name" { + type = string + + description = "Name of Autoscaling Group for the Proxy VMs." + +} + +variable "autoscaling_group_scaling" { + type = object({ + min_size = number + max_size = number + desired_capacity = number + }) + + description = "Minimum, maximum and desired size of EC2 instance in the Auto Scaling Group." + + default = { + min_size = 3 + max_size = 6 + desired_capacity = 3 + } +} + +variable "proxy_subnet_ids" { + type = list(any) + + description = "The IDs of the subnet where the proxy VMs will run" + +} + +# ------- Internal Network Load Balancer ------- +variable "network_load_balancer_name" { + type = string + + description = "Name of Network Load Balancer for the Proxy." + +} + +variable "lb_subnet_ids" { + type = list(any) + + description = "The IDs of the subnet for the Network Load Balancer" + +} + +variable "target_group_proxy_name" { + type = string + + description = "Name of Target Group for the Proxy." + +} + +# ------- Route table updates ------- +variable "route_tables_to_update" { + description = "List of any route tables to update to point to the Network interface of the Proxy VM" + type = list(object({ + route_tables = list(string) + destination_cidr_block = string + })) + + default = [] +} \ No newline at end of file