WIP: introduce VLAN for private networking

main.tf

@@ -8,6 +8,13 @@ module "sshkey" {
   cluster_name = var.cluster_name
 }
 
+module "network" {
+  source = "./modules/network"
+
+  project_id = var.metal_project_id
+  metal_metro      = var.metal_metro
+}
+
 module "bastion" {
   source     = "./modules/bastion"
   depends_on = [module.sshkey]
@@ -21,7 +28,7 @@ module "bastion" {
   cluster_basedomain   = var.cluster_basedomain
   ocp_version          = var.ocp_version
   ocp_version_zstream  = var.ocp_version_zstream
-  //depends              = [module.prepare_openshift.finished]
+  vlan = var.metal_vlan
 }
 
 module "dns_lb" {

modules/bastion/README.md

@@ -2,3 +2,5 @@
 # Nginx via Terraform on Equinix Metal
 
 This is a very basic [Terraform](http://terraform.io) template that will deploy [Nginx](http://nginx.org) on [Equinix Metal](https://metal.equinix.com) baremetal.
+
+Nginx will host IPXE and Ignition config files for each node defined.

modules/bastion/assets/cloudinit_centos_8.cfg.tpl

@@ -0,0 +1,59 @@
+## template: jinja
+#cloud-config
+package_update: true
+package_upgrade: true
+package_reboot_if_required: true
+write_files:
+- path: /etc/sysctl.d/10-ip-forwarding.conf
+  content: |
+    net.ipv4.ip_forward=1
+- path: /etc/iptables/rules.v4
+  content: |
+    # Generated by iptables-save v1.8.4 on Tue Aug 24 19:16:37 2021
+    *filter
+    :INPUT ACCEPT [0:0]
+    :FORWARD ACCEPT [0:0]
+    :OUTPUT ACCEPT [0:0]
+    COMMIT
+    # Completed on Tue Aug 24 19:16:37 2021
+    # Generated by iptables-save v1.8.4 on Tue Aug 24 19:16:37 2021
+    *nat
+    :PREROUTING ACCEPT [0:0]
+    :INPUT ACCEPT [0:0]
+    :OUTPUT ACCEPT [0:0]
+    :POSTROUTING ACCEPT [0:0]
+    -A POSTROUTING -o bond0 -j MASQUERADE
+    COMMIT
+    # Completed on Tue Aug 24 19:16:37 2021
+- path: /etc/iptables/rules.v6
+  content: |
+    # Generated by ip6tables-save v1.8.4 on Tue Aug 24 19:16:37 2021
+    *filter
+    :INPUT ACCEPT [0:0]
+    :FORWARD ACCEPT [0:0]
+    :OUTPUT ACCEPT [0:0]
+    COMMIT
+- path: /etc/network/interfaces
+  append: true
+  content: |
+
+    auto bond0.${metal_vlan_id}
+    iface bond0.${metal_vlan_id} inet static
+        pre-up sleep 5
+        address ${address}
+        netmask ${netmask}
+        vlan-raw-device bond0
+- path: /etc/dnsmasq.d/openshift.config
+  append: true
+  content: |
+    bind-interfaces
+    interface=bond0.${metal_vlan_id}
+    dhcp-range=${host_dhcp_start},${host_dhcp_end},${lease_time}
+packages:
+  - iptables
+  - iptables-services
+  - dnsmasq
+runcmd:
+- sysctl -p /etc/sysctl.d/10-ip-forwarding.conf
+- systemctl restart networking
+- systemctl restart dnsmasq

modules/bastion/assets/ipxe.tpl

@@ -8,5 +8,5 @@ set coreos-img $${coreos-url}/rhcos-$${release}.$${zstream}-$${arch}-live-rootfs
 set console console=ttyS1,115200n8
 
 kernel $${coreos-url}/rhcos-$${release}.$${zstream}-$${arch}-live-kernel-$${arch} initrd=main coreos.live.rootfs_url=$${coreos-img} coreos.inst.ignition_url=http://${ bastion_ip }:8080/${ node_type }-append.ign random.trust_cpu=on rd.luks.options=discard console=tty1 console=ttyS1,115200n8 coreos.inst.persistent-kargs="console=tty1 console=ttyS1,115200n8" coreos.inst.install_dev=/dev/sda
-initrd --name main $${coreos-url}/rhcos-$${release}.$${zstream}-$${arch}-live-initramfs.$${arch}.img
+initrd --name main $${coreos-url}/rhcos-$${release}.$${zstream}-$${arch}-live-initramfs.$${arch}.img 
 boot

modules/bastion/assets/user_data_centos_8.sh

@@ -16,4 +16,5 @@ systemctl enable nfs-server.service
 systemctl start nfs-server.service
 
 mkdir -p /mnt/nfs/ocp
-chmod -R 777 /mnt/nfs/ocp
+chmod -R 777 /mnt/nfs/ocp
+

modules/bastion/main.tf

@@ -16,7 +16,38 @@ resource "equinix_metal_device" "lb" {
   operating_system = var.operating_system
   billing_cycle    = var.billing_cycle
   project_id       = var.project_id
-  user_data        = file("${path.module}/assets/user_data_${var.operating_system}.sh")
+  user_data        = data.cloudinit_config.lb.rendered
+}
+
+
+resource "equinix_metal_port" "lb_bond0" {
+  port_id  = [for p in equinix_metal_device.lb.ports : p.id if p.name == "bond0"][0]
+  layer2   = false
+  bonded   = true
+  vlan_ids = [var.vlan]
+}
+
+
+data "cloudinit_config" "lb" {
+  gzip          = false
+  base64_encode = false
+
+  part {
+    filename = "cloud-config.cfg"
+    content  = templatefile("${path.module}/assets/cloudinit_${var.operating_system}.cfg.tpl",
+    {
+      metal_vlan_id   = 1000,
+      address         = cidrhost(var.cluster_subnet, 2),
+      netmask         = cidrnetmask(var.cluster_subnet),
+      host_dhcp_start = cidrhost(var.cluster_subnet, 3),
+      host_dhcp_end   = cidrhost(var.cluster_subnet, 15),
+      lease_time      = "infinite",
+    })
+  }
+  part {
+    filename = "user-data"
+    content  = file("${path.module}/assets/user_data_${var.operating_system}.sh")
+  }
 }
 
 resource "null_resource" "dircheck" {
@@ -72,7 +103,7 @@ resource "null_resource" "ipxe_files" {
 
     content = templatefile("${path.module}/assets/ipxe.tpl", {
       node_type           = each.value
-      bastion_ip          = equinix_metal_device.lb.access_public_ipv4
+      bastion_ip          = equinix_metal_device.lb.access_private_ipv4
       ocp_version         = var.ocp_version
       ocp_version_zstream = var.ocp_version_zstream
     })
@@ -107,7 +138,7 @@ resource "null_resource" "ignition_append_files" {
 
     content = templatefile("${path.module}/assets/ignition-append.json.tpl", {
       node_type          = each.value
-      bastion_ip         = equinix_metal_device.lb.access_public_ipv4
+      bastion_ip         = equinix_metal_device.lb.access_private_ipv4
       cluster_name       = var.cluster_name
       cluster_basedomain = var.cluster_basedomain
     })

modules/bastion/outputs.tf

@@ -2,6 +2,10 @@ output "lb_ip" {
   value = equinix_metal_device.lb.access_public_ipv4
 }
 
+output "node_ip" {
+  value = equinix_metal_device.lb.access_private_ipv4
+}
+
 output "finished" {
   depends_on = [
     null_resource.ipxe_files,

modules/bastion/variables.tf

@@ -42,3 +42,15 @@ variable "nodes" {
   type        = list(string)
   default     = ["bootstrap", "master", "worker"]
 }
+
+variable "vlan" {
+  description = "VLAN to use"
+  type        = number
+  default     = 1000
+}
+
+variable "cluster_subnet" {
+  description = "The subnet to use for the cluster"
+  type        = string
+  default     = "192.168.100.0/22"
+}

modules/network/main.tf

@@ -0,0 +1,60 @@
+locals {
+  vlan_id                 = var.create_vlan ? element(equinix_metal_vlan.openshift[*].id, 0) : element(data.equinix_metal_vlan.openshift[*].id, 0)
+  vxlan                   = var.create_vlan ? element(equinix_metal_vlan.openshift[*].vxlan, 0) : element(data.equinix_metal_vlan.openshift[*].vxlan, 0)
+  vrf_id                  = var.create_vrf ? element(equinix_metal_vrf.openshift[*].id, 0) : element(data.equinix_metal_vrf.openshift[*].id, 0)
+}
+
+
+# This generates a random suffix to avoid VRF name
+# collisions when multiple clusters are deployed to
+# an existing Metal project
+resource "random_string" "vrf_name_suffix" {
+  length  = 5
+  special = false
+}
+
+
+resource "equinix_metal_vlan" "openshift" {
+  count       = var.create_vlan ? 1 : 0
+  project_id  = var.project_id
+  description = var.metal_vlan_description
+  metro       = var.metal_metro
+}
+
+
+data "equinix_metal_vlan" "openshift" {
+  count      = var.create_vlan ? 0 : 1
+  project_id = var.project_id
+  vxlan      = var.metal_vlan_id
+}
+
+resource "equinix_metal_vrf" "openshift" {
+  count       = var.create_vrf ? 1 : 0
+  description = "VRF with ASN 65000 and a pool of address space that includes 192.168.100.0/25"
+  name        = "openshift-vrf-${random_string.vrf_name_suffix.result}"
+  metro       = var.metal_metro
+  local_asn   = "65000"
+  ip_ranges   = [var.cluster_subnet]
+  project_id  = var.project_id
+}
+
+data "equinix_metal_vrf" "openshift" {
+  count  = var.create_vrf ? 0 : 1
+  vrf_id = var.vrf_id
+}
+
+resource "equinix_metal_reserved_ip_block" "openshift" {
+  description = "Reserved IP block (${var.cluster_subnet}) taken from on of the ranges in the VRF's pool of address space."
+  project_id  = var.project_id
+  metro       = var.metal_metro
+  type        = "vrf"
+  vrf_id      = local.vrf_id
+  cidr        = split("/", var.cluster_subnet)[1]
+  network     = cidrhost(var.cluster_subnet, 0)
+}
+
+resource "equinix_metal_gateway" "gateway" {
+  project_id        = var.project_id
+  vlan_id           = local.vlan_id
+  ip_reservation_id = equinix_metal_reserved_ip_block.openshift.id
+}

modules/network/outputs.tf

@@ -0,0 +1,11 @@
+output "vrf_id" {
+value = local.vrf_id
+}
+
+output "vlan_id" {
+value = local.vlan_id
+}
+
+output "reserved_ip_block_id" {
+value = equinix_metal_reserved_ip_block.openshift.id
+}

modules/network/variables.tf

@@ -0,0 +1,28 @@
+variable "cluster_subnet" { default = "192.168.100.0/22" }
+variable "metal_metro" { default = "da" }
+variable "create_vrf" { default = true }
+variable "project_id" {}
+
+variable "create_vlan" {
+  type        = bool
+  default     = true
+  description = "Whether to create a new VLAN for this project."
+}
+variable "metal_vlan_id" {
+  type        = number
+  default     = null
+  description = "ID of the VLAN you wish to use."
+}
+
+variable "metal_vlan_description" {
+  type        = string
+  default     = "openshift-demo"
+  description = "Description to add to created VLAN."
+}
+
+
+variable "vrf_id" {
+  type        = string
+  default     = null
+  description = "ID of the VRF you wish to use."
+}

modules/network/versions.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_providers {
+    equinix = {
+      source = "equinix/equinix"
+      version = "~> 1.14"
+    }
+    null = {
+      source = "hashicorp/null"
+    }
+  }
+  required_version = ">= 1.0.0"
+  provider_meta "equinix" {
+    module_name = "equinix-metal-openshift-on-baremetal/bastion"
+  }
+}

modules/node/main.tf

@@ -8,5 +8,9 @@ resource "equinix_metal_device" "node" {
   count            = var.node_count
   billing_cycle    = "hourly"
   project_id       = var.project_id
+  ip_address {
+    type = "private_ipv4"
+    cidr = 30
+  }
 }
 

modules/node/outputs.tf

@@ -1,5 +1,5 @@
 output "node_ip" {
-  value = equinix_metal_device.node.*.access_public_ipv4
+  value = equinix_metal_device.node.*.access_private_ipv4
 }
 
 output "finished" {

variables.tf

@@ -32,6 +32,11 @@ variable "metal_project_id" {
   description = "Your Equinix Metal Project ID"
 }
 
+variable "metal_vlan" {
+  description = "Your Equinix Metal VLAN (ex. 1000)"
+  default = 1000
+}
+
 variable "bastion_operating_system" {
   description = "Your preferred bastion operating systems (RHEL or CentOS)"
   default     = "rhel_7"