Merge pull request #2441 from jhixson74/master_azure_bring_your_own_vnet

Azure: Allow customer provisioned virtual networks & subnets

data/data/azure/dns/dns.tf

@@ -12,7 +12,7 @@ resource "azureprivatedns_zone_virtual_network_link" "network" {
   name                  = "${var.cluster_id}-network-link"
   resource_group_name   = var.resource_group_name
   private_dns_zone_name = azureprivatedns_zone.private.name
-  virtual_network_id    = var.virtual_network
+  virtual_network_id    = var.virtual_network_id
 }
 
 resource "azureprivatedns_a_record" "apiint_internal" {

data/data/azure/dns/variables.tf

@@ -34,7 +34,7 @@ variable "internal_lb_ipaddress" {
   type        = string
 }
 
-variable "virtual_network" {
+variable "virtual_network_id" {
   description = "The ID for Virtual Network that will be linked to the Private DNS zone."
   type        = string
 }
@@ -54,4 +54,3 @@ variable "resource_group_name" {
   type        = string
   description = "Resource group for the deployment"
 }
-

data/data/azure/main.tf

@@ -30,7 +30,7 @@ module "bootstrap" {
   identity            = azurerm_user_assigned_identity.main.id
   cluster_id          = var.cluster_id
   ignition            = var.ignition_bootstrap
-  subnet_id           = module.vnet.public_subnet_id
+  subnet_id           = module.vnet.master_subnet_id
   elb_backend_pool_id = module.vnet.public_lb_backend_pool_id
   ilb_backend_pool_id = module.vnet.internal_lb_backend_pool_id
   tags                = local.tags
@@ -45,6 +45,12 @@ module "vnet" {
   cluster_id          = var.cluster_id
   region              = var.azure_region
   dns_label           = var.cluster_id
+
+  preexisting_network         = var.azure_preexisting_network
+  network_resource_group_name = var.azure_network_resource_group_name
+  virtual_network_name        = var.azure_virtual_network
+  master_subnet               = var.azure_control_plane_subnet
+  worker_subnet               = var.azure_compute_subnet
 }
 
 module "master" {
@@ -60,7 +66,7 @@ module "master" {
   external_lb_id      = module.vnet.public_lb_id
   elb_backend_pool_id = module.vnet.public_lb_backend_pool_id
   ilb_backend_pool_id = module.vnet.internal_lb_backend_pool_id
-  subnet_id           = module.vnet.public_subnet_id
+  subnet_id           = module.vnet.master_subnet_id
   instance_count      = var.master_count
   storage_account     = azurerm_storage_account.cluster
   os_volume_type      = var.azure_master_root_volume_type
@@ -72,7 +78,7 @@ module "dns" {
   cluster_domain                  = var.cluster_domain
   cluster_id                      = var.cluster_id
   base_domain                     = var.base_domain
-  virtual_network                 = module.vnet.network_id
+  virtual_network_id              = module.vnet.virtual_network_id
   external_lb_fqdn                = module.vnet.public_lb_pip_fqdn
   internal_lb_ipaddress           = module.vnet.internal_lb_ip_address
   resource_group_name             = azurerm_resource_group.main.name
@@ -93,6 +99,12 @@ resource "azurerm_resource_group" "main" {
   tags     = local.tags
 }
 
+data "azurerm_resource_group" "network" {
+  count = var.azure_preexisting_network ? 1 : 0
+
+  name = var.azure_network_resource_group_name
+}
+
 resource "azurerm_storage_account" "cluster" {
   name                     = "cluster${random_string.storage_suffix.result}"
   resource_group_name      = azurerm_resource_group.main.name
@@ -114,6 +126,14 @@ resource "azurerm_role_assignment" "main" {
   principal_id         = azurerm_user_assigned_identity.main.principal_id
 }
 
+resource "azurerm_role_assignment" "network" {
+  count = var.azure_preexisting_network ? 1 : 0
+
+  scope                = data.azurerm_resource_group.network[0].id
+  role_definition_name = "Contributor"
+  principal_id         = azurerm_user_assigned_identity.main.principal_id
+}
+
 # copy over the vhd to cluster resource group and create an image using that
 resource "azurerm_storage_container" "vhd" {
   name                 = "vhd"

data/data/azure/master/master.tf

@@ -5,7 +5,8 @@ locals {
 }
 
 resource "azurerm_network_interface" "master" {
-  count               = var.instance_count
+  count = var.instance_count
+
   name                = "${var.cluster_id}-master${count.index}-nic"
   location            = var.region
   resource_group_name = var.resource_group_name

data/data/azure/variables-azure.tf

@@ -76,8 +76,33 @@ variable "azure_tenant_id" {
   description = "The tenant ID that should be used to interact with Azure API"
 }
 
-
 variable "azure_master_availability_zones" {
   type        = list(string)
   description = "The availability zones in which to create the masters. The length of this list must match master_count."
 }
+
+variable "azure_preexisting_network" {
+  type        = bool
+  default     = false
+  description = "Specifies whether an existing network should be used or a new one created for installation."
+}
+
+variable "azure_network_resource_group_name" {
+  type        = string
+  description = "The name of the network resource group, either existing or to be created."
+}
+
+variable "azure_virtual_network" {
+  type        = string
+  description = "The name of the virtual network, either existing or to be created."
+}
+
+variable "azure_control_plane_subnet" {
+  type        = string
+  description = "The name of the subnet for the control plane, either existing or to be created."
+}
+
+variable "azure_compute_subnet" {
+  type        = string
+  description = "The name of the subnet for worker nodes, either existing or to be created"
+}

data/data/azure/vnet/common.tf

@@ -1,13 +1,37 @@
 # Canonical internal state definitions for this module.
 # read only: only locals and data source definitions allowed. No resources or module blocks in this file
 
+data "azurerm_subnet" "preexisting_master_subnet" {
+  count = var.preexisting_network ? 1 : 0
+
+  resource_group_name  = var.network_resource_group_name
+  virtual_network_name = var.virtual_network_name
+  name                 = var.master_subnet
+}
+
+data "azurerm_subnet" "preexisting_worker_subnet" {
+  count = var.preexisting_network ? 1 : 0
+
+  resource_group_name  = var.network_resource_group_name
+  virtual_network_name = var.virtual_network_name
+  name                 = var.worker_subnet
+}
+
+data "azurerm_virtual_network" "preexisting_virtual_network" {
+  count = var.preexisting_network ? 1 : 0
+
+  resource_group_name = var.network_resource_group_name
+  name                = var.virtual_network_name
+}
+
 // Only reference data sources which are guaranteed to exist at any time (above) in this locals{} block
 locals {
   master_subnet_cidr = cidrsubnet(var.vnet_cidr, 3, 0) #master subnet is a smaller subnet within the vnet. i.e from /21 to /24
-  node_subnet_cidr   = cidrsubnet(var.vnet_cidr, 3, 1) #node subnet is a smaller subnet within the vnet. i.e from /21 to /24
-}
+  worker_subnet_cidr = cidrsubnet(var.vnet_cidr, 3, 1) #node subnet is a smaller subnet within the vnet. i.e from /21 to /24
+
+  master_subnet_id = var.preexisting_network ? data.azurerm_subnet.preexisting_master_subnet[0].id : azurerm_subnet.master_subnet[0].id
+  worker_subnet_id = var.preexisting_network ? data.azurerm_subnet.preexisting_worker_subnet[0].id : azurerm_subnet.worker_subnet[0].id
 
-data "azurerm_virtual_network" "cluster_vnet" {
-  name                = azurerm_virtual_network.cluster_vnet.name
-  resource_group_name = var.resource_group_name
+  virtual_network    = var.preexisting_network ? data.azurerm_virtual_network.preexisting_virtual_network[0].name : azurerm_virtual_network.cluster_vnet[0].name
+  virtual_network_id = var.preexisting_network ? data.azurerm_virtual_network.preexisting_virtual_network[0].id : azurerm_virtual_network.cluster_vnet[0].id
 }

data/data/azure/vnet/internal-lb.tf

@@ -10,7 +10,7 @@ resource "azurerm_lb" "internal" {
 
   frontend_ip_configuration {
     name                          = local.internal_lb_frontend_ip_configuration_name
-    subnet_id                     = azurerm_subnet.master_subnet.id
+    subnet_id                     = local.master_subnet_id
     private_ip_address_allocation = "Dynamic"
   }
 }

data/data/azure/vnet/nsg.tf

@@ -5,7 +5,9 @@ resource "azurerm_network_security_group" "master" {
 }
 
 resource "azurerm_subnet_network_security_group_association" "master" {
-  subnet_id                 = azurerm_subnet.master_subnet.id
+  count = var.preexisting_network ? 0 : 1
+
+  subnet_id                 = azurerm_subnet.master_subnet[0].id
   network_security_group_id = azurerm_network_security_group.master.id
 }
 
@@ -16,7 +18,9 @@ resource "azurerm_network_security_group" "worker" {
 }
 
 resource "azurerm_subnet_network_security_group_association" "worker" {
-  subnet_id                 = azurerm_subnet.node_subnet.id
+  count = var.preexisting_network ? 0 : 1
+
+  subnet_id                 = azurerm_subnet.worker_subnet[0].id
   network_security_group_id = azurerm_network_security_group.worker.id
 }
 

data/data/azure/vnet/outputs.tf

@@ -2,14 +2,6 @@ output "cluster-pip" {
   value = azurerm_public_ip.cluster_public_ip.ip_address
 }
 
-output "network_id" {
-  value = data.azurerm_virtual_network.cluster_vnet.id
-}
-
-output "public_subnet_id" {
-  value = azurerm_subnet.master_subnet.id
-}
-
 output "public_lb_backend_pool_id" {
   value = azurerm_lb_backend_address_pool.master_public_lb_pool.id
 }
@@ -33,3 +25,15 @@ output "internal_lb_ip_address" {
 output "master_nsg_name" {
   value = azurerm_network_security_group.master.name
 }
+
+output "virtual_network_id" {
+  value = local.virtual_network_id
+}
+
+output "master_subnet_id" {
+  value = local.master_subnet_id
+}
+
+output "worker_subnet_id" {
+  value = local.worker_subnet_id
+}

data/data/azure/vnet/variables.tf

@@ -36,3 +36,29 @@ variable "dns_label" {
   type        = string
   description = "The label used to build the dns name. i.e. <label>.<region>.cloudapp.azure.com"
 }
+
+variable "preexisting_network" {
+  type        = bool
+  description = "This value determines if a vnet already exists or not. If true, then will not create a new vnet, subnet, or nsg's"
+  default     = false
+}
+
+variable "network_resource_group_name" {
+  type        = string
+  description = "This is the name of the network resource group for new or existing network resources"
+}
+
+variable "virtual_network_name" {
+  type        = string
+  description = "This is the name of the virtual network, new or existing"
+}
+
+variable "master_subnet" {
+  type        = string
+  description = "This is the name of the subnet used for the control plane, new or existing"
+}
+
+variable "worker_subnet" {
+  type        = string
+  description = "This is the name of the subnet used for the compute nodes, new or existing"
+}

data/data/azure/vnet/vnet.tf

@@ -1,5 +1,7 @@
 resource "azurerm_virtual_network" "cluster_vnet" {
-  name                = "${var.cluster_id}-vnet"
+  count = var.preexisting_network ? 0 : 1
+
+  name                = var.virtual_network_name
   resource_group_name = var.resource_group_name
   location            = var.region
   address_space       = [var.vnet_cidr]
@@ -12,16 +14,19 @@ resource "azurerm_route_table" "route_table" {
 }
 
 resource "azurerm_subnet" "master_subnet" {
+  count = var.preexisting_network ? 0 : 1
+
   resource_group_name  = var.resource_group_name
   address_prefix       = local.master_subnet_cidr
-  virtual_network_name = data.azurerm_virtual_network.cluster_vnet.name
-  name                 = "${var.cluster_id}-master-subnet"
+  virtual_network_name = local.virtual_network
+  name                 = var.master_subnet
 }
 
-resource "azurerm_subnet" "node_subnet" {
+resource "azurerm_subnet" "worker_subnet" {
+  count = var.preexisting_network ? 0 : 1
+
   resource_group_name  = var.resource_group_name
-  address_prefix       = local.node_subnet_cidr
-  virtual_network_name = data.azurerm_virtual_network.cluster_vnet.name
-  name                 = "${var.cluster_id}-worker-subnet"
+  address_prefix       = local.worker_subnet_cidr
+  virtual_network_name = local.virtual_network
+  name                 = var.worker_subnet
 }
-

pkg/asset/cluster/tfvars.go

@@ -176,11 +176,24 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 		for i, m := range masters {
 			masterConfigs[i] = m.Spec.ProviderSpec.Value.Object.(*azureprovider.AzureMachineProviderSpec)
 		}
+		workers, err := workersAsset.MachineSets()
+		if err != nil {
+			return err
+		}
+		workerConfigs := make([]*azureprovider.AzureMachineProviderSpec, len(masters))
+		for i, w := range workers {
+			workerConfigs[i] = w.Spec.Template.Spec.ProviderSpec.Value.Object.(*azureprovider.AzureMachineProviderSpec)
+		}
+		preexistingnetwork := installConfig.Config.Azure.VirtualNetwork != ""
 		data, err := azuretfvars.TFVars(
-			auth,
-			installConfig.Config.Azure.BaseDomainResourceGroupName,
-			string(*rhcosImage),
-			masterConfigs,
+			azuretfvars.TFVarsSources{
+				Auth: auth,
+				BaseDomainResourceGroupName: installConfig.Config.Azure.BaseDomainResourceGroupName,
+				MasterConfigs:               masterConfigs,
+				WorkerConfigs:               workerConfigs,
+				ImageURL:                    string(*rhcosImage),
+				PreexistingNetwork:          preexistingnetwork,
+			},
 		)
 		if err != nil {
 			return errors.Wrapf(err, "failed to get %s Terraform variables", platform)

pkg/asset/machines/azure/machines.go

@@ -84,6 +84,12 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 	if len(mpool.Zones) > 0 && azIdx != nil {
 		az = &mpool.Zones[*azIdx]
 	}
+
+	networkResourceGroup, virtualNetwork, subnet, err := getNetworkInfo(platform, clusterID, role)
+	if err != nil {
+		return nil, err
+	}
+
 	return &azureprovider.AzureMachineProviderSpec{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "azureproviderconfig.openshift.io/v1beta1",
@@ -103,15 +109,31 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 				StorageAccountType: "Premium_LRS",
 			},
 		},
-		Zone:            az,
-		Subnet:          fmt.Sprintf("%s-%s-subnet", clusterID, role),
-		ManagedIdentity: fmt.Sprintf("%s-identity", clusterID),
-		Vnet:            fmt.Sprintf("%s-vnet", clusterID),
-		ResourceGroup:   fmt.Sprintf("%s-rg", clusterID),
+		Zone:                 az,
+		Subnet:               subnet,
+		ManagedIdentity:      fmt.Sprintf("%s-identity", clusterID),
+		Vnet:                 virtualNetwork,
+		ResourceGroup:        fmt.Sprintf("%s-rg", clusterID),
+		NetworkResourceGroup: networkResourceGroup,
 	}, nil
 }
 
 // ConfigMasters sets the PublicIP flag and assigns a set of load balancers to the given machines
 func ConfigMasters(machines []machineapi.Machine, clusterID string) {
 	//TODO
 }
+
+func getNetworkInfo(platform *azure.Platform, clusterID, role string) (string, string, string, error) {
+	if platform.VirtualNetwork == "" {
+		return fmt.Sprintf("%s-rg", clusterID), fmt.Sprintf("%s-vnet", clusterID), fmt.Sprintf("%s-%s-subnet", clusterID, role), nil
+	}
+
+	switch role {
+	case "worker":
+		return platform.NetworkResourceGroupName, platform.VirtualNetwork, platform.ComputeSubnet, nil
+	case "master":
+		return platform.NetworkResourceGroupName, platform.VirtualNetwork, platform.ControlPlaneSubnet, nil
+	default:
+		return "", "", "", fmt.Errorf("unrecognized machine role %s", role)
+	}
+}