Exclude subnets for node ip selection

Adds ability to configure subnets describing what ranges shall be
excluded from node ip selection. External subnet exclusions and internal
subnet exlusions are separately configureable.

add documentation for cloud config Nodes values

Signed-off-by: Tyler Schultz <[email protected]>
Co-authored-by: Aidan Obley <[email protected]>
Co-authored-by: Christian Ang <[email protected]>

docs/book/cloud_config.md

@@ -47,9 +47,17 @@ Here's the entire cloud config spec using example values:
 [Labels]
   region = k8s-region
   zone = k8s-zone
+
+[Nodes]
+  internal-network-subnet-cidr = "192.0.2.0/24"
+  external-network-subnet-cidr = "198.51.100.0/24"
+  internal-vm-network-name = "Internal K8s Traffic"
+  external-vm-network-name = "External/Outbound Traffic"
+  exclude-internal-network-subnet-cidr = "192.0.2.0/24,fe80::1/128"
+  exclude-external-network-subnet-cidr = "192.1.2.0/24,fe80::2/128"
 ```
 
-There are 3 sections in the cloud config file, let's break down the fields in each section:
+There are 4 sections in the cloud config file, let's break down the fields in each section:
 
 ### Global
 
@@ -171,6 +179,61 @@ on your Nodes and PersistentVolumes based on the value of the tags specified her
   zone = k8s-zone
 ```
 
+### Nodes
+
+The Nodes section defines the way that the Node IPs are selected from the
+addresses assigned to the Node in kube-api.
+
+Addresses in the optional `exclude-internal-network-subnet-cidr` and optional
+`exclude-external-network-subnet-cidr` are removed from consideration for any
+match before any selection happens.
+
+If provided, the `internal-network-subnet-cidr` and
+`external-network-subnet-cidr` matching will be attempted first. Addresses that
+fall within each of the provided CIDRs will be selected.
+
+If provided, and the subnet matching method does not select a matching address,
+the `internal-vm-network-name` and `external-vm-network-name` matching will be
+attempted. Addresses belonging to networks that match the name in vSphere will
+be selected.
+
+If these methods are unsuccessful at selecting an address, or if these other
+configurations were not provided, default selection will select the first
+address that is not a Localhost address.
+
+```bash
+[Nodes]
+  # If set, the vSphere cloud provider will select the first address that falls
+  # within the provided subnet and assign that value to the Internal network for
+  # the node.
+  internal-network-subnet-cidr = "192.0.2.0/24"
+
+  # If set, the vSphere cloud provider will select the first address that falls
+  # within the provided subnet and assign that value to the External network for
+  # the node.
+  external-network-subnet-cidr = "198.51.100.0/24"
+
+  # If set, the vSphere cloud provider will select the first address found in
+  # the VM network matching the provided name and assign that value to the
+  # Internal network for the node.
+  internal-vm-network-name = "Internal K8s Traffic"
+
+  # If set, the vSphere cloud provider will select the first address found in
+  # the VM network matching the provided name and assign that value to the
+  # External network for the node.
+  external-vm-network-name = "External/Outbound Traffic"
+
+  # If set, the vSphere cloud provider will never select addresses for the
+  # Internal network that fall within the provided subnet ranges. This
+  # configuration has the highest precedence. See notes above for details.
+  exclude-internal-network-subnet-cidr = "192.0.2.0/24,fe80::1/128"
+
+  # If set, the vSphere cloud provider will never select addresses for the
+  # External network that fall within the provided subnet ranges. This
+  # configuration has the highest precedence. See notes above for details.
+  exclude-external-network-subnet-cidr = "192.1.2.0/24,fe80::2/128"
+```
+
 ### Storing vCenter Credentials in a Kubernetes Secret
 
 ## FAQ

pkg/cloudprovider/vsphere/config/config_ini_legacy.go

@@ -35,10 +35,12 @@ func (cci *CPIConfigINI) CreateConfig() *CPIConfig {
 	cfg := &CPIConfig{
 		*cci.CommonConfigINI.CreateConfig(),
 		Nodes{
-			InternalNetworkSubnetCIDR: cci.Nodes.InternalNetworkSubnetCIDR,
-			ExternalNetworkSubnetCIDR: cci.Nodes.ExternalNetworkSubnetCIDR,
-			InternalVMNetworkName:     cci.Nodes.InternalVMNetworkName,
-			ExternalVMNetworkName:     cci.Nodes.ExternalVMNetworkName,
+			InternalNetworkSubnetCIDR:        cci.Nodes.InternalNetworkSubnetCIDR,
+			ExternalNetworkSubnetCIDR:        cci.Nodes.ExternalNetworkSubnetCIDR,
+			InternalVMNetworkName:            cci.Nodes.InternalVMNetworkName,
+			ExternalVMNetworkName:            cci.Nodes.ExternalVMNetworkName,
+			ExcludeInternalNetworkSubnetCIDR: cci.Nodes.ExcludeInternalNetworkSubnetCIDR,
+			ExcludeExternalNetworkSubnetCIDR: cci.Nodes.ExcludeExternalNetworkSubnetCIDR,
 		},
 	}
 

pkg/cloudprovider/vsphere/config/config_ini_legacy_test.go

@@ -52,6 +52,21 @@ internal-vm-network-name = "Internal K8s Traffic"
 external-vm-network-name = "External/Outbound Traffic"
 `
 
+const excludeSubnetINIConfig = `
+[Global]
+server = 0.0.0.0
+port = 443
+user = user
+password = password
+insecure-flag = true
+datacenters = us-west
+ca-file = /some/path/to/a/ca.pem
+
+[Nodes]
+exclude-internal-network-subnet-cidr = "192.0.2.0/24,fe80::1/128"
+exclude-external-network-subnet-cidr = "192.1.2.0/24,fe80::2/128"
+`
+
 func TestReadINIConfigSubnetCidr(t *testing.T) {
 	_, err := ReadCPIConfigINI(nil)
 	if err == nil {
@@ -94,3 +109,18 @@ func TestReadINIConfigNetworkName(t *testing.T) {
 		t.Errorf("incorrect internal vm network name: %s", cfg.Nodes.ExternalVMNetworkName)
 	}
 }
+
+func TestReadINIConfigExcludeSubnetCidr(t *testing.T) {
+	cfg, err := ReadCPIConfigINI([]byte(excludeSubnetINIConfig))
+	if err != nil {
+		t.Fatalf("Should succeed when a valid config is provided: %s", err)
+	}
+
+	if cfg.Nodes.ExcludeInternalNetworkSubnetCIDR != "192.0.2.0/24,fe80::1/128" {
+		t.Errorf("incorrect exclude internal network subnet cidrs: %s", cfg.Nodes.ExcludeInternalNetworkSubnetCIDR)
+	}
+
+	if cfg.Nodes.ExcludeExternalNetworkSubnetCIDR != "192.1.2.0/24,fe80::2/128" {
+		t.Errorf("incorrect exclude external network subnet cidrs: %s", cfg.Nodes.ExcludeExternalNetworkSubnetCIDR)
+	}
+}

pkg/cloudprovider/vsphere/config/config_yaml.go

@@ -36,10 +36,12 @@ func (ccy *CPIConfigYAML) CreateConfig() *CPIConfig {
 	cfg := &CPIConfig{
 		*ccy.CommonConfigYAML.CreateConfig(),
 		Nodes{
-			InternalNetworkSubnetCIDR: ccy.Nodes.InternalNetworkSubnetCIDR,
-			ExternalNetworkSubnetCIDR: ccy.Nodes.ExternalNetworkSubnetCIDR,
-			InternalVMNetworkName:     ccy.Nodes.InternalVMNetworkName,
-			ExternalVMNetworkName:     ccy.Nodes.ExternalVMNetworkName,
+			InternalNetworkSubnetCIDR:        ccy.Nodes.InternalNetworkSubnetCIDR,
+			ExternalNetworkSubnetCIDR:        ccy.Nodes.ExternalNetworkSubnetCIDR,
+			InternalVMNetworkName:            ccy.Nodes.InternalVMNetworkName,
+			ExternalVMNetworkName:            ccy.Nodes.ExternalVMNetworkName,
+			ExcludeInternalNetworkSubnetCIDR: ccy.Nodes.ExcludeInternalNetworkSubnetCIDR,
+			ExcludeExternalNetworkSubnetCIDR: ccy.Nodes.ExcludeExternalNetworkSubnetCIDR,
 		},
 	}
 

pkg/cloudprovider/vsphere/config/config_yaml_test.go

@@ -57,6 +57,22 @@ nodes:
   externalVmNetworkName: External/Outbound Traffic
 `
 
+const excludeSubnetCidrYAMLConfig = `
+global:
+  server: 0.0.0.0
+  port: 443
+  user: user
+  password: password
+  insecureFlag: true
+  datacenters:
+    - us-west
+  caFile: /some/path/to/a/ca.pem
+
+nodes:
+  excludeInternalNetworkSubnetCidr: "192.0.2.0/24,fe80::1/128"
+  excludeExternalNetworkSubnetCidr: "192.1.2.0/24,fe80::2/128"
+`
+
 func TestReadYAMLConfigSubnetCidr(t *testing.T) {
 	_, err := ReadCPIConfigYAML(nil)
 	if err == nil {
@@ -99,3 +115,18 @@ func TestReadYAMLConfigNetworkName(t *testing.T) {
 		t.Errorf("incorrect internal vm network name: %s", cfg.Nodes.ExternalVMNetworkName)
 	}
 }
+
+func TestReadYAMLConfigExcludeSubnetCidr(t *testing.T) {
+	cfg, err := ReadCPIConfigYAML([]byte(excludeSubnetCidrYAMLConfig))
+	if err != nil {
+		t.Fatalf("Should succeed when a valid config is provided: %s", err)
+	}
+
+	if cfg.Nodes.ExcludeInternalNetworkSubnetCIDR != "192.0.2.0/24,fe80::1/128" {
+		t.Errorf("incorrect exclude internal network subnet cidrs: %s", cfg.Nodes.ExcludeInternalNetworkSubnetCIDR)
+	}
+
+	if cfg.Nodes.ExcludeExternalNetworkSubnetCIDR != "192.1.2.0/24,fe80::2/128" {
+		t.Errorf("incorrect exclude external network subnet cidrs: %s", cfg.Nodes.ExcludeExternalNetworkSubnetCIDR)
+	}
+}

pkg/cloudprovider/vsphere/config/types_common.go

@@ -38,6 +38,11 @@ type Nodes struct {
 	// only have a single IP address assigned to it.
 	InternalVMNetworkName string
 	ExternalVMNetworkName string
+	// IP addresses in these subnet ranges will be excluded when selecting
+	// the IP address from the VirtualMachine's VM for use in the
+	// status.addresses fields.
+	ExcludeInternalNetworkSubnetCIDR string
+	ExcludeExternalNetworkSubnetCIDR string
 }
 
 // CPIConfig is used to read and store information (related only to the CPI) from the cloud configuration file

pkg/cloudprovider/vsphere/config/types_ini_legacy.go

@@ -37,6 +37,11 @@ type NodesINI struct {
 	// only have a single IP address assigned to it.
 	InternalVMNetworkName string `gcfg:"internal-vm-network-name"`
 	ExternalVMNetworkName string `gcfg:"external-vm-network-name"`
+	// IP addresses in these subnet ranges will be excluded when selecting
+	// the IP address from the VirtualMachine's VM for use in the
+	// status.addresses fields.
+	ExcludeInternalNetworkSubnetCIDR string `gcfg:"exclude-internal-network-subnet-cidr"`
+	ExcludeExternalNetworkSubnetCIDR string `gcfg:"exclude-external-network-subnet-cidr"`
 }
 
 // CPIConfigINI is the INI representation

pkg/cloudprovider/vsphere/config/types_yaml.go

@@ -41,6 +41,11 @@ type NodesYAML struct {
 	// only have a single IP address assigned to it.
 	InternalVMNetworkName string `yaml:"internalVmNetworkName"`
 	ExternalVMNetworkName string `yaml:"externalVmNetworkName"`
+	// IP addresses in these subnet ranges will be excluded when selecting
+	// the IP address from the VirtualMachine's VM for use in the
+	// status.addresses fields.
+	ExcludeInternalNetworkSubnetCIDR string `yaml:"excludeInternalNetworkSubnetCidr"`
+	ExcludeExternalNetworkSubnetCIDR string `yaml:"excludeExternalNetworkSubnetCidr"`
 }
 
 // CPIConfigYAML is the YAML representation

pkg/cloudprovider/vsphere/nodemanager.go

@@ -241,6 +241,8 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error {
 
 	var internalNetworkSubnet *net.IPNet
 	var externalNetworkSubnet *net.IPNet
+	var excludeInternalNetworkSubnets []*net.IPNet
+	var excludeExternalNetworkSubnets []*net.IPNet
 	var internalVMNetworkName string
 	var externalVMNetworkName string
 
@@ -257,6 +259,14 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error {
 				return err
 			}
 		}
+		excludeInternalNetworkSubnets, err = parseCIDRs(nm.cfg.Nodes.ExcludeInternalNetworkSubnetCIDR)
+		if err != nil {
+			return err
+		}
+		excludeExternalNetworkSubnets, err = parseCIDRs(nm.cfg.Nodes.ExcludeExternalNetworkSubnetCIDR)
+		if err != nil {
+			return err
+		}
 		internalVMNetworkName = nm.cfg.Nodes.InternalVMNetworkName
 		externalVMNetworkName = nm.cfg.Nodes.ExternalVMNetworkName
 	}
@@ -301,6 +311,8 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error {
 			ipFamily,
 			internalNetworkSubnet,
 			externalNetworkSubnet,
+			excludeInternalNetworkSubnets,
+			excludeExternalNetworkSubnets,
 			internalVMNetworkName,
 			externalVMNetworkName,
 		)
@@ -355,12 +367,17 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error {
 //
 // The returned ipAddrNetworkNames will match the given ipFamily.
 //
+// ipAddrNetworkNames that are contained in the excludeInternalNetworkSubnets
+// will never be returned as an internal address, and similarly addresses
+// contained in the exludedExternalNetworkSubnets will never be returned
+// as an external address - no matter the method of discovery described below.
+//
 // The returned ipAddrNetworkNames will be selected first by attempting to
 // match the given internalNetworkSubnet and externalNetworkSubnet. Subnet
 // matching has the highest precedence.
 //
 // If subnet matches are not found, or if subnets are not provided, then an
-// attempt is made to select ipAddrNetworkNames that match the givent network
+// attempt is made to select ipAddrNetworkNames that match the given network
 // names. Network name matching has the second highest precedence.
 //
 // If ipAddrNetworkNames are not found by subnet nor network name matching, then
@@ -370,32 +387,36 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error {
 // If either of these IPs cannot be discovered, nil will be returned instead.
 func discoverIPs(ipAddrNetworkNames []*ipAddrNetworkName, ipFamily string,
 	internalNetworkSubnet, externalNetworkSubnet *net.IPNet,
+	excludeInternalNetworkSubnets, excludeExternalNetworkSubnets []*net.IPNet,
 	internalVMNetworkName, externalVMNetworkName string) (internal *ipAddrNetworkName, external *ipAddrNetworkName) {
 
 	ipFamilyMatches := collectMatchesForIPFamily(ipAddrNetworkNames, ipFamily)
 
 	var discoveredInternal *ipAddrNetworkName
 	var discoveredExternal *ipAddrNetworkName
 
-	if len(ipFamilyMatches) != 0 {
-		discoveredInternal = findSubnetMatch(ipFamilyMatches, internalNetworkSubnet)
+	filteredInternalMatches := filterSubnetExclusions(ipFamilyMatches, excludeInternalNetworkSubnets)
+	filteredExternalMatches := filterSubnetExclusions(ipFamilyMatches, excludeExternalNetworkSubnets)
+
+	if len(filteredInternalMatches) > 0 || len(filteredExternalMatches) > 0 {
+		discoveredInternal = findSubnetMatch(filteredInternalMatches, internalNetworkSubnet)
 		if discoveredInternal != nil {
 			klog.V(2).Infof("Adding Internal IP by AddressMatching: %s", discoveredInternal.ipAddr)
 		}
-		discoveredExternal = findSubnetMatch(ipFamilyMatches, externalNetworkSubnet)
+		discoveredExternal = findSubnetMatch(filteredExternalMatches, externalNetworkSubnet)
 		if discoveredExternal != nil {
 			klog.V(2).Infof("Adding External IP by AddressMatching: %s", discoveredExternal.ipAddr)
 		}
 
 		if discoveredInternal == nil && internalVMNetworkName != "" {
-			discoveredInternal = findNetworkNameMatch(ipFamilyMatches, internalVMNetworkName)
+			discoveredInternal = findNetworkNameMatch(filteredInternalMatches, internalVMNetworkName)
 			if discoveredInternal != nil {
 				klog.V(2).Infof("Adding Internal IP by NetworkName: %s", discoveredInternal.ipAddr)
 			}
 		}
 
 		if discoveredExternal == nil && externalVMNetworkName != "" {
-			discoveredExternal = findNetworkNameMatch(ipFamilyMatches, externalVMNetworkName)
+			discoveredExternal = findNetworkNameMatch(filteredExternalMatches, externalVMNetworkName)
 			if discoveredExternal != nil {
 				klog.V(2).Infof("Adding External IP by NetworkName: %s", discoveredExternal.ipAddr)
 			}
@@ -405,10 +426,16 @@ func discoverIPs(ipAddrNetworkNames []*ipAddrNetworkName, ipFamily string,
 		// address selection behavior which is to only support a single address and
 		// return the first one found
 		if discoveredInternal == nil && discoveredExternal == nil {
-			klog.V(5).Info("Default address selection. Single NIC, Single IP Address")
-			klog.V(2).Infof("Adding IP: %s", ipFamilyMatches[0].ipAddr)
-			discoveredInternal = ipFamilyMatches[0]
-			discoveredExternal = ipFamilyMatches[0]
+			klog.V(5).Info("Default address selection.")
+			if len(filteredInternalMatches) > 0 {
+				klog.V(2).Infof("Adding Internal IP: %s", filteredInternalMatches[0].ipAddr)
+				discoveredInternal = filteredInternalMatches[0]
+			}
+
+			if len(filteredExternalMatches) > 0 {
+				klog.V(2).Infof("Adding External IP: %s", filteredExternalMatches[0].ipAddr)
+				discoveredExternal = filteredExternalMatches[0]
+			}
 		} else {
 			// At least one of the Internal or External addresses has been found.
 			// Minimally the Internal needs to exist for the node to function correctly.
@@ -438,6 +465,24 @@ func collectNonVNICDevices(guestNicInfos []types.GuestNicInfo) []types.GuestNicI
 	return toReturn
 }
 
+// parseCIDRs converts a comma delimited string of CIDRs to
+// a slice of IPNet pointers.
+func parseCIDRs(cidrsString string) ([]*net.IPNet, error) {
+	if cidrsString != "" {
+		cidrStringSlice := strings.Split(cidrsString, ",")
+		subnets := make([]*net.IPNet, len(cidrStringSlice))
+		for i, cidrString := range cidrStringSlice {
+			_, ipNet, err := net.ParseCIDR(cidrString)
+			if err != nil {
+				return nil, err
+			}
+			subnets[i] = ipNet
+		}
+		return subnets, nil
+	}
+	return nil, nil
+}
+
 // toIPAddrNetworkNames maps an array of GuestNicInfo to and array of *ipAddrNetworkName.
 func toIPAddrNetworkNames(guestNicInfos []types.GuestNicInfo) []*ipAddrNetworkName {
 	var candidates []*ipAddrNetworkName
@@ -536,6 +581,18 @@ func excludeLocalhostIPs(ipAddrNetworkNames []*ipAddrNetworkName) []*ipAddrNetwo
 	})
 }
 
+func filterSubnetExclusions(ipAddrNetworkNames []*ipAddrNetworkName, exlusionSubnets []*net.IPNet) []*ipAddrNetworkName {
+	return filter(ipAddrNetworkNames, func(i *ipAddrNetworkName) bool {
+		for _, exlusionSubnet := range exlusionSubnets {
+			if exlusionSubnet.Contains(i.ip()) {
+				klog.V(4).Infof("IP is excluded %q because it is contained in exlusion subnet %q", i.ipAddr, exlusionSubnet.String())
+				return false
+			}
+		}
+		return true
+	})
+}
+
 // GetNode gets the NodeInfo by UUID
 func (nm *NodeManager) GetNode(UUID string, node *pb.Node) error {
 	nodeInfo, err := nm.FindNodeInfo(UUID)