Merge pull request #567 from tylerschultz/backport-node-ip-exclusion

Backport node ip exclusion to 1.22 branch

docs/book/cloud_config.md

@@ -47,9 +47,17 @@ Here's the entire cloud config spec using example values:
 [Labels]
   region = k8s-region
   zone = k8s-zone
+
+[Nodes]
+  internal-network-subnet-cidr = "192.0.2.0/24"
+  external-network-subnet-cidr = "198.51.100.0/24"
+  internal-vm-network-name = "Internal K8s Traffic"
+  external-vm-network-name = "External/Outbound Traffic"
+  exclude-internal-network-subnet-cidr = "192.0.2.0/24,fe80::1/128"
+  exclude-external-network-subnet-cidr = "192.1.2.0/24,fe80::2/128"
 ```
 
-There are 3 sections in the cloud config file, let's break down the fields in each section:
+There are 4 sections in the cloud config file, let's break down the fields in each section:
 
 ### Global
 
@@ -171,6 +179,61 @@ on your Nodes and PersistentVolumes based on the value of the tags specified her
   zone = k8s-zone
 ```
 
+### Nodes
+
+The Nodes section defines the way that the Node IPs are selected from the
+addresses assigned to the Node in kube-api.
+
+Addresses in the optional `exclude-internal-network-subnet-cidr` and optional
+`exclude-external-network-subnet-cidr` are removed from consideration for any
+match before any selection happens.
+
+If provided, the `internal-network-subnet-cidr` and
+`external-network-subnet-cidr` matching will be attempted first. Addresses that
+fall within each of the provided CIDRs will be selected.
+
+If provided, and the subnet matching method does not select a matching address,
+the `internal-vm-network-name` and `external-vm-network-name` matching will be
+attempted. Addresses belonging to networks that match the name in vSphere will
+be selected.
+
+If these methods are unsuccessful at selecting an address, or if these other
+configurations were not provided, default selection will select the first
+address that is not a Localhost address.
+
+```bash
+[Nodes]
+  # If set, the vSphere cloud provider will select the first address that falls
+  # within the provided subnet and assign that value to the Internal network for
+  # the node.
+  internal-network-subnet-cidr = "192.0.2.0/24"
+
+  # If set, the vSphere cloud provider will select the first address that falls
+  # within the provided subnet and assign that value to the External network for
+  # the node.
+  external-network-subnet-cidr = "198.51.100.0/24"
+
+  # If set, the vSphere cloud provider will select the first address found in
+  # the VM network matching the provided name and assign that value to the
+  # Internal network for the node.
+  internal-vm-network-name = "Internal K8s Traffic"
+
+  # If set, the vSphere cloud provider will select the first address found in
+  # the VM network matching the provided name and assign that value to the
+  # External network for the node.
+  external-vm-network-name = "External/Outbound Traffic"
+
+  # If set, the vSphere cloud provider will never select addresses for the
+  # Internal network that fall within the provided subnet ranges. This
+  # configuration has the highest precedence. See notes above for details.
+  exclude-internal-network-subnet-cidr = "192.0.2.0/24,fe80::1/128"
+
+  # If set, the vSphere cloud provider will never select addresses for the
+  # External network that fall within the provided subnet ranges. This
+  # configuration has the highest precedence. See notes above for details.
+  exclude-external-network-subnet-cidr = "192.1.2.0/24,fe80::2/128"
+```
+
 ### Storing vCenter Credentials in a Kubernetes Secret
 
 ## FAQ

pkg/cloudprovider/vsphere/config/config_ini_legacy.go

@@ -35,10 +35,12 @@ func (cci *CPIConfigINI) CreateConfig() *CPIConfig {
 	cfg := &CPIConfig{
 		*cci.CommonConfigINI.CreateConfig(),
 		Nodes{
-			InternalNetworkSubnetCIDR: cci.Nodes.InternalNetworkSubnetCIDR,
-			ExternalNetworkSubnetCIDR: cci.Nodes.ExternalNetworkSubnetCIDR,
-			InternalVMNetworkName:     cci.Nodes.InternalVMNetworkName,
-			ExternalVMNetworkName:     cci.Nodes.ExternalVMNetworkName,
+			InternalNetworkSubnetCIDR:        cci.Nodes.InternalNetworkSubnetCIDR,
+			ExternalNetworkSubnetCIDR:        cci.Nodes.ExternalNetworkSubnetCIDR,
+			InternalVMNetworkName:            cci.Nodes.InternalVMNetworkName,
+			ExternalVMNetworkName:            cci.Nodes.ExternalVMNetworkName,
+			ExcludeInternalNetworkSubnetCIDR: cci.Nodes.ExcludeInternalNetworkSubnetCIDR,
+			ExcludeExternalNetworkSubnetCIDR: cci.Nodes.ExcludeExternalNetworkSubnetCIDR,
 		},
 	}
 

pkg/cloudprovider/vsphere/config/config_ini_legacy_test.go

@@ -52,6 +52,21 @@ internal-vm-network-name = "Internal K8s Traffic"
 external-vm-network-name = "External/Outbound Traffic"
 `
 
+const excludeSubnetINIConfig = `
+[Global]
+server = 0.0.0.0
+port = 443
+user = user
+password = password
+insecure-flag = true
+datacenters = us-west
+ca-file = /some/path/to/a/ca.pem
+
+[Nodes]
+exclude-internal-network-subnet-cidr = "192.0.2.0/24,fe80::1/128"
+exclude-external-network-subnet-cidr = "192.1.2.0/24,fe80::2/128"
+`
+
 func TestReadINIConfigSubnetCidr(t *testing.T) {
 	_, err := ReadCPIConfigINI(nil)
 	if err == nil {
@@ -94,3 +109,18 @@ func TestReadINIConfigNetworkName(t *testing.T) {
 		t.Errorf("incorrect internal vm network name: %s", cfg.Nodes.ExternalVMNetworkName)
 	}
 }
+
+func TestReadINIConfigExcludeSubnetCidr(t *testing.T) {
+	cfg, err := ReadCPIConfigINI([]byte(excludeSubnetINIConfig))
+	if err != nil {
+		t.Fatalf("Should succeed when a valid config is provided: %s", err)
+	}
+
+	if cfg.Nodes.ExcludeInternalNetworkSubnetCIDR != "192.0.2.0/24,fe80::1/128" {
+		t.Errorf("incorrect exclude internal network subnet cidrs: %s", cfg.Nodes.ExcludeInternalNetworkSubnetCIDR)
+	}
+
+	if cfg.Nodes.ExcludeExternalNetworkSubnetCIDR != "192.1.2.0/24,fe80::2/128" {
+		t.Errorf("incorrect exclude external network subnet cidrs: %s", cfg.Nodes.ExcludeExternalNetworkSubnetCIDR)
+	}
+}

pkg/cloudprovider/vsphere/config/config_yaml.go

@@ -36,10 +36,12 @@ func (ccy *CPIConfigYAML) CreateConfig() *CPIConfig {
 	cfg := &CPIConfig{
 		*ccy.CommonConfigYAML.CreateConfig(),
 		Nodes{
-			InternalNetworkSubnetCIDR: ccy.Nodes.InternalNetworkSubnetCIDR,
-			ExternalNetworkSubnetCIDR: ccy.Nodes.ExternalNetworkSubnetCIDR,
-			InternalVMNetworkName:     ccy.Nodes.InternalVMNetworkName,
-			ExternalVMNetworkName:     ccy.Nodes.ExternalVMNetworkName,
+			InternalNetworkSubnetCIDR:        ccy.Nodes.InternalNetworkSubnetCIDR,
+			ExternalNetworkSubnetCIDR:        ccy.Nodes.ExternalNetworkSubnetCIDR,
+			InternalVMNetworkName:            ccy.Nodes.InternalVMNetworkName,
+			ExternalVMNetworkName:            ccy.Nodes.ExternalVMNetworkName,
+			ExcludeInternalNetworkSubnetCIDR: ccy.Nodes.ExcludeInternalNetworkSubnetCIDR,
+			ExcludeExternalNetworkSubnetCIDR: ccy.Nodes.ExcludeExternalNetworkSubnetCIDR,
 		},
 	}
 

pkg/cloudprovider/vsphere/config/config_yaml_test.go

@@ -57,6 +57,22 @@ nodes:
   externalVmNetworkName: External/Outbound Traffic
 `
 
+const excludeSubnetCidrYAMLConfig = `
+global:
+  server: 0.0.0.0
+  port: 443
+  user: user
+  password: password
+  insecureFlag: true
+  datacenters:
+    - us-west
+  caFile: /some/path/to/a/ca.pem
+
+nodes:
+  excludeInternalNetworkSubnetCidr: "192.0.2.0/24,fe80::1/128"
+  excludeExternalNetworkSubnetCidr: "192.1.2.0/24,fe80::2/128"
+`
+
 func TestReadYAMLConfigSubnetCidr(t *testing.T) {
 	_, err := ReadCPIConfigYAML(nil)
 	if err == nil {
@@ -99,3 +115,18 @@ func TestReadYAMLConfigNetworkName(t *testing.T) {
 		t.Errorf("incorrect internal vm network name: %s", cfg.Nodes.ExternalVMNetworkName)
 	}
 }
+
+func TestReadYAMLConfigExcludeSubnetCidr(t *testing.T) {
+	cfg, err := ReadCPIConfigYAML([]byte(excludeSubnetCidrYAMLConfig))
+	if err != nil {
+		t.Fatalf("Should succeed when a valid config is provided: %s", err)
+	}
+
+	if cfg.Nodes.ExcludeInternalNetworkSubnetCIDR != "192.0.2.0/24,fe80::1/128" {
+		t.Errorf("incorrect exclude internal network subnet cidrs: %s", cfg.Nodes.ExcludeInternalNetworkSubnetCIDR)
+	}
+
+	if cfg.Nodes.ExcludeExternalNetworkSubnetCIDR != "192.1.2.0/24,fe80::2/128" {
+		t.Errorf("incorrect exclude external network subnet cidrs: %s", cfg.Nodes.ExcludeExternalNetworkSubnetCIDR)
+	}
+}

pkg/cloudprovider/vsphere/config/types_common.go

@@ -38,6 +38,11 @@ type Nodes struct {
 	// only have a single IP address assigned to it.
 	InternalVMNetworkName string
 	ExternalVMNetworkName string
+	// IP addresses in these subnet ranges will be excluded when selecting
+	// the IP address from the VirtualMachine's VM for use in the
+	// status.addresses fields.
+	ExcludeInternalNetworkSubnetCIDR string
+	ExcludeExternalNetworkSubnetCIDR string
 }
 
 // CPIConfig is used to read and store information (related only to the CPI) from the cloud configuration file

pkg/cloudprovider/vsphere/config/types_ini_legacy.go

@@ -37,6 +37,11 @@ type NodesINI struct {
 	// only have a single IP address assigned to it.
 	InternalVMNetworkName string `gcfg:"internal-vm-network-name"`
 	ExternalVMNetworkName string `gcfg:"external-vm-network-name"`
+	// IP addresses in these subnet ranges will be excluded when selecting
+	// the IP address from the VirtualMachine's VM for use in the
+	// status.addresses fields.
+	ExcludeInternalNetworkSubnetCIDR string `gcfg:"exclude-internal-network-subnet-cidr"`
+	ExcludeExternalNetworkSubnetCIDR string `gcfg:"exclude-external-network-subnet-cidr"`
 }
 
 // CPIConfigINI is the INI representation

pkg/cloudprovider/vsphere/config/types_yaml.go

@@ -41,6 +41,11 @@ type NodesYAML struct {
 	// only have a single IP address assigned to it.
 	InternalVMNetworkName string `yaml:"internalVmNetworkName"`
 	ExternalVMNetworkName string `yaml:"externalVmNetworkName"`
+	// IP addresses in these subnet ranges will be excluded when selecting
+	// the IP address from the VirtualMachine's VM for use in the
+	// status.addresses fields.
+	ExcludeInternalNetworkSubnetCIDR string `yaml:"excludeInternalNetworkSubnetCidr"`
+	ExcludeExternalNetworkSubnetCIDR string `yaml:"excludeExternalNetworkSubnetCidr"`
 }
 
 // CPIConfigYAML is the YAML representation

pkg/cloudprovider/vsphere/nodemanager.go

@@ -241,6 +241,8 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error {
 
 	var internalNetworkSubnet *net.IPNet
 	var externalNetworkSubnet *net.IPNet
+	var excludeInternalNetworkSubnets []*net.IPNet
+	var excludeExternalNetworkSubnets []*net.IPNet
 	var internalVMNetworkName string
 	var externalVMNetworkName string
 
@@ -257,6 +259,14 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error {
 				return err
 			}
 		}
+		excludeInternalNetworkSubnets, err = parseCIDRs(nm.cfg.Nodes.ExcludeInternalNetworkSubnetCIDR)
+		if err != nil {
+			return err
+		}
+		excludeExternalNetworkSubnets, err = parseCIDRs(nm.cfg.Nodes.ExcludeExternalNetworkSubnetCIDR)
+		if err != nil {
+			return err
+		}
 		internalVMNetworkName = nm.cfg.Nodes.InternalVMNetworkName
 		externalVMNetworkName = nm.cfg.Nodes.ExternalVMNetworkName
 	}
@@ -301,6 +311,8 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error {
 			ipFamily,
 			internalNetworkSubnet,
 			externalNetworkSubnet,
+			excludeInternalNetworkSubnets,
+			excludeExternalNetworkSubnets,
 			internalVMNetworkName,
 			externalVMNetworkName,
 		)
@@ -355,12 +367,17 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error {
 //
 // The returned ipAddrNetworkNames will match the given ipFamily.
 //
+// ipAddrNetworkNames that are contained in the excludeInternalNetworkSubnets
+// will never be returned as an internal address, and similarly addresses
+// contained in the exludedExternalNetworkSubnets will never be returned
+// as an external address - no matter the method of discovery described below.
+//
 // The returned ipAddrNetworkNames will be selected first by attempting to
 // match the given internalNetworkSubnet and externalNetworkSubnet. Subnet
 // matching has the highest precedence.
 //
 // If subnet matches are not found, or if subnets are not provided, then an
-// attempt is made to select ipAddrNetworkNames that match the givent network
+// attempt is made to select ipAddrNetworkNames that match the given network
 // names. Network name matching has the second highest precedence.
 //
 // If ipAddrNetworkNames are not found by subnet nor network name matching, then
@@ -370,32 +387,36 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error {
 // If either of these IPs cannot be discovered, nil will be returned instead.
 func discoverIPs(ipAddrNetworkNames []*ipAddrNetworkName, ipFamily string,
 	internalNetworkSubnet, externalNetworkSubnet *net.IPNet,
+	excludeInternalNetworkSubnets, excludeExternalNetworkSubnets []*net.IPNet,
 	internalVMNetworkName, externalVMNetworkName string) (internal *ipAddrNetworkName, external *ipAddrNetworkName) {
 
 	ipFamilyMatches := collectMatchesForIPFamily(ipAddrNetworkNames, ipFamily)
 
 	var discoveredInternal *ipAddrNetworkName
 	var discoveredExternal *ipAddrNetworkName
 
-	if len(ipFamilyMatches) != 0 {
-		discoveredInternal = findSubnetMatch(ipFamilyMatches, internalNetworkSubnet)
+	filteredInternalMatches := filterSubnetExclusions(ipFamilyMatches, excludeInternalNetworkSubnets)
+	filteredExternalMatches := filterSubnetExclusions(ipFamilyMatches, excludeExternalNetworkSubnets)
+
+	if len(filteredInternalMatches) > 0 || len(filteredExternalMatches) > 0 {
+		discoveredInternal = findSubnetMatch(filteredInternalMatches, internalNetworkSubnet)
 		if discoveredInternal != nil {
 			klog.V(2).Infof("Adding Internal IP by AddressMatching: %s", discoveredInternal.ipAddr)
 		}
-		discoveredExternal = findSubnetMatch(ipFamilyMatches, externalNetworkSubnet)
+		discoveredExternal = findSubnetMatch(filteredExternalMatches, externalNetworkSubnet)
 		if discoveredExternal != nil {
 			klog.V(2).Infof("Adding External IP by AddressMatching: %s", discoveredExternal.ipAddr)
 		}
 
 		if discoveredInternal == nil && internalVMNetworkName != "" {
-			discoveredInternal = findNetworkNameMatch(ipFamilyMatches, internalVMNetworkName)
+			discoveredInternal = findNetworkNameMatch(filteredInternalMatches, internalVMNetworkName)
 			if discoveredInternal != nil {
 				klog.V(2).Infof("Adding Internal IP by NetworkName: %s", discoveredInternal.ipAddr)
 			}
 		}
 
 		if discoveredExternal == nil && externalVMNetworkName != "" {
-			discoveredExternal = findNetworkNameMatch(ipFamilyMatches, externalVMNetworkName)
+			discoveredExternal = findNetworkNameMatch(filteredExternalMatches, externalVMNetworkName)
 			if discoveredExternal != nil {
 				klog.V(2).Infof("Adding External IP by NetworkName: %s", discoveredExternal.ipAddr)
 			}
@@ -405,10 +426,16 @@ func discoverIPs(ipAddrNetworkNames []*ipAddrNetworkName, ipFamily string,
 		// address selection behavior which is to only support a single address and
 		// return the first one found
 		if discoveredInternal == nil && discoveredExternal == nil {
-			klog.V(5).Info("Default address selection. Single NIC, Single IP Address")
-			klog.V(2).Infof("Adding IP: %s", ipFamilyMatches[0].ipAddr)
-			discoveredInternal = ipFamilyMatches[0]
-			discoveredExternal = ipFamilyMatches[0]
+			klog.V(5).Info("Default address selection.")
+			if len(filteredInternalMatches) > 0 {
+				klog.V(2).Infof("Adding Internal IP: %s", filteredInternalMatches[0].ipAddr)
+				discoveredInternal = filteredInternalMatches[0]
+			}
+
+			if len(filteredExternalMatches) > 0 {
+				klog.V(2).Infof("Adding External IP: %s", filteredExternalMatches[0].ipAddr)
+				discoveredExternal = filteredExternalMatches[0]
+			}
 		} else {
 			// At least one of the Internal or External addresses has been found.
 			// Minimally the Internal needs to exist for the node to function correctly.
@@ -438,6 +465,24 @@ func collectNonVNICDevices(guestNicInfos []types.GuestNicInfo) []types.GuestNicI
 	return toReturn
 }
 
+// parseCIDRs converts a comma delimited string of CIDRs to
+// a slice of IPNet pointers.
+func parseCIDRs(cidrsString string) ([]*net.IPNet, error) {
+	if cidrsString != "" {
+		cidrStringSlice := strings.Split(cidrsString, ",")
+		subnets := make([]*net.IPNet, len(cidrStringSlice))
+		for i, cidrString := range cidrStringSlice {
+			_, ipNet, err := net.ParseCIDR(cidrString)
+			if err != nil {
+				return nil, err
+			}
+			subnets[i] = ipNet
+		}
+		return subnets, nil
+	}
+	return nil, nil
+}
+
 // toIPAddrNetworkNames maps an array of GuestNicInfo to and array of *ipAddrNetworkName.
 func toIPAddrNetworkNames(guestNicInfos []types.GuestNicInfo) []*ipAddrNetworkName {
 	var candidates []*ipAddrNetworkName
@@ -536,6 +581,18 @@ func excludeLocalhostIPs(ipAddrNetworkNames []*ipAddrNetworkName) []*ipAddrNetwo
 	})
 }
 
+func filterSubnetExclusions(ipAddrNetworkNames []*ipAddrNetworkName, exlusionSubnets []*net.IPNet) []*ipAddrNetworkName {
+	return filter(ipAddrNetworkNames, func(i *ipAddrNetworkName) bool {
+		for _, exlusionSubnet := range exlusionSubnets {
+			if exlusionSubnet.Contains(i.ip()) {
+				klog.V(4).Infof("IP is excluded %q because it is contained in exlusion subnet %q", i.ipAddr, exlusionSubnet.String())
+				return false
+			}
+		}
+		return true
+	})
+}
+
 // GetNode gets the NodeInfo by UUID
 func (nm *NodeManager) GetNode(UUID string, node *pb.Node) error {
 	nodeInfo, err := nm.FindNodeInfo(UUID)