Skip to content

Commit

Permalink
Add Nutanix CCM ignore node IPs list (#9068)
Browse files Browse the repository at this point in the history
* BUGFIX Add Add Nutanix CCM ignore node IPs list

* Fix comments and linter errors
 - fix kube-vip validation
 - fix kube-vip add to ignore node list
 - add unit-test

* Fix comments
  • Loading branch information
adiantum authored Dec 19, 2024
1 parent 7e4cf0e commit 586f904
Showing 70 changed files with 1,233 additions and 96 deletions.
Original file line number Diff line number Diff line change
@@ -42,6 +42,13 @@ spec:
bundle for users that configured their Prism Central with certificates
from non-publicly trusted CAs
type: string
ccmExcludeNodeIPs:
description: CcmExcludeIPs is the optional list of IP addresses that
should be excluded from the CCM IP pool for nodes. List should be
valid IP addresses and IP address ranges.
items:
type: string
type: array
credentialRef:
description: CredentialRef is the reference to the secret name that
contains the credentials for the Nutanix Prism Central. The namespace
5 changes: 5 additions & 0 deletions pkg/api/v1alpha1/nutanixdatacenterconfig_types.go
Original file line number Diff line number Diff line change
@@ -47,6 +47,11 @@ type NutanixDatacenterConfigSpec struct {
// FailureDomains is the optional list of failure domains for the Nutanix Datacenter.
// +optional
FailureDomains []NutanixDatacenterFailureDomain `json:"failureDomains,omitempty"`

// CcmExcludeIPs is the optional list of IP addresses that should be excluded from the CCM IP pool for nodes.
// List should be valid IP addresses and IP address ranges.
// +optional
CcmExcludeNodeIPs []string `json:"ccmExcludeNodeIPs,omitempty"`
}

// NutanixDatacenterFailureDomain defines the failure domain for the Nutanix Datacenter.
5 changes: 5 additions & 0 deletions pkg/api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion pkg/providers/nutanix/config/cp-template.yaml
Original file line number Diff line number Diff line change
@@ -647,7 +647,8 @@ data:
"enableCustomLabeling": false,
"topologyDiscovery": {
"type": "Prism"
}
},
"ignoredNodeIPs": [{{ range $i, $ip := .ccmIgnoredNodeIPs }}{{ if $i }}, {{ end }}"{{ $ip }}"{{ end }}]
}
---
apiVersion: rbac.authorization.k8s.io/v1
91 changes: 91 additions & 0 deletions pkg/providers/nutanix/template.go
Original file line number Diff line number Diff line change
@@ -4,6 +4,8 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"net"
"strings"

"sigs.k8s.io/yaml"

@@ -176,9 +178,12 @@ func buildTemplateMapCP(

failureDomains := generateNutanixFailureDomains(datacenterSpec.FailureDomains)

ccmIgnoredNodeIPs := generateCcmIgnoredNodeIPsList(clusterSpec)

values := map[string]interface{}{
"auditPolicy": auditPolicy,
"apiServerExtraArgs": apiServerExtraArgs.ToPartialYaml(),
"ccmIgnoredNodeIPs": ccmIgnoredNodeIPs,
"cloudProviderImage": versionsBundle.Nutanix.CloudProvider.VersionedImage(),
"clusterName": clusterSpec.Cluster.Name,
"controlPlaneEndpointIp": clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host,
@@ -568,3 +573,89 @@ func generateNutanixFailureDomains(eksNutanixFailureDomains []v1alpha1.NutanixDa
}
return failureDomains
}

func incrementIP(ip net.IP) {
for i := len(ip) - 1; i >= 0; i-- {
ip[i]++
if ip[i] > 0 {
break
}
}
}

func compareIP(ip1, ip2 net.IP) (int, error) {
if len(ip1) != len(ip2) {
return -1, fmt.Errorf("IP addresses are not the same protocol")
}

for i := 0; i < len(ip1); i++ {
if ip1[i] < ip2[i] {
return -1, nil
}
if ip1[i] > ip2[i] {
return 1, nil
}
}

return 0, nil
}

func addCIDRToIgnoredNodeIPsList(cidr string, result []string) []string {
ip, ipNet, _ := net.ParseCIDR(cidr)

// Add all ip addresses in the range to the list
for ip := ip.Mask(ipNet.Mask); ipNet.Contains(ip); incrementIP(ip) {
if ip != nil {
result = append(result, ip.String())
}
}

return result
}

func addIPRangeToIgnoredNodeIPsList(ipRangeStr string, result []string) []string {
// Parse the range
ipRange := strings.Split(ipRangeStr, "-")

// Parse the start and end of the range
start := net.ParseIP(strings.TrimSpace(ipRange[0]))
end := net.ParseIP(strings.TrimSpace(ipRange[1]))

cmp, _ := compareIP(start, end)
if cmp >= 0 {
// swap start and end if start is greater than end
start, end = end, start
}

// Add all ip addresses in the range to the list
for ip := start; !ip.Equal(end); incrementIP(ip) {
result = append(result, ip.String())
}

result = append(result, end.String())

return result
}

func addIPAddressToIgnoredNodeIPsList(ipAddrStr string, result []string) []string {
result = append(result, ipAddrStr)
return result
}

func generateCcmIgnoredNodeIPsList(clusterSpec *cluster.Spec) []string {
// Add the kube-vip IP address to the list
result := []string{clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host}

for _, IPAddrOrRange := range clusterSpec.NutanixDatacenter.Spec.CcmExcludeNodeIPs {
addrOrRange := strings.TrimSpace(IPAddrOrRange)
if strings.Contains(addrOrRange, "/") {
result = addCIDRToIgnoredNodeIPsList(addrOrRange, result)
} else if strings.Contains(addrOrRange, "-") {
result = addIPRangeToIgnoredNodeIPsList(addrOrRange, result)
} else {
result = addIPAddressToIgnoredNodeIPsList(addrOrRange, result)
}
}

return result
}
41 changes: 41 additions & 0 deletions pkg/providers/nutanix/template_test.go
Original file line number Diff line number Diff line change
@@ -848,6 +848,47 @@ func TestTemplateBuilderGPUs(t *testing.T) {
}
}

func TestTemplateBuilderCcmExcludeNodeIPs(t *testing.T) {
for _, tc := range []struct {
Input string
Output string
ChangeFn func(clusterSpec *cluster.Spec) *cluster.Spec
}{
{
Input: "testdata/eksa-cluster-ccm-exclude-node-ips.yaml",
Output: "testdata/expected_cluster_ccm_exclude_node_ips.yaml",
ChangeFn: func(clusterSpec *cluster.Spec) *cluster.Spec {
excludeNodeIPs := []string{
"127.100.200.101",
"10.10.10.10-10.10.10.13",
"10.123.0.0/29",
}
clusterSpec.NutanixDatacenter.Spec.CcmExcludeNodeIPs = excludeNodeIPs

return clusterSpec
},
},
} {
clusterSpec := test.NewFullClusterSpec(t, tc.Input)

machineCfg := clusterSpec.NutanixMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)

t.Setenv(constants.EksaNutanixUsernameKey, "admin")
t.Setenv(constants.EksaNutanixPasswordKey, "password")
creds := GetCredsFromEnv()

clusterSpec = tc.ChangeFn(clusterSpec)

bldr := NewNutanixTemplateBuilder(&clusterSpec.NutanixDatacenter.Spec, &machineCfg.Spec, nil,
map[string]anywherev1.NutanixMachineConfigSpec{}, creds, time.Now)

cpSpec, err := bldr.GenerateCAPISpecControlPlane(clusterSpec)
assert.NoError(t, err)
assert.NotNil(t, cpSpec)
test.AssertContentToFile(t, string(cpSpec), tc.Output)
}
}

func minimalNutanixConfigSpec(t *testing.T) (*anywherev1.NutanixDatacenterConfig, *anywherev1.NutanixMachineConfig, map[string]anywherev1.NutanixMachineConfigSpec) {
dcConf := &anywherev1.NutanixDatacenterConfig{}
err := yaml.Unmarshal([]byte(nutanixDatacenterConfigSpec), dcConf)
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ spec:
name: test
count: 1
endpoint:
host: test
host: 10.199.199.1
certSANs: ["foo.bar"]
machineGroupRef:
name: test
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ spec:
name: test
count: 1
endpoint:
host: test
host: 10.199.199.1
certSANs: ["11.11.11.11"]
machineGroupRef:
name: test
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ spec:
name: test
count: 1
endpoint:
host: test
host: 10.199.199.1
machineGroupRef:
name: test
kind: NutanixMachineConfig
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ spec:
name: test
count: 1
endpoint:
host: test
host: 10.199.199.1
machineGroupRef:
name: test
kind: NutanixMachineConfig
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ spec:
name: test
count: 1
endpoint:
host: test
host: 10.199.199.1
machineGroupRef:
name: test
kind: NutanixMachineConfig
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ spec:
name: eksa-unit-test
count: 3
endpoint:
host: test-ip
host: 10.199.199.1
machineGroupRef:
name: eksa-unit-test
kind: NutanixMachineConfig
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ spec:
name: eksa-unit-test
count: 3
endpoint:
host: test-ip
host: 10.199.199.1
machineGroupRef:
name: eksa-unit-test
kind: NutanixMachineConfig
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ spec:
name: eksa-unit-test
count: 3
endpoint:
host: test-ip
host: 10.199.199.1
machineGroupRef:
name: eksa-unit-test
kind: NutanixMachineConfig
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ spec:
name: eksa-unit-test
count: 3
endpoint:
host: test-ip
host: 10.199.199.1
machineGroupRef:
name: eksa-unit-test
kind: NutanixMachineConfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: anywhere.eks.amazonaws.com/v1alpha1
kind: NutanixDatacenterConfig
metadata:
name: eksa-unit-test
namespace: default
spec:
endpoint: "prism.nutanix.com"
port: 9440
credentialRef:
kind: Secret
name: "nutanix-credentials"
ccmExcludeNodeIPs:
- 10.0.0.1
- 10.0.0.0/24
- 10.0.0.10-10.0.0.30
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: anywhere.eks.amazonaws.com/v1alpha1
kind: NutanixDatacenterConfig
metadata:
name: eksa-unit-test
namespace: default
spec:
endpoint: "prism.nutanix.com"
port: 9440
credentialRef:
kind: Secret
name: "nutanix-credentials"
ccmExcludeNodeIPs:
- 10.0.0.1
- 10.0.0.0/24
- 10.0.0.10-10.0.0.30
- 10.100.0.0//16
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: anywhere.eks.amazonaws.com/v1alpha1
kind: NutanixDatacenterConfig
metadata:
name: eksa-unit-test
namespace: default
spec:
endpoint: "prism.nutanix.com"
port: 9440
credentialRef:
kind: Secret
name: "nutanix-credentials"
ccmExcludeNodeIPs:
- 10.0.0.1
- 10.0.0.0/24
- 10.0.0.10-10.0.0.30
- 244.244.01
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: anywhere.eks.amazonaws.com/v1alpha1
kind: NutanixDatacenterConfig
metadata:
name: eksa-unit-test
namespace: default
spec:
endpoint: "prism.nutanix.com"
port: 9440
credentialRef:
kind: Secret
name: "nutanix-credentials"
ccmExcludeNodeIPs:
- 10.0.0.1
- 10.0.0.0/24
- 10.0.0.10-10.0.0.30
- 10.100.0.10-10.100.10.10-10.100.20.30
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: anywhere.eks.amazonaws.com/v1alpha1
kind: NutanixDatacenterConfig
metadata:
name: eksa-unit-test
namespace: default
spec:
endpoint: "prism.nutanix.com"
port: 9440
credentialRef:
kind: Secret
name: "nutanix-credentials"
ccmExcludeNodeIPs:
- 10.0.0.1
- 10.0.0.0/24
- 10.0.0.10-10.0.0.30
- 192.179.1.1-10.1.1.1
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
apiVersion: anywhere.eks.amazonaws.com/v1alpha1
kind: NutanixDatacenterConfig
metadata:
name: eksa-unit-test
namespace: default
spec:
endpoint: "prism.nutanix.com"
port: 9440
credentialRef:
kind: Secret
name: "nutanix-credentials"
ccmExcludeNodeIPs:
- 10.0.0.1
- 10.0.0.0/24
- 10.0.0.10-10.0.0.30
- 10.0.10.0-::1

Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ spec:
name: eksa-unit-test
count: 3
endpoint:
host: test-ip
host: 10.199.199.1
machineGroupRef:
name: eksa-unit-test
kind: NutanixMachineConfig
Loading

0 comments on commit 586f904

Please sign in to comment.