feat: create a dummy NC to store secondary IPs in nodesubnet deployments with Cilium

cns/NetworkContainerContract.go

@@ -99,10 +99,13 @@ const (
 	Managed        = "Managed"
 	CRD            = "CRD"
 	MultiTenantCRD = "MultiTenantCRD"
+	AzureHost      = "AzureHost"
 )
 
-var ErrInvalidNCID = errors.New("invalid NetworkContainerID")
-var ErrInvalidIP = errors.New("invalid IP")
+var (
+	ErrInvalidNCID = errors.New("invalid NetworkContainerID")
+	ErrInvalidIP   = errors.New("invalid IP")
+)
 
 // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary.
 type CreateNetworkContainerRequest struct {

cns/nodesubnet/initialization.go

@@ -0,0 +1,36 @@
+package nodesubnet
+
+import (
+	"context"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	cnstypes "github.com/Azure/azure-container-networking/cns/types"
+	"github.com/pkg/errors"
+	"golang.org/x/exp/maps"
+)
+
+type ipamReconciler interface {
+	ReconcileIPAMStateForNodeSubnet(ncRequests []*cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.PodInfo) cnstypes.ResponseCode
+}
+
+func ReconcileInitialCNSState(_ context.Context, ipamReconciler ipamReconciler, podInfoByIPProvider cns.PodInfoByIPProvider) (int, error) {
+	// Get previous PodInfo state from podInfoByIPProvider
+	podInfoByIP, err := podInfoByIPProvider.PodInfoByIP()
+	if err != nil {
+		return 0, errors.Wrap(err, "provider failed to provide PodInfoByIP")
+	}
+
+	logger.Printf("Reconciling initial CNS state with %d IPs", len(podInfoByIP))
+
+	// Create a network container request that holds all the IPs from PodInfoByIP
+	secondaryIPs := maps.Keys(podInfoByIP)
+	ncRequest := CreateNodeSubnetNCRequest(secondaryIPs)
+	responseCode := ipamReconciler.ReconcileIPAMStateForNodeSubnet([]*cns.CreateNetworkContainerRequest{ncRequest}, podInfoByIP)
+
+	if responseCode != cnstypes.Success {
+		return 0, errors.Errorf("failed to reconcile initial CNS state: %d", responseCode)
+	}
+
+	return len(secondaryIPs), nil
+}

cns/nodesubnet/initialization_test.go

@@ -0,0 +1,114 @@
+package nodesubnet_test
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/cnireconciler"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/nodesubnet"
+	"github.com/Azure/azure-container-networking/cns/restserver"
+	"github.com/Azure/azure-container-networking/cns/types"
+	"github.com/Azure/azure-container-networking/store"
+)
+
+func getMockStore() store.KeyValueStore {
+	mockStore := store.NewMockStore("")
+	endpointState := map[string]*restserver.EndpointInfo{
+		"12e65d89e58cb23c784e97840cf76866bfc9902089bdc8e87e9f64032e312b0b": {
+			PodName:      "coredns-54b69f46b8-ldmwr",
+			PodNamespace: "kube-system",
+			IfnameToIPMap: map[string]*restserver.IPInfo{
+				"eth0": {
+					IPv4: []net.IPNet{
+						{
+							IP:   net.IPv4(10, 10, 0, 52),
+							Mask: net.CIDRMask(24, 32),
+						},
+					},
+				},
+			},
+		},
+		"1fc5176913a3a1a7facfb823dde3b4ded404041134fef4f4a0c8bba140fc0413": {
+			PodName:      "load-test-7f7d49687d-wxc9p",
+			PodNamespace: "load-test",
+			IfnameToIPMap: map[string]*restserver.IPInfo{
+				"eth0": {
+					IPv4: []net.IPNet{
+						{
+							IP:   net.IPv4(10, 10, 0, 63),
+							Mask: net.CIDRMask(24, 32),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	err := mockStore.Write(restserver.EndpointStoreKey, endpointState)
+	if err != nil {
+		return nil
+	}
+	return mockStore
+}
+
+type MockIpamStateReconciler struct{}
+
+func (m *MockIpamStateReconciler) ReconcileIPAMStateForNodeSubnet(ncRequests []*cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.PodInfo) types.ResponseCode {
+	if len(ncRequests) == 1 && len(ncRequests[0].SecondaryIPConfigs) == len(podInfoByIP) {
+		return types.Success
+	}
+
+	return types.UnexpectedError
+}
+
+func TestNewCNSPodInfoProvider(t *testing.T) {
+	tests := []struct {
+		name       string
+		store      store.KeyValueStore
+		wantErr    bool
+		reconciler *MockIpamStateReconciler
+		exp        int
+	}{
+		{
+			name:       "happy_path",
+			store:      getMockStore(),
+			wantErr:    false,
+			reconciler: &MockIpamStateReconciler{},
+			exp:        2,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, cancel := testContext(t)
+			defer cancel()
+
+			podInfoByIPProvider, err := cnireconciler.NewCNSPodInfoProvider(tt.store)
+			checkErr(t, err, false)
+
+			got, err := nodesubnet.ReconcileInitialCNSState(ctx, tt.reconciler, podInfoByIPProvider)
+			checkErr(t, err, tt.wantErr)
+			if got != tt.exp {
+				t.Errorf("got %d IPs reconciled, expected %d", got, tt.exp)
+			}
+		})
+	}
+}
+
+// testContext creates a context from the provided testing.T that will be
+// canceled if the test suite is terminated.
+func testContext(t *testing.T) (context.Context, context.CancelFunc) {
+	if deadline, ok := t.Deadline(); ok {
+		return context.WithDeadline(context.Background(), deadline)
+	}
+	return context.WithCancel(context.Background())
+}
+
+func init() {
+	logger.InitLogger("testlogs", 0, 0, "./")
+}

cns/nodesubnet/nodesubnet_nc.go

@@ -0,0 +1,40 @@
+package nodesubnet
+
+import (
+	"strconv"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
+)
+
+const (
+	// ID for fake NC that we create to store NodeSubnet IPS
+	NodeSubnetNCID          = "55022629-3854-499b-7133-5e6887959f4ea" // md5sum of "NodeSubnetNC_IPv4"
+	NodeSubnetNCVersion     = 0
+	NodeSubnetHostVersion   = "0"
+	NodeSubnetNCStatus      = v1alpha.NCUpdateSuccess
+	NodeSubnetHostPrimaryIP = ""
+)
+
+// CreateNodeSubnetNCRequest generates a CreateNetworkContainerRequest that simply stores the static secondary IPs.
+func CreateNodeSubnetNCRequest(secondaryIPs []string) *cns.CreateNetworkContainerRequest {
+	secondaryIPConfigs := map[string]cns.SecondaryIPConfig{}
+
+	for _, secondaryIP := range secondaryIPs {
+		// iterate through all secondary IP addresses add them to the request as secondary IPConfigs.
+		secondaryIPConfigs[secondaryIP] = cns.SecondaryIPConfig{
+			IPAddress: secondaryIP,
+			NCVersion: NodeSubnetNCVersion,
+		}
+	}
+
+	return &cns.CreateNetworkContainerRequest{
+		HostPrimaryIP:        NodeSubnetHostPrimaryIP,
+		SecondaryIPConfigs:   secondaryIPConfigs,
+		NetworkContainerid:   NodeSubnetNCID,
+		NetworkContainerType: cns.Docker,                                 // Using docker as the NC type for NodeSubnet to match Swift. (The NC is not real)
+		Version:              strconv.FormatInt(NodeSubnetNCVersion, 10), //nolint:gomnd // it's decimal
+		IPConfiguration:      cns.IPConfiguration{},
+		NCStatus:             NodeSubnetNCStatus,
+	}
+}

cns/nodesubnet/nodesubnet_nc_test.go

@@ -0,0 +1,54 @@
+package nodesubnet_test
+
+import (
+	"testing"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/nodesubnet"
+	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestCreateNodeSubnetNCRequest_EmptySecondaryIPs(t *testing.T) {
+	secondaryIPs := []string{}
+	expectedRequest := &cns.CreateNetworkContainerRequest{
+		HostPrimaryIP:        nodesubnet.NodeSubnetHostPrimaryIP,
+		SecondaryIPConfigs:   map[string]cns.SecondaryIPConfig{},
+		NetworkContainerid:   nodesubnet.NodeSubnetNCID,
+		NetworkContainerType: cns.Docker,
+		Version:              "0",
+		IPConfiguration:      cns.IPConfiguration{},
+		NCStatus:             v1alpha.NCUpdateSuccess,
+	}
+
+	request := nodesubnet.CreateNodeSubnetNCRequest(secondaryIPs)
+	if !cmp.Equal(request, expectedRequest) {
+		t.Errorf("Unexepected diff in NodeSubnetNCRequest: %v", cmp.Diff(request, expectedRequest))
+	}
+}
+
+func TestCreateNodeSubnetNCRequest_NonEmptySecondaryIPs(t *testing.T) {
+	secondaryIPs := []string{"10.0.0.1", "10.0.0.2"}
+	expectedRequest := &cns.CreateNetworkContainerRequest{
+		HostPrimaryIP: nodesubnet.NodeSubnetHostPrimaryIP,
+		SecondaryIPConfigs: map[string]cns.SecondaryIPConfig{
+			"10.0.0.1": {IPAddress: "10.0.0.1", NCVersion: nodesubnet.NodeSubnetNCVersion},
+			"10.0.0.2": {IPAddress: "10.0.0.2", NCVersion: nodesubnet.NodeSubnetNCVersion},
+		},
+		NetworkContainerid:   nodesubnet.NodeSubnetNCID,
+		NetworkContainerType: cns.Docker,
+		Version:              "0",
+		IPConfiguration:      cns.IPConfiguration{},
+		NCStatus:             v1alpha.NCUpdateSuccess,
+	}
+
+	request := nodesubnet.CreateNodeSubnetNCRequest(secondaryIPs)
+	if !cmp.Equal(request, expectedRequest) {
+		t.Errorf("Unexepected diff in NodeSubnetNCRequest: %v", cmp.Diff(request, expectedRequest))
+	}
+}
+
+func init() {
+	logger.InitLogger("testlogs", 0, 0, "./")
+}

cns/restserver/internalapi.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/nodesubnet"
 	"github.com/Azure/azure-container-networking/cns/types"
 	"github.com/Azure/azure-container-networking/common"
 	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
@@ -275,22 +276,7 @@ func (service *HTTPRestService) syncHostNCVersion(ctx context.Context, channelMo
 	return len(programmedNCs), nil
 }
 
-func (service *HTTPRestService) ReconcileIPAMState(ncReqs []*cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.PodInfo, nnc *v1alpha.NodeNetworkConfig) types.ResponseCode {
-	logger.Printf("Reconciling CNS IPAM state with nc requests: [%+v], PodInfo [%+v], NNC: [%+v]", ncReqs, podInfoByIP, nnc)
-	// if no nc reqs, there is no CRD state yet
-	if len(ncReqs) == 0 {
-		logger.Printf("CNS starting with no NC state, podInfoMap count %d", len(podInfoByIP))
-		return types.Success
-	}
-
-	// first step in reconciliation is to create all the NCs in CNS, no IP assignment yet.
-	for _, ncReq := range ncReqs {
-		returnCode := service.CreateOrUpdateNetworkContainerInternal(ncReq)
-		if returnCode != types.Success {
-			return returnCode
-		}
-	}
-
+func (service *HTTPRestService) ReconcileIPAssignment(podInfoByIP map[string]cns.PodInfo, ncReqs []*cns.CreateNetworkContainerRequest) types.ResponseCode {
 	// index all the secondary IP configs for all the nc reqs, for easier lookup later on.
 	allSecIPsIdx := make(map[string]*cns.CreateNetworkContainerRequest)
 	for i := range ncReqs {
@@ -321,6 +307,7 @@ func (service *HTTPRestService) ReconcileIPAMState(ncReqs []*cns.CreateNetworkCo
 	//   }
 	//
 	// such that we can iterate over pod interfaces, and assign all IPs for it at once.
+
 	podKeyToPodIPs, err := newPodKeyToPodIPsMap(podInfoByIP)
 	if err != nil {
 		logger.Errorf("could not transform pods indexed by IP address to pod IPs indexed by interface: %v", err)
@@ -378,12 +365,69 @@ func (service *HTTPRestService) ReconcileIPAMState(ncReqs []*cns.CreateNetworkCo
 		}
 	}
 
+	return types.Success
+}
+
+func (service *HTTPRestService) CreateNCs(ncReqs []*cns.CreateNetworkContainerRequest) types.ResponseCode {
+	for _, ncReq := range ncReqs {
+		returnCode := service.CreateOrUpdateNetworkContainerInternal(ncReq)
+		if returnCode != types.Success {
+			return returnCode
+		}
+	}
+
+	return types.Success
+}
+
+func (service *HTTPRestService) ReconcileIPAMStateForSwift(ncReqs []*cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.PodInfo, nnc *v1alpha.NodeNetworkConfig) types.ResponseCode {
+	logger.Printf("Reconciling CNS IPAM state with nc requests: [%+v], PodInfo [%+v], NNC: [%+v]", ncReqs, podInfoByIP, nnc)
+	// if no nc reqs, there is no CRD state yet
+	if len(ncReqs) == 0 {
+		logger.Printf("CNS starting with no NC state, podInfoMap count %d", len(podInfoByIP))
+		return types.Success
+	}
+
+	// first step in reconciliation is to create all the NCs in CNS, no IP assignment yet.
+	if returnCode := service.CreateNCs(ncReqs); returnCode != types.Success {
+		return returnCode
+	}
+
+	logger.Debugf("ncReqs created successfully, now save IPs")
+	// now reconcile IPAM state.
+	if returnCode := service.ReconcileIPAssignment(podInfoByIP, ncReqs); returnCode != types.Success {
+		return returnCode
+	}
+
 	if err := service.MarkExistingIPsAsPendingRelease(nnc.Spec.IPsNotInUse); err != nil {
 		logger.Errorf("[Azure CNS] Error. Failed to mark IPs as pending %v", nnc.Spec.IPsNotInUse)
 		return types.UnexpectedError
 	}
 
-	return 0
+	return types.Success
+}
+
+// todo: there is some redundancy between this funcation and ReconcileIPAMStateForNodeSubnet. The difference is that this one
+// doesn't include the NNC parameter. We may want to unify the common parts.
+func (service *HTTPRestService) ReconcileIPAMStateForNodeSubnet(ncReqs []*cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.PodInfo) types.ResponseCode {
+	logger.Printf("Reconciling CNS IPAM state with nc requests: [%+v], PodInfo [%+v]", ncReqs, podInfoByIP)
+
+	if len(ncReqs) != 1 {
+		logger.Errorf("Nodesubnet should always have 1 NC to hold secondary IPs")
+		return types.NetworkContainerNotSpecified
+	}
+
+	// first step in reconciliation is to create all the NCs in CNS, no IP assignment yet.
+	if returnCode := service.CreateNCs(ncReqs); returnCode != types.Success {
+		return returnCode
+	}
+
+	logger.Debugf("ncReqs created successfully, now save IPs")
+	// now reconcile IPAM state.
+	if returnCode := service.ReconcileIPAssignment(podInfoByIP, ncReqs); returnCode != types.Success {
+		return returnCode
+	}
+
+	return types.Success
 }
 
 var (
@@ -526,11 +570,19 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req *cns.
 		return types.UnsupportedOrchestratorType
 	}
 
-	// Validate PrimaryCA must never be empty
-	err := validateIPSubnet(req.IPConfiguration.IPSubnet)
-	if err != nil {
-		logger.Errorf("[Azure CNS] Error. PrimaryCA is invalid, NC Req: %v", req)
-		return types.InvalidPrimaryIPConfig
+	if req.NetworkContainerid == nodesubnet.NodeSubnetNCID {
+		// For NodeSubnet scenarios, Validate PrimaryCA must be empty
+		if req.IPConfiguration.IPSubnet.IPAddress != "" {
+			logger.Errorf("[Azure CNS] Error. PrimaryCA is invalid, NC Req: %v", req)
+			return types.InvalidPrimaryIPConfig
+		}
+	} else {
+		// For Swift scenarios, Validate PrimaryCA must never be empty
+		err := validateIPSubnet(req.IPConfiguration.IPSubnet)
+		if err != nil {
+			logger.Errorf("[Azure CNS] Error. PrimaryCA is invalid, NC Req: %v", req)
+			return types.InvalidPrimaryIPConfig
+		}
 	}
 
 	// Validate SecondaryIPConfig