From f564b4790bf061e101f22575cdd67adbc486a4f7 Mon Sep 17 00:00:00 2001 From: rahulsawra98 <104886535+rahulsawra98@users.noreply.github.com> Date: Wed, 19 Jul 2023 11:46:57 +0530 Subject: [PATCH 1/5] feat(): vpnkeyrotation apis, webhook, ITs, UTs and service layer (#151) feat(): vpnkeyrotation apis, webhook, ITs, UTs and service layer (#151) Signed-off-by: rahulsawra98 90 + // Expected Error: + //{ + // Type: "FieldValueInvalid", + // Message: "Invalid value: 90: spec.rotationInterval in body should be less than or equal to 90", + // Field: "spec.rotationInterval", + // }, + s.Spec.RotationInterval = 100 + Expect(k8sClient.Create(ctx, s)).Should(Not(Succeed())) + + // RotationInterval < 30 + s.Spec.RotationInterval = 20 + // Expected Error: + // { + // Type: "FieldValueInvalid", + // Message: "Invalid value: 30: spec.rotationInterval in body should be greater than or equal to 30", + // Field: "spec.rotationInterval", + // }, + Expect(k8sClient.Create(ctx, s)).Should(Not(Succeed())) + }) + It("Should create slice with default rotation interval=30days if not specified", func() { + createdSliceConfig := &v1alpha1.SliceConfig{} + + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + }, createdSliceConfig) + + Expect(err).To(BeNil()) + Expect(createdSliceConfig.Spec.RotationInterval).Should(Equal(30)) + }) + It("Should Create VPNKeyRotationConfig Once Slice Is Created", func() { + // it should create vpnkeyrotationconfig + createdVpnKeyConfig := &v1alpha1.VpnKeyRotation{} + getKey := types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, createdVpnKeyConfig) + return err == nil + }, timeout, interval).Should(BeTrue()) + + }) + It("Should Update VpnKeyRotationConfig with correct clusters(2) gateway mapping", func() { + createdVpnKeyConfig := &v1alpha1.VpnKeyRotation{} + getKey := types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, createdVpnKeyConfig) + if err != nil { + return false + } + return createdVpnKeyConfig.Spec.ClusterGatewayMapping != nil + }, timeout, interval).Should(BeTrue()) + + expectedMap := map[string][]string{ + "worker-1": {sliceName + "-worker-1-worker-2"}, + "worker-2": {sliceName + "-worker-2-worker-1"}, + } + // check if map is contructed correctly + Expect(createdVpnKeyConfig.Spec.ClusterGatewayMapping).To(Equal(expectedMap)) + }) + It("Should update vpnkeyrotationconfig with certificateCreation and Expiry TS", func() { + createdVpnKeyConfig := &v1alpha1.VpnKeyRotation{} + getKey := types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + } + // check if creation TS is not zero + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, createdVpnKeyConfig) + if err != nil { + return false + } + return !createdVpnKeyConfig.Spec.CertificateCreationTime.IsZero() + }, timeout, interval).Should(BeTrue()) + + // check if expiry TS is not zero + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, createdVpnKeyConfig) + if err != nil { + return false + } + return !createdVpnKeyConfig.Spec.CertificateExpiryTime.IsZero() + }, timeout, interval).Should(BeTrue()) + }) + + It("Should recreate/retrigger jobs for cert creation once it expires", func() { + createdVpnKeyConfig := &v1alpha1.VpnKeyRotation{} + + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + }, createdVpnKeyConfig) + + Expect(err).To(BeNil()) + + // expire it + time := time.Now().Add(-1 * time.Hour) + createdVpnKeyConfig.Spec.CertificateExpiryTime = &metav1.Time{Time: time} + err = k8sClient.Update(ctx, createdVpnKeyConfig) + Expect(err).To(BeNil()) + + // expect new jobs to be created + job := batchv1.JobList{} + o := map[string]string{ + "SLICE_NAME": sliceName, + } + listOpts := []client.ListOption{ + client.MatchingLabels( + o, + ), + } + Eventually(func() bool { + err = k8sClient.List(ctx, &job, listOpts...) + if err != nil { + return false + } + return len(job.Items) > 0 + }, timeout, interval).Should(BeTrue()) + }) + // cluster onboarding tests + It("Should Update VPNKeyRotation Config in case a new cluster is added", func() { + // update sliceconfig + createdSliceConfig := &v1alpha1.SliceConfig{} + + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + }, createdSliceConfig) + + Expect(err).To(BeNil()) + createdSliceConfig.Spec.Clusters = append(createdSliceConfig.Spec.Clusters, "worker-3") + Expect(k8sClient.Update(ctx, createdSliceConfig)).Should(Succeed()) + + createdVpnKeyConfig := &v1alpha1.VpnKeyRotation{} + + Eventually(func() []string { + err = k8sClient.Get(ctx, types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + }, createdVpnKeyConfig) + + if err != nil { + return []string{""} + } + return createdVpnKeyConfig.Spec.Clusters + }, timeout, interval).Should(Equal([]string{"worker-1", "worker-2", "worker-3"})) + }) + It("Should Update Cluster(3) Gateway Mapping", func() { + createdVpnKeyConfig := &v1alpha1.VpnKeyRotation{} + getKey := types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, createdVpnKeyConfig) + if err != nil { + return false + } + return createdVpnKeyConfig.Spec.ClusterGatewayMapping != nil + }, timeout, interval).Should(BeTrue()) + + // the length of map should be 3 + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, createdVpnKeyConfig) + if err != nil { + return false + } + return len(createdVpnKeyConfig.Spec.ClusterGatewayMapping) == 3 + }, timeout, interval).Should(BeTrue()) + }) + + It("Should Update VPNKey Rotation Config in case a cluster is de-boarded", func() { + // update sliceconfig + createdSliceConfig := &v1alpha1.SliceConfig{} + + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + }, createdSliceConfig) + + Expect(err).To(BeNil()) + createdSliceConfig.Spec.Clusters = []string{"worker-1", "worker-2"} + Expect(k8sClient.Update(ctx, createdSliceConfig)).Should(Succeed()) + + createdVpnKeyConfig := &v1alpha1.VpnKeyRotation{} + + Eventually(func() []string { + err = k8sClient.Get(ctx, types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + }, createdVpnKeyConfig) + + if err != nil { + return []string{""} + } + return createdVpnKeyConfig.Spec.Clusters + }, timeout, interval).Should(Equal([]string{"worker-1", "worker-2"})) + }) + It("Should Update Cluster(2) Gateway Mapping", func() { + createdVpnKeyConfig := &v1alpha1.VpnKeyRotation{} + getKey := types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, createdVpnKeyConfig) + if err != nil { + return false + } + return createdVpnKeyConfig.Spec.ClusterGatewayMapping != nil + }, timeout, interval).Should(BeTrue()) + + // the length of map should be 2 + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, createdVpnKeyConfig) + if err != nil { + return false + } + return len(createdVpnKeyConfig.Spec.ClusterGatewayMapping) == 2 + }, timeout, interval).Should(BeTrue()) + }) + }) + Context("Webhook Tests", func() { + var slice *v1alpha1.SliceConfig + var cluster1 *v1alpha1.Cluster + var cluster2 *v1alpha1.Cluster + var cluster3 *v1alpha1.Cluster + os.Setenv("KUBESLICE_CONTROLLER_MANAGER_NAMESPACE", controlPlaneNamespace) + ctx := context.Background() + + slice = &v1alpha1.SliceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: sliceName, + Namespace: sliceNamespace, + }, + Spec: v1alpha1.SliceConfigSpec{ + Clusters: []string{"worker-1", "worker-2"}, + MaxClusters: 4, + SliceSubnet: "10.1.0.0/16", + SliceGatewayProvider: v1alpha1.WorkerSliceGatewayProvider{ + SliceGatewayType: "OpenVPN", + SliceCaType: "Local", + }, + SliceIpamType: "Local", + SliceType: "Application", + QosProfileDetails: &v1alpha1.QOSProfile{ + BandwidthCeilingKbps: 5120, + DscpClass: "AF11", + }, + }, + } + cluster1 = &v1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + Namespace: "kubeslice-cisco", + }, + Spec: v1alpha1.ClusterSpec{ + NodeIPs: []string{"11.11.11.12"}, + }, + } + cluster2 = &v1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-2", + Namespace: "kubeslice-cisco", + }, + Spec: v1alpha1.ClusterSpec{ + NodeIPs: []string{"11.11.11.13"}, + }, + } + cluster3 = &v1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-3", + Namespace: "kubeslice-cisco", + }, + Spec: v1alpha1.ClusterSpec{ + NodeIPs: []string{"11.11.11.14"}, + }, + } + BeforeAll(func() { + ns := v1.Namespace{} + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: "kubeslice-cisco", + }, &ns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(k8sClient.Create(ctx, cluster1)).Should(Succeed()) + // update cluster status + getKey := types.NamespacedName{ + Namespace: cluster1.Namespace, + Name: cluster1.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, cluster1) + return err == nil + }, timeout, interval).Should(BeTrue()) + cluster1.Status.CniSubnet = []string{"192.168.0.0/24"} + cluster1.Status.RegistrationStatus = v1alpha1.RegistrationStatusRegistered + Expect(k8sClient.Status().Update(ctx, cluster1)).Should(Succeed()) + + Expect(k8sClient.Create(ctx, cluster2)).Should(Succeed()) + // update cluster status + getKey = types.NamespacedName{ + Namespace: cluster2.Namespace, + Name: cluster2.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, cluster2) + return err == nil + }, timeout, interval).Should(BeTrue()) + cluster2.Status.CniSubnet = []string{"192.168.1.0/24"} + cluster2.Status.RegistrationStatus = v1alpha1.RegistrationStatusRegistered + Expect(k8sClient.Status().Update(ctx, cluster2)).Should(Succeed()) + + Expect(k8sClient.Create(ctx, cluster3)).Should(Succeed()) + // update cluster status + getKey = types.NamespacedName{ + Namespace: cluster3.Namespace, + Name: cluster3.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, cluster3) + return err == nil + }, timeout, interval).Should(BeTrue()) + cluster3.Status.CniSubnet = []string{"10.1.1.1/16"} + cluster3.Status.RegistrationStatus = v1alpha1.RegistrationStatusRegistered + Expect(k8sClient.Status().Update(ctx, cluster3)).Should(Succeed()) + + // it should create sliceconfig + Expect(k8sClient.Create(ctx, slice)).Should(Succeed()) + + }) + AfterAll(func() { + // update sliceconfig tor remove clusters + createdSliceConfig := v1alpha1.SliceConfig{} + getKey := types.NamespacedName{ + Namespace: sliceNamespace, + Name: sliceName, + } + Expect(k8sClient.Get(ctx, getKey, &createdSliceConfig)).Should(Succeed()) + createdSliceConfig.Spec.Clusters = []string{} + Expect(k8sClient.Update(ctx, &createdSliceConfig)).Should(Succeed()) + // wait till workersliceconfigs are deleted + workerSliceConfigList := workerv1alpha1.WorkerSliceConfigList{} + ls := map[string]string{ + "original-slice-name": slice.Name, + } + listOpts := []client.ListOption{ + client.MatchingLabels(ls), + } + Eventually(func() bool { + err := k8sClient.List(ctx, &workerSliceConfigList, listOpts...) + if err != nil { + return false + } + return len(workerSliceConfigList.Items) == 0 + }, timeout, interval).Should(BeTrue()) + + // it should delete sliceconfig + Expect(k8sClient.Delete(ctx, slice)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, cluster1)).Should(Succeed()) + clusterGetKey := types.NamespacedName{ + Namespace: cluster1.Namespace, + Name: cluster1.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, clusterGetKey, cluster1) + return errors.IsNotFound(err) + }, timeout, interval).Should(BeTrue()) + + Expect(k8sClient.Delete(ctx, cluster2)).Should(Succeed()) + clusterGetKey = types.NamespacedName{ + Namespace: cluster2.Namespace, + Name: cluster2.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, clusterGetKey, cluster2) + return errors.IsNotFound(err) + }, timeout, interval).Should(BeTrue()) + Expect(k8sClient.Delete(ctx, cluster3)).Should(Succeed()) + clusterGetKey = types.NamespacedName{ + Namespace: cluster3.Namespace, + Name: cluster3.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, clusterGetKey, cluster3) + return errors.IsNotFound(err) + }, timeout, interval).Should(BeTrue()) + + }) + It("Should not allow creating vpn keyrotation config if sliceconfig is not present", func() { + vpnkeyRotation := v1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-vpn", + Namespace: slice.Namespace, + }, + Spec: v1alpha1.VpnKeyRotationSpec{ + SliceName: "demo-vpn", + }, + } + Expect(k8sClient.Create(ctx, &vpnkeyRotation)).ShouldNot(Succeed()) + }) + It("Should not allow deleting vpnkeyrotation config, if slice is present and raise an event", func() { + // get vpnkey rotation config + createdVpnKeyConfig := &v1alpha1.VpnKeyRotation{} + getKey := types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, createdVpnKeyConfig) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // should fail + Expect(k8sClient.Delete(ctx, createdVpnKeyConfig)).ShouldNot(Succeed()) + // check for event + eventList := &v1.EventList{} + Eventually(func() bool { + err := k8sClient.List(ctx, eventList, client.InNamespace("kubeslice-cisco")) + return err == nil && len(eventList.Items) > 0 && eventFound(eventList, string(events.EventIllegalVPNKeyRotationConfigDelete)) + }, timeout, interval).Should(BeTrue()) + + }) + }) + Context("SliceConfig RenewBefore Test Case", func() { + var project *v1alpha1.Project + var slice *v1alpha1.SliceConfig + var cluster1 *v1alpha1.Cluster + var cluster2 *v1alpha1.Cluster + var cluster3 *v1alpha1.Cluster + os.Setenv("KUBESLICE_CONTROLLER_MANAGER_NAMESPACE", controlPlaneNamespace) + ctx := context.Background() + project = &v1alpha1.Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cisco", + Namespace: controlPlaneNamespace, + }, + } + slice = &v1alpha1.SliceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-1", + Namespace: sliceNamespace, + }, + Spec: v1alpha1.SliceConfigSpec{ + Clusters: []string{"worker-1", "worker-2"}, + MaxClusters: 4, + SliceSubnet: "10.1.0.0/16", + SliceGatewayProvider: v1alpha1.WorkerSliceGatewayProvider{ + SliceGatewayType: "OpenVPN", + SliceCaType: "Local", + }, + SliceIpamType: "Local", + SliceType: "Application", + QosProfileDetails: &v1alpha1.QOSProfile{ + BandwidthCeilingKbps: 5120, + DscpClass: "AF11", + }, + }, + } + cluster1 = &v1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + Namespace: "kubeslice-cisco", + }, + Spec: v1alpha1.ClusterSpec{ + NodeIPs: []string{"11.11.11.12"}, + }, + } + cluster2 = &v1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-2", + Namespace: "kubeslice-cisco", + }, + Spec: v1alpha1.ClusterSpec{ + NodeIPs: []string{"11.11.11.13"}, + }, + } + cluster3 = &v1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-3", + Namespace: "kubeslice-cisco", + }, + Spec: v1alpha1.ClusterSpec{ + NodeIPs: []string{"11.11.11.14"}, + }, + } + BeforeAll(func() { + ns := v1.Namespace{} + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: "kubeslice-cisco", + }, &ns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(k8sClient.Create(ctx, cluster1)).Should(Succeed()) + // update cluster status + getKey := types.NamespacedName{ + Namespace: cluster1.Namespace, + Name: cluster1.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, cluster1) + return err == nil + }, timeout, interval).Should(BeTrue()) + cluster1.Status.CniSubnet = []string{"192.168.0.0/24"} + cluster1.Status.RegistrationStatus = v1alpha1.RegistrationStatusRegistered + Expect(k8sClient.Status().Update(ctx, cluster1)).Should(Succeed()) + + Expect(k8sClient.Create(ctx, cluster2)).Should(Succeed()) + // update cluster status + getKey = types.NamespacedName{ + Namespace: cluster2.Namespace, + Name: cluster2.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, cluster2) + return err == nil + }, timeout, interval).Should(BeTrue()) + cluster2.Status.CniSubnet = []string{"192.168.1.0/24"} + cluster2.Status.RegistrationStatus = v1alpha1.RegistrationStatusRegistered + Expect(k8sClient.Status().Update(ctx, cluster2)).Should(Succeed()) + + Expect(k8sClient.Create(ctx, cluster3)).Should(Succeed()) + // update cluster status + getKey = types.NamespacedName{ + Namespace: cluster3.Namespace, + Name: cluster3.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, cluster3) + return err == nil + }, timeout, interval).Should(BeTrue()) + cluster3.Status.CniSubnet = []string{"10.1.1.1/16"} + cluster3.Status.RegistrationStatus = v1alpha1.RegistrationStatusRegistered + Expect(k8sClient.Status().Update(ctx, cluster3)).Should(Succeed()) + + // it should create sliceconfig + Expect(k8sClient.Create(ctx, slice)).Should(Succeed()) + + }) + AfterAll(func() { + // update sliceconfig tor remove clusters + createdSliceConfig := v1alpha1.SliceConfig{} + getKey := types.NamespacedName{ + Namespace: sliceNamespace, + Name: slice.Name, + } + Expect(k8sClient.Get(ctx, getKey, &createdSliceConfig)).Should(Succeed()) + createdSliceConfig.Spec.Clusters = []string{} + Expect(k8sClient.Update(ctx, &createdSliceConfig)).Should(Succeed()) + // wait till workersliceconfigs are deleted + workerSliceConfigList := workerv1alpha1.WorkerSliceConfigList{} + ls := map[string]string{ + "original-slice-name": slice.Name, + } + listOpts := []client.ListOption{ + client.MatchingLabels(ls), + } + Eventually(func() bool { + err := k8sClient.List(ctx, &workerSliceConfigList, listOpts...) + if err != nil { + return false + } + return len(workerSliceConfigList.Items) == 0 + }, timeout, interval).Should(BeTrue()) + + // it should delete sliceconfig + Expect(k8sClient.Delete(ctx, slice)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, cluster1)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, cluster2)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, cluster3)).Should(Succeed()) + + Expect(k8sClient.Delete(ctx, project)).Should(Succeed()) + }) + // NOTE:since there would be no job conrtoller present - the secrets(certs) will not be created + It("should create new jobs for cert creation once a valid renewBefore is set", func() { + By("fetching workerslice gatways") + workerSliceGwList := workerv1alpha1.WorkerSliceGatewayList{} + ls := map[string]string{ + "original-slice-name": slice.Name, + } + listOpts := []client.ListOption{ + client.MatchingLabels(ls), + } + Eventually(func() bool { + err := k8sClient.List(ctx, &workerSliceGwList, listOpts...) + if err != nil { + return false + } + return len(workerSliceGwList.Items) == 2 + }, timeout, interval).Should(BeTrue()) + + By("checking if jobs are created") + job := batchv1.JobList{} + o := map[string]string{ + "SLICE_NAME": slice.Name, + } + listOpts = []client.ListOption{ + client.MatchingLabels( + o, + ), + } + Eventually(func() bool { + err := k8sClient.List(ctx, &job, listOpts...) + if err != nil { + return false + } + fmt.Println("len of jobs", len(job.Items)) + return len(job.Items) == 1 + }, timeout, interval).Should(BeTrue()) + + createdSliceConfig := v1alpha1.SliceConfig{} + getKey := types.NamespacedName{ + Namespace: sliceNamespace, + Name: slice.Name, + } + Expect(k8sClient.Get(ctx, getKey, &createdSliceConfig)).Should(Succeed()) + now := metav1.Now() + createdSliceConfig.Spec.RenewBefore = &now + // update the sliceconfig + Expect(k8sClient.Update(ctx, &createdSliceConfig)).Should(Succeed()) + + // should update vpnkeyrotation config CertExpiryTS to now + // get vpnkey rotation config + createdVpnKeyConfig := &v1alpha1.VpnKeyRotation{} + getKey = types.NamespacedName{ + Namespace: slice.Namespace, + Name: slice.Name, + } + Eventually(func() bool { + err := k8sClient.Get(ctx, getKey, createdVpnKeyConfig) + return err == nil + }, timeout, interval).Should(BeTrue()) + + exyr, exmon, exday := now.Date() + gotyr, gotmonth, gotday := createdVpnKeyConfig.Spec.CertificateExpiryTime.Date() + Expect(exyr).To(Equal(gotyr)) + Expect(exmon).To(Equal(gotmonth)) + Expect(exday).To(Equal(gotday)) + + // should create new job to create new certs + By("checking if jobs are created") + job = batchv1.JobList{} + o = map[string]string{ + "SLICE_NAME": slice.Name, + } + listOpts = []client.ListOption{ + client.MatchingLabels( + o, + ), + } + Eventually(func() bool { + err := k8sClient.List(ctx, &job, listOpts...) + if err != nil { + return false + } + fmt.Println("len of jobs", len(job.Items)) + return len(job.Items) == 2 + }, timeout, interval).Should(BeTrue()) + }) + }) +}) + +func eventFound(events *v1.EventList, eventTitle string) bool { + for _, event := range events.Items { + if event.Labels["eventTitle"] == eventTitle { + return true + } + } + return false +} diff --git a/events/events_generated.go b/events/events_generated.go index 48cf1544..c006bab3 100644 --- a/events/events_generated.go +++ b/events/events_generated.go @@ -654,6 +654,70 @@ var EventsMap = map[events.EventName]*events.EventSchema{ ReportingController: "controller", Message: "Slice gateway job got created.", }, + "VPNKeyRotationConfigCreated": { + Name: "VPNKeyRotationConfigCreated", + Reason: "VPNKeyRotationConfigCreated", + Action: "CreateVPNKeyRotationConfig", + Type: events.EventTypeNormal, + ReportingController: "controller", + Message: "VPNKeyRotationConfig got created.", + }, + "VPNKeyRotationConfigCreationFailed": { + Name: "VPNKeyRotationConfigCreationFailed", + Reason: "VPNKeyRotationConfigCreationFailed", + Action: "CreateVPNKeyRotationConfig", + Type: events.EventTypeWarning, + ReportingController: "controller", + Message: "VPNKeyRotationConfig creation failed.", + }, + "VPNKeyRotationStart": { + Name: "VPNKeyRotationStart", + Reason: "VPNKeyRotationStart", + Action: "StartedVPNKeyRotationProcess", + Type: events.EventTypeNormal, + ReportingController: "controller", + Message: "VPNKeyRotation Process started , new certs will be created!", + }, + "VPNKeyRotationConfigUpdated": { + Name: "VPNKeyRotationConfigUpdated", + Reason: "VPNKeyRotationConfigUpdated", + Action: "UpdatedVPNKeyRotationConfig", + Type: events.EventTypeNormal, + ReportingController: "controller", + Message: "VPNKeyRotation Config Updated with CreationTS and ExpiryTS!", + }, + "CertificateJobCreationFailed": { + Name: "CertificateJobCreationFailed", + Reason: "CertificateJobCreationFailed", + Action: "VPNKeyRotation", + Type: events.EventTypeWarning, + ReportingController: "controller", + Message: "Failed creating certificate creation jobs!", + }, + "CertificatesRenewNow": { + Name: "CertificatesRenewNow", + Reason: "CertificatesRenewNow", + Action: "RenewBeforeInSliceConfig", + Type: events.EventTypeNormal, + ReportingController: "controller", + Message: "Certificates to be renewed Now!", + }, + "IllegalVPNKeyRotationConfigDelete": { + Name: "IllegalVPNKeyRotationConfigDelete", + Reason: "IllegalVPNKeyRotationConfigDelete", + Action: "DeleteVPNKeyRotationConfig", + Type: events.EventTypeWarning, + ReportingController: "controller", + Message: "Illegaly trying to delete VPNKeyRotationConfig", + }, + "CertificateJobFailed": { + Name: "CertificateJobFailed", + Reason: "CertificateJobFailed", + Action: "Failed CertCreationJob", + Type: events.EventTypeWarning, + ReportingController: "controller", + Message: "Warning - Certificate Creation job Failed", + }, } var ( @@ -736,4 +800,12 @@ var ( EventWorkerSliceGatewayCreated events.EventName = "WorkerSliceGatewayCreated" EventSliceGatewayJobCreationFailed events.EventName = "SliceGatewayJobCreationFailed" EventSliceGatewayJobCreated events.EventName = "SliceGatewayJobCreated" + EventVPNKeyRotationConfigCreated events.EventName = "VPNKeyRotationConfigCreated" + EventVPNKeyRotationConfigCreationFailed events.EventName = "VPNKeyRotationConfigCreationFailed" + EventVPNKeyRotationStart events.EventName = "VPNKeyRotationStart" + EventVPNKeyRotationConfigUpdated events.EventName = "VPNKeyRotationConfigUpdated" + EventCertificateJobCreationFailed events.EventName = "CertificateJobCreationFailed" + EventCertificatesRenewNow events.EventName = "CertificatesRenewNow" + EventIllegalVPNKeyRotationConfigDelete events.EventName = "IllegalVPNKeyRotationConfigDelete" + EventCertificateJobFailed events.EventName = "CertificateJobFailed" ) diff --git a/go.mod b/go.mod index 663d7457..d9f14286 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/kubeslice/kubeslice-controller -go 1.18 +go 1.19 require ( github.com/dailymotion/allure-go v0.7.0 @@ -23,6 +23,7 @@ require ( ) require ( + bou.ke/monkey v1.0.2 cloud.google.com/go v0.81.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.18 // indirect @@ -53,8 +54,8 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/onsi/ginkgo/v2 v2.9.5 - github.com/onsi/gomega v1.27.6 + github.com/onsi/ginkgo/v2 v2.9.7 + github.com/onsi/gomega v1.27.8 github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect diff --git a/go.sum b/go.sum index dcf1d2c4..4b185f94 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +bou.ke/monkey v1.0.2 h1:kWcnsrCNUatbxncxR/ThdYqbytgOIArtYWqcQLQzKLI= +bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -381,13 +383,13 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= +github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= diff --git a/integration.dockerfile b/integration.dockerfile new file mode 100644 index 00000000..e645342f --- /dev/null +++ b/integration.dockerfile @@ -0,0 +1,24 @@ +FROM golang:1.19 as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +COPY Makefile Makefile + +# Copy the go source +COPY apis/ apis/ +COPY config/ config +COPY controllers/ controllers/ +COPY events/ events/ +COPY hack/ hack/ +COPY metrics/ metrics/ +COPY service/ service/ +COPY util/ util/ + +# Download dependencies +RUN make envtest +RUN make controller-gen + +# CMD ["make", "test-local"] +CMD ["make", "int-test"] \ No newline at end of file diff --git a/main.go b/main.go index 673d8c6e..5f7f8203 100644 --- a/main.go +++ b/main.go @@ -21,24 +21,26 @@ import ( "fmt" "os" - ossEvents "github.com/kubeslice/kubeslice-controller/events" "github.com/kubeslice/kubeslice-monitoring/pkg/events" - "github.com/kubeslice/kubeslice-controller/metrics" + ossEvents "github.com/kubeslice/kubeslice-controller/events" + "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "github.com/kubeslice/kubeslice-controller/metrics" + controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" workerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1" "github.com/kubeslice/kubeslice-controller/controllers/controller" "github.com/kubeslice/kubeslice-controller/controllers/worker" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" //+kubebuilder:scaffold:imports ) @@ -69,10 +71,11 @@ func main() { wsi := service.WithWorkerServiceImportService(mr) se := service.WithServiceExportConfigService(wsi, mr) wsgrs := service.WithWorkerSliceGatewayRecyclerService() - sc := service.WithSliceConfigService(ns, acs, wsgs, wscs, wsi, se, wsgrs, mr) + vpn := service.WithVpnKeyRotationService(wsgs, wscs) + sc := service.WithSliceConfigService(ns, acs, wsgs, wscs, wsi, se, wsgrs, mr, vpn) sqcs := service.WithSliceQoSConfigService(wscs, mr) p := service.WithProjectService(ns, acs, c, sc, se, sqcs, mr) - initialize(service.WithServices(wscs, p, c, sc, se, wsgs, wsi, sqcs, wsgrs)) + initialize(service.WithServices(wscs, p, c, sc, se, wsgs, wsi, sqcs, wsgrs, vpn)) } func initialize(services *service.Services) { @@ -249,6 +252,16 @@ func initialize(services *service.Services) { setupLog.Error(err, "unable to create controller", "controller", "SliceQoSConfig") os.Exit(1) } + if err = (&controller.VpnKeyRotationReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: controllerLog.With("name", "VpnKeyRotationConfig"), + VpnKeyRotationService: services.VpnKeyRotationService, + EventRecorder: &eventRecorder, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "VpnKeyRotationConfig") + os.Exit(1) + } if os.Getenv("ENABLE_WEBHOOKS") != "false" { if err = (&controllerv1alpha1.Project{}).SetupWebhookWithManager(mgr, service.ValidateProjectCreate, service.ValidateProjectUpdate, service.ValidateProjectDelete); err != nil { @@ -279,6 +292,10 @@ func initialize(services *service.Services) { setupLog.Error(err, "unable to create webhook", "webhook", "SliceQoSConfig") os.Exit(1) } + if err = (&controllerv1alpha1.VpnKeyRotation{}).SetupWebhookWithManager(mgr, service.ValidateVpnKeyRotationCreate, service.ValidateVpnKeyRotationDelete); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "VpnKeyRotation") + os.Exit(1) + } } //+kubebuilder:scaffold:builder @@ -301,9 +318,9 @@ func initialize(services *service.Services) { //All Controller RBACs goes here. -//+kubebuilder:rbac:groups=controller.kubeslice.io,resources=projects;clusters;sliceconfigs;serviceexportconfigs;sliceqosconfigs,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=controller.kubeslice.io,resources=projects/status;clusters/status;sliceconfigs/status;serviceexportconfigs/status;sliceqosconfigs/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=controller.kubeslice.io,resources=projects/finalizers;clusters/finalizers;sliceconfigs/finalizers;serviceexportconfigs/finalizers;sliceqosconfigs/finalizers,verbs=update +//+kubebuilder:rbac:groups=controller.kubeslice.io,resources=projects;clusters;sliceconfigs;serviceexportconfigs;sliceqosconfigs;vpnkeyrotations,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=controller.kubeslice.io,resources=projects/status;clusters/status;sliceconfigs/status;serviceexportconfigs/status;sliceqosconfigs/status;vpnkeyrotations/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=controller.kubeslice.io,resources=projects/finalizers;clusters/finalizers;sliceconfigs/finalizers;serviceexportconfigs/finalizers;sliceqosconfigs/finalizers;vpnkeyrotations/finalizers,verbs=update //+kubebuilder:rbac:groups=worker.kubeslice.io,resources=workersliceconfigs;workerserviceimports;workerslicegateways;workerslicegwrecyclers,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=worker.kubeslice.io,resources=workersliceconfigs/status;workerserviceimports/status;workerslicegateways/status;workerslicegwrecyclers/status,verbs=get;update;patch diff --git a/service/bootstrap.go b/service/bootstrap.go index 542453bd..666f3bfa 100644 --- a/service/bootstrap.go +++ b/service/bootstrap.go @@ -28,6 +28,7 @@ type Services struct { WorkerServiceImportService IWorkerServiceImportService SliceQoSConfigService ISliceQoSConfigService WorkerSliceGatewayRecyclerService IWorkerSliceGatewayRecyclerService + VpnKeyRotationService IVpnKeyRotationService } // bootstrapping Services @@ -41,6 +42,7 @@ func WithServices( wsis IWorkerServiceImportService, sqcs ISliceQoSConfigService, wsgrs IWorkerSliceGatewayRecyclerService, + vpn IVpnKeyRotationService, ) *Services { return &Services{ ProjectService: ps, @@ -52,6 +54,7 @@ func WithServices( WorkerServiceImportService: wsis, SliceQoSConfigService: sqcs, WorkerSliceGatewayRecyclerService: wsgrs, + VpnKeyRotationService: vpn, } } @@ -101,6 +104,7 @@ func WithSliceConfigService( se IServiceExportConfigService, wsgrs IWorkerSliceGatewayRecyclerService, mf metrics.IMetricRecorder, + vpn IVpnKeyRotationService, ) ISliceConfigService { return &SliceConfigService{ ns: ns, @@ -111,6 +115,7 @@ func WithSliceConfigService( se: se, wsgrs: wsgrs, mf: mf, + vpn: vpn, } } @@ -199,3 +204,11 @@ func WithSliceQoSConfigService(wsc IWorkerSliceConfigService, mf metrics.IMetric func WithMetricsRecorder() metrics.IMetricRecorder { return &metrics.MetricRecorder{} } + +// bootstrapping Vpn Key Rotation service +func WithVpnKeyRotationService(w IWorkerSliceGatewayService, ws IWorkerSliceConfigService) IVpnKeyRotationService { + return &VpnKeyRotationService{ + wsgs: w, + wscs: ws, + } +} diff --git a/service/cluster_service.go b/service/cluster_service.go index 919be473..552a0293 100644 --- a/service/cluster_service.go +++ b/service/cluster_service.go @@ -238,6 +238,9 @@ func (c *ClusterService) ReconcileCluster(ctx context.Context, req ctrl.Request) if err != nil { return ctrl.Result{}, err } + if secret.Data == nil { + secret.Data = make(map[string][]byte) + } secret.Data["controllerEndpoint"] = []byte(ControllerEndpoint) secret.Data["clusterName"] = []byte(cluster.Name) err = util.UpdateResource(ctx, &secret) diff --git a/service/job_service.go b/service/job_service.go index 43f2398d..86651066 100644 --- a/service/job_service.go +++ b/service/job_service.go @@ -62,6 +62,9 @@ func (j *JobService) CreateJob(ctx context.Context, namespace string, jobImage s ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, + Labels: map[string]string{ + "SLICE_NAME": environment["SLICE_NAME"], + }, }, Spec: batchv1.JobSpec{ TTLSecondsAfterFinished: &tTLSecondsAfterFinished, diff --git a/service/kube_slice_resource_names.go b/service/kube_slice_resource_names.go index b557a92d..d567df97 100644 --- a/service/kube_slice_resource_names.go +++ b/service/kube_slice_resource_names.go @@ -48,6 +48,7 @@ const ( resourceSecrets = "secrets" resourceEvents = "events" ResourceStatusSuffix = "/status" + resourceVpnKeyRotationConfigs = "vpnkeyrotations" ) // metric kind @@ -127,15 +128,16 @@ const ( // Finalizers const ( - ProjectFinalizer = "controller.kubeslice.io/project-finalizer" - ClusterFinalizer = "controller.kubeslice.io/cluster-finalizer" - ClusterDeregisterFinalizer = "worker.kubeslice.io/cluster-deregister-finalizer" - SliceConfigFinalizer = "controller.kubeslice.io/slice-configuration-finalizer" - serviceExportConfigFinalizer = "controller.kubeslice.io/service-export-finalizer" - WorkerSliceConfigFinalizer = "worker.kubeslice.io/worker-slice-configuration-finalizer" - WorkerSliceGatewayFinalizer = "worker.kubeslice.io/worker-slice-gateway-finalizer" - WorkerServiceImportFinalizer = "worker.kubeslice.io/worker-service-import-finalizer" - SliceQoSConfigFinalizer = "controller.kubeslice.io/slice-qos-config-finalizer" + ProjectFinalizer = "controller.kubeslice.io/project-finalizer" + ClusterFinalizer = "controller.kubeslice.io/cluster-finalizer" + ClusterDeregisterFinalizer = "worker.kubeslice.io/cluster-deregister-finalizer" + SliceConfigFinalizer = "controller.kubeslice.io/slice-configuration-finalizer" + serviceExportConfigFinalizer = "controller.kubeslice.io/service-export-finalizer" + WorkerSliceConfigFinalizer = "worker.kubeslice.io/worker-slice-configuration-finalizer" + WorkerSliceGatewayFinalizer = "worker.kubeslice.io/worker-slice-gateway-finalizer" + WorkerServiceImportFinalizer = "worker.kubeslice.io/worker-service-import-finalizer" + SliceQoSConfigFinalizer = "controller.kubeslice.io/slice-qos-config-finalizer" + VPNKeyRotationConfigFinalizer = "controller.kubeslice.io/vpn-key-rotation-config-finalizer" ) // ControllerEndpoint @@ -213,6 +215,11 @@ var ( APIGroups: []string{apiGroupKubeSliceWorker}, Resources: []string{resourceWorkerSliceGwRecycler}, }, + { + Verbs: []string{verbUpdate, verbPatch, verbGet, verbList, verbWatch}, + APIGroups: []string{apiGroupKubeSliceControllers}, + Resources: []string{resourceVpnKeyRotationConfigs}, + }, { Verbs: []string{verbUpdate, verbPatch, verbGet, verbList, verbWatch}, APIGroups: []string{apiGroupKubeSliceWorker}, @@ -221,7 +228,7 @@ var ( { Verbs: []string{verbUpdate, verbPatch, verbGet}, APIGroups: []string{apiGroupKubeSliceControllers}, - Resources: []string{resourceCluster + ResourceStatusSuffix}, + Resources: []string{resourceCluster + ResourceStatusSuffix, resourceVpnKeyRotationConfigs + ResourceStatusSuffix}, }, { Verbs: []string{verbUpdate, verbPatch, verbGet}, diff --git a/service/mocks/IVpnKeyRotationService.go b/service/mocks/IVpnKeyRotationService.go new file mode 100644 index 00000000..17ce1d8d --- /dev/null +++ b/service/mocks/IVpnKeyRotationService.go @@ -0,0 +1,96 @@ +// Code generated by mockery v2.22.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + reconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" +) + +// IVpnKeyRotationService is an autogenerated mock type for the IVpnKeyRotationService type +type IVpnKeyRotationService struct { + mock.Mock +} + +// CreateMinimalVpnKeyRotationConfig provides a mock function with given fields: ctx, sliceName, namespace, r +func (_m *IVpnKeyRotationService) CreateMinimalVpnKeyRotationConfig(ctx context.Context, sliceName string, namespace string, r int) error { + ret := _m.Called(ctx, sliceName, namespace, r) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, int) error); ok { + r0 = rf(ctx, sliceName, namespace, r) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ReconcileClusters provides a mock function with given fields: ctx, sliceName, namespace, clusters +func (_m *IVpnKeyRotationService) ReconcileClusters(ctx context.Context, sliceName string, namespace string, clusters []string) (*v1alpha1.VpnKeyRotation, error) { + ret := _m.Called(ctx, sliceName, namespace, clusters) + + var r0 *v1alpha1.VpnKeyRotation + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, []string) (*v1alpha1.VpnKeyRotation, error)); ok { + return rf(ctx, sliceName, namespace, clusters) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, []string) *v1alpha1.VpnKeyRotation); ok { + r0 = rf(ctx, sliceName, namespace, clusters) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1alpha1.VpnKeyRotation) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, []string) error); ok { + r1 = rf(ctx, sliceName, namespace, clusters) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ReconcileVpnKeyRotation provides a mock function with given fields: ctx, req +func (_m *IVpnKeyRotationService) ReconcileVpnKeyRotation(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + ret := _m.Called(ctx, req) + + var r0 reconcile.Result + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, reconcile.Request) (reconcile.Result, error)); ok { + return rf(ctx, req) + } + if rf, ok := ret.Get(0).(func(context.Context, reconcile.Request) reconcile.Result); ok { + r0 = rf(ctx, req) + } else { + r0 = ret.Get(0).(reconcile.Result) + } + + if rf, ok := ret.Get(1).(func(context.Context, reconcile.Request) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewIVpnKeyRotationService interface { + mock.TestingT + Cleanup(func()) +} + +// NewIVpnKeyRotationService creates a new instance of IVpnKeyRotationService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewIVpnKeyRotationService(t mockConstructorTestingTNewIVpnKeyRotationService) *IVpnKeyRotationService { + mock := &IVpnKeyRotationService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/service/mocks/IWorkerSliceGatewayService.go b/service/mocks/IWorkerSliceGatewayService.go index fb6632a9..cbceba23 100644 --- a/service/mocks/IWorkerSliceGatewayService.go +++ b/service/mocks/IWorkerSliceGatewayService.go @@ -10,6 +10,8 @@ import ( reconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" + util "github.com/kubeslice/kubeslice-controller/util" + v1alpha1 "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1" ) @@ -18,6 +20,20 @@ type IWorkerSliceGatewayService struct { mock.Mock } +// BuildNetworkAddresses provides a mock function with given fields: sliceSubnet, sourceClusterName, destinationClusterName, clusterMap, clusterCidr +func (_m *IWorkerSliceGatewayService) BuildNetworkAddresses(sliceSubnet string, sourceClusterName string, destinationClusterName string, clusterMap map[string]int, clusterCidr string) util.WorkerSliceGatewayNetworkAddresses { + ret := _m.Called(sliceSubnet, sourceClusterName, destinationClusterName, clusterMap, clusterCidr) + + var r0 util.WorkerSliceGatewayNetworkAddresses + if rf, ok := ret.Get(0).(func(string, string, string, map[string]int, string) util.WorkerSliceGatewayNetworkAddresses); ok { + r0 = rf(sliceSubnet, sourceClusterName, destinationClusterName, clusterMap, clusterCidr) + } else { + r0 = ret.Get(0).(util.WorkerSliceGatewayNetworkAddresses) + } + + return r0 +} + // CreateMinimumWorkerSliceGateways provides a mock function with given fields: ctx, sliceName, clusterNames, namespace, label, clusterMap, sliceSubnet, clusterCidr func (_m *IWorkerSliceGatewayService) CreateMinimumWorkerSliceGateways(ctx context.Context, sliceName string, clusterNames []string, namespace string, label map[string]string, clusterMap map[string]int, sliceSubnet string, clusterCidr string) (reconcile.Result, error) { ret := _m.Called(ctx, sliceName, clusterNames, namespace, label, clusterMap, sliceSubnet, clusterCidr) @@ -56,6 +72,20 @@ func (_m *IWorkerSliceGatewayService) DeleteWorkerSliceGatewaysByLabel(ctx conte return r0 } +// GenerateCerts provides a mock function with given fields: ctx, sliceName, namespace, serverGateway, clientGateway, gatewayAddresses +func (_m *IWorkerSliceGatewayService) GenerateCerts(ctx context.Context, sliceName string, namespace string, serverGateway *v1alpha1.WorkerSliceGateway, clientGateway *v1alpha1.WorkerSliceGateway, gatewayAddresses util.WorkerSliceGatewayNetworkAddresses) error { + ret := _m.Called(ctx, sliceName, namespace, serverGateway, clientGateway, gatewayAddresses) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, *v1alpha1.WorkerSliceGateway, *v1alpha1.WorkerSliceGateway, util.WorkerSliceGatewayNetworkAddresses) error); ok { + r0 = rf(ctx, sliceName, namespace, serverGateway, clientGateway, gatewayAddresses) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // ListWorkerSliceGateways provides a mock function with given fields: ctx, ownerLabel, namespace func (_m *IWorkerSliceGatewayService) ListWorkerSliceGateways(ctx context.Context, ownerLabel map[string]string, namespace string) ([]v1alpha1.WorkerSliceGateway, error) { ret := _m.Called(ctx, ownerLabel, namespace) diff --git a/service/slice_config_service.go b/service/slice_config_service.go index 8e74f7e6..9e9d9d4d 100644 --- a/service/slice_config_service.go +++ b/service/slice_config_service.go @@ -44,6 +44,7 @@ type SliceConfigService struct { se IServiceExportConfigService wsgrs IWorkerSliceGatewayRecyclerService mf metrics.IMetricRecorder + vpn IVpnKeyRotationService } // ReconcileSliceConfig is a function to reconcile the sliceconfig @@ -143,7 +144,19 @@ func (s *SliceConfigService) ReconcileSliceConfig(ctx context.Context, req ctrl. } logger.Infof("sliceConfig %v reconciled", req.NamespacedName) - // Step 5: Create ServiceImport Objects + // Step 5: Create VPNKeyRotation CR + // TODO(rahul): handle change in rotation interval + if err := s.vpn.CreateMinimalVpnKeyRotationConfig(ctx, sliceConfig.Name, sliceConfig.Namespace, sliceConfig.Spec.RotationInterval); err != nil { + // register an event + util.RecordEvent(ctx, eventRecorder, sliceConfig, nil, events.EventVPNKeyRotationConfigCreationFailed) + return ctrl.Result{}, err + } + // Step 6: update cluster info into vpnkeyrotation Cconfig + if _, err := s.vpn.ReconcileClusters(ctx, sliceConfig.Name, sliceConfig.Namespace, sliceConfig.Spec.Clusters); err != nil { + return ctrl.Result{}, err + } + + // Step 7: Create ServiceImport Objects serviceExports := &v1alpha1.ServiceExportConfigList{} _, err = s.getServiceExportBySliceName(ctx, req.Namespace, sliceConfig.Name, serviceExports) if err != nil { diff --git a/service/slice_config_service_test.go b/service/slice_config_service_test.go index a81b143a..a09d414b 100644 --- a/service/slice_config_service_test.go +++ b/service/slice_config_service_test.go @@ -93,10 +93,12 @@ func SliceConfigReconciliationCompleteHappyCase(t *testing.T) { arg.Name = requestObj.Namespace arg.Labels[util.LabelName] = fmt.Sprintf(util.LabelValue, "Project", requestObj.Namespace) }).Once() + clientMock.On("Get", ctx, mock.Anything, mock.Anything).Return(nil) clusterMap := map[string]int{ "cluster-1": 1, "cluster-2": 2, } + workerSliceConfigMock.On("CreateMinimalWorkerSliceConfig", ctx, mock.Anything, requestObj.Namespace, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(clusterMap, nil).Once() workerSliceGatewayMock.On("CreateMinimumWorkerSliceGateways", ctx, mock.Anything, mock.Anything, requestObj.Namespace, mock.Anything, clusterMap, mock.Anything, mock.Anything).Return(ctrl.Result{}, nil).Once() label := map[string]string{ @@ -662,6 +664,7 @@ func setupSliceConfigTest(name string, namespace string) (*mocks.IWorkerSliceGat workerServiceImportMock := &mocks.IWorkerServiceImportService{} workerSliceGatewayRecyclerMock := &mocks.IWorkerSliceGatewayRecyclerService{} mMock := &metricMock.IMetricRecorder{} + vpn := mocks.IVpnKeyRotationService{} sliceConfigService := SliceConfigService{ sgs: workerSliceGatewayMock, ms: workerSliceConfigMock, @@ -669,6 +672,7 @@ func setupSliceConfigTest(name string, namespace string) (*mocks.IWorkerSliceGat si: workerServiceImportMock, wsgrs: workerSliceGatewayRecyclerMock, mf: mMock, + vpn: &vpn, } namespacedName := types.NamespacedName{ Name: name, @@ -687,6 +691,8 @@ func setupSliceConfigTest(name string, namespace string) (*mocks.IWorkerSliceGat Component: util.ComponentController, Slice: util.NotApplicable, }) + vpn.On("CreateMinimalVpnKeyRotationConfig", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + vpn.On("ReconcileClusters", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) ctx := util.PrepareKubeSliceControllersRequestContext(context.Background(), clientMock, scheme, "SliceConfigServiceTest", &eventRecorder) return workerSliceGatewayMock, workerSliceConfigMock, serviceExportConfigMock, workerServiceImportMock, workerSliceGatewayRecyclerMock, clientMock, sliceConfig, ctx, sliceConfigService, requestObj, mMock } diff --git a/service/slice_config_webhook_validation.go b/service/slice_config_webhook_validation.go index 12c8647c..fba314e9 100644 --- a/service/slice_config_webhook_validation.go +++ b/service/slice_config_webhook_validation.go @@ -22,8 +22,11 @@ import ( "regexp" "strconv" "strings" + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" workerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1" @@ -93,6 +96,12 @@ func ValidateSliceConfigUpdate(ctx context.Context, sliceConfig *controllerv1alp if err := preventMaxClusterCountUpdate(ctx, sliceConfig, old); err != nil { return apierrors.NewInvalid(schema.GroupKind{Group: apiGroupKubeSliceControllers, Kind: "SliceConfig"}, sliceConfig.Name, field.ErrorList{err}) } + if err := validateRenewNowInSliceConfig(ctx, sliceConfig, old); err != nil { + return apierrors.NewInvalid(schema.GroupKind{Group: apiGroupKubeSliceControllers, Kind: "SliceConfig"}, sliceConfig.Name, field.ErrorList{err}) + } + if _, err := validateRotationIntervalInSliceConfig(ctx, sliceConfig, old); err != nil { + return apierrors.NewInvalid(schema.GroupKind{Group: apiGroupKubeSliceControllers, Kind: "SliceConfig"}, sliceConfig.Name, field.ErrorList{err}) + } return nil } @@ -107,6 +116,82 @@ func ValidateSliceConfigDelete(ctx context.Context, sliceConfig *controllerv1alp return nil } +func validateRenewNowInSliceConfig(ctx context.Context, sliceConfig *controllerv1alpha1.SliceConfig, old runtime.Object) *field.Error { + oldSliceConfig := old.(*controllerv1alpha1.SliceConfig) + // nochange detected + if sliceConfig.Spec.RenewBefore.Equal(oldSliceConfig.Spec.RenewBefore) { + return nil + } + // change detected + vpnKeyRotation := controllerv1alpha1.VpnKeyRotation{} + exists, _ := util.GetResourceIfExist(ctx, types.NamespacedName{ + Namespace: sliceConfig.Namespace, + Name: sliceConfig.Name, + }, &vpnKeyRotation) + if exists { + for gateway := range vpnKeyRotation.Status.CurrentRotationState { + status, ok := vpnKeyRotation.Status.CurrentRotationState[gateway] + if ok { + if status.Status != controllerv1alpha1.Complete { + return &field.Error{ + Type: field.ErrorTypeForbidden, + Field: "Field: RenewBefore", + Detail: fmt.Sprintf("Certs Renewal status for %s gateway is not in Complete state", gateway), + } + } + } + } + } + // check if we are past and its a correct time + if !time.Now().After(sliceConfig.Spec.RenewBefore.Time) { + return &field.Error{ + Type: field.ErrorTypeForbidden, + Field: "Field: RenewBefore", + Detail: "Renewal Time inappropriate for sliceconfig", + } + } + + vpnKeyRotation.Spec.CertificateExpiryTime = sliceConfig.Spec.RenewBefore + err := util.UpdateResource(ctx, &vpnKeyRotation) + if err != nil { + return &field.Error{ + Type: field.ErrorTypeForbidden, + Field: "Field: RenewBefore", + Detail: "Failed to Update Renewal Time, Please try again!", + } + } + return nil +} + +func validateRotationIntervalInSliceConfig(ctx context.Context, sliceConfig *controllerv1alpha1.SliceConfig, old runtime.Object) (*controllerv1alpha1.VpnKeyRotation, *field.Error) { + oldSliceConfig := old.(*controllerv1alpha1.SliceConfig) + // nochange detected + if sliceConfig.Spec.RotationInterval == oldSliceConfig.Spec.RotationInterval { + return nil, nil + } + // change detected + vpnKeyRotation := controllerv1alpha1.VpnKeyRotation{} + exists, _ := util.GetResourceIfExist(ctx, types.NamespacedName{ + Namespace: sliceConfig.Namespace, + Name: sliceConfig.Name, + }, &vpnKeyRotation) + if exists { + vpnKeyRotation.Spec.RotationInterval = sliceConfig.Spec.RotationInterval + // update the new expiry TS + expiryTS := metav1.NewTime(vpnKeyRotation.Spec.CertificateCreationTime.AddDate(0, 0, vpnKeyRotation.Spec.RotationInterval).Add(-1 * time.Hour)) + vpnKeyRotation.Spec.CertificateExpiryTime = &expiryTS + err := util.UpdateResource(ctx, &vpnKeyRotation) + if err != nil { + return nil, &field.Error{ + Type: field.ErrorTypeForbidden, + Field: "Field: RenewBefore", + Detail: "Failed to Update Renewal Time, Please try again!", + } + } + } + return &vpnKeyRotation, nil +} + // checkNamespaceDeboardingStatus checks if the namespace is deboarding func checkNamespaceDeboardingStatus(ctx context.Context, sliceConfig *controllerv1alpha1.SliceConfig) *field.Error { workerSlices := &workerv1alpha1.WorkerSliceConfigList{} @@ -263,6 +348,9 @@ func preventUpdate(ctx context.Context, sc *controllerv1alpha1.SliceConfig, old if sliceConfig.Spec.SliceIpamType != sc.Spec.SliceIpamType { return field.Invalid(field.NewPath("Spec").Child("SliceIpamType"), sc.Spec.SliceIpamType, "cannot be updated") } + if sliceConfig.Spec.VPNConfig.Cipher != sc.Spec.VPNConfig.Cipher { + return field.Invalid(field.NewPath("Spec").Child("VPNConfig").Child("Cipher"), sc.Spec.VPNConfig.Cipher, "cannot be updated") + } return nil } diff --git a/service/slice_config_webhook_validation_test.go b/service/slice_config_webhook_validation_test.go index b5526f1d..76dffc4a 100644 --- a/service/slice_config_webhook_validation_test.go +++ b/service/slice_config_webhook_validation_test.go @@ -20,8 +20,10 @@ import ( "context" "fmt" "testing" + "time" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" "github.com/dailymotion/allure-go" controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" @@ -107,6 +109,14 @@ var SliceConfigWebhookValidationTestBed = map[string]func(*testing.T){ "SliceConfigWebhookValidation_ValidateQosProfileStandardQosProfileNameDoesNotExist": ValidateQosProfileStandardQosProfileNameDoesNotExist, "SliceConfigWebhookValidation_ValidateMaxCluster": ValidateMaxCluster, "SliceConfigWebhookValidation_ValidateMaxClusterForParticipatingCluster": ValidateMaxClusterForParticipatingCluster, + "TestValidateCertsRotationInterval_Positive": TestValidateCertsRotationInterval_Positive, + "TestValidateCertsRotationInterval_Negative": TestValidateCertsRotationInterval_Negative, + "TestValidateCertsRotationInterval_inProgressClusterStatus": TestValidateCertsRotationInterval_NegativeClusterStatus, + "TestValidateCertsRotationInterval_PositiveClusterStatus": TestValidateCertsRotationInterval_PositiveClusterStatus, + "TestValidateRotationInterval_Change_Decreased": TestValidateRotationInterval_Change_Decreased, + "TestValidateRotationInterval_Change_Increased": TestValidateRotationInterval_Change_Increased, + "TestValidateRotationInterval_NoChange": TestValidateRotationInterval_NoChange, + "SliceConfigWebhookValidation_UpdateValidateSliceConfigUpdatingVPNCipher": UpdateValidateSliceConfigUpdatingVPNCipher, } func CreateValidateProjectNamespaceDoesNotExist(t *testing.T) { @@ -698,6 +708,9 @@ func CreateValidateSliceConfigWithoutErrors(t *testing.T) { func UpdateValidateSliceConfigUpdatingSliceSubnet(t *testing.T) { oldSliceConfig := controllerv1alpha1.SliceConfig{} oldSliceConfig.Spec.SliceSubnet = "192.168.1.0/16" + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } name := "slice_config" namespace := "namespace" clientMock, newSliceConfig, ctx := setupSliceConfigWebhookValidationTest(name, namespace) @@ -709,8 +722,28 @@ func UpdateValidateSliceConfigUpdatingSliceSubnet(t *testing.T) { clientMock.AssertExpectations(t) } +func UpdateValidateSliceConfigUpdatingVPNCipher(t *testing.T) { + oldSliceConfig := controllerv1alpha1.SliceConfig{} + oldSliceConfig.Spec.SliceSubnet = "192.168.1.0/16" + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-128-CBC", + } + name := "slice_config" + namespace := "namespace" + clientMock, newSliceConfig, ctx := setupSliceConfigWebhookValidationTest(name, namespace) + newSliceConfig.Spec.SliceSubnet = "192.168.1.0/16" + err := ValidateSliceConfigUpdate(ctx, newSliceConfig, runtime.Object(&oldSliceConfig)) + require.NotNil(t, err) + require.Contains(t, err.Error(), "Spec.VPNConfig.Cipher: Invalid value:") + require.Contains(t, err.Error(), "cannot be updated") + clientMock.AssertExpectations(t) +} + func UpdateValidateSliceConfigUpdatingSliceType(t *testing.T) { oldSliceConfig := controllerv1alpha1.SliceConfig{} + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } oldSliceConfig.Spec.SliceType = "TYPE_1" name := "slice_config" namespace := "namespace" @@ -725,6 +758,9 @@ func UpdateValidateSliceConfigUpdatingSliceType(t *testing.T) { func UpdateValidateSliceConfigUpdatingSliceGatewayType(t *testing.T) { oldSliceConfig := controllerv1alpha1.SliceConfig{} + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } oldSliceConfig.Spec.SliceGatewayProvider.SliceGatewayType = "TYPE_1" name := "slice_config" namespace := "namespace" @@ -739,6 +775,9 @@ func UpdateValidateSliceConfigUpdatingSliceGatewayType(t *testing.T) { func UpdateValidateSliceConfigUpdatingSliceCaType(t *testing.T) { oldSliceConfig := controllerv1alpha1.SliceConfig{} + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } oldSliceConfig.Spec.SliceGatewayProvider.SliceCaType = "TYPE_1" name := "slice_config" namespace := "namespace" @@ -753,6 +792,9 @@ func UpdateValidateSliceConfigUpdatingSliceCaType(t *testing.T) { func UpdateValidateSliceConfigUpdatingSliceIpamType(t *testing.T) { oldSliceConfig := controllerv1alpha1.SliceConfig{} + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } oldSliceConfig.Spec.SliceIpamType = "TYPE_1" name := "slice_config" namespace := "namespace" @@ -868,7 +910,6 @@ func UpdateValidateSliceConfigWithNewClusterUnhealthy(t *testing.T) { }).Once() err := ValidateSliceConfigUpdate(ctx, newSliceConfig, runtime.Object(oldSliceConfig)) t.Log(err.Error()) - require.NotNil(t, err) require.Contains(t, err.Error(), "Spec.Clusters: Invalid value:") require.Contains(t, err.Error(), "cluster health is not normal") require.Contains(t, err.Error(), newSliceConfig.Spec.Clusters[2]) @@ -1612,7 +1653,256 @@ func ValidateMaxClusterForParticipatingCluster(t *testing.T) { require.Contains(t, err.Error(), "participating clusters cannot be greater than MaxClusterCount") clientMock.AssertExpectations(t) } +func TestValidateCertsRotationInterval_Positive(t *testing.T) { + now := metav1.Now() + name := "slice_config" + namespace := "randomNamespace" + clientMock, sliceConfig, ctx := setupSliceConfigWebhookValidationTest(name, namespace) + sliceConfig.Spec.RenewBefore = &now + expiry := metav1.Now().Add(30) + clientMock.On("Get", ctx, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(2).(*controllerv1alpha1.VpnKeyRotation) + arg.ObjectMeta = metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + } + arg.Spec = controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: name, + CertificateCreationTime: &now, + CertificateExpiryTime: &metav1.Time{Time: expiry}, + } + }).Once() + + clientMock.On("Update", mock.Anything, mock.Anything).Return(nil) + oldSliceConfig := controllerv1alpha1.SliceConfig{} + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } + err := validateRenewNowInSliceConfig(ctx, sliceConfig, &oldSliceConfig) + require.Nil(t, err) +} + +func TestValidateCertsRotationInterval_Negative(t *testing.T) { + name := "slice_config" + namespace := "randomNamespace" + clientMock, sliceConfig, ctx := setupSliceConfigWebhookValidationTest(name, namespace) + // RenewBefore is 1 hour after, decline + renewBefore := metav1.Time{Time: metav1.Now().Add(time.Hour * 1)} + sliceConfig.Spec.RenewBefore = &renewBefore + expiry := metav1.Now().Add(30) + now := metav1.Now() + clientMock.On("Get", ctx, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(2).(*controllerv1alpha1.VpnKeyRotation) + arg.ObjectMeta = metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + } + arg.Spec = controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: name, + CertificateCreationTime: &now, + CertificateExpiryTime: &metav1.Time{Time: expiry}, + } + }).Once() + oldSliceConfig := controllerv1alpha1.SliceConfig{} + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } + err := validateRenewNowInSliceConfig(ctx, sliceConfig, &oldSliceConfig) + require.NotNil(t, err) +} + +func TestValidateCertsRotationInterval_NegativeClusterStatus(t *testing.T) { + name := "slice_config" + namespace := "randomNamespace" + clientMock, sliceConfig, ctx := setupSliceConfigWebhookValidationTest(name, namespace) + now := metav1.Now() + sliceConfig.Spec.RenewBefore = &now + expiry := metav1.Now().Add(30) + clientMock.On("Get", ctx, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(2).(*controllerv1alpha1.VpnKeyRotation) + arg.ObjectMeta = metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + } + arg.Spec = controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: name, + CertificateCreationTime: &now, + CertificateExpiryTime: &metav1.Time{Time: expiry}, + ClusterGatewayMapping: map[string][]string{ + "cluster-1": {"gateway-1"}, + "cluster-2": {"gateway-2"}, + }, + } + arg.Status = controllerv1alpha1.VpnKeyRotationStatus{ + CurrentRotationState: map[string]controllerv1alpha1.StatusOfKeyRotation{ + + "gateway-1": controllerv1alpha1.StatusOfKeyRotation{ + Status: controllerv1alpha1.Complete, + LastUpdatedTimestamp: metav1.Now(), + }, + "gateway-2": controllerv1alpha1.StatusOfKeyRotation{ + Status: controllerv1alpha1.InProgress, + LastUpdatedTimestamp: metav1.Now(), + }, + }, + } + }).Once() + oldSliceConfig := controllerv1alpha1.SliceConfig{} + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } + err := validateRenewNowInSliceConfig(ctx, sliceConfig, &oldSliceConfig) + require.NotNil(t, err) + require.Equal(t, err.Type, field.ErrorTypeForbidden) +} +func TestValidateCertsRotationInterval_PositiveClusterStatus(t *testing.T) { + name := "slice_config" + namespace := "randomNamespace" + clientMock, sliceConfig, ctx := setupSliceConfigWebhookValidationTest(name, namespace) + now := metav1.Now() + sliceConfig.Spec.RenewBefore = &now + expiry := metav1.Now().Add(30) + + clientMock.On("Get", ctx, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(2).(*controllerv1alpha1.VpnKeyRotation) + arg.ObjectMeta = metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + } + arg.Spec = controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: name, + CertificateCreationTime: &now, + CertificateExpiryTime: &metav1.Time{expiry}, + ClusterGatewayMapping: map[string][]string{ + "cluster-1": {"gateway-1"}, + "cluster-2": {"gateway-2"}, + }, + } + arg.Status = controllerv1alpha1.VpnKeyRotationStatus{ + CurrentRotationState: map[string]controllerv1alpha1.StatusOfKeyRotation{ + + "gateway-1": controllerv1alpha1.StatusOfKeyRotation{ + Status: controllerv1alpha1.Complete, + LastUpdatedTimestamp: metav1.Now(), + }, + "gateway-2": controllerv1alpha1.StatusOfKeyRotation{ + Status: controllerv1alpha1.Complete, + LastUpdatedTimestamp: metav1.Now(), + }, + }, + } + }).Once() + clientMock.On("Update", mock.Anything, mock.Anything).Return(nil) + oldSliceConfig := controllerv1alpha1.SliceConfig{} + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } + err := validateRenewNowInSliceConfig(ctx, sliceConfig, &oldSliceConfig) + require.Nil(t, err) +} + +// rotationInterval updates TC +func TestValidateRotationInterval_NoChange(t *testing.T) { + name := "slice_config" + namespace := "randomNamespace" + clientMock, sliceConfig, ctx := setupSliceConfigWebhookValidationTest(name, namespace) + sliceConfig.Spec.RotationInterval = 30 + + clientMock.On("Get", ctx, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(2).(*controllerv1alpha1.VpnKeyRotation) + arg.ObjectMeta = metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + } + arg.Spec = controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: name, + RotationInterval: 30, + } + }).Once() + clientMock.On("Update", mock.Anything, mock.Anything).Return(nil) + oldSliceConfig := controllerv1alpha1.SliceConfig{ + Spec: controllerv1alpha1.SliceConfigSpec{ + RotationInterval: 30, + }, + } + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } + _, err := validateRotationIntervalInSliceConfig(ctx, sliceConfig, &oldSliceConfig) + require.Nil(t, err) +} +func TestValidateRotationInterval_Change_Increased(t *testing.T) { + name := "slice_config" + namespace := "randomNamespace" + clientMock, sliceConfig, ctx := setupSliceConfigWebhookValidationTest(name, namespace) + sliceConfig.Spec.RotationInterval = 45 + now := metav1.Now() + expiry := metav1.Now().Add(30) + + clientMock.On("Get", ctx, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(2).(*controllerv1alpha1.VpnKeyRotation) + arg.ObjectMeta = metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + } + arg.Spec = controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: name, + RotationInterval: 30, + CertificateCreationTime: &now, + CertificateExpiryTime: &metav1.Time{expiry}, + } + }).Once() + clientMock.On("Update", mock.Anything, mock.Anything).Return(nil) + oldSliceConfig := controllerv1alpha1.SliceConfig{ + Spec: controllerv1alpha1.SliceConfigSpec{ + RotationInterval: 30, + }, + } + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } + expectedResp := metav1.NewTime(now.AddDate(0, 0, 45).Add(-1 * time.Hour)) + gotResp, err := validateRotationIntervalInSliceConfig(ctx, sliceConfig, &oldSliceConfig) + require.Nil(t, err) + require.Equal(t, &expectedResp, gotResp.Spec.CertificateExpiryTime) +} +func TestValidateRotationInterval_Change_Decreased(t *testing.T) { + name := "slice_config" + namespace := "randomNamespace" + clientMock, sliceConfig, ctx := setupSliceConfigWebhookValidationTest(name, namespace) + // new interval + sliceConfig.Spec.RotationInterval = 30 + now := metav1.Now() + expiry := metav1.Now().Add(45) + + clientMock.On("Get", ctx, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(2).(*controllerv1alpha1.VpnKeyRotation) + arg.ObjectMeta = metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + } + arg.Spec = controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: name, + RotationInterval: 45, + CertificateCreationTime: &now, + CertificateExpiryTime: &metav1.Time{expiry}, + } + }).Once() + clientMock.On("Update", mock.Anything, mock.Anything).Return(nil) + oldSliceConfig := controllerv1alpha1.SliceConfig{ + Spec: controllerv1alpha1.SliceConfigSpec{ + RotationInterval: 45, + }, + } + oldSliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } + expectedResp := metav1.NewTime(now.AddDate(0, 0, 30).Add(-1 * time.Hour)) + gotResp, err := validateRotationIntervalInSliceConfig(ctx, sliceConfig, &oldSliceConfig) + require.Nil(t, err) + require.Equal(t, &expectedResp, gotResp.Spec.CertificateExpiryTime) +} func setupSliceConfigWebhookValidationTest(name string, namespace string) (*utilMock.Client, *controllerv1alpha1.SliceConfig, context.Context) { clientMock := &utilMock.Client{} sliceConfig := &controllerv1alpha1.SliceConfig{ @@ -1621,6 +1911,10 @@ func setupSliceConfigWebhookValidationTest(name string, namespace string) (*util Namespace: namespace, }, } + sliceConfig.Spec.VPNConfig = &controllerv1alpha1.VPNConfiguration{ + Cipher: "AES-256-CBC", + } + ctx := util.PrepareKubeSliceControllersRequestContext(context.Background(), clientMock, nil, "SliceConfigWebhookValidationServiceTest", nil) return clientMock, sliceConfig, ctx } diff --git a/service/vpn_key_rotation_service.go b/service/vpn_key_rotation_service.go new file mode 100644 index 00000000..19780a32 --- /dev/null +++ b/service/vpn_key_rotation_service.go @@ -0,0 +1,453 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "context" + "fmt" + "reflect" + "sync/atomic" + "time" + + controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + workerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1" + "github.com/kubeslice/kubeslice-controller/events" + "github.com/kubeslice/kubeslice-controller/util" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +type IVpnKeyRotationService interface { + CreateMinimalVpnKeyRotationConfig(ctx context.Context, sliceName, namespace string, r int) error + ReconcileClusters(ctx context.Context, sliceName, namespace string, clusters []string) (*controllerv1alpha1.VpnKeyRotation, error) + ReconcileVpnKeyRotation(ctx context.Context, req ctrl.Request) (ctrl.Result, error) +} + +type VpnKeyRotationService struct { + wsgs IWorkerSliceGatewayService + wscs IWorkerSliceConfigService + jobCreationInProgress atomic.Bool +} + +// JobStatus represents the status of a job. +type JobStatus int + +const ( + JobStatusComplete JobStatus = iota + JobStatusError + JobStatusSuspended + JobStatusListError + JobStatusRunning + JobNotCreated +) + +// String returns the string representation of JobStatus. +func (status JobStatus) String() string { + switch status { + case JobStatusComplete: + return "JobStatusComplete" + case JobStatusError: + return "JobStatusError" + case JobStatusSuspended: + return "JobStatusSuspended" + case JobStatusListError: + return "JobStatusListError" + case JobStatusRunning: + return "JobStatusRunning" + default: + return fmt.Sprintf("Unknown JobStatus: %d", status) + } +} + +// CreateMinimalVpnKeyRotationConfig creates minimal VPNKeyRotationCR if not found +func (v *VpnKeyRotationService) CreateMinimalVpnKeyRotationConfig(ctx context.Context, sliceName, namespace string, r int) error { + logger := util.CtxLogger(ctx). + With("name", "CreateMinimalVpnKeyRotationConfig"). + With("reconciler", "VpnKeyRotationConfig") + + vpnKeyRotationConfig := controllerv1alpha1.VpnKeyRotation{} + found, err := util.GetResourceIfExist(ctx, types.NamespacedName{ + Namespace: namespace, + Name: sliceName, + }, &vpnKeyRotationConfig) + if err != nil { + logger.Errorf("error fetching vpnKeyRotationConfig %s. Err: %s ", sliceName, err.Error()) + return err + } + if !found { + vpnKeyRotationConfig = controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: sliceName, + Namespace: namespace, + Labels: map[string]string{ + "kubeslice-slice": sliceName, + }, + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + RotationInterval: r, + SliceName: sliceName, + RotationCount: 1, + }, + } + if err := util.CreateResource(ctx, &vpnKeyRotationConfig); err != nil { + return err + } + logger.Debugf("created vpnKeyRotationConfig %s ", sliceName) + } + return nil +} + +// ReconcileClusters checks whether any cluster is added/removed and updates it in vpnkeyrotation config +// the first arg is returned for testing purposes +func (v *VpnKeyRotationService) ReconcileClusters(ctx context.Context, sliceName, namespace string, clusters []string) (*controllerv1alpha1.VpnKeyRotation, error) { + logger := util.CtxLogger(ctx). + With("name", "ReconcileClusters"). + With("reconciler", "VpnKeyRotationConfig") + + vpnKeyRotationConfig := controllerv1alpha1.VpnKeyRotation{} + found, err := util.GetResourceIfExist(ctx, types.NamespacedName{ + Namespace: namespace, + Name: sliceName, + }, &vpnKeyRotationConfig) + if err != nil { + logger.Errorf("error fetching vpnKeyRotationConfig %s. Err: %s ", sliceName, err.Error()) + return nil, err + } + if found { + if !reflect.DeepEqual(vpnKeyRotationConfig.Spec.Clusters, clusters) { + vpnKeyRotationConfig.Spec.Clusters = clusters + return &vpnKeyRotationConfig, util.UpdateResource(ctx, &vpnKeyRotationConfig) + } + } + return &vpnKeyRotationConfig, nil +} + +func (v *VpnKeyRotationService) ReconcileVpnKeyRotation(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // Step 0: Get VpnKeyRotation resource + logger := util.CtxLogger(ctx). + With("name", "ReconcileVpnKeyRotation"). + With("reconciler", "VpnKeyRotationConfig") + + logger.Infof("Starting Recoincilation of VpnKeyRotation with name %s in namespace %s", + req.Name, req.Namespace) + vpnKeyRotationConfig := &controllerv1alpha1.VpnKeyRotation{} + found, err := util.GetResourceIfExist(ctx, req.NamespacedName, vpnKeyRotationConfig) + if err != nil { + logger.Errorf("Err: %s", err.Error()) + return ctrl.Result{}, err + } + if !found { + logger.Infof("Vpn Key Rotation Config %v not found, returning from reconciler loop.", req.NamespacedName) + return ctrl.Result{}, nil + } + // get slice config + s, err := v.getSliceConfig(ctx, req.Name, req.Namespace) + if err != nil { + logger.Errorf("Err getting sliceconfig: %s", err.Error()) + return ctrl.Result{}, err + } + if vpnKeyRotationConfig.GetOwnerReferences() == nil { + if err := controllerutil.SetControllerReference(s, vpnKeyRotationConfig, util.GetKubeSliceControllerRequestContext(ctx).Scheme); err != nil { + logger.Errorf("failed to set SliceConfig as owner of vpnKeyRotationConfig. Err %s", err.Error()) + return ctrl.Result{}, err + } + } + // Step 1: Build map of clusterName: gateways + clusterGatewayMapping, err := v.constructClusterGatewayMapping(ctx, s) + if err != nil { + logger.Errorf("Err constructing clusterGatewayMapping: %s", err.Error()) + return ctrl.Result{}, err + } + copyVpnConfig := vpnKeyRotationConfig.DeepCopy() + + toUpdate := false + if !reflect.DeepEqual(copyVpnConfig.Spec.ClusterGatewayMapping, clusterGatewayMapping) { + copyVpnConfig.Spec.ClusterGatewayMapping = clusterGatewayMapping + toUpdate = true + } + if !reflect.DeepEqual(copyVpnConfig.Spec.RotationInterval, s.Spec.RotationInterval) { + copyVpnConfig.Spec.RotationInterval = s.Spec.RotationInterval + toUpdate = true + } + if !reflect.DeepEqual(copyVpnConfig.Spec.SliceName, s.Name) { + copyVpnConfig.Spec.SliceName = s.Name + toUpdate = true + } + if !reflect.DeepEqual(copyVpnConfig.Spec.Clusters, s.Spec.Clusters) { + copyVpnConfig.Spec.Clusters = s.Spec.Clusters + toUpdate = true + } + if toUpdate { + logger.Debugf("vpnkeyrotation config %s deviated from sliceconfig %s", copyVpnConfig.Name, s.Name) + if err := util.UpdateResource(ctx, copyVpnConfig); err != nil { + logger.Errorf("Err updating clusterGatewayMapping in vpnconfig: %s", err.Error()) + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + // Step 2: TODO Update Certificate Creation TimeStamp and Expiry Timestamp if + // a. The Creation TS and Expiry TS is empty + // b. The Current TS is pass the expiry TS + res, copyVpnConfig, err := v.reconcileVpnKeyRotationConfig(ctx, copyVpnConfig, s) + if err != nil { + logger.Errorf("Err: %s", err.Error()) + return res, err + } + if res.RequeueAfter > 0 { + return res, nil + } + // always returns error but but if err==nil and copyVpnConfig==nil this means dont requeue + if copyVpnConfig == nil { + return ctrl.Result{}, nil + } + expiryTime := copyVpnConfig.Spec.CertificateExpiryTime.Time + remainingDuration := expiryTime.Sub(metav1.Now().Time) + logger.Debugf("vpnkeyrotation config reconciler will requeue after %s", remainingDuration) + return ctrl.Result{RequeueAfter: remainingDuration}, nil +} + +func (v *VpnKeyRotationService) reconcileVpnKeyRotationConfig(ctx context.Context, copyVpnConfig *controllerv1alpha1.VpnKeyRotation, s *controllerv1alpha1.SliceConfig) (ctrl.Result, *controllerv1alpha1.VpnKeyRotation, error) { + logger := util.CtxLogger(ctx) + + //Load Event Recorder with project name, vpnkeyrotation(slice) name and namespace + eventRecorder := util.CtxEventRecorder(ctx). + WithProject(util.GetProjectName(s.Namespace)). + WithNamespace(s.Namespace). + WithSlice(s.Name) + + now := metav1.Now() + // Check if it's the first time creation + if copyVpnConfig.Spec.CertificateCreationTime.IsZero() && copyVpnConfig.Spec.CertificateExpiryTime.IsZero() { + // verify jobs are completed + status, err := v.verifyAllJobsAreCompleted(ctx, copyVpnConfig.Spec.SliceName) + if err != nil { + return ctrl.Result{}, nil, err + } + // requeue after 1 minute if job is still running + if status == JobStatusRunning { + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil, nil + } + if status == JobStatusError || status == JobStatusSuspended { + // register an event + util.RecordEvent(ctx, eventRecorder, copyVpnConfig, nil, events.EventCertificateJobFailed) + return ctrl.Result{}, nil, nil + } + if status == JobNotCreated { + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil, nil + } + + copyVpnConfig.Spec.CertificateCreationTime = &now + expiryTS := metav1.NewTime(now.AddDate(0, 0, copyVpnConfig.Spec.RotationInterval).Add(-1 * time.Hour)) + copyVpnConfig.Spec.CertificateExpiryTime = &expiryTS + if err := util.UpdateResource(ctx, copyVpnConfig); err != nil { + return ctrl.Result{}, nil, err + } + //register an event + util.RecordEvent(ctx, eventRecorder, copyVpnConfig, nil, events.EventVPNKeyRotationConfigUpdated) + + } else { + if now.After(copyVpnConfig.Spec.CertificateExpiryTime.Time) { + if !v.jobCreationInProgress.Load() { + if err := v.triggerJobsForCertCreation(ctx, copyVpnConfig, s); err != nil { + logger.Error("error creating new certs", err) + // register an event + util.RecordEvent(ctx, eventRecorder, copyVpnConfig, nil, events.EventCertificateJobCreationFailed) + return ctrl.Result{}, nil, err + } + v.jobCreationInProgress.Store(true) + logger.Debugf("jobs triggered for creating new certs for slice %s", s.Name) + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil, nil + } + // verify jobs are completed + status, err := v.verifyAllJobsAreCompleted(ctx, copyVpnConfig.Spec.SliceName) + if err != nil { + return ctrl.Result{}, nil, err + } + logger.Debugf("certs job status for sliceconfig %s = %s ", s.Name, status.String()) + // requeue after 1 minute if job is still running + if status == JobStatusRunning { + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil, nil + } + if status == JobStatusError || status == JobStatusSuspended { + // register an event + util.RecordEvent(ctx, eventRecorder, copyVpnConfig, nil, events.EventCertificateJobFailed) + return ctrl.Result{}, nil, nil + } + if status == JobNotCreated { + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil, nil + } + copyVpnConfig.Spec.CertificateCreationTime = &now + expiryTS := metav1.NewTime(now.AddDate(0, 0, copyVpnConfig.Spec.RotationInterval).Add(-1 * time.Hour)) + copyVpnConfig.Spec.CertificateExpiryTime = &expiryTS + copyVpnConfig.Spec.RotationCount = copyVpnConfig.Spec.RotationCount + 1 + if err := util.UpdateResource(ctx, copyVpnConfig); err != nil { + return ctrl.Result{}, nil, err + } + // restore the variable jobCreationInProgress to false + v.jobCreationInProgress.Store(false) + //register an event + util.RecordEvent(ctx, eventRecorder, copyVpnConfig, nil, events.EventVPNKeyRotationStart) + } + } + return ctrl.Result{}, copyVpnConfig, nil +} + +func (v *VpnKeyRotationService) constructClusterGatewayMapping(ctx context.Context, s *controllerv1alpha1.SliceConfig) (map[string][]string, error) { + var clusterGatewayMapping = make(map[string][]string, 0) + for _, cluster := range s.Spec.Clusters { + // list workerslicegateways + o := map[string]string{ + "worker-cluster": cluster, + "original-slice-name": s.Name, + } + workerSliceGatewaysList, err := v.listWorkerSliceGateways(ctx, o) + if err != nil { + return nil, err + } + vl := v.fetchGatewayNames(workerSliceGatewaysList) + clusterGatewayMapping[cluster] = vl + } + return clusterGatewayMapping, nil +} + +func (v *VpnKeyRotationService) triggerJobsForCertCreation(ctx context.Context, vpnKeyRotationConfig *controllerv1alpha1.VpnKeyRotation, s *controllerv1alpha1.SliceConfig) error { + o := map[string]string{ + "original-slice-name": vpnKeyRotationConfig.Spec.SliceName, + } + workerSliceGatewaysList, err := v.listWorkerSliceGateways(ctx, o) + if err != nil { + return err + } + // fire certificate creation jobs for each gateway pair + for _, gateway := range workerSliceGatewaysList.Items { + if gateway.Spec.GatewayHostType == "Server" { + cl, err := v.listClientPairGateway(workerSliceGatewaysList, gateway.Spec.RemoteGatewayConfig.GatewayName) + if err != nil { + return err + } + // construct clustermap + clusterCidr := util.FindCIDRByMaxClusters(s.Spec.MaxClusters) + completeResourceName := fmt.Sprintf(util.LabelValue, util.GetObjectKind(s), s.GetName()) + ownershipLabel := util.GetOwnerLabel(completeResourceName) + workerSliceConfigs, err := v.wscs.ListWorkerSliceConfigs(ctx, ownershipLabel, s.Namespace) + if err != nil { + return err + } + clusterMap := v.wscs.ComputeClusterMap(s.Spec.Clusters, workerSliceConfigs) + // contruct gw address + gatewayAddresses := v.wsgs.BuildNetworkAddresses(s.Spec.SliceSubnet, gateway.Spec.LocalGatewayConfig.ClusterName, gateway.Spec.RemoteGatewayConfig.ClusterName, clusterMap, clusterCidr) + // call GenerateCerts() + if err := v.wsgs.GenerateCerts(ctx, s.Name, s.Namespace, &gateway, cl, gatewayAddresses); err != nil { + return err + } + } + } + return nil +} + +func (v *VpnKeyRotationService) listWorkerSliceGateways(ctx context.Context, labels map[string]string) (*workerv1alpha1.WorkerSliceGatewayList, error) { + workerSliceGatewaysList := workerv1alpha1.WorkerSliceGatewayList{} + // list workerslicegateways + listOpts := []client.ListOption{ + client.MatchingLabels( + labels, + ), + } + if err := util.ListResources(ctx, &workerSliceGatewaysList, listOpts...); err != nil { + return nil, err + } + return &workerSliceGatewaysList, nil +} + +// getSliceConfig +func (v *VpnKeyRotationService) getSliceConfig(ctx context.Context, name, namespace string) (*controllerv1alpha1.SliceConfig, error) { + s := controllerv1alpha1.SliceConfig{} + found, err := util.GetResourceIfExist(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, &s) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("sliceconfig %s not found", name) + } + return &s, nil +} + +func (v *VpnKeyRotationService) listClientPairGateway(wl *workerv1alpha1.WorkerSliceGatewayList, clientGatewayName string) (*workerv1alpha1.WorkerSliceGateway, error) { + for _, gateway := range wl.Items { + if gateway.Name == clientGatewayName { + return &gateway, nil + } + } + return nil, fmt.Errorf("cannot find gateway %s", clientGatewayName) +} + +// verifyAllJobsAreCompleted checks if all the jobs are in complete state +func (v *VpnKeyRotationService) verifyAllJobsAreCompleted(ctx context.Context, sliceName string) (JobStatus, error) { + jobs := batchv1.JobList{} + o := map[string]string{ + "SLICE_NAME": sliceName, + } + listOpts := []client.ListOption{ + client.MatchingLabels(o), + } + if err := util.ListResources(ctx, &jobs, listOpts...); err != nil { + return JobStatusListError, err + } + + if len(jobs.Items) == 0 { + return JobNotCreated, nil + } + + for _, job := range jobs.Items { + for _, condition := range job.Status.Conditions { + if condition.Type == batchv1.JobFailed && condition.Status == corev1.ConditionTrue { + return JobStatusError, nil + } + + if condition.Type == batchv1.JobSuspended && condition.Status == corev1.ConditionTrue { + return JobStatusSuspended, nil + } + } + } + + for _, job := range jobs.Items { + if job.Status.Active > 0 { + return JobStatusRunning, nil + } + } + + return JobStatusComplete, nil +} + +// fetchGatewayNames fetches gateway names from the list of workerv1alpha1.WorkerSliceGatewayList +func (v *VpnKeyRotationService) fetchGatewayNames(gl *workerv1alpha1.WorkerSliceGatewayList) []string { + var gatewayNames []string + for _, g := range gl.Items { + if g.DeletionTimestamp.IsZero() { + gatewayNames = append(gatewayNames, g.Name) + } + } + return gatewayNames +} diff --git a/service/vpn_key_rotation_service_test.go b/service/vpn_key_rotation_service_test.go new file mode 100644 index 00000000..2e98ee9f --- /dev/null +++ b/service/vpn_key_rotation_service_test.go @@ -0,0 +1,1365 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "bou.ke/monkey" + controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + workerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1" + ossEvents "github.com/kubeslice/kubeslice-controller/events" + "github.com/kubeslice/kubeslice-controller/service/mocks" + "github.com/kubeslice/kubeslice-controller/util" + utilMock "github.com/kubeslice/kubeslice-controller/util/mocks" + "github.com/kubeslice/kubeslice-monitoring/pkg/events" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + kubeerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + ctrl "sigs.k8s.io/controller-runtime" +) + +type createMinimalVpnKeyRotationConfigTestCase struct { + name string + sliceName string + namespace string + expectedErr error + getArg1, getArg2, getArg3 interface{} + getRet1 interface{} + createArg1, createArg2 interface{} + createRet1 interface{} +} + +func setupTestCase() (context.Context, *utilMock.Client, VpnKeyRotationService, *mocks.IWorkerSliceGatewayService, *mocks.IWorkerSliceConfigService) { + clientMock := &utilMock.Client{} + scheme := runtime.NewScheme() + utilruntime.Must(controllerv1alpha1.AddToScheme(scheme)) + wg := &mocks.IWorkerSliceGatewayService{} + ws := &mocks.IWorkerSliceConfigService{} + eventRecorder := events.NewEventRecorder(clientMock, scheme, ossEvents.EventsMap, events.EventRecorderOptions{ + Version: "v1alpha1", + Cluster: util.ClusterController, + Component: util.ComponentController, + Slice: util.NotApplicable, + }) + return util.PrepareKubeSliceControllersRequestContext(context.Background(), clientMock, scheme, "ClusterTestController", &eventRecorder), clientMock, VpnKeyRotationService{ + wsgs: wg, + wscs: ws, + }, wg, ws +} + +func Test_CreateMinimalVpnKeyRotationConfig(t *testing.T) { + testCases := []createMinimalVpnKeyRotationConfigTestCase{ + { + name: "should create vpnkeyrotation config successfully", + sliceName: "demo-slice", + namespace: "demo-namespace", + expectedErr: nil, + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: kubeerrors.NewNotFound(util.Resource("VpnKeyRotationConfigTest"), "VpnKeyRotationConfig not found"), + createArg1: mock.Anything, + createArg2: mock.Anything, + createRet1: nil, + }, + { + name: "should return error if creating vpnkeyrotation config fails", + sliceName: "demo-slice", + namespace: "demo-namespace", + expectedErr: errors.New("Failed to create vpnkeyrotation"), + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: kubeerrors.NewNotFound(util.Resource("VpnKeyRotationConfigTest"), "VpnKeyRotationConfig not found"), + createArg1: mock.Anything, + createArg2: mock.Anything, + createRet1: errors.New("Failed to create vpnkeyrotation"), + }, + } + + for _, tc := range testCases { + runCreateMinimalVpnKeyRotationConfigTestCase(t, tc) + } +} + +func runCreateMinimalVpnKeyRotationConfigTestCase(t *testing.T, tc createMinimalVpnKeyRotationConfigTestCase) { + ctx, clientMock, vpn, _, _ := setupTestCase() + clientMock. + On("Get", tc.getArg1, tc.getArg2, tc.getArg3). + Return(tc.getRet1).Once() + + clientMock. + On("Create", tc.createArg1, tc.createArg2). + Return(tc.createRet1).Once() + + gotErr := vpn.CreateMinimalVpnKeyRotationConfig(ctx, tc.sliceName, tc.namespace, 90) + require.Equal(t, gotErr, tc.expectedErr) + clientMock.AssertExpectations(t) +} + +type reconcileClustersTestCase struct { + name string + sliceName string + namespace string + expectedErr error + expectedResp *controllerv1alpha1.VpnKeyRotation + existingClusters []string + addclusters []string + getArg1, getArg2, getArg3 interface{} + getRet1 interface{} + updateArg1, updateArg2 interface{} + updateRet1 interface{} +} + +func Test_ReconcileClusters(t *testing.T) { + testCases := []reconcileClustersTestCase{ + { + name: "should update vpnkeyrotation CR with cluster names sucessfully", + sliceName: "demo-slice", + namespace: "demo-ns", + expectedErr: nil, + expectedResp: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-slice", + Namespace: "demo-ns", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + Clusters: []string{"worker-1", "worker-2"}, + SliceName: "demo-slice", + }, + }, + existingClusters: []string{}, + addclusters: []string{"worker-1", "worker-2"}, + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: nil, + updateArg1: mock.Anything, + updateArg2: mock.Anything, + updateRet1: nil, + }, + { + name: "should update cluster list in vpnkeyrotation CR when a cluster is added", + sliceName: "demo-slice", + namespace: "demo-ns", + expectedErr: nil, + expectedResp: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-slice", + Namespace: "demo-ns", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + Clusters: []string{"worker-1", "worker-2", "worker-3"}, + SliceName: "demo-slice", + }, + }, + existingClusters: []string{"worker-1", "worker-2"}, + addclusters: []string{"worker-1", "worker-2", "worker-3"}, + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: nil, + updateArg1: mock.Anything, + updateArg2: mock.Anything, + updateRet1: nil, + }, + } + + for _, tc := range testCases { + runReconcileClustersTestCase(t, tc) + } +} + +func runReconcileClustersTestCase(t *testing.T, tc reconcileClustersTestCase) { + ctx, clientMock, vpn, _, _ := setupTestCase() + + clientMock. + On("Get", tc.getArg1, tc.getArg2, tc.getArg3). + Return(tc.getRet1).Run(func(args mock.Arguments) { + arg := args.Get(2).(*controllerv1alpha1.VpnKeyRotation) + arg.Name = tc.sliceName + arg.Namespace = tc.namespace + arg.Spec = controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: tc.sliceName, + Clusters: tc.existingClusters, + } + }).Once() + + clientMock. + On("Update", tc.updateArg1, tc.updateArg2).Return(tc.updateRet1).Once() + + gotResp, gotErr := vpn.ReconcileClusters(ctx, tc.sliceName, tc.namespace, tc.addclusters) + require.Equal(t, gotErr, tc.expectedErr) + + require.Equal(t, gotResp, tc.expectedResp) + clientMock.AssertExpectations(t) +} + +type constructClusterGatewayMappingTestCase struct { + name string + sliceConfig *controllerv1alpha1.SliceConfig + expectedResp map[string][]string + expectedErr error + listArg1, listArg2, listArg3 interface{} + listRet1 interface{} +} + +func Test_ConstructClusterGatewayMapping(t *testing.T) { + testCases := []constructClusterGatewayMappingTestCase{ + { + name: "should return error in case list fails", + sliceConfig: &controllerv1alpha1.SliceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-namespace", + }, + Spec: controllerv1alpha1.SliceConfigSpec{ + Clusters: []string{"worker-1", "worker-2"}, + }, + }, + expectedResp: nil, + expectedErr: errors.New("workerslicegateway not found"), + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: errors.New("workerslicegateway not found"), + }, + { + name: "should return a valid constructed map", + sliceConfig: &controllerv1alpha1.SliceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-namespace", + }, + Spec: controllerv1alpha1.SliceConfigSpec{ + Clusters: []string{"worker-1", "worker-2"}, + }, + }, + expectedResp: map[string][]string{ + "worker-1": {"test-slice-worker-1-worker-2"}, + "worker-2": {"test-slice-worker-2-worker-1"}, + }, + expectedErr: nil, + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: nil, + }, + } + for _, tc := range testCases { + runClusterGatewayMappingTestCase(t, tc) + } +} + +func runClusterGatewayMappingTestCase(t *testing.T, tc constructClusterGatewayMappingTestCase) { + ctx, clientMock, vpn, _, _ := setupTestCase() + + if tc.expectedErr != nil { + clientMock. + On("List", tc.listArg1, tc.listArg2, tc.listArg3). + Return(tc.listRet1).Once() + } else { + clientMock. + On("List", tc.listArg1, tc.listArg2, tc.listArg3). + Return(tc.listRet1).Run(func(args mock.Arguments) { + w := args.Get(1).(*workerv1alpha1.WorkerSliceGatewayList) + w.Items = append(w.Items, workerv1alpha1.WorkerSliceGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-1-worker-2", + Labels: map[string]string{ + "worker-cluster": "worker-1", + "original-slice-name": tc.sliceConfig.Name, + }, + }, + }) + }).Once() + + clientMock. + On("List", tc.listArg1, tc.listArg2, tc.listArg3). + Return(tc.listRet1).Run(func(args mock.Arguments) { + w := args.Get(1).(*workerv1alpha1.WorkerSliceGatewayList) + w.Items = append(w.Items, workerv1alpha1.WorkerSliceGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-2-worker-1", + Labels: map[string]string{ + "worker-cluster": "worker-2", + "original-slice-name": tc.sliceConfig.Name, + }, + }, + }) + }).Once() + } + + gotResp, gotErr := vpn.constructClusterGatewayMapping(ctx, tc.sliceConfig) + require.Equal(t, gotErr, tc.expectedErr) + + require.Equal(t, gotResp, tc.expectedResp) + clientMock.AssertExpectations(t) +} + +type getSliceConfigTestCase struct { + name string + sliceName string + namespace string + expectedResp *controllerv1alpha1.SliceConfig + expectedErr error + getArg1, getArg2, getArg3 interface{} + getRet1 interface{} +} + +func Test_getSliceConfig(t *testing.T) { + testCases := []getSliceConfigTestCase{ + { + name: "it should return error in sliceconfig not found", + sliceName: "test-slice", + namespace: "test-ns", + expectedResp: nil, + expectedErr: errors.New("sliceconfig not found"), + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: errors.New("sliceconfig not found"), + }, + { + name: "it should return sliceconfig successfully", + sliceName: "test-slice", + namespace: "test-ns", + expectedResp: &controllerv1alpha1.SliceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + }, + expectedErr: nil, + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: nil, + }, + } + for _, tc := range testCases { + runGetSliceConfigTestCase(t, tc) + } +} + +func runGetSliceConfigTestCase(t *testing.T, tc getSliceConfigTestCase) { + ctx, clientMock, vpn, _, _ := setupTestCase() + + clientMock. + On("Get", tc.getArg1, tc.getArg2, tc.getArg3). + Return(tc.getRet1).Run(func(args mock.Arguments) { + arg := args.Get(2).(*controllerv1alpha1.SliceConfig) + arg.Name = tc.sliceName + arg.Namespace = tc.namespace + }).Once() + + gotResp, gotErr := vpn.getSliceConfig(ctx, tc.sliceName, tc.namespace) + require.Equal(t, gotErr, tc.expectedErr) + + require.Equal(t, gotResp, tc.expectedResp) + clientMock.AssertExpectations(t) +} + +type reconcileVpnKeyRotationConfigTestCase struct { + name string + arg1 *controllerv1alpha1.VpnKeyRotation + arg2 *controllerv1alpha1.SliceConfig + expectedErr error + expectedResp *controllerv1alpha1.VpnKeyRotation + updateArg1, updateArg2, updateArg3 interface{} + updateRet1 interface{} + now metav1.Time + reconcileResult ctrl.Result +} + +func Test_reconcileVpnKeyRotationConfig(t *testing.T) { + ts := metav1.NewTime(time.Date(2021, 06, 16, 20, 34, 58, 651387237, time.UTC)) + expiryTs := metav1.NewTime(ts.AddDate(0, 0, 30).Add(-1 * time.Hour)) + newTs := metav1.NewTime(time.Date(2021, 07, 16, 20, 34, 58, 651387237, time.UTC)) + testCases := []reconcileVpnKeyRotationConfigTestCase{ + { + name: "should update CertCreation TS and Expiry TS when it is nil", + arg1: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice", + Clusters: []string{"worker-1", "worker-2"}, + RotationInterval: 30, + RotationCount: 1, + }, + }, + arg2: &controllerv1alpha1.SliceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + }, + expectedErr: nil, + expectedResp: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice", + Clusters: []string{"worker-1", "worker-2"}, + RotationInterval: 30, + CertificateCreationTime: &ts, + CertificateExpiryTime: &expiryTs, + RotationCount: 1, + }, + }, + updateArg1: mock.Anything, + updateArg2: mock.Anything, + updateArg3: mock.Anything, + updateRet1: nil, + now: ts, + reconcileResult: ctrl.Result{}, + }, + { + name: "should requeue after firing new jobs", + arg1: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice", + Clusters: []string{"worker-1", "worker-2"}, + RotationInterval: 30, + CertificateCreationTime: &ts, + CertificateExpiryTime: &expiryTs, + RotationCount: 1, + }, + }, + arg2: &controllerv1alpha1.SliceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + }, + expectedErr: nil, + expectedResp: nil, + updateArg1: mock.Anything, + updateArg2: mock.Anything, + updateArg3: mock.Anything, + updateRet1: nil, + now: newTs, + reconcileResult: ctrl.Result{RequeueAfter: 30 * time.Second}, + }, + } + for _, tc := range testCases { + runReconcileVpnKeyRotationConfig(t, &tc) + } +} +func runReconcileVpnKeyRotationConfig(t *testing.T, tc *reconcileVpnKeyRotationConfigTestCase) { + ctx, clientMock, vpn, wg, ws := setupTestCase() + + // Mocking metav1.Now() with a fixed time value + patch := monkey.Patch(metav1.Now, func() metav1.Time { + return tc.now + }) + defer patch.Unpatch() + // NOTE: Monkey pathcing sometimes requires the inlining to be disabled + // use go test -gcflags=-l + // setup Expectations + gwList := &workerv1alpha1.WorkerSliceGatewayList{} + clientMock. + On("List", mock.Anything, gwList, mock.Anything). + Return(nil).Run(func(args mock.Arguments) { + w := args.Get(1).(*workerv1alpha1.WorkerSliceGatewayList) + w.Items = append(w.Items, + workerv1alpha1.WorkerSliceGateway{ + Spec: workerv1alpha1.WorkerSliceGatewaySpec{ + GatewayHostType: "Server", + RemoteGatewayConfig: workerv1alpha1.SliceGatewayConfig{ + GatewayName: "test-slice-worker-2-worker-1", + }, + }, + }, + workerv1alpha1.WorkerSliceGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-2-worker-1", + }, + Spec: workerv1alpha1.WorkerSliceGatewaySpec{ + GatewayHostType: "Client", + RemoteGatewayConfig: workerv1alpha1.SliceGatewayConfig{ + GatewayName: "test-slice-worker-1-worker-2", + }, + }, + }) + }).Times(1) + + jobList := &batchv1.JobList{} + + clientMock. + On("List", mock.Anything, jobList, mock.Anything). + Return(nil).Run(func(args mock.Arguments) { + w := args.Get(1).(*batchv1.JobList) + w.Items = append(w.Items, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "job-1", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "job-2", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }, + }, + }, + }) + }).Times(2) + + clientMock.On("Create", ctx, mock.AnythingOfType("*v1.Event")).Return(nil).Once() + + workerSliceConfigs := workerv1alpha1.WorkerSliceConfigList{ + Items: []workerv1alpha1.WorkerSliceConfig{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-config-1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-config-2", + }, + }, + }, + } + + ws.On("ListWorkerSliceConfigs", mock.Anything, mock.Anything, mock.Anything).Return(workerSliceConfigs.Items, nil).Once() + + clusterMap := map[string]int{ + "cluster-1": 1, + "cluster-2": 2, + } + + ws.On("ComputeClusterMap", mock.Anything, mock.Anything).Return(clusterMap).Once() + + gwAddress := util.WorkerSliceGatewayNetworkAddresses{} + + wg.On("BuildNetworkAddresses", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(gwAddress).Once() + + wg.On("GenerateCerts", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + + clientMock. + On("Update", tc.updateArg1, tc.updateArg2).Return(tc.updateRet1).Once() + + reconcileResult, gotResp, gotErr := vpn.reconcileVpnKeyRotationConfig(ctx, tc.arg1, tc.arg2) + require.Equal(t, gotErr, tc.expectedErr) + + require.Equal(t, tc.expectedResp, gotResp) + require.Equal(t, tc.reconcileResult, reconcileResult) +} + +type listClientPairGatewayTesCase struct { + name string + arg1 *workerv1alpha1.WorkerSliceGatewayList + arg2 string + expectedResp *workerv1alpha1.WorkerSliceGateway + expectedErr error +} + +func Test_listClientPairGateway(t *testing.T) { + testCases := []listClientPairGatewayTesCase{ + { + name: "it should return correct workerslicegw", + arg1: &workerv1alpha1.WorkerSliceGatewayList{ + Items: []workerv1alpha1.WorkerSliceGateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-1-worker-2", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-2-worker-1", + }, + }, + }, + }, + arg2: "test-slice-worker-1-worker-2", + expectedResp: &workerv1alpha1.WorkerSliceGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-1-worker-2", + }, + }, + expectedErr: nil, + }, + { + name: "it should return error", + arg1: &workerv1alpha1.WorkerSliceGatewayList{ + Items: []workerv1alpha1.WorkerSliceGateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-1-worker-2", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-2-worker-1", + }, + }, + }, + }, + arg2: "test-slice-worker-1-worker-3", + expectedResp: nil, + expectedErr: fmt.Errorf("cannot find gateway %s", "test-slice-worker-1-worker-3"), + }, + } + for _, tc := range testCases { + runlistClientPairGatewayTestCase(t, tc) + } +} + +func runlistClientPairGatewayTestCase(t *testing.T, tc listClientPairGatewayTesCase) { + _, _, vpn, _, _ := setupTestCase() + + gotResp, gotErr := vpn.listClientPairGateway(tc.arg1, tc.arg2) + require.Equal(t, gotErr, tc.expectedErr) + require.Equal(t, gotResp, tc.expectedResp) +} + +type verifyAllJobsAreCompletedTestCase struct { + name string + arg1 string + expectedResp JobStatus + completionType corev1.ConditionStatus + failedStatus corev1.ConditionStatus + listArg1, listArg2, listArg3 interface{} + listRet1 interface{} +} + +func Test_verifyAllJobsAreCompleted(t *testing.T) { + testCases := []verifyAllJobsAreCompletedTestCase{ + { + name: "should return JobStatusComplete if all jobs are in completed", + arg1: "test-slice", + expectedResp: JobStatusComplete, + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: nil, + completionType: corev1.ConditionTrue, + failedStatus: corev1.ConditionFalse, + }, + { + name: "should return JobStatusRunning if all jobs are not in complete state", + arg1: "test-slice", + expectedResp: JobStatusRunning, + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: nil, + completionType: corev1.ConditionFalse, + failedStatus: corev1.ConditionFalse, + }, + { + name: "should return JobStatusListError if listing job fails", + arg1: "test-slice", + expectedResp: JobStatusListError, + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: fmt.Errorf("cannot list jobs"), + }, + { + name: "should return JobStatusError if jobs are in failed state", + arg1: "test-slice", + expectedResp: JobStatusError, + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: nil, + failedStatus: corev1.ConditionTrue, + }, + } + for _, tc := range testCases { + runVerifyAllJobsAreCompleted(t, tc) + } +} + +func runVerifyAllJobsAreCompleted(t *testing.T, tc verifyAllJobsAreCompletedTestCase) { + ctx, clientMock, vpn, _, _ := setupTestCase() + + if tc.failedStatus == corev1.ConditionTrue { + clientMock. + On("List", tc.listArg1, tc.listArg2, tc.listArg3). + Return(tc.listRet1).Run(func(args mock.Arguments) { + w := args.Get(1).(*batchv1.JobList) + w.Items = append(w.Items, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "job-1", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "job-2", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobFailed, + Status: tc.failedStatus, + }, + }, + }, + }) + }).Once() + } + + if tc.completionType == corev1.ConditionTrue { + clientMock. + On("List", tc.listArg1, tc.listArg2, tc.listArg3). + Return(tc.listRet1).Run(func(args mock.Arguments) { + w := args.Get(1).(*batchv1.JobList) + w.Items = append(w.Items, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "job-1", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "job-2", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: tc.completionType, + }, + }, + }, + }) + }).Once() + } else { + clientMock. + On("List", tc.listArg1, tc.listArg2, tc.listArg3). + Return(tc.listRet1).Run(func(args mock.Arguments) { + w := args.Get(1).(*batchv1.JobList) + w.Items = append(w.Items, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "job-1", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Active: 1, + }, + }, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "job-2", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Active: 1, + }, + }) + }).Once() + } + gotResp, _ := vpn.verifyAllJobsAreCompleted(ctx, tc.arg1) + + require.Equal(t, gotResp, tc.expectedResp) +} + +type triggerJobsForCertCreationTestCase struct { + name string + arg1 *controllerv1alpha1.VpnKeyRotation + arg2 *controllerv1alpha1.SliceConfig + expectedResp error + listArg1, listArg2, listArg3 interface{} + listRet1 interface{} + listWorkerSliceConfigsArg1, listWorkerSliceConfigsArg2, listWorkerSliceConfigsArg3 interface{} + listWorkerSliceConfigsRet1 interface{} + workerSliceGateways []workerv1alpha1.WorkerSliceGateway + generateCertsRet1 interface{} +} + +func Test_triggerJobsForCertCreation(t *testing.T) { + testCases := []triggerJobsForCertCreationTestCase{ + { + name: "should return error in case listing workerslicegateways failed", + arg1: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice", + }, + }, + arg2: &controllerv1alpha1.SliceConfig{}, + expectedResp: fmt.Errorf("Error listing workerslicegateways"), + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: fmt.Errorf("Error listing workerslicegateways"), + }, + { + name: "should return error in case client workerslicegateway is not present", + arg1: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice", + }, + }, + arg2: &controllerv1alpha1.SliceConfig{}, + expectedResp: fmt.Errorf("cannot find gateway %s", "test-slice-worker2-worker-1"), + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: nil, + workerSliceGateways: []workerv1alpha1.WorkerSliceGateway{ + { + Spec: workerv1alpha1.WorkerSliceGatewaySpec{ + GatewayHostType: "Server", + RemoteGatewayConfig: workerv1alpha1.SliceGatewayConfig{ + GatewayName: "test-slice-worker2-worker-1", + }, + }, + }, + }, + }, + { + name: "should return error in case listing workersliceconfigs failed", + arg1: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice", + }, + }, + arg2: &controllerv1alpha1.SliceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + }, + Spec: controllerv1alpha1.SliceConfigSpec{ + MaxClusters: 3, + }, + }, + expectedResp: fmt.Errorf("error listing workersliceconfigs"), + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: nil, + listWorkerSliceConfigsArg1: mock.Anything, + listWorkerSliceConfigsArg2: mock.Anything, + listWorkerSliceConfigsArg3: mock.Anything, + listWorkerSliceConfigsRet1: fmt.Errorf("error listing workersliceconfigs"), + workerSliceGateways: []workerv1alpha1.WorkerSliceGateway{ + { + Spec: workerv1alpha1.WorkerSliceGatewaySpec{ + GatewayHostType: "Server", + RemoteGatewayConfig: workerv1alpha1.SliceGatewayConfig{ + GatewayName: "test-slice-worker-2-worker-1", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-2-worker-1", + }, + Spec: workerv1alpha1.WorkerSliceGatewaySpec{ + GatewayHostType: "Client", + RemoteGatewayConfig: workerv1alpha1.SliceGatewayConfig{ + GatewayName: "test-slice-worker-1-worker-2", + }, + }, + }, + }, + }, + { + name: "should return error in case generating certs failed", + arg1: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice", + }, + }, + arg2: &controllerv1alpha1.SliceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + }, + Spec: controllerv1alpha1.SliceConfigSpec{ + MaxClusters: 3, + }, + }, + expectedResp: fmt.Errorf("error generating Certs"), + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: nil, + listWorkerSliceConfigsArg1: mock.Anything, + listWorkerSliceConfigsArg2: mock.Anything, + listWorkerSliceConfigsArg3: mock.Anything, + listWorkerSliceConfigsRet1: nil, + workerSliceGateways: []workerv1alpha1.WorkerSliceGateway{ + { + Spec: workerv1alpha1.WorkerSliceGatewaySpec{ + GatewayHostType: "Server", + RemoteGatewayConfig: workerv1alpha1.SliceGatewayConfig{ + GatewayName: "test-slice-worker-2-worker-1", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-2-worker-1", + }, + Spec: workerv1alpha1.WorkerSliceGatewaySpec{ + GatewayHostType: "Client", + RemoteGatewayConfig: workerv1alpha1.SliceGatewayConfig{ + GatewayName: "test-slice-worker-1-worker-2", + }, + }, + }, + }, + generateCertsRet1: fmt.Errorf("error generating Certs"), + }, + { + name: "should return error in case generating certs failed", + arg1: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice", + }, + }, + arg2: &controllerv1alpha1.SliceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + }, + Spec: controllerv1alpha1.SliceConfigSpec{ + MaxClusters: 3, + }, + }, + expectedResp: nil, + listArg1: mock.Anything, + listArg2: mock.Anything, + listArg3: mock.Anything, + listRet1: nil, + listWorkerSliceConfigsArg1: mock.Anything, + listWorkerSliceConfigsArg2: mock.Anything, + listWorkerSliceConfigsArg3: mock.Anything, + listWorkerSliceConfigsRet1: nil, + workerSliceGateways: []workerv1alpha1.WorkerSliceGateway{ + { + Spec: workerv1alpha1.WorkerSliceGatewaySpec{ + GatewayHostType: "Server", + RemoteGatewayConfig: workerv1alpha1.SliceGatewayConfig{ + GatewayName: "test-slice-worker-2-worker-1", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-2-worker-1", + }, + Spec: workerv1alpha1.WorkerSliceGatewaySpec{ + GatewayHostType: "Client", + RemoteGatewayConfig: workerv1alpha1.SliceGatewayConfig{ + GatewayName: "test-slice-worker-1-worker-2", + }, + }, + }, + }, + generateCertsRet1: nil, + }, + } + for _, tc := range testCases { + runTriggerJobsForCertCreation(t, tc) + } +} + +func runTriggerJobsForCertCreation(t *testing.T, tc triggerJobsForCertCreationTestCase) { + ctx, clientMock, vpn, wg, ws := setupTestCase() + + clientMock. + On("List", tc.listArg1, tc.listArg2, tc.listArg3). + Return(tc.listRet1).Run(func(args mock.Arguments) { + w := args.Get(1).(*workerv1alpha1.WorkerSliceGatewayList) + w.Items = tc.workerSliceGateways + }).Once() + + workerSliceConfigs := workerv1alpha1.WorkerSliceConfigList{ + Items: []workerv1alpha1.WorkerSliceConfig{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-slice-config-1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-slice-config-2", + }, + }, + }, + } + + ws.On("ListWorkerSliceConfigs", tc.listWorkerSliceConfigsArg1, tc.listWorkerSliceConfigsArg2, tc.listWorkerSliceConfigsArg3).Return(workerSliceConfigs.Items, tc.listWorkerSliceConfigsRet1).Once() + + clusterMap := map[string]int{ + "cluster-1": 1, + "cluster-2": 2, + } + + ws.On("ComputeClusterMap", tc.listWorkerSliceConfigsArg1, tc.listWorkerSliceConfigsArg2).Return(clusterMap).Once() + + gwAddress := util.WorkerSliceGatewayNetworkAddresses{} + + wg.On("BuildNetworkAddresses", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(gwAddress).Once() + + wg.On("GenerateCerts", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.generateCertsRet1).Once() + + gotResp := vpn.triggerJobsForCertCreation(ctx, tc.arg1, tc.arg2) + require.Equal(t, gotResp, tc.expectedResp) + + clientMock.AssertExpectations(t) +} + +type reconcileVpnKeyRotationTestCase struct { + name string + request ctrl.Request + expectedResponse ctrl.Result + expectedError error + getArg1, getArg2, getArg3 interface{} + getRet1 interface{} + getRet2 interface{} + completionType corev1.ConditionStatus + clusterGatewayMapping map[string][]string +} + +func Test_reconcileVpnKeyRotation(t *testing.T) { + testCases := []reconcileVpnKeyRotationTestCase{ + { + name: "should return error in case vpnkeyrotation config not found", + request: ctrl.Request{NamespacedName: types.NamespacedName{ + Namespace: "test-ns", + Name: "test-slice", + }}, + expectedResponse: ctrl.Result{}, + expectedError: fmt.Errorf("vpnkeyrotation config not found"), + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: fmt.Errorf("vpnkeyrotation config not found"), + }, + { + name: "should return error in case slice config not found", + request: ctrl.Request{NamespacedName: types.NamespacedName{ + Namespace: "test-ns", + Name: "test-slice", + }}, + expectedResponse: ctrl.Result{}, + expectedError: fmt.Errorf("slice config not found"), + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: nil, + getRet2: fmt.Errorf("slice config not found"), + }, + { + name: "should successfully requeue after building clusterGatewayMapping", + request: ctrl.Request{NamespacedName: types.NamespacedName{ + Namespace: "test-ns", + Name: "test-slice", + }}, + expectedResponse: ctrl.Result{Requeue: true}, + expectedError: nil, + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: nil, + getRet2: nil, + }, + { + name: "should wait and requeue till all the jobs are in comlpletion state", + request: ctrl.Request{NamespacedName: types.NamespacedName{ + Namespace: "test-ns", + Name: "test-slice", + }}, + expectedResponse: ctrl.Result{RequeueAfter: 30 * time.Second}, + expectedError: nil, + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: nil, + getRet2: nil, + completionType: corev1.ConditionFalse, + clusterGatewayMapping: map[string][]string{ + "worker-1": {"test-slice-worker-1-worker-2"}, + "worker-2": {"test-slice-worker-2-worker-1"}, + }, + }, + { + name: "should successfully requeue before 1 hour of expiry", + request: ctrl.Request{NamespacedName: types.NamespacedName{ + Namespace: "test-ns", + Name: "test-slice", + }}, + expectedResponse: ctrl.Result{RequeueAfter: metav1.NewTime(time.Date(2021, 06, 16, 20, 34, 58, 651387237, time.UTC)).AddDate(0, 0, 30).Add(-1 * time.Hour).Sub(time.Date(2021, 06, 16, 20, 34, 58, 651387237, time.UTC))}, + expectedError: nil, + getArg1: mock.Anything, + getArg2: mock.Anything, + getArg3: mock.Anything, + getRet1: nil, + getRet2: nil, + completionType: corev1.ConditionTrue, + clusterGatewayMapping: map[string][]string{ + "worker-1": {"test-slice-worker-1-worker-2"}, + "worker-2": {"test-slice-worker-2-worker-1"}, + }, + }, + } + for _, tc := range testCases { + runReconcileVpnKeyRotation(t, tc) + } +} + +func runReconcileVpnKeyRotation(t *testing.T, tc reconcileVpnKeyRotationTestCase) { + ctx, clientMock, vpn, _, _ := setupTestCase() + + clientMock. + On("Get", tc.getArg1, tc.getArg2, tc.getArg3). + Return(tc.getRet1).Run(func(args mock.Arguments) { + v := args.Get(2).(*controllerv1alpha1.VpnKeyRotation) + v.ObjectMeta = metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + } + v.Spec = controllerv1alpha1.VpnKeyRotationSpec{ + Clusters: []string{"worker-1", "worker-2"}, + RotationInterval: 30, + SliceName: "test-slice", + ClusterGatewayMapping: tc.clusterGatewayMapping, + } + }).Once() + + clientMock. + On("Get", tc.getArg1, tc.getArg2, tc.getArg3). + Return(tc.getRet2).Run(func(args mock.Arguments) { + s := args.Get(2).(*controllerv1alpha1.SliceConfig) + s.ObjectMeta = metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + } + s.Spec = controllerv1alpha1.SliceConfigSpec{ + Clusters: []string{"worker-1", "worker-2"}, + RotationInterval: 30, + } + }).Once() + + clientMock. + On("List", mock.Anything, mock.Anything, mock.Anything). + Return(nil).Run(func(args mock.Arguments) { + w := args.Get(1).(*workerv1alpha1.WorkerSliceGatewayList) + w.Items = append(w.Items, workerv1alpha1.WorkerSliceGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-1-worker-2", + Labels: map[string]string{ + "worker-cluster": "worker-1", + "original-slice-name": "test-slice", + }, + }, + }) + }).Once() + + clientMock. + On("List", mock.Anything, mock.Anything, mock.Anything). + Return(nil).Run(func(args mock.Arguments) { + w := args.Get(1).(*workerv1alpha1.WorkerSliceGatewayList) + w.Items = append(w.Items, workerv1alpha1.WorkerSliceGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-2-worker-1", + Labels: map[string]string{ + "worker-cluster": "worker-2", + "original-slice-name": "test-slice", + }, + }, + }) + }).Once() + + clientMock. + On("Update", mock.Anything, mock.Anything).Return(nil).Once() + + clientMock.On("Create", ctx, mock.AnythingOfType("*v1.Event")).Return(nil).Once() + + if tc.completionType == corev1.ConditionTrue { + clientMock. + On("List", mock.Anything, mock.Anything, mock.Anything). + Return(nil).Run(func(args mock.Arguments) { + w := args.Get(1).(*batchv1.JobList) + w.Items = append(w.Items, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-1-worker-2", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-2-worker-1", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: tc.completionType, + }, + }, + }, + }) + }).Once() + } else if tc.completionType == corev1.ConditionFalse { + clientMock. + On("List", mock.Anything, mock.Anything, mock.Anything). + Return(nil).Run(func(args mock.Arguments) { + w := args.Get(1).(*batchv1.JobList) + w.Items = append(w.Items, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-1-worker-2", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Active: 1, + }, + }, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice-worker-2-worker-1", + Labels: map[string]string{ + "SLICE_NAME": "test-slice", + }, + }, + Status: batchv1.JobStatus{ + Active: 1, + }, + }) + }).Once() + } + + clientMock. + On("Update", mock.Anything, mock.Anything).Return(nil).Once() + + // Mocking metav1.Now() with a fixed time value + patch := monkey.Patch(metav1.Now, func() metav1.Time { + return metav1.NewTime(time.Date(2021, 06, 16, 20, 34, 58, 651387237, time.UTC)) + }) + defer patch.Unpatch() + + gotResp, gotErr := vpn.ReconcileVpnKeyRotation(ctx, tc.request) + + require.Equal(t, tc.expectedError, gotErr) + require.Equal(t, tc.expectedResponse, gotResp) + +} diff --git a/service/vpn_key_rotation_webhook_validation.go b/service/vpn_key_rotation_webhook_validation.go new file mode 100644 index 00000000..e93c93a9 --- /dev/null +++ b/service/vpn_key_rotation_webhook_validation.go @@ -0,0 +1,55 @@ +package service + +import ( + "context" + "fmt" + + controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + "github.com/kubeslice/kubeslice-controller/events" + "github.com/kubeslice/kubeslice-controller/util" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func ValidateVpnKeyRotationCreate(ctx context.Context, r *controllerv1alpha1.VpnKeyRotation) error { + if r.Spec.SliceName == "" { + return fmt.Errorf("invalid config,.spec.sliceName could not be empty") + } + if r.Name != r.Spec.SliceName { + return fmt.Errorf("invalid config, name should match with slice name") + } + slice := &controllerv1alpha1.SliceConfig{} + found, err := util.GetResourceIfExist(ctx, client.ObjectKey{ + Name: r.Spec.SliceName, + Namespace: r.Namespace, + }, slice) + if err != nil { + return err + } + if !found { + return fmt.Errorf("invalid config, sliceconfig %s not present", r.Spec.SliceName) + } + return nil +} + +func ValidateVpnKeyRotationDelete(ctx context.Context, r *controllerv1alpha1.VpnKeyRotation) error { + slice := &controllerv1alpha1.SliceConfig{} + found, err := util.GetResourceIfExist(ctx, client.ObjectKey{ + Name: r.Spec.SliceName, + Namespace: r.Namespace, + }, slice) + if err != nil { + return err + } + if found && slice.ObjectMeta.DeletionTimestamp.IsZero() { + //Load Event Recorder with project name, vpnkeyrotation(slice) name and namespace + eventRecorder := util.CtxEventRecorder(ctx). + WithProject(util.GetProjectName(r.Namespace)). + WithNamespace(r.Namespace). + WithSlice(r.Name) + //Register an event for worker slice config deleted forcefully + util.RecordEvent(ctx, eventRecorder, r, slice, events.EventIllegalVPNKeyRotationConfigDelete) + return fmt.Errorf("vpnkeyrotation config %s not allowed to delete unless sliceconfig is deleted", r.Name) + } + // if not found or timestamp is non-zero,this means slice is deleted/under deletion. + return nil +} diff --git a/service/vpn_key_rotation_webhook_validation_test.go b/service/vpn_key_rotation_webhook_validation_test.go new file mode 100644 index 00000000..54adc7f6 --- /dev/null +++ b/service/vpn_key_rotation_webhook_validation_test.go @@ -0,0 +1,159 @@ +package service + +import ( + "context" + "fmt" + "testing" + + controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + ossEvents "github.com/kubeslice/kubeslice-controller/events" + "github.com/kubeslice/kubeslice-controller/util" + utilMock "github.com/kubeslice/kubeslice-controller/util/mocks" + "github.com/kubeslice/kubeslice-monitoring/pkg/events" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +type validateVpnKeyRotationCreateTestCase struct { + name string + arg *controllerv1alpha1.VpnKeyRotation + expectedErr error + sliceConfigPresent bool +} + +func Test_validateVpnKeyRotationCreate(t *testing.T) { + testCase := []validateVpnKeyRotationCreateTestCase{ + { + name: "should return nil if name matches the original slice name", + arg: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-slice", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice", + }, + }, + expectedErr: nil, + sliceConfigPresent: true, + }, + { + name: "should return error if slicename is empty", + arg: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-slice", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "", + }, + }, + expectedErr: fmt.Errorf("invalid config,.spec.sliceName could not be empty"), + sliceConfigPresent: true, + }, + { + name: "should return error if name does not macthes original slice name", + arg: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-slice", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice-1", + }, + }, + expectedErr: fmt.Errorf("invalid config, name should match with slice name"), + sliceConfigPresent: true, + }, + { + name: "should return error if sliceconfig is not present", + arg: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-slice", + }, + Spec: controllerv1alpha1.VpnKeyRotationSpec{ + SliceName: "test-slice", + }, + }, + expectedErr: fmt.Errorf("sliceconfig test-slice not found"), + sliceConfigPresent: false, + }, + } + for _, tc := range testCase { + runValidateVpnKeyRotationCreateTest(t, tc) + } +} + +func runValidateVpnKeyRotationCreateTest(t *testing.T, tc validateVpnKeyRotationCreateTestCase) { + ctx, clientMock := setupValidationTestCase() + if tc.sliceConfigPresent { + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(nil) + } else { + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("sliceconfig test-slice not found")) + } + gotErr := ValidateVpnKeyRotationCreate(ctx, tc.arg) + require.Equal(t, tc.expectedErr, gotErr) +} + +func setupValidationTestCase() (context.Context, *utilMock.Client) { + clientMock := &utilMock.Client{} + scheme := runtime.NewScheme() + utilruntime.Must(controllerv1alpha1.AddToScheme(scheme)) + eventRecorder := events.NewEventRecorder(clientMock, scheme, ossEvents.EventsMap, events.EventRecorderOptions{ + Version: "v1alpha1", + Cluster: util.ClusterController, + Component: util.ComponentController, + Slice: util.NotApplicable, + }) + return util.PrepareKubeSliceControllersRequestContext(context.Background(), clientMock, scheme, "ClusterTestController", &eventRecorder), clientMock +} + +type validateVpnKeyRotationDeleteTestCase struct { + name string + arg *controllerv1alpha1.VpnKeyRotation + expectedErr error + deletionTs v1.Time +} + +func Test_validateVpnKeyRotationDelete(t *testing.T) { + testCase := []validateVpnKeyRotationDeleteTestCase{ + { + name: "should return nil if slice is under deletion", + arg: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + }, + deletionTs: v1.Now(), + expectedErr: nil, + }, + { + name: "should return error if slice is not deleted/under deletion", + arg: &controllerv1alpha1.VpnKeyRotation{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-slice", + Namespace: "test-ns", + }, + }, + expectedErr: fmt.Errorf("vpnkeyrotation config %s not allowed to delete unless sliceconfig is deleted", "test-slice"), + }, + } + for _, tc := range testCase { + runValidateVpnKeyRotationDeleteTest(t, tc) + } +} + +func runValidateVpnKeyRotationDeleteTest(t *testing.T, tc validateVpnKeyRotationDeleteTestCase) { + ctx, clientMock := setupValidationTestCase() + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(2).(*controllerv1alpha1.SliceConfig) + arg.ObjectMeta = v1.ObjectMeta{ + DeletionTimestamp: &tc.deletionTs, + } + }) + clientMock.On("Create", ctx, mock.AnythingOfType("*v1.Event")).Return(nil).Once() + + gotErr := ValidateVpnKeyRotationDelete(ctx, tc.arg) + require.Equal(t, tc.expectedErr, gotErr) +} diff --git a/service/worker_slice_gateway_service.go b/service/worker_slice_gateway_service.go index c176570e..8f1cd7a0 100644 --- a/service/worker_slice_gateway_service.go +++ b/service/worker_slice_gateway_service.go @@ -18,13 +18,15 @@ package service import ( "context" + "errors" "fmt" - "github.com/kubeslice/kubeslice-controller/metrics" + "os" "reflect" "strings" "time" "github.com/kubeslice/kubeslice-controller/events" + "github.com/kubeslice/kubeslice-controller/metrics" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -48,6 +50,11 @@ type IWorkerSliceGatewayService interface { ListWorkerSliceGateways(ctx context.Context, ownerLabel map[string]string, namespace string) ([]v1alpha1.WorkerSliceGateway, error) DeleteWorkerSliceGatewaysByLabel(ctx context.Context, label map[string]string, namespace string) error NodeIpReconciliationOfWorkerSliceGateways(ctx context.Context, cluster *controllerv1alpha1.Cluster, namespace string) error + GenerateCerts(ctx context.Context, sliceName string, namespace string, + serverGateway *v1alpha1.WorkerSliceGateway, clientGateway *v1alpha1.WorkerSliceGateway, + gatewayAddresses util.WorkerSliceGatewayNetworkAddresses) error + BuildNetworkAddresses(sliceSubnet, sourceClusterName, destinationClusterName string, + clusterMap map[string]int, clusterCidr string) util.WorkerSliceGatewayNetworkAddresses } // WorkerSliceGatewayService is a schema for interfaces JobService, WorkerSliceConfigService, SecretService @@ -59,15 +66,6 @@ type WorkerSliceGatewayService struct { } // WorkerSliceGatewayNetworkAddresses is a schema for WorkerSlice gateway network parameters -type WorkerSliceGatewayNetworkAddresses struct { - ServerNetwork string - ClientNetwork string - ServerSubnet string - ClientSubnet string - ServerVpnNetwork string - ServerVpnAddress string - ClientVpnAddress string -} // ReconcileWorkerSliceGateways is a function to reconcile/restore the worker slice gateways func (s *WorkerSliceGatewayService) ReconcileWorkerSliceGateways(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -430,7 +428,7 @@ func (s *WorkerSliceGatewayService) createMinimumGatewaysIfNotExists(ctx context for j := i + 1; j < noClusters; j++ { sourceCluster, destinationCluster := clusterMapping[clusterNames[i]], clusterMapping[clusterNames[j]] gatewayNumber := s.calculateGatewayNumber(clusterMap[sourceCluster.Name], clusterMap[destinationCluster.Name]) - gatewayAddresses := s.buildNetworkAddresses(sliceSubnet, sourceCluster.Name, destinationCluster.Name, clusterMap, clusterCidr) + gatewayAddresses := s.BuildNetworkAddresses(sliceSubnet, sourceCluster.Name, destinationCluster.Name, clusterMap, clusterCidr) err := s.createMinimumGateWayPairIfNotExists(ctx, sourceCluster, destinationCluster, sliceName, namespace, ownerLabel, gatewayNumber, gatewayAddresses) if err != nil { return ctrl.Result{}, err @@ -444,7 +442,7 @@ func (s *WorkerSliceGatewayService) createMinimumGatewaysIfNotExists(ctx context // createMinimumGateWayPairIfNotExists is a function to create the pair of gatways between 2 clusters if not exists func (s *WorkerSliceGatewayService) createMinimumGateWayPairIfNotExists(ctx context.Context, sourceCluster *controllerv1alpha1.Cluster, destinationCluster *controllerv1alpha1.Cluster, sliceName string, namespace string, - label map[string]string, gatewayNumber int, gatewayAddresses WorkerSliceGatewayNetworkAddresses) error { + label map[string]string, gatewayNumber int, gatewayAddresses util.WorkerSliceGatewayNetworkAddresses) error { serverGatewayName := fmt.Sprintf(gatewayName, sliceName, sourceCluster.Name, destinationCluster.Name) clientGatewayName := fmt.Sprintf(gatewayName, sliceName, destinationCluster.Name, sourceCluster.Name) gateway := v1alpha1.WorkerSliceGateway{} @@ -525,7 +523,8 @@ func (s *WorkerSliceGatewayService) createMinimumGateWayPairIfNotExists(ctx cont "object_kind": metricKindWorkerSliceGateway, }, ) - err = s.generateCerts(ctx, sliceName, namespace, serverGatewayObject, clientGatewayObject, gatewayAddresses) + + err = s.GenerateCerts(ctx, sliceName, namespace, serverGatewayObject, clientGatewayObject, gatewayAddresses) if err != nil { return err } @@ -534,9 +533,9 @@ func (s *WorkerSliceGatewayService) createMinimumGateWayPairIfNotExists(ctx cont } // buildNetworkAddresses - function generates the object of WorkerSliceGatewayNetworkAddresses -func (s *WorkerSliceGatewayService) buildNetworkAddresses(sliceSubnet, sourceClusterName, destinationClusterName string, - clusterMap map[string]int, clusterCidr string) WorkerSliceGatewayNetworkAddresses { - gatewayAddresses := WorkerSliceGatewayNetworkAddresses{} +func (s *WorkerSliceGatewayService) BuildNetworkAddresses(sliceSubnet, sourceClusterName, destinationClusterName string, + clusterMap map[string]int, clusterCidr string) util.WorkerSliceGatewayNetworkAddresses { + gatewayAddresses := util.WorkerSliceGatewayNetworkAddresses{} ipr := strings.Split(sliceSubnet, ".") serverSubnet := fmt.Sprintf(util.GetClusterPrefixPool(sliceSubnet, clusterMap[sourceClusterName], clusterCidr)) clientSubnet := fmt.Sprintf(util.GetClusterPrefixPool(sliceSubnet, clusterMap[destinationClusterName], clusterCidr)) @@ -604,9 +603,21 @@ func (s *WorkerSliceGatewayService) buildMinimumGateway(sourceCluster, destinati } // generateCerts is a function to generate the certificates between serverGateway and clientGateway -func (s *WorkerSliceGatewayService) generateCerts(ctx context.Context, sliceName string, namespace string, +func (s *WorkerSliceGatewayService) GenerateCerts(ctx context.Context, sliceName string, namespace string, serverGateway *v1alpha1.WorkerSliceGateway, clientGateway *v1alpha1.WorkerSliceGateway, - gatewayAddresses WorkerSliceGatewayNetworkAddresses) error { + gatewayAddresses util.WorkerSliceGatewayNetworkAddresses) error { + sliceConfig := &controllerv1alpha1.SliceConfig{} + found, err := util.GetResourceIfExist(ctx, client.ObjectKey{ + Name: sliceName, + Namespace: namespace, + }, sliceConfig) + if err != nil { + return err + } + if !found { + errMsg := fmt.Sprintf("sliceConfig for %v not found in %v.", sliceName, namespace) + return errors.New(errMsg) + } cpr := s.buildCertPairRequest(sliceName, serverGateway, clientGateway, gatewayAddresses) //Load Event Recorder with project name, slice name and namespace @@ -626,8 +637,15 @@ func (s *WorkerSliceGatewayService) generateCerts(ctx context.Context, sliceName environment["CLIENT_SLICEGATEWAY_NAME"] = clientGateway.Name environment["SLICE_NAME"] = sliceName environment["CERT_GEN_REQUESTS"], _ = util.EncodeToBase64(&cpr) + if nil == sliceConfig.Spec.VPNConfig { + environment["VPN_CIPHER"] = "AES-256-CBC" + } else { + environment["VPN_CIPHER"] = sliceConfig.Spec.VPNConfig.Cipher + } + + jobNamespace = os.Getenv("KUBESLICE_CONTROLLER_MANAGER_NAMESPACE") util.CtxLogger(ctx).Info("jobNamespace", jobNamespace) //todo:remove - _, err := s.js.CreateJob(ctx, jobNamespace, JobImage, environment) + _, err = s.js.CreateJob(ctx, jobNamespace, JobImage, environment) if err != nil { //Register an event for gateway job creation failure util.RecordEvent(ctx, eventRecorder, serverGateway, clientGateway, events.EventSliceGatewayJobCreationFailed) @@ -657,7 +675,7 @@ func (s *WorkerSliceGatewayService) generateCerts(ctx context.Context, sliceName // buildCertPairRequest is a function to generate the pair between server-cluster and client-cluster func (s *WorkerSliceGatewayService) buildCertPairRequest(sliceName string, gateway1, gateway2 *v1alpha1.WorkerSliceGateway, - gatewayAddresses WorkerSliceGatewayNetworkAddresses) CertPairRequestMap { + gatewayAddresses util.WorkerSliceGatewayNetworkAddresses) CertPairRequestMap { clusterName := gateway1.Spec.LocalGatewayConfig.ClusterName serverNumber := gateway1.Spec.GatewayNumber serverId, clientId := gateway1.Name, gateway2.Name diff --git a/service/worker_slice_gateway_service_test.go b/service/worker_slice_gateway_service_test.go index c0f68d0a..563d4175 100644 --- a/service/worker_slice_gateway_service_test.go +++ b/service/worker_slice_gateway_service_test.go @@ -402,8 +402,9 @@ func testCreateMinimumWorkerSliceGatewaysNotExists(t *testing.T) { clientMock.On("Create", ctx, mock.Anything).Return(nil).Once() clientMock.On("Create", ctx, mock.AnythingOfType("*v1.Event")).Return(nil).Once() mMock.On("RecordCounterMetric", mock.Anything, mock.Anything).Return().Once() - jobMock.On("CreateJob", ctx, jobNamespace, JobImage, mock.Anything).Return(ctrl.Result{}, nil).Once() + jobMock.On("CreateJob", ctx, mock.Anything, JobImage, mock.Anything).Return(ctrl.Result{}, nil).Once() clientMock.On("Update", ctx, mock.AnythingOfType("*v1.Event")).Return(nil).Once() + clientMock.On("Get", ctx, mock.Anything, mock.Anything).Return(nil).Once() mMock.On("RecordCounterMetric", mock.Anything, mock.Anything).Return().Once() result, err := workerSliceGatewayService.CreateMinimumWorkerSliceGateways(ctx, "red", clusterNames, requestObj.Namespace, label, clusterMap, "10.10.10.10/16", "/16") expectedResult := ctrl.Result{} diff --git a/unit_tests.dockerfile b/unit_tests.dockerfile index 8fca50f4..5b743317 100644 --- a/unit_tests.dockerfile +++ b/unit_tests.dockerfile @@ -1,12 +1,13 @@ -FROM golang:1.18 as builder +FROM golang:1.19 as builder COPY . /build WORKDIR /build/service ENV ALLURE_RESULTS_PATH=/build -RUN go test --coverprofile=coverage.out; exit 0 +RUN go test -gcflags=-l --coverprofile=coverage.out; exit 0 RUN mkdir -p coverage-report RUN go tool cover -html=coverage.out -o coverage-report/report.html + FROM scratch as exporter COPY --from=builder /build/allure-results /allure-results COPY --from=builder /build/service/coverage-report /coverage-report diff --git a/util/common.go b/util/common.go index 08e8f47a..fa524213 100644 --- a/util/common.go +++ b/util/common.go @@ -28,6 +28,16 @@ import ( "go.uber.org/zap/zapcore" ) +type WorkerSliceGatewayNetworkAddresses struct { + ServerNetwork string + ClientNetwork string + ServerSubnet string + ClientSubnet string + ServerVpnNetwork string + ServerVpnAddress string + ClientVpnAddress string +} + // AppendHyphenToString is a function add hyphen at the end of string func AppendHyphenToString(stringToAppend string) string { if strings.HasSuffix(stringToAppend, "-") { From 1f1546d85f5a4419433003ec4b4b9afdb7053a62 Mon Sep 17 00:00:00 2001 From: Md Imran Date: Wed, 27 Sep 2023 14:25:39 +0530 Subject: [PATCH 2/5] Update trivy.yml Signed-off-by: Md Imran --- .github/workflows/trivy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 97d1db79..a7077385 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -17,7 +17,7 @@ jobs: - name: Run Trivy vulnerability scanner in repo mode uses: aquasecurity/trivy-action@master with: - scan-type: 'fs' + scan-type: 'repo' ignore-unfixed: true format: 'sarif' output: 'trivy-results.sarif' From 3e82a326f426863c4ca93108ed96a92dd1c4e444 Mon Sep 17 00:00:00 2001 From: Md Imran Date: Wed, 27 Sep 2023 14:28:02 +0530 Subject: [PATCH 3/5] Create codeql.yml Signed-off-by: Md Imran --- .github/workflows/codeql.yml | 80 ++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..5c587017 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,80 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "master" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "master" ] + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" From 21b09014bb524a1ecc8f181c3869c560201d65eb Mon Sep 17 00:00:00 2001 From: Md Imran Date: Wed, 27 Sep 2023 14:50:20 +0530 Subject: [PATCH 4/5] Update trivy.yml Signed-off-by: Md Imran --- .github/workflows/trivy.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index a7077385..3d203e04 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -1,13 +1,21 @@ name: trivy on: push: - branches: - - master - pull_request: - branches: - - master + branches: [ "master" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "master" ] + schedule: + - cron: '29 19 * * 6' +permissions: + contents: read + jobs: build: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status name: Build runs-on: ubuntu-20.04 steps: @@ -26,4 +34,4 @@ jobs: - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v2 with: - sarif_file: 'trivy-results.sarif' + sarif_file: 'trivy-results.sarif' \ No newline at end of file From 4522d5155850f5a12b811dc4ab4f8e0e885966d6 Mon Sep 17 00:00:00 2001 From: Md Imran Date: Wed, 27 Sep 2023 14:52:29 +0530 Subject: [PATCH 5/5] Delete codeql.yml Signed-off-by: Md Imran --- .github/workflows/codeql.yml | 80 ------------------------------------ 1 file changed, 80 deletions(-) delete mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 5c587017..00000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,80 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ "master" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "master" ] - -jobs: - analyze: - name: Analyze - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners - # Consider using larger runners for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ] - # Use only 'java' to analyze code written in Java, Kotlin or both - # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:${{matrix.language}}"