From 77eaad31f7c0cc1833b443134ef2bc6879d1eb0a Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 6 Sep 2024 13:39:11 -0400 Subject: [PATCH 1/5] supply additional labels and annotations to fs-minio pods --- pkg/kotsutil/kots.go | 34 ++++++++++++++++++++ pkg/snapshot/filesystem_minio.go | 53 ++++++++++++++++++++++++-------- 2 files changed, 74 insertions(+), 13 deletions(-) diff --git a/pkg/kotsutil/kots.go b/pkg/kotsutil/kots.go index 73ee3b5c49..99b8639b80 100644 --- a/pkg/kotsutil/kots.go +++ b/pkg/kotsutil/kots.go @@ -8,6 +8,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "k8s.io/client-go/kubernetes" "os" "path" "path/filepath" @@ -1140,6 +1141,8 @@ type InstallationParams struct { WithMinio bool AppVersionLabel string RequestedChannelSlug string + AdditionalAnnotations map[string]string + AdditionalLabels map[string]string } func GetInstallationParams(configMapName string) (InstallationParams, error) { @@ -1150,6 +1153,12 @@ func GetInstallationParams(configMapName string) (InstallationParams, error) { return autoConfig, errors.Wrap(err, "failed to get k8s clientset") } + return GetInstallationParamsWithClientset(clientset, configMapName) +} + +func GetInstallationParamsWithClientset(clientset kubernetes.Interface, configMapName string) (InstallationParams, error) { + autoConfig := InstallationParams{} + isKurl, err := kurl.IsKurl(clientset) if err != nil { return autoConfig, errors.Wrap(err, "failed to check if cluster is kurl") @@ -1183,6 +1192,31 @@ func GetInstallationParams(configMapName string) (InstallationParams, error) { autoConfig.EnableImageDeletion = isKurl } + autoConfig.AdditionalAnnotations = make(map[string]string) + allAnnotations := strings.Split(kotsadmConfigMap.Data["additional-annotations"], ",") + for _, annotation := range allAnnotations { + if annotation == "" { + continue + } + parts := strings.Split(annotation, "=") + if len(parts) != 2 { + return autoConfig, errors.Errorf("invalid additional annotation %q", annotation) + } + autoConfig.AdditionalAnnotations[parts[0]] = parts[1] + } + autoConfig.AdditionalLabels = make(map[string]string) + allLabels := strings.Split(kotsadmConfigMap.Data["additional-labels"], ",") + for _, label := range allLabels { + if label == "" { + continue + } + parts := strings.Split(label, "=") + if len(parts) != 2 { + return autoConfig, errors.Errorf("invalid additional label %q", label) + } + autoConfig.AdditionalLabels[parts[0]] = parts[1] + } + return autoConfig, nil } diff --git a/pkg/snapshot/filesystem_minio.go b/pkg/snapshot/filesystem_minio.go index afe5970f86..4464610df6 100644 --- a/pkg/snapshot/filesystem_minio.go +++ b/pkg/snapshot/filesystem_minio.go @@ -19,6 +19,7 @@ import ( kotsadmresources "github.com/replicatedhq/kots/pkg/kotsadm/resources" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" kotsadmversion "github.com/replicatedhq/kots/pkg/kotsadm/version" + "github.com/replicatedhq/kots/pkg/kotsutil" "github.com/replicatedhq/kots/pkg/kurl" "github.com/replicatedhq/kots/pkg/logger" kotss3 "github.com/replicatedhq/kots/pkg/s3" @@ -300,6 +301,11 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe } } + globalOptions, err := kotsutil.GetInstallationParamsWithClientset(clientset, kotsadmtypes.KotsadmConfigMap) + if err != nil { + return nil, errors.Wrap(err, "failed to get global options") + } + env := []corev1.EnvVar{ { Name: "MINIO_UPDATE", @@ -329,13 +335,28 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe }, } + podAnnotations := map[string]string{ + "kots.io/fs-minio-creds-secret-checksum": secretChecksum, + } + for k, v := range globalOptions.AdditionalAnnotations { + podAnnotations[k] = v + } + podLabels := map[string]string{ + "app": "kotsadm-fs-minio", + } + for k, v := range globalOptions.AdditionalLabels { + podLabels[k] = v + } + return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "Deployment", }, ObjectMeta: metav1.ObjectMeta{ - Name: FileSystemMinioDeploymentName, + Name: FileSystemMinioDeploymentName, + Labels: globalOptions.AdditionalLabels, + Annotations: globalOptions.AdditionalAnnotations, }, Spec: appsv1.DeploymentSpec{ Replicas: pointer.Int32Ptr(1), @@ -349,12 +370,8 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "kotsadm-fs-minio", - }, - Annotations: map[string]string{ - "kots.io/fs-minio-creds-secret-checksum": secretChecksum, - }, + Labels: podLabels, + Annotations: podAnnotations, }, Spec: corev1.PodSpec{ SecurityContext: &securityContext, @@ -837,18 +854,28 @@ func fileSystemMinioConfigPod(clientset kubernetes.Interface, deployOptions File } } + globalOptions, err := kotsutil.GetInstallationParamsWithClientset(clientset, kotsadmtypes.KotsadmConfigMap) + if err != nil { + return nil, errors.Wrap(err, "failed to get global options") + } + podLabels := map[string]string{ + "app": "kotsadm-fs-minio", + "check": podCheckTag, + } + for k, v := range globalOptions.AdditionalLabels { + podLabels[k] = v + } + pod := &corev1.Pod{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Pod", }, ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: deployOptions.Namespace, - Labels: map[string]string{ - "app": "kotsadm-fs-minio", - "check": podCheckTag, - }, + Name: podName, + Namespace: deployOptions.Namespace, + Labels: podLabels, + Annotations: globalOptions.AdditionalAnnotations, }, Spec: corev1.PodSpec{ SecurityContext: &securityContext, From b35e491a6671a0b9ab6a6430c13c77401672a882 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 6 Sep 2024 16:11:19 -0400 Subject: [PATCH 2/5] use correct namespace --- pkg/kotsutil/kots.go | 6 +++--- pkg/snapshot/filesystem_minio.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/kotsutil/kots.go b/pkg/kotsutil/kots.go index 99b8639b80..c4a736b938 100644 --- a/pkg/kotsutil/kots.go +++ b/pkg/kotsutil/kots.go @@ -1153,10 +1153,10 @@ func GetInstallationParams(configMapName string) (InstallationParams, error) { return autoConfig, errors.Wrap(err, "failed to get k8s clientset") } - return GetInstallationParamsWithClientset(clientset, configMapName) + return GetInstallationParamsWithClientset(clientset, configMapName, util.PodNamespace) } -func GetInstallationParamsWithClientset(clientset kubernetes.Interface, configMapName string) (InstallationParams, error) { +func GetInstallationParamsWithClientset(clientset kubernetes.Interface, configMapName string, namespace string) (InstallationParams, error) { autoConfig := InstallationParams{} isKurl, err := kurl.IsKurl(clientset) @@ -1164,7 +1164,7 @@ func GetInstallationParamsWithClientset(clientset kubernetes.Interface, configMa return autoConfig, errors.Wrap(err, "failed to check if cluster is kurl") } - kotsadmConfigMap, err := clientset.CoreV1().ConfigMaps(util.PodNamespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) + kotsadmConfigMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) if err != nil { if kuberneteserrors.IsNotFound(err) { return autoConfig, nil diff --git a/pkg/snapshot/filesystem_minio.go b/pkg/snapshot/filesystem_minio.go index 4464610df6..9cac5ba903 100644 --- a/pkg/snapshot/filesystem_minio.go +++ b/pkg/snapshot/filesystem_minio.go @@ -301,7 +301,7 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe } } - globalOptions, err := kotsutil.GetInstallationParamsWithClientset(clientset, kotsadmtypes.KotsadmConfigMap) + globalOptions, err := kotsutil.GetInstallationParamsWithClientset(clientset, kotsadmtypes.KotsadmConfigMap, deployOptions.Namespace) if err != nil { return nil, errors.Wrap(err, "failed to get global options") } @@ -854,7 +854,7 @@ func fileSystemMinioConfigPod(clientset kubernetes.Interface, deployOptions File } } - globalOptions, err := kotsutil.GetInstallationParamsWithClientset(clientset, kotsadmtypes.KotsadmConfigMap) + globalOptions, err := kotsutil.GetInstallationParamsWithClientset(clientset, kotsadmtypes.KotsadmConfigMap, deployOptions.Namespace) if err != nil { return nil, errors.Wrap(err, "failed to get global options") } From 835f82aab0dc9a1f4843f0a6cfc8637c1ab14f89 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 6 Sep 2024 16:39:44 -0400 Subject: [PATCH 3/5] add GetInstallationParamsWithClientset unit test --- pkg/kotsutil/kots_test.go | 75 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/pkg/kotsutil/kots_test.go b/pkg/kotsutil/kots_test.go index 5bc98f2ace..846f8aab98 100644 --- a/pkg/kotsutil/kots_test.go +++ b/pkg/kotsutil/kots_test.go @@ -3,9 +3,11 @@ package kotsutil_test import ( "encoding/base64" "io/ioutil" + "k8s.io/client-go/kubernetes" "os" "path/filepath" "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,6 +25,7 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" applicationv1beta1 "sigs.k8s.io/application/api/v1beta1" ) @@ -1255,3 +1258,75 @@ func TestFindChannelInLicense(t *testing.T) { }) } } + +func TestGetInstallationParamsWithClientset(t *testing.T) { + type args struct { + configMapName string + namespace string + clientSet kubernetes.Interface + } + tests := []struct { + name string + args args + want kotsutil.InstallationParams + wantErr assert.ErrorAssertionFunc + }{ + { + name: "basic test", + args: args{ + configMapName: "kotsadm", + namespace: "test-namespace", + clientSet: fake.NewClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "additional-annotations": "abc/xyz=test-annotation1,test.annotation/two=test.value/two/test", + "additional-labels": "xyz=label2,abc=123", + "app-version-label": "", + "ensure-rbac": "true", + "initial-app-images-pushed": "false", + "kots-install-id": "2liAJUuyAi3Gnyhvi5Arv5BJRZ4", + "minio-enabled-snapshots": "true", + "registry-is-read-only": "false", + "requested-channel-slug": "stable", + "skip-compatibility-check": "false", + "skip-preflights": "false", + "skip-rbac-check": "false", + "strict-security-context": "false", + "use-minimal-rbac": "false", + "wait-duration": "2m0s", + "with-minio": "true", + }, + }), + }, + want: kotsutil.InstallationParams{ + AdditionalAnnotations: map[string]string{"abc/xyz": "test-annotation1", "test.annotation/two": "test.value/two/test"}, + AdditionalLabels: map[string]string{"abc": "123", "xyz": "label2"}, + AppVersionLabel: "", + EnsureRBAC: true, + KotsadmRegistry: "", + SkipImagePush: false, + SkipPreflights: false, + SkipCompatibilityCheck: false, + RegistryIsReadOnly: false, + EnableImageDeletion: false, + SkipRBACCheck: false, + UseMinimalRBAC: false, + StrictSecurityContext: false, + WaitDuration: time.Minute * 2, + WithMinio: true, + RequestedChannelSlug: "stable", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := require.New(t) + got, err := kotsutil.GetInstallationParamsWithClientset(tt.args.clientSet, tt.args.configMapName, tt.args.namespace) + req.NoError(err) + req.Equal(tt.want, got) + }) + } +} From 8be83bc0cde1e03b0b6ecda2d93e3b37bdb98913 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 6 Sep 2024 17:03:18 -0400 Subject: [PATCH 4/5] add unit test for kotsadm-fs-minio creation --- pkg/kotsutil/kots_test.go | 47 ++++++++++++++++ pkg/snapshot/filesystem_minio_test.go | 80 +++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 pkg/snapshot/filesystem_minio_test.go diff --git a/pkg/kotsutil/kots_test.go b/pkg/kotsutil/kots_test.go index 846f8aab98..c51dfce811 100644 --- a/pkg/kotsutil/kots_test.go +++ b/pkg/kotsutil/kots_test.go @@ -1320,6 +1320,53 @@ func TestGetInstallationParamsWithClientset(t *testing.T) { RequestedChannelSlug: "stable", }, }, + { + name: "no labels or annotations", + args: args{ + configMapName: "kotsadm", + namespace: "test-namespace", + clientSet: fake.NewClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "app-version-label": "", + "ensure-rbac": "true", + "initial-app-images-pushed": "false", + "kots-install-id": "2liAJUuyAi3Gnyhvi5Arv5BJRZ4", + "minio-enabled-snapshots": "true", + "registry-is-read-only": "false", + "requested-channel-slug": "stable", + "skip-compatibility-check": "false", + "skip-preflights": "false", + "skip-rbac-check": "false", + "strict-security-context": "false", + "use-minimal-rbac": "false", + "wait-duration": "2m0s", + "with-minio": "true", + }, + }), + }, + want: kotsutil.InstallationParams{ + AdditionalAnnotations: map[string]string{}, + AdditionalLabels: map[string]string{}, + AppVersionLabel: "", + EnsureRBAC: true, + KotsadmRegistry: "", + SkipImagePush: false, + SkipPreflights: false, + SkipCompatibilityCheck: false, + RegistryIsReadOnly: false, + EnableImageDeletion: false, + SkipRBACCheck: false, + UseMinimalRBAC: false, + StrictSecurityContext: false, + WaitDuration: time.Minute * 2, + WithMinio: true, + RequestedChannelSlug: "stable", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/snapshot/filesystem_minio_test.go b/pkg/snapshot/filesystem_minio_test.go new file mode 100644 index 0000000000..61b378d747 --- /dev/null +++ b/pkg/snapshot/filesystem_minio_test.go @@ -0,0 +1,80 @@ +package snapshot + +import ( + "context" + "github.com/replicatedhq/kots/pkg/kotsadm/types" + kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" + kotssnapshottypes "github.com/replicatedhq/kots/pkg/snapshot/types" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "testing" +) + +func Test_ensureFileSystemMinioDeployment(t *testing.T) { + hostPathString := "/test/hostpath" + type args struct { + clientSet kubernetes.Interface + deployOptions FileSystemDeployOptions + registryConfig types.RegistryConfig + marshalledSecret []byte + } + tests := []struct { + name string + args args + validate func(t *testing.T, clientset kubernetes.Interface) + }{ + { + name: "new hostpath deployment", + args: args{ + clientSet: fake.NewClientset( + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: kotsadmtypes.KotsadmConfigMap, + Namespace: "test-namespace", + }, + Data: map[string]string{ + "additional-annotations": "abc/xyz=test-annotation1,test.annotation/two=test.value/two/test", + "additional-labels": "xyz=label2,abc=123", + }, + }, + ), + deployOptions: FileSystemDeployOptions{ + Namespace: "test-namespace", + IsOpenShift: false, + ForceReset: false, + FileSystemConfig: kotssnapshottypes.FileSystemConfig{ + HostPath: &hostPathString, + }, + }, + }, + validate: func(t *testing.T, clientset kubernetes.Interface) { + createdDeployment, err := clientset.AppsV1().Deployments("test-namespace").Get(context.Background(), "kotsadm-fs-minio", metav1.GetOptions{}) + require.NoError(t, err) + require.NotNil(t, createdDeployment) + + // check annotations + require.Equal(t, map[string]string{"abc/xyz": "test-annotation1", "test.annotation/two": "test.value/two/test"}, createdDeployment.Annotations) + // check labels + require.Equal(t, map[string]string{"xyz": "label2", "abc": "123"}, createdDeployment.Labels) + + // check pod template annotations + require.Equal(t, map[string]string{"abc/xyz": "test-annotation1", "test.annotation/two": "test.value/two/test", "kots.io/fs-minio-creds-secret-checksum": "d41d8cd98f00b204e9800998ecf8427e"}, createdDeployment.Spec.Template.Annotations) + // check pod template labels + require.Equal(t, map[string]string{"xyz": "label2", "abc": "123", "app": "kotsadm-fs-minio"}, createdDeployment.Spec.Template.Labels) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + req := require.New(t) + + err := ensureFileSystemMinioDeployment(ctx, tt.args.clientSet, tt.args.deployOptions, tt.args.registryConfig, tt.args.marshalledSecret) + req.NoError(err) + tt.validate(t, tt.args.clientSet) + }) + } +} From 3ac602b684470b5eba8992ebabc1468bc19b67c7 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 6 Sep 2024 17:04:35 -0400 Subject: [PATCH 5/5] update imports --- pkg/kotsutil/kots.go | 2 +- pkg/kotsutil/kots_test.go | 2 +- pkg/snapshot/filesystem_minio_test.go | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/kotsutil/kots.go b/pkg/kotsutil/kots.go index c4a736b938..9e6d5e010a 100644 --- a/pkg/kotsutil/kots.go +++ b/pkg/kotsutil/kots.go @@ -8,7 +8,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "k8s.io/client-go/kubernetes" "os" "path" "path/filepath" @@ -43,6 +42,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" serializer "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" applicationv1beta1 "sigs.k8s.io/application/api/v1beta1" ) diff --git a/pkg/kotsutil/kots_test.go b/pkg/kotsutil/kots_test.go index c51dfce811..d6b200fbd0 100644 --- a/pkg/kotsutil/kots_test.go +++ b/pkg/kotsutil/kots_test.go @@ -3,7 +3,6 @@ package kotsutil_test import ( "encoding/base64" "io/ioutil" - "k8s.io/client-go/kubernetes" "os" "path/filepath" "testing" @@ -25,6 +24,7 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" applicationv1beta1 "sigs.k8s.io/application/api/v1beta1" ) diff --git a/pkg/snapshot/filesystem_minio_test.go b/pkg/snapshot/filesystem_minio_test.go index 61b378d747..62f8e020a3 100644 --- a/pkg/snapshot/filesystem_minio_test.go +++ b/pkg/snapshot/filesystem_minio_test.go @@ -2,6 +2,8 @@ package snapshot import ( "context" + "testing" + "github.com/replicatedhq/kots/pkg/kotsadm/types" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" kotssnapshottypes "github.com/replicatedhq/kots/pkg/snapshot/types" @@ -10,7 +12,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" - "testing" ) func Test_ensureFileSystemMinioDeployment(t *testing.T) {