From 25b04781686354dc9153fbb3a647d43596e3cbd0 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 4 Sep 2024 18:20:51 -0400 Subject: [PATCH] begin adding labels and annotations --- cmd/kots/cli/install.go | 21 +++++++ pkg/identity/deploy/deploy.go | 4 +- pkg/kotsadm/main.go | 3 + pkg/kotsadm/objects/configmaps_objects.go | 12 ++++ pkg/kotsadm/objects/distribution_objects.go | 19 ++++-- pkg/kotsadm/objects/kotsadm_objects.go | 64 ++++++++++++++------- pkg/kotsadm/objects/minio_objects.go | 28 ++++++--- pkg/kotsadm/objects/rqlite_objects.go | 19 ++++-- pkg/kotsadm/types/constants.go | 1 + pkg/kotsadm/types/deployoptions.go | 2 + 10 files changed, 129 insertions(+), 44 deletions(-) diff --git a/cmd/kots/cli/install.go b/cmd/kots/cli/install.go index 5b8f7a1e5b..6b7c39f759 100644 --- a/cmd/kots/cli/install.go +++ b/cmd/kots/cli/install.go @@ -259,6 +259,23 @@ func InstallCmd() *cobra.Command { simultaneousUploads, _ := strconv.Atoi(v.GetString("airgap-upload-parallelism")) + additionalLabels := map[string]string{} + additionalAnnotations := map[string]string{} + for _, label := range v.GetStringSlice("additional-labels") { + parts := strings.Split(label, "=") + if len(parts) != 2 { + return errors.Errorf("additional-labels flag is not in the correct format. Must be key=value") + } + additionalLabels[parts[0]] = parts[1] + } + for _, annotation := range v.GetStringSlice("additional-annotations") { + parts := strings.Split(annotation, "=") + if len(parts) != 2 { + return errors.Errorf("additional-annotations flag is not in the correct format. Must be key=value") + } + additionalAnnotations[parts[0]] = parts[1] + } + deployOptions := kotsadmtypes.DeployOptions{ Namespace: namespace, Context: v.GetString("context"), @@ -288,6 +305,8 @@ func InstallCmd() *cobra.Command { IncludeMinioSnapshots: v.GetBool("with-minio"), StrictSecurityContext: v.GetBool("strict-security-context"), RequestedChannelSlug: preferredChannelSlug, + AdditionalLabels: additionalLabels, + AdditionalAnnotations: additionalAnnotations, RegistryConfig: *registryConfig, @@ -529,6 +548,8 @@ func InstallCmd() *cobra.Command { cmd.Flags().Bool("skip-compatibility-check", false, "set to true to skip compatibility checks between the current kots version and the app") cmd.Flags().String("app-version-label", "", "the application version label to install. if not specified, the latest version will be installed") cmd.Flags().Bool("exclude-admin-console", false, "set to true to exclude the admin console and only install the application") + cmd.Flags().StringArray("additional-annotations", []string{}, "additional annotations to add to kotsadm pods") + cmd.Flags().StringArray("additional-labels", []string{}, "additional labels to add to kotsadm pods") registryFlags(cmd.Flags()) diff --git a/pkg/identity/deploy/deploy.go b/pkg/identity/deploy/deploy.go index 3f007351ad..c2fdad8b3c 100644 --- a/pkg/identity/deploy/deploy.go +++ b/pkg/identity/deploy/deploy.go @@ -371,6 +371,7 @@ var ( dexMemoryResource = resource.MustParse("50Mi") ) +// TODO: add labels/annotations func deploymentResource(issuerURL, configChecksum string, options Options) (*appsv1.Deployment, error) { // TODO: use GetAdminConsoleImage function dexVersion, err := imageutil.GetTag(image.Dex) @@ -417,7 +418,8 @@ func deploymentResource(issuerURL, configChecksum string, options Options) (*app ObjectMeta: metav1.ObjectMeta{ Name: types.DeploymentName(options.NamePrefix), Namespace: options.Namespace, - Labels: kotsadmtypes.GetKotsadmLabels(AdditionalLabels(options.NamePrefix, options.AdditionalLabels)), + // TODO add additional labels/annotations + Labels: kotsadmtypes.GetKotsadmLabels(AdditionalLabels(options.NamePrefix, options.AdditionalLabels)), }, Spec: appsv1.DeploymentSpec{ Replicas: pointer.Int32Ptr(2), diff --git a/pkg/kotsadm/main.go b/pkg/kotsadm/main.go index 7ddeb1a42c..dfb113a7fb 100644 --- a/pkg/kotsadm/main.go +++ b/pkg/kotsadm/main.go @@ -791,6 +791,7 @@ func ensureDisasterRecoveryLabels(deployOptions *types.DeployOptions, clientset } for _, pod := range pods.Items { if _, ok := pod.ObjectMeta.Labels[types.BackupLabel]; !ok { + // TODO add additional labels/annotations pod.ObjectMeta.Labels = types.GetKotsadmLabels(pod.ObjectMeta.Labels) // remove existing velero exclude label/annotation (if exists) @@ -812,6 +813,7 @@ func ensureDisasterRecoveryLabels(deployOptions *types.DeployOptions, clientset for _, deployment := range deployments.Items { if _, ok := deployment.ObjectMeta.Labels[types.BackupLabel]; !ok { // ensure labels + // TODO add additional labels/annotations deployment.ObjectMeta.Labels = types.GetKotsadmLabels(deployment.ObjectMeta.Labels) deployment.Spec.Template.ObjectMeta.Labels = types.GetKotsadmLabels(deployment.Spec.Template.ObjectMeta.Labels) @@ -837,6 +839,7 @@ func ensureDisasterRecoveryLabels(deployOptions *types.DeployOptions, clientset for _, statefulSet := range statefulSets.Items { if _, ok := statefulSet.ObjectMeta.Labels[types.BackupLabel]; !ok { // ensure labels + // TODO add additional labels/annotations statefulSet.ObjectMeta.Labels = types.GetKotsadmLabels(statefulSet.ObjectMeta.Labels) statefulSet.Spec.Template.ObjectMeta.Labels = types.GetKotsadmLabels(statefulSet.Spec.Template.ObjectMeta.Labels) diff --git a/pkg/kotsadm/objects/configmaps_objects.go b/pkg/kotsadm/objects/configmaps_objects.go index 2082570f6b..20446b7175 100644 --- a/pkg/kotsadm/objects/configmaps_objects.go +++ b/pkg/kotsadm/objects/configmaps_objects.go @@ -3,6 +3,7 @@ package kotsadm import ( _ "embed" "fmt" + "strings" "github.com/replicatedhq/kots/pkg/kotsadm/types" kotsadmversion "github.com/replicatedhq/kots/pkg/kotsadm/version" @@ -11,6 +12,15 @@ import ( ) func KotsadmConfigMap(deployOptions types.DeployOptions) *corev1.ConfigMap { + additionalAnnotationsArray := []string{} + for key, value := range deployOptions.AdditionalAnnotations { + additionalAnnotationsArray = append(additionalAnnotationsArray, fmt.Sprintf("%s=%s", key, value)) + } + additionalLabelsArray := []string{} + for key, value := range deployOptions.AdditionalLabels { + additionalLabelsArray = append(additionalLabelsArray, fmt.Sprintf("%s=%s", key, value)) + } + data := map[string]string{ "kots-install-id": fmt.Sprintf("%v", deployOptions.InstallID), "initial-app-images-pushed": fmt.Sprintf("%v", deployOptions.AppImagesPushed), @@ -26,6 +36,8 @@ func KotsadmConfigMap(deployOptions types.DeployOptions) *corev1.ConfigMap { "with-minio": fmt.Sprintf("%v", deployOptions.IncludeMinio), "app-version-label": deployOptions.AppVersionLabel, "requested-channel-slug": deployOptions.RequestedChannelSlug, + "additional-annotations": strings.Join(additionalAnnotationsArray, ","), + "additional-labels": strings.Join(additionalLabelsArray, ","), } if kotsadmversion.KotsadmPullSecret(deployOptions.Namespace, deployOptions.RegistryConfig) != nil { diff --git a/pkg/kotsadm/objects/distribution_objects.go b/pkg/kotsadm/objects/distribution_objects.go index 147f7b67d2..f3f09f2272 100644 --- a/pkg/kotsadm/objects/distribution_objects.go +++ b/pkg/kotsadm/objects/distribution_objects.go @@ -84,15 +84,23 @@ func DistributionStatefulset(deployOptions types.DeployOptions, size resource.Qu securityContext = k8sutil.SecurePodContext(1000, 1000, deployOptions.StrictSecurityContext) } + podLabels := map[string]string{ + "app": "kotsadm-storage-registry", + } + for k, v := range deployOptions.AdditionalLabels { + podLabels[k] = v + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "StatefulSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm-storage-registry", - Namespace: deployOptions.Namespace, - Labels: types.GetKotsadmLabels(), + Name: "kotsadm-storage-registry", + Namespace: deployOptions.Namespace, + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(deployOptions.AdditionalLabels), }, Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ @@ -124,9 +132,8 @@ func DistributionStatefulset(deployOptions types.DeployOptions, size resource.Qu ServiceName: "kotsadm-storage-registry", Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: types.GetKotsadmLabels(map[string]string{ - "app": "kotsadm-storage-registry", - }), + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(podLabels), }, Spec: corev1.PodSpec{ SecurityContext: securityContext, diff --git a/pkg/kotsadm/objects/kotsadm_objects.go b/pkg/kotsadm/objects/kotsadm_objects.go index f091d31f02..348ab9a139 100644 --- a/pkg/kotsadm/objects/kotsadm_objects.go +++ b/pkg/kotsadm/objects/kotsadm_objects.go @@ -344,15 +344,31 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }) } + podAnnotations := map[string]string{ + "backup.velero.io/backup-volumes": "backup", + "pre.hook.backup.velero.io/command": `["/backup.sh"]`, + "pre.hook.backup.velero.io/timeout": "10m", + } + for k, v := range deployOptions.AdditionalAnnotations { + podAnnotations[k] = v + } + podLabels := map[string]string{ + "app": "kotsadm", + } + for k, v := range deployOptions.AdditionalLabels { + podLabels[k] = v + } + deployment := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "Deployment", }, ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm", - Namespace: deployOptions.Namespace, - Labels: types.GetKotsadmLabels(), + Name: "kotsadm", + Namespace: deployOptions.Namespace, + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(deployOptions.AdditionalLabels), }, Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -362,14 +378,8 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: types.GetKotsadmLabels(map[string]string{ - "app": "kotsadm", - }), - Annotations: map[string]string{ - "backup.velero.io/backup-volumes": "backup", - "pre.hook.backup.velero.io/command": `["/backup.sh"]`, - "pre.hook.backup.velero.io/timeout": "10m", - }, + Labels: types.GetKotsadmLabels(podLabels), + Annotations: podAnnotations, }, Spec: corev1.PodSpec{ Affinity: &corev1.Affinity{ @@ -841,15 +851,31 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit storageClassName = &deployOptions.StorageClassName } + podAnnotations := map[string]string{ + "backup.velero.io/backup-volumes": "backup", + "pre.hook.backup.velero.io/command": `["/backup.sh"]`, + "pre.hook.backup.velero.io/timeout": "10m", + } + for k, v := range deployOptions.AdditionalAnnotations { + podAnnotations[k] = v + } + podLabels := map[string]string{ + "app": "kotsadm", + } + for k, v := range deployOptions.AdditionalLabels { + podLabels[k] = v + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "StatefulSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm", - Namespace: deployOptions.Namespace, - Labels: types.GetKotsadmLabels(), + Name: "kotsadm", + Namespace: deployOptions.Namespace, + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(deployOptions.AdditionalLabels), }, Spec: appsv1.StatefulSetSpec{ ServiceName: "kotsadm", @@ -860,14 +886,8 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: types.GetKotsadmLabels(map[string]string{ - "app": "kotsadm", - }), - Annotations: map[string]string{ - "backup.velero.io/backup-volumes": "backup", - "pre.hook.backup.velero.io/command": `["/backup.sh"]`, - "pre.hook.backup.velero.io/timeout": "10m", - }, + Labels: types.GetKotsadmLabels(podLabels), + Annotations: podAnnotations, }, Spec: corev1.PodSpec{ Affinity: &corev1.Affinity{ diff --git a/pkg/kotsadm/objects/minio_objects.go b/pkg/kotsadm/objects/minio_objects.go index 9e8f4bea71..26f5a19620 100644 --- a/pkg/kotsadm/objects/minio_objects.go +++ b/pkg/kotsadm/objects/minio_objects.go @@ -82,15 +82,29 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity) initContainers = append(initContainers, migrateToMinioXlInitContainers(deployOptions, resourceRequirements)...) } + podAnnotations := map[string]string{ + "backup.velero.io/backup-volumes": "kotsadm-minio,minio-config-dir,minio-cert-dir", + } + for k, v := range deployOptions.AdditionalAnnotations { + podAnnotations[k] = v + } + podLabels := map[string]string{ + "app": "kotsadm-minio", + } + for k, v := range deployOptions.AdditionalLabels { + podLabels[k] = v + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "StatefulSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm-minio", - Namespace: deployOptions.Namespace, - Labels: types.GetKotsadmLabels(), + Name: "kotsadm-minio", + Namespace: deployOptions.Namespace, + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(deployOptions.AdditionalLabels), }, Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ @@ -122,12 +136,8 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity) }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: types.GetKotsadmLabels(map[string]string{ - "app": "kotsadm-minio", - }), - Annotations: map[string]string{ - "backup.velero.io/backup-volumes": "kotsadm-minio,minio-config-dir,minio-cert-dir", - }, + Labels: types.GetKotsadmLabels(podLabels), + Annotations: podAnnotations, }, Spec: corev1.PodSpec{ Affinity: &corev1.Affinity{ diff --git a/pkg/kotsadm/objects/rqlite_objects.go b/pkg/kotsadm/objects/rqlite_objects.go index 80c3359ccb..a3d8cefed4 100644 --- a/pkg/kotsadm/objects/rqlite_objects.go +++ b/pkg/kotsadm/objects/rqlite_objects.go @@ -50,15 +50,23 @@ func RqliteStatefulset(deployOptions types.DeployOptions, size resource.Quantity storageClassName = &deployOptions.StorageClassName } + podLabels := map[string]string{ + "app": "kotsadm-rqlite", + } + for k, v := range deployOptions.AdditionalLabels { + podLabels[k] = v + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "StatefulSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm-rqlite", - Namespace: deployOptions.Namespace, - Labels: types.GetKotsadmLabels(), + Name: "kotsadm-rqlite", + Namespace: deployOptions.Namespace, + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(deployOptions.AdditionalLabels), }, Spec: appsv1.StatefulSetSpec{ ServiceName: "kotsadm-rqlite-headless", @@ -89,9 +97,8 @@ func RqliteStatefulset(deployOptions types.DeployOptions, size resource.Quantity }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: types.GetKotsadmLabels(map[string]string{ - "app": "kotsadm-rqlite", - }), + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(podLabels), }, Spec: corev1.PodSpec{ SecurityContext: securityContext, diff --git a/pkg/kotsadm/types/constants.go b/pkg/kotsadm/types/constants.go index d7564d2998..d8be327a99 100644 --- a/pkg/kotsadm/types/constants.go +++ b/pkg/kotsadm/types/constants.go @@ -28,6 +28,7 @@ const DefaultSupportBundleSpecKey = "default" const ClusterSpecificSupportBundleSpecKey = "cluster-specific" const VendorSpecificSupportBundleSpecKey = "vendor" +// TODO: additional labels in many places func GetKotsadmLabels(additionalLabels ...map[string]string) map[string]string { labels := map[string]string{ KotsadmKey: KotsadmLabelValue, diff --git a/pkg/kotsadm/types/deployoptions.go b/pkg/kotsadm/types/deployoptions.go index 14d5be2404..35a061d600 100644 --- a/pkg/kotsadm/types/deployoptions.go +++ b/pkg/kotsadm/types/deployoptions.go @@ -57,6 +57,8 @@ type DeployOptions struct { AdditionalNamespaces []string IsGKEAutopilot bool RequestedChannelSlug string + AdditionalAnnotations map[string]string + AdditionalLabels map[string]string IdentityConfig kotsv1beta1.IdentityConfig IngressConfig kotsv1beta1.IngressConfig