feat: add cluster addons support

Makefile

@@ -99,7 +99,7 @@ example-klog: install
 # INTEGRATION TESTS
 
 .PHONY: integration
-integration: integration-basic integration-external-policies
+integration: integration-basic integration-cluster-addons integration-external-policies
 
 .PHONY: integration-reset
 integration-reset:
@@ -113,6 +113,13 @@ integration-basic: integration-reset
 	@terraform -chdir=./tests/basic plan
 	@terraform -chdir=./tests/basic apply -auto-approve
 
+.PHONY: integration-cluster-addons
+integration-cluster-addons: integration-reset
+	@terraform -chdir=./tests/cluster-addons init
+	@terraform -chdir=./tests/cluster-addons validate
+	@terraform -chdir=./tests/cluster-addons plan
+	@terraform -chdir=./tests/cluster-addons apply -auto-approve
+
 .PHONY: integration-external-policies
 integration-external-policies: integration-reset
 	@terraform -chdir=./tests/external-policies init

docs/data-sources/cluster.md

@@ -126,6 +126,7 @@ The following arguments are supported:
 - `annotations` - (Computed) - Map(String) - Annotations is an unstructured key value map stored with a resource that may be<br />set by external tools to store and retrieve arbitrary metadata. They are not<br />queryable and should be preserved when modifying objects.
 - `name` - (Required) - String - Name defines the cluster name.
 - `admin_ssh_key` - (Computed) - String - AdminSshKey defines the cluster admin ssh key.
+- `cluster_addons` - (Computed) - List(String) - ClusterAddons defines the cluster addons.
 - `secrets` - (Computed) - [cluster_secrets](#cluster_secrets) - Secrets defines the cluster secret.
 
 ## Nested resources

docs/resources/cluster.md

@@ -211,6 +211,7 @@ The following arguments are supported:
 - `annotations` - (Optional) - Map(String) - Annotations is an unstructured key value map stored with a resource that may be<br />set by external tools to store and retrieve arbitrary metadata. They are not<br />queryable and should be preserved when modifying objects.
 - `name` - (Required) - (Force new) - String - Name defines the cluster name.
 - `admin_ssh_key` - (Optional) - (Sensitive) - String - AdminSshKey defines the cluster admin ssh key.
+- `cluster_addons` - (Optional) - List(String) - ClusterAddons defines the cluster addons.
 - `secrets` - (Optional) - [cluster_secrets](#cluster_secrets) - Secrets defines the cluster secret.
 - `revision` - (Computed) - Int - Revision is incremented every time the resource changes, this is useful for triggering cluster updater.
 

pkg/api/resources/Cluster.go

@@ -6,9 +6,12 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/pkg/apis/kops"
+	kopsutil "k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/client/simple"
+	"k8s.io/kops/pkg/clusteraddons"
 	"k8s.io/kops/pkg/resources"
 	"k8s.io/kops/pkg/resources/ops"
+	"k8s.io/kops/pkg/wellknownoperators"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
 )
 
@@ -27,6 +30,8 @@ type Cluster struct {
 	Name string
 	// AdminSshKey defines the cluster admin ssh key
 	AdminSshKey string
+	// ClusterAddons defines the cluster addons
+	ClusterAddons []string
 	// Secrets defines the cluster secret
 	Secrets *ClusterSecrets
 	// Revision is incremented every time the resource changes, this is useful for triggering cluster updater
@@ -88,7 +93,7 @@ func GetCluster(name string, clientset simple.Clientset) (*Cluster, error) {
 	return cluster, nil
 }
 
-func CreateCluster(name string, labels map[string]string, annotations map[string]string, adminSshKey string, secrets *ClusterSecrets, spec kops.ClusterSpec, clientset simple.Clientset) (*Cluster, error) {
+func CreateCluster(name string, labels map[string]string, annotations map[string]string, adminSshKey string, secrets *ClusterSecrets, clusterAddons []string, spec kops.ClusterSpec, clientset simple.Clientset) (*Cluster, error) {
 	kc := makeKopsCluster(name, labels, annotations, spec)
 	cloud, err := cloudup.BuildCloud(kc)
 	if err != nil {
@@ -97,6 +102,27 @@ func CreateCluster(name string, labels map[string]string, annotations map[string
 	if err := cloudup.PerformAssignments(kc, cloud); err != nil {
 		return nil, err
 	}
+	// TODO: deep validate ?
+	// TODO: assets builder ?
+	channel, err := cloudup.ChannelForCluster(kc)
+	if err != nil {
+		return nil, err
+	}
+	kubernetesVersion, err := kopsutil.ParseKubernetesVersion(kc.Spec.KubernetesVersion)
+	if err != nil {
+		return nil, err
+	}
+	addons, err := wellknownoperators.CreateAddons(channel, kubernetesVersion, kc)
+	if err != nil {
+		return nil, err
+	}
+	for _, addon := range clusterAddons {
+		addon, err := clusteraddons.ParseClusterAddon([]byte(addon))
+		if err != nil {
+			return nil, err
+		}
+		addons = append(addons, addon.Objects...)
+	}
 	_, err = clientset.CreateCluster(context.Background(), kc)
 	if err != nil {
 		return nil, err
@@ -105,6 +131,10 @@ func CreateCluster(name string, labels map[string]string, annotations map[string
 	if err != nil {
 		return nil, err
 	}
+	addonsClient := clientset.AddonsFor(kc)
+	if err := addonsClient.Replace(addons); err != nil {
+		return nil, err
+	}
 	if adminSshKey != "" {
 		sshCredentialStore, err := clientset.SSHCredentialStore(kc)
 		if err != nil {
@@ -133,7 +163,7 @@ func CreateCluster(name string, labels map[string]string, annotations map[string
 	return makeCluster(adminSshKey, secrets, kc), nil
 }
 
-func UpdateCluster(name string, labels map[string]string, annotations map[string]string, adminSshKey string, secrets *ClusterSecrets, spec kops.ClusterSpec, clientset simple.Clientset) (*Cluster, error) {
+func UpdateCluster(name string, labels map[string]string, annotations map[string]string, adminSshKey string, secrets *ClusterSecrets, clusterAddons []string, spec kops.ClusterSpec, clientset simple.Clientset) (*Cluster, error) {
 	kc := makeKopsCluster(name, labels, annotations, spec)
 	cloud, err := cloudup.BuildCloud(kc)
 	if err != nil {
@@ -142,10 +172,35 @@ func UpdateCluster(name string, labels map[string]string, annotations map[string
 	if err := cloudup.PerformAssignments(kc, cloud); err != nil {
 		return nil, err
 	}
+	// TODO: deep validate ?
+	// TODO: assets builder ?
+	channel, err := cloudup.ChannelForCluster(kc)
+	if err != nil {
+		return nil, err
+	}
+	kubernetesVersion, err := kopsutil.ParseKubernetesVersion(kc.Spec.KubernetesVersion)
+	if err != nil {
+		return nil, err
+	}
+	addons, err := wellknownoperators.CreateAddons(channel, kubernetesVersion, kc)
+	if err != nil {
+		return nil, err
+	}
+	for _, addon := range clusterAddons {
+		addon, err := clusteraddons.ParseClusterAddon([]byte(addon))
+		if err != nil {
+			return nil, err
+		}
+		addons = append(addons, addon.Objects...)
+	}
 	kc, err = clientset.UpdateCluster(context.Background(), kc, nil)
 	if err != nil {
 		return nil, err
 	}
+	addonsClient := clientset.AddonsFor(kc)
+	if err := addonsClient.Replace(addons); err != nil {
+		return nil, err
+	}
 	sshCredentialStore, err := clientset.SSHCredentialStore(kc)
 	if err != nil {
 		return nil, err

pkg/resources/Cluster.go

@@ -30,7 +30,16 @@ func Cluster() *schema.Resource {
 
 func ClusterCreate(c context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics {
 	in := resourcesschema.ExpandResourceCluster(d.Get("").(map[string]interface{}))
-	if cluster, err := resources.CreateCluster(in.Name, in.Labels, in.Annotations, in.AdminSshKey, in.Secrets, in.ClusterSpec, config.Clientset(m)); err != nil {
+	if cluster, err := resources.CreateCluster(
+		in.Name,
+		in.Labels,
+		in.Annotations,
+		in.AdminSshKey,
+		in.Secrets,
+		in.ClusterAddons,
+		in.ClusterSpec,
+		config.Clientset(m),
+	); err != nil {
 		return diag.FromErr(err)
 	} else {
 		d.SetId(cluster.Name)
@@ -40,7 +49,16 @@ func ClusterCreate(c context.Context, d *schema.ResourceData, m interface{}) dia
 
 func ClusterUpdate(c context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics {
 	in := resourcesschema.ExpandResourceCluster(d.Get("").(map[string]interface{}))
-	if cluster, err := resources.UpdateCluster(in.Name, in.Labels, in.Annotations, in.AdminSshKey, in.Secrets, in.ClusterSpec, config.Clientset(m)); err != nil {
+	if cluster, err := resources.UpdateCluster(
+		in.Name,
+		in.Labels,
+		in.Annotations,
+		in.AdminSshKey,
+		in.Secrets,
+		in.ClusterAddons,
+		in.ClusterSpec,
+		config.Clientset(m),
+	); err != nil {
 		return diag.FromErr(err)
 	} else {
 		d.SetId(cluster.Name)

pkg/schemas/resources/DataSource_Cluster.generated.go

@@ -91,6 +91,7 @@ func DataSourceCluster() *schema.Resource {
 			"annotations":                       ComputedMap(String()),
 			"name":                              RequiredString(),
 			"admin_ssh_key":                     ComputedString(),
+			"cluster_addons":                    ComputedList(String()),
 			"secrets":                           ComputedStruct(DataSourceClusterSecrets()),
 		},
 	}
@@ -181,6 +182,18 @@ func ExpandDataSourceCluster(in map[string]interface{}) resources.Cluster {
 		AdminSshKey: func(in interface{}) string {
 			return string(ExpandString(in))
 		}(in["admin_ssh_key"]),
+		ClusterAddons: func(in interface{}) []string {
+			return func(in interface{}) []string {
+				if in == nil {
+					return nil
+				}
+				var out []string
+				for _, in := range in.([]interface{}) {
+					out = append(out, string(ExpandString(in)))
+				}
+				return out
+			}(in)
+		}(in["cluster_addons"]),
 		Secrets: func(in interface{}) *resources.ClusterSecrets {
 			return func(in interface{}) *resources.ClusterSecrets {
 				if in == nil {
@@ -234,6 +247,15 @@ func FlattenDataSourceClusterInto(in resources.Cluster, out map[string]interface
 	out["admin_ssh_key"] = func(in string) interface{} {
 		return FlattenString(string(in))
 	}(in.AdminSshKey)
+	out["cluster_addons"] = func(in []string) interface{} {
+		return func(in []string) []interface{} {
+			var out []interface{}
+			for _, in := range in {
+				out = append(out, FlattenString(string(in)))
+			}
+			return out
+		}(in)
+	}(in.ClusterAddons)
 	out["secrets"] = func(in *resources.ClusterSecrets) interface{} {
 		return func(in *resources.ClusterSecrets) interface{} {
 			if in == nil {

pkg/schemas/resources/DataSource_Cluster.generated_test.go

@@ -101,6 +101,7 @@ func TestExpandDataSourceCluster(t *testing.T) {
 					"annotations":                       func() map[string]interface{} { return nil }(),
 					"name":                              "",
 					"admin_ssh_key":                     "",
+					"cluster_addons":                    func() []interface{} { return nil }(),
 					"secrets":                           nil,
 				},
 			},
@@ -197,6 +198,7 @@ func TestFlattenDataSourceClusterInto(t *testing.T) {
 		"annotations":                       func() map[string]interface{} { return nil }(),
 		"name":                              "",
 		"admin_ssh_key":                     "",
+		"cluster_addons":                    func() []interface{} { return nil }(),
 		"secrets":                           nil,
 	}
 	type args struct {
@@ -1050,6 +1052,17 @@ func TestFlattenDataSourceClusterInto(t *testing.T) {
 			},
 			want: _default,
 		},
+		{
+			name: "ClusterAddons - default",
+			args: args{
+				in: func() resources.Cluster {
+					subject := resources.Cluster{}
+					subject.ClusterAddons = nil
+					return subject
+				}(),
+			},
+			want: _default,
+		},
 		{
 			name: "Secrets - default",
 			args: args{
@@ -1153,6 +1166,7 @@ func TestFlattenDataSourceCluster(t *testing.T) {
 		"annotations":                       func() map[string]interface{} { return nil }(),
 		"name":                              "",
 		"admin_ssh_key":                     "",
+		"cluster_addons":                    func() []interface{} { return nil }(),
 		"secrets":                           nil,
 	}
 	type args struct {
@@ -2006,6 +2020,17 @@ func TestFlattenDataSourceCluster(t *testing.T) {
 			},
 			want: _default,
 		},
+		{
+			name: "ClusterAddons - default",
+			args: args{
+				in: func() resources.Cluster {
+					subject := resources.Cluster{}
+					subject.ClusterAddons = nil
+					return subject
+				}(),
+			},
+			want: _default,
+		},
 		{
 			name: "Secrets - default",
 			args: args{

pkg/schemas/resources/Resource_Cluster.generated.go

@@ -91,6 +91,7 @@ func ResourceCluster() *schema.Resource {
 			"annotations":                       OptionalMap(String()),
 			"name":                              ForceNew(RequiredString()),
 			"admin_ssh_key":                     Sensitive(OptionalString()),
+			"cluster_addons":                    OptionalList(String()),
 			"secrets":                           OptionalStruct(ResourceClusterSecrets()),
 			"revision":                          ComputedInt(),
 		},
@@ -182,6 +183,18 @@ func ExpandResourceCluster(in map[string]interface{}) resources.Cluster {
 		AdminSshKey: func(in interface{}) string {
 			return string(ExpandString(in))
 		}(in["admin_ssh_key"]),
+		ClusterAddons: func(in interface{}) []string {
+			return func(in interface{}) []string {
+				if in == nil {
+					return nil
+				}
+				var out []string
+				for _, in := range in.([]interface{}) {
+					out = append(out, string(ExpandString(in)))
+				}
+				return out
+			}(in)
+		}(in["cluster_addons"]),
 		Secrets: func(in interface{}) *resources.ClusterSecrets {
 			return func(in interface{}) *resources.ClusterSecrets {
 				if in == nil {
@@ -238,6 +251,15 @@ func FlattenResourceClusterInto(in resources.Cluster, out map[string]interface{}
 	out["admin_ssh_key"] = func(in string) interface{} {
 		return FlattenString(string(in))
 	}(in.AdminSshKey)
+	out["cluster_addons"] = func(in []string) interface{} {
+		return func(in []string) []interface{} {
+			var out []interface{}
+			for _, in := range in {
+				out = append(out, FlattenString(string(in)))
+			}
+			return out
+		}(in)
+	}(in.ClusterAddons)
 	out["secrets"] = func(in *resources.ClusterSecrets) interface{} {
 		return func(in *resources.ClusterSecrets) interface{} {
 			if in == nil {

pkg/schemas/resources/Resource_Cluster.generated_test.go

@@ -101,6 +101,7 @@ func TestExpandResourceCluster(t *testing.T) {
 					"annotations":                       func() map[string]interface{} { return nil }(),
 					"name":                              "",
 					"admin_ssh_key":                     "",
+					"cluster_addons":                    func() []interface{} { return nil }(),
 					"secrets":                           nil,
 					"revision":                          0,
 				},
@@ -198,6 +199,7 @@ func TestFlattenResourceClusterInto(t *testing.T) {
 		"annotations":                       func() map[string]interface{} { return nil }(),
 		"name":                              "",
 		"admin_ssh_key":                     "",
+		"cluster_addons":                    func() []interface{} { return nil }(),
 		"secrets":                           nil,
 		"revision":                          0,
 	}
@@ -1052,6 +1054,17 @@ func TestFlattenResourceClusterInto(t *testing.T) {
 			},
 			want: _default,
 		},
+		{
+			name: "ClusterAddons - default",
+			args: args{
+				in: func() resources.Cluster {
+					subject := resources.Cluster{}
+					subject.ClusterAddons = nil
+					return subject
+				}(),
+			},
+			want: _default,
+		},
 		{
 			name: "Secrets - default",
 			args: args{
@@ -1166,6 +1179,7 @@ func TestFlattenResourceCluster(t *testing.T) {
 		"annotations":                       func() map[string]interface{} { return nil }(),
 		"name":                              "",
 		"admin_ssh_key":                     "",
+		"cluster_addons":                    func() []interface{} { return nil }(),
 		"secrets":                           nil,
 		"revision":                          0,
 	}
@@ -2020,6 +2034,17 @@ func TestFlattenResourceCluster(t *testing.T) {
 			},
 			want: _default,
 		},
+		{
+			name: "ClusterAddons - default",
+			args: args{
+				in: func() resources.Cluster {
+					subject := resources.Cluster{}
+					subject.ClusterAddons = nil
+					return subject
+				}(),
+			},
+			want: _default,
+		},
 		{
 			name: "Secrets - default",
 			args: args{