diff --git a/Makefile b/Makefile
index 51154145..4774b325 100644
--- a/Makefile
+++ b/Makefile
@@ -99,7 +99,7 @@ example-klog: install
# INTEGRATION TESTS
.PHONY: integration
-integration: integration-basic integration-external-policies
+integration: integration-basic integration-cluster-addons integration-external-policies
.PHONY: integration-reset
integration-reset:
@@ -113,6 +113,13 @@ integration-basic: integration-reset
@terraform -chdir=./tests/basic plan
@terraform -chdir=./tests/basic apply -auto-approve
+.PHONY: integration-cluster-addons
+integration-cluster-addons: integration-reset
+ @terraform -chdir=./tests/cluster-addons init
+ @terraform -chdir=./tests/cluster-addons validate
+ @terraform -chdir=./tests/cluster-addons plan
+ @terraform -chdir=./tests/cluster-addons apply -auto-approve
+
.PHONY: integration-external-policies
integration-external-policies: integration-reset
@terraform -chdir=./tests/external-policies init
diff --git a/docs/data-sources/cluster.md b/docs/data-sources/cluster.md
index e4688a2d..308ffefa 100644
--- a/docs/data-sources/cluster.md
+++ b/docs/data-sources/cluster.md
@@ -126,6 +126,7 @@ The following arguments are supported:
- `annotations` - (Computed) - Map(String) - Annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
- `name` - (Required) - String - Name defines the cluster name.
- `admin_ssh_key` - (Computed) - String - AdminSshKey defines the cluster admin ssh key.
+- `cluster_addons` - (Computed) - List(String) - ClusterAddons defines the cluster addons.
- `secrets` - (Computed) - [cluster_secrets](#cluster_secrets) - Secrets defines the cluster secret.
## Nested resources
diff --git a/docs/resources/cluster.md b/docs/resources/cluster.md
index 31d39d1c..f80d3cda 100644
--- a/docs/resources/cluster.md
+++ b/docs/resources/cluster.md
@@ -211,6 +211,7 @@ The following arguments are supported:
- `annotations` - (Optional) - Map(String) - Annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
- `name` - (Required) - (Force new) - String - Name defines the cluster name.
- `admin_ssh_key` - (Optional) - (Sensitive) - String - AdminSshKey defines the cluster admin ssh key.
+- `cluster_addons` - (Optional) - List(String) - ClusterAddons defines the cluster addons.
- `secrets` - (Optional) - [cluster_secrets](#cluster_secrets) - Secrets defines the cluster secret.
- `revision` - (Computed) - Int - Revision is incremented every time the resource changes, this is useful for triggering cluster updater.
diff --git a/pkg/api/resources/Cluster.go b/pkg/api/resources/Cluster.go
index cdf5d1af..bd29d1d2 100644
--- a/pkg/api/resources/Cluster.go
+++ b/pkg/api/resources/Cluster.go
@@ -6,9 +6,12 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kops/pkg/apis/kops"
+ kopsutil "k8s.io/kops/pkg/apis/kops/util"
"k8s.io/kops/pkg/client/simple"
+ "k8s.io/kops/pkg/clusteraddons"
"k8s.io/kops/pkg/resources"
"k8s.io/kops/pkg/resources/ops"
+ "k8s.io/kops/pkg/wellknownoperators"
"k8s.io/kops/upup/pkg/fi/cloudup"
)
@@ -27,6 +30,8 @@ type Cluster struct {
Name string
// AdminSshKey defines the cluster admin ssh key
AdminSshKey string
+ // ClusterAddons defines the cluster addons
+ ClusterAddons []string
// Secrets defines the cluster secret
Secrets *ClusterSecrets
// Revision is incremented every time the resource changes, this is useful for triggering cluster updater
@@ -88,7 +93,7 @@ func GetCluster(name string, clientset simple.Clientset) (*Cluster, error) {
return cluster, nil
}
-func CreateCluster(name string, labels map[string]string, annotations map[string]string, adminSshKey string, secrets *ClusterSecrets, spec kops.ClusterSpec, clientset simple.Clientset) (*Cluster, error) {
+func CreateCluster(name string, labels map[string]string, annotations map[string]string, adminSshKey string, secrets *ClusterSecrets, clusterAddons []string, spec kops.ClusterSpec, clientset simple.Clientset) (*Cluster, error) {
kc := makeKopsCluster(name, labels, annotations, spec)
cloud, err := cloudup.BuildCloud(kc)
if err != nil {
@@ -97,6 +102,27 @@ func CreateCluster(name string, labels map[string]string, annotations map[string
if err := cloudup.PerformAssignments(kc, cloud); err != nil {
return nil, err
}
+ // TODO: deep validate ?
+ // TODO: assets builder ?
+ channel, err := cloudup.ChannelForCluster(kc)
+ if err != nil {
+ return nil, err
+ }
+ kubernetesVersion, err := kopsutil.ParseKubernetesVersion(kc.Spec.KubernetesVersion)
+ if err != nil {
+ return nil, err
+ }
+ addons, err := wellknownoperators.CreateAddons(channel, kubernetesVersion, kc)
+ if err != nil {
+ return nil, err
+ }
+ for _, addon := range clusterAddons {
+ addon, err := clusteraddons.ParseClusterAddon([]byte(addon))
+ if err != nil {
+ return nil, err
+ }
+ addons = append(addons, addon.Objects...)
+ }
_, err = clientset.CreateCluster(context.Background(), kc)
if err != nil {
return nil, err
@@ -105,6 +131,10 @@ func CreateCluster(name string, labels map[string]string, annotations map[string
if err != nil {
return nil, err
}
+ addonsClient := clientset.AddonsFor(kc)
+ if err := addonsClient.Replace(addons); err != nil {
+ return nil, err
+ }
if adminSshKey != "" {
sshCredentialStore, err := clientset.SSHCredentialStore(kc)
if err != nil {
@@ -133,7 +163,7 @@ func CreateCluster(name string, labels map[string]string, annotations map[string
return makeCluster(adminSshKey, secrets, kc), nil
}
-func UpdateCluster(name string, labels map[string]string, annotations map[string]string, adminSshKey string, secrets *ClusterSecrets, spec kops.ClusterSpec, clientset simple.Clientset) (*Cluster, error) {
+func UpdateCluster(name string, labels map[string]string, annotations map[string]string, adminSshKey string, secrets *ClusterSecrets, clusterAddons []string, spec kops.ClusterSpec, clientset simple.Clientset) (*Cluster, error) {
kc := makeKopsCluster(name, labels, annotations, spec)
cloud, err := cloudup.BuildCloud(kc)
if err != nil {
@@ -142,10 +172,35 @@ func UpdateCluster(name string, labels map[string]string, annotations map[string
if err := cloudup.PerformAssignments(kc, cloud); err != nil {
return nil, err
}
+ // TODO: deep validate ?
+ // TODO: assets builder ?
+ channel, err := cloudup.ChannelForCluster(kc)
+ if err != nil {
+ return nil, err
+ }
+ kubernetesVersion, err := kopsutil.ParseKubernetesVersion(kc.Spec.KubernetesVersion)
+ if err != nil {
+ return nil, err
+ }
+ addons, err := wellknownoperators.CreateAddons(channel, kubernetesVersion, kc)
+ if err != nil {
+ return nil, err
+ }
+ for _, addon := range clusterAddons {
+ addon, err := clusteraddons.ParseClusterAddon([]byte(addon))
+ if err != nil {
+ return nil, err
+ }
+ addons = append(addons, addon.Objects...)
+ }
kc, err = clientset.UpdateCluster(context.Background(), kc, nil)
if err != nil {
return nil, err
}
+ addonsClient := clientset.AddonsFor(kc)
+ if err := addonsClient.Replace(addons); err != nil {
+ return nil, err
+ }
sshCredentialStore, err := clientset.SSHCredentialStore(kc)
if err != nil {
return nil, err
diff --git a/pkg/resources/Cluster.go b/pkg/resources/Cluster.go
index 455e9746..7a0627ea 100644
--- a/pkg/resources/Cluster.go
+++ b/pkg/resources/Cluster.go
@@ -30,7 +30,16 @@ func Cluster() *schema.Resource {
func ClusterCreate(c context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics {
in := resourcesschema.ExpandResourceCluster(d.Get("").(map[string]interface{}))
- if cluster, err := resources.CreateCluster(in.Name, in.Labels, in.Annotations, in.AdminSshKey, in.Secrets, in.ClusterSpec, config.Clientset(m)); err != nil {
+ if cluster, err := resources.CreateCluster(
+ in.Name,
+ in.Labels,
+ in.Annotations,
+ in.AdminSshKey,
+ in.Secrets,
+ in.ClusterAddons,
+ in.ClusterSpec,
+ config.Clientset(m),
+ ); err != nil {
return diag.FromErr(err)
} else {
d.SetId(cluster.Name)
@@ -40,7 +49,16 @@ func ClusterCreate(c context.Context, d *schema.ResourceData, m interface{}) dia
func ClusterUpdate(c context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics {
in := resourcesschema.ExpandResourceCluster(d.Get("").(map[string]interface{}))
- if cluster, err := resources.UpdateCluster(in.Name, in.Labels, in.Annotations, in.AdminSshKey, in.Secrets, in.ClusterSpec, config.Clientset(m)); err != nil {
+ if cluster, err := resources.UpdateCluster(
+ in.Name,
+ in.Labels,
+ in.Annotations,
+ in.AdminSshKey,
+ in.Secrets,
+ in.ClusterAddons,
+ in.ClusterSpec,
+ config.Clientset(m),
+ ); err != nil {
return diag.FromErr(err)
} else {
d.SetId(cluster.Name)
diff --git a/pkg/schemas/resources/DataSource_Cluster.generated.go b/pkg/schemas/resources/DataSource_Cluster.generated.go
index 75d7d4dc..ea15acac 100644
--- a/pkg/schemas/resources/DataSource_Cluster.generated.go
+++ b/pkg/schemas/resources/DataSource_Cluster.generated.go
@@ -91,6 +91,7 @@ func DataSourceCluster() *schema.Resource {
"annotations": ComputedMap(String()),
"name": RequiredString(),
"admin_ssh_key": ComputedString(),
+ "cluster_addons": ComputedList(String()),
"secrets": ComputedStruct(DataSourceClusterSecrets()),
},
}
@@ -181,6 +182,18 @@ func ExpandDataSourceCluster(in map[string]interface{}) resources.Cluster {
AdminSshKey: func(in interface{}) string {
return string(ExpandString(in))
}(in["admin_ssh_key"]),
+ ClusterAddons: func(in interface{}) []string {
+ return func(in interface{}) []string {
+ if in == nil {
+ return nil
+ }
+ var out []string
+ for _, in := range in.([]interface{}) {
+ out = append(out, string(ExpandString(in)))
+ }
+ return out
+ }(in)
+ }(in["cluster_addons"]),
Secrets: func(in interface{}) *resources.ClusterSecrets {
return func(in interface{}) *resources.ClusterSecrets {
if in == nil {
@@ -234,6 +247,15 @@ func FlattenDataSourceClusterInto(in resources.Cluster, out map[string]interface
out["admin_ssh_key"] = func(in string) interface{} {
return FlattenString(string(in))
}(in.AdminSshKey)
+ out["cluster_addons"] = func(in []string) interface{} {
+ return func(in []string) []interface{} {
+ var out []interface{}
+ for _, in := range in {
+ out = append(out, FlattenString(string(in)))
+ }
+ return out
+ }(in)
+ }(in.ClusterAddons)
out["secrets"] = func(in *resources.ClusterSecrets) interface{} {
return func(in *resources.ClusterSecrets) interface{} {
if in == nil {
diff --git a/pkg/schemas/resources/DataSource_Cluster.generated_test.go b/pkg/schemas/resources/DataSource_Cluster.generated_test.go
index bb581551..dac6cf15 100644
--- a/pkg/schemas/resources/DataSource_Cluster.generated_test.go
+++ b/pkg/schemas/resources/DataSource_Cluster.generated_test.go
@@ -101,6 +101,7 @@ func TestExpandDataSourceCluster(t *testing.T) {
"annotations": func() map[string]interface{} { return nil }(),
"name": "",
"admin_ssh_key": "",
+ "cluster_addons": func() []interface{} { return nil }(),
"secrets": nil,
},
},
@@ -197,6 +198,7 @@ func TestFlattenDataSourceClusterInto(t *testing.T) {
"annotations": func() map[string]interface{} { return nil }(),
"name": "",
"admin_ssh_key": "",
+ "cluster_addons": func() []interface{} { return nil }(),
"secrets": nil,
}
type args struct {
@@ -1050,6 +1052,17 @@ func TestFlattenDataSourceClusterInto(t *testing.T) {
},
want: _default,
},
+ {
+ name: "ClusterAddons - default",
+ args: args{
+ in: func() resources.Cluster {
+ subject := resources.Cluster{}
+ subject.ClusterAddons = nil
+ return subject
+ }(),
+ },
+ want: _default,
+ },
{
name: "Secrets - default",
args: args{
@@ -1153,6 +1166,7 @@ func TestFlattenDataSourceCluster(t *testing.T) {
"annotations": func() map[string]interface{} { return nil }(),
"name": "",
"admin_ssh_key": "",
+ "cluster_addons": func() []interface{} { return nil }(),
"secrets": nil,
}
type args struct {
@@ -2006,6 +2020,17 @@ func TestFlattenDataSourceCluster(t *testing.T) {
},
want: _default,
},
+ {
+ name: "ClusterAddons - default",
+ args: args{
+ in: func() resources.Cluster {
+ subject := resources.Cluster{}
+ subject.ClusterAddons = nil
+ return subject
+ }(),
+ },
+ want: _default,
+ },
{
name: "Secrets - default",
args: args{
diff --git a/pkg/schemas/resources/Resource_Cluster.generated.go b/pkg/schemas/resources/Resource_Cluster.generated.go
index ee65a4c0..d4de2bd1 100644
--- a/pkg/schemas/resources/Resource_Cluster.generated.go
+++ b/pkg/schemas/resources/Resource_Cluster.generated.go
@@ -91,6 +91,7 @@ func ResourceCluster() *schema.Resource {
"annotations": OptionalMap(String()),
"name": ForceNew(RequiredString()),
"admin_ssh_key": Sensitive(OptionalString()),
+ "cluster_addons": OptionalList(String()),
"secrets": OptionalStruct(ResourceClusterSecrets()),
"revision": ComputedInt(),
},
@@ -182,6 +183,18 @@ func ExpandResourceCluster(in map[string]interface{}) resources.Cluster {
AdminSshKey: func(in interface{}) string {
return string(ExpandString(in))
}(in["admin_ssh_key"]),
+ ClusterAddons: func(in interface{}) []string {
+ return func(in interface{}) []string {
+ if in == nil {
+ return nil
+ }
+ var out []string
+ for _, in := range in.([]interface{}) {
+ out = append(out, string(ExpandString(in)))
+ }
+ return out
+ }(in)
+ }(in["cluster_addons"]),
Secrets: func(in interface{}) *resources.ClusterSecrets {
return func(in interface{}) *resources.ClusterSecrets {
if in == nil {
@@ -238,6 +251,15 @@ func FlattenResourceClusterInto(in resources.Cluster, out map[string]interface{}
out["admin_ssh_key"] = func(in string) interface{} {
return FlattenString(string(in))
}(in.AdminSshKey)
+ out["cluster_addons"] = func(in []string) interface{} {
+ return func(in []string) []interface{} {
+ var out []interface{}
+ for _, in := range in {
+ out = append(out, FlattenString(string(in)))
+ }
+ return out
+ }(in)
+ }(in.ClusterAddons)
out["secrets"] = func(in *resources.ClusterSecrets) interface{} {
return func(in *resources.ClusterSecrets) interface{} {
if in == nil {
diff --git a/pkg/schemas/resources/Resource_Cluster.generated_test.go b/pkg/schemas/resources/Resource_Cluster.generated_test.go
index 1e98f2fb..04125fc2 100644
--- a/pkg/schemas/resources/Resource_Cluster.generated_test.go
+++ b/pkg/schemas/resources/Resource_Cluster.generated_test.go
@@ -101,6 +101,7 @@ func TestExpandResourceCluster(t *testing.T) {
"annotations": func() map[string]interface{} { return nil }(),
"name": "",
"admin_ssh_key": "",
+ "cluster_addons": func() []interface{} { return nil }(),
"secrets": nil,
"revision": 0,
},
@@ -198,6 +199,7 @@ func TestFlattenResourceClusterInto(t *testing.T) {
"annotations": func() map[string]interface{} { return nil }(),
"name": "",
"admin_ssh_key": "",
+ "cluster_addons": func() []interface{} { return nil }(),
"secrets": nil,
"revision": 0,
}
@@ -1052,6 +1054,17 @@ func TestFlattenResourceClusterInto(t *testing.T) {
},
want: _default,
},
+ {
+ name: "ClusterAddons - default",
+ args: args{
+ in: func() resources.Cluster {
+ subject := resources.Cluster{}
+ subject.ClusterAddons = nil
+ return subject
+ }(),
+ },
+ want: _default,
+ },
{
name: "Secrets - default",
args: args{
@@ -1166,6 +1179,7 @@ func TestFlattenResourceCluster(t *testing.T) {
"annotations": func() map[string]interface{} { return nil }(),
"name": "",
"admin_ssh_key": "",
+ "cluster_addons": func() []interface{} { return nil }(),
"secrets": nil,
"revision": 0,
}
@@ -2020,6 +2034,17 @@ func TestFlattenResourceCluster(t *testing.T) {
},
want: _default,
},
+ {
+ name: "ClusterAddons - default",
+ args: args{
+ in: func() resources.Cluster {
+ subject := resources.Cluster{}
+ subject.ClusterAddons = nil
+ return subject
+ }(),
+ },
+ want: _default,
+ },
{
name: "Secrets - default",
args: args{
diff --git a/tests/cluster-addons/cluster.tf b/tests/cluster-addons/cluster.tf
new file mode 100644
index 00000000..59bdf72d
--- /dev/null
+++ b/tests/cluster-addons/cluster.tf
@@ -0,0 +1,116 @@
+resource "kops_cluster" "cluster" {
+ name = local.clusterName
+ admin_ssh_key = file("${path.module}/../id_rsa.pub")
+ kubernetes_version = "1.19.12"
+ dns_zone = local.dnsZone
+ network_id = local.vpcId
+ cluster_addons = [
+ file("${path.module}/kubescheduler.yaml")
+ ]
+
+ cloud_provider {
+ aws {}
+ }
+
+ api {
+ dns {}
+ }
+
+ authorization {
+ rbac {}
+ }
+
+ iam {
+ allow_container_registry = true
+ }
+
+ networking {
+ calico {}
+ }
+
+ topology {
+ masters = "private"
+ nodes = "private"
+ dns {
+ type = "Private"
+ }
+ }
+
+ # private subnets
+ subnet {
+ name = "private-0"
+ type = "Private"
+ provider_id = local.privateSubnets[0].subnetId
+ zone = local.privateSubnets[0].zone
+ }
+ subnet {
+ name = "utility-0"
+ type = "Utility"
+ provider_id = local.utilitySubnets[0].subnetId
+ zone = local.utilitySubnets[0].zone
+ }
+
+ # etcd clusters
+ etcd_cluster {
+ name = "main"
+ member {
+ name = "master-0"
+ instance_group = "master-0"
+ }
+ }
+ etcd_cluster {
+ name = "events"
+ member {
+ name = "master-0"
+ instance_group = "master-0"
+ }
+ }
+
+ kubelet {
+ anonymous_auth {
+ value = false
+ }
+ }
+}
+
+resource "kops_instance_group" "master-0" {
+ cluster_name = kops_cluster.cluster.id
+ name = "master-0"
+ role = "Master"
+ min_size = 1
+ max_size = 1
+ machine_type = local.masterType
+ subnets = ["private-0"]
+}
+
+resource "kops_instance_group" "node-0" {
+ cluster_name = kops_cluster.cluster.id
+ name = "node-0"
+ role = "Node"
+ min_size = 1
+ max_size = 2
+ machine_type = local.nodeType
+ subnets = ["private-0"]
+}
+
+resource "kops_cluster_updater" "updater" {
+ cluster_name = kops_cluster.cluster.id
+
+ keepers = {
+ cluster = kops_cluster.cluster.revision
+ master-0 = kops_instance_group.master-0.revision
+ node-0 = kops_instance_group.node-0.revision
+ }
+
+ apply {
+ skip = true
+ }
+
+ rolling_update {
+ skip = true
+ }
+
+ validate {
+ skip = true
+ }
+}
diff --git a/tests/cluster-addons/kubescheduler.yaml b/tests/cluster-addons/kubescheduler.yaml
new file mode 100644
index 00000000..db608408
--- /dev/null
+++ b/tests/cluster-addons/kubescheduler.yaml
@@ -0,0 +1,6 @@
+# This is an example "addon" object to provide additional/custom kube-scheduler configuration
+apiVersion: kubescheduler.config.k8s.io/v1beta2
+kind: KubeSchedulerConfiguration
+clientConnection:
+ burst: 100
+ qps: 3.1
diff --git a/tests/cluster-addons/locals.tf b/tests/cluster-addons/locals.tf
new file mode 100644
index 00000000..fc45b958
--- /dev/null
+++ b/tests/cluster-addons/locals.tf
@@ -0,0 +1,13 @@
+locals {
+ masterType = "t3.medium"
+ nodeType = "t3.medium"
+ clusterName = "cluster.example.com"
+ dnsZone = "example.com"
+ vpcId = "vpc-12345678"
+ privateSubnets = [
+ { subnetId = "subnet-1", zone = "us-test-1a" }
+ ]
+ utilitySubnets = [
+ { subnetId = "subnet-2", zone = "us-test-1a" }
+ ]
+}
diff --git a/tests/cluster-addons/provider.tf b/tests/cluster-addons/provider.tf
new file mode 100644
index 00000000..4f2b5cff
--- /dev/null
+++ b/tests/cluster-addons/provider.tf
@@ -0,0 +1,17 @@
+terraform {
+ required_providers {
+ kops = {
+ source = "github/eddycharly/kops"
+ version = "0.0.1"
+ }
+ }
+}
+
+provider "kops" {
+ state_store = "file://./store/"
+ mock = true
+ aws {
+ region = "us-test-1"
+ }
+ feature_flags = [ "ClusterAddons" ]
+}