diff --git a/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go.tmpl b/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go.tmpl index 7071bf4aed10..b9582f8e1fe7 100644 --- a/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go.tmpl +++ b/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go.tmpl @@ -40,6 +40,7 @@ var SharedCryptoKey = map[string]string{ type BootstrappedKMS struct { *cloudkms.KeyRing *cloudkms.CryptoKey + CryptoKeyVersions []*cloudkms.CryptoKeyVersion } func BootstrapKMSKey(t *testing.T) BootstrappedKMS { @@ -77,6 +78,7 @@ func BootstrapKMSKeyWithPurposeInLocationAndName(t *testing.T, purpose, location return BootstrappedKMS{ &cloudkms.KeyRing{}, &cloudkms.CryptoKey{}, + nil, } } @@ -111,8 +113,8 @@ func BootstrapKMSKeyWithPurposeInLocationAndName(t *testing.T, purpose, location if transport_tpg.IsGoogleApiErrorWithCode(err, 404) { algos := map[string]string{ "ENCRYPT_DECRYPT": "GOOGLE_SYMMETRIC_ENCRYPTION", - "ASYMMETRIC_SIGN": "RSA_SIGN_PKCS1_4096_SHA512", - "ASYMMETRIC_DECRYPT": "RSA_DECRYPT_OAEP_4096_SHA512", + "ASYMMETRIC_SIGN": "RSA_SIGN_PKCS1_4096_SHA256", + "ASYMMETRIC_DECRYPT": "RSA_DECRYPT_OAEP_4096_SHA256", } template := cloudkms.CryptoKeyVersionTemplate{ Algorithm: algos[purpose], @@ -138,9 +140,16 @@ func BootstrapKMSKeyWithPurposeInLocationAndName(t *testing.T, purpose, location t.Fatalf("Unable to bootstrap KMS key. CryptoKey is nil!") } + // TODO(b/372305432): Use the pagination properly. + ckvResp, err := kmsClient.Projects.Locations.KeyRings.CryptoKeys.CryptoKeyVersions.List(keyName).Do() + if err != nil { + t.Fatalf("Unable to list cryptoKeyVersions: %v", err) + } + return BootstrappedKMS{ keyRing, cryptoKey, + ckvResp.CryptoKeyVersions, } } diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster.go.tmpl b/mmv1/third_party/terraform/services/container/resource_container_cluster.go.tmpl index eebf01f54c98..0d759da387d9 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster.go.tmpl +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster.go.tmpl @@ -2190,6 +2190,62 @@ func ResourceContainerCluster() *schema.Resource { }, }, }, + "user_managed_keys_config": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Description: `The custom keys configuration of the cluster.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "cluster_ca": { + Type: schema.TypeString, + Optional: true, + Description: `The Certificate Authority Service caPool to use for the cluster CA in this cluster.`, + }, + "etcd_api_ca": { + Type: schema.TypeString, + Optional: true, + Description: `The Certificate Authority Service caPool to use for the etcd API CA in this cluster.`, + }, + "etcd_peer_ca": { + Type: schema.TypeString, + Optional: true, + Description: `The Certificate Authority Service caPool to use for the etcd peer CA in this cluster.`, + }, + "aggregation_ca": { + Type: schema.TypeString, + Optional: true, + Description: `The Certificate Authority Service caPool to use for the aggreation CA in this cluster.`, + }, + "service_account_signing_keys": { + Type: schema.TypeSet, + Optional: true, + Description: `The Cloud KMS cryptoKeyVersions to use for signing service account JWTs issued by this cluster.`, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "service_account_verification_keys": { + Type: schema.TypeSet, + Optional: true, + Description: `The Cloud KMS cryptoKeyVersions to use for verifying service account JWTs issued by this cluster.`, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "control_plane_disk_encryption_key": { + Type: schema.TypeString, + Optional: true, + Description: `The Cloud KMS cryptoKey to use for Confidential Hyperdisk on the control plane nodes.`, + }, + "gkeops_etcd_backup_encryption_key": { + Type: schema.TypeString, + Optional: true, + Description: `Resource path of the Cloud KMS cryptoKey to use for encryption of internal etcd backups.`, + }, + }, + }, + }, {{- if ne $.TargetVersionName "ga" }} "workload_alts_config": { Type: schema.TypeList, @@ -2517,6 +2573,10 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er cluster.Fleet = expandFleet(v) } + if v, ok := d.GetOk("user_managed_keys_config"); ok { + cluster.UserManagedKeysConfig = expandUserManagedKeysConfig(v) + } + if err := validateNodePoolAutoConfig(cluster); err != nil { return err } @@ -3019,6 +3079,9 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro if err := d.Set("fleet", flattenFleet(cluster.Fleet)); err != nil { return err } + if err := d.Set("user_managed_keys_config", flattenUserManagedKeysConfig(cluster.UserManagedKeysConfig)); err != nil { + return err + } if err := d.Set("enable_k8s_beta_apis", flattenEnableK8sBetaApis(cluster.EnableK8sBetaApis)); err != nil { return err } @@ -4188,6 +4251,20 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er log.Printf("[INFO] GKE cluster %s fleet config has been updated", d.Id()) } + if d.HasChange("user_managed_keys_config") { + req := &container.UpdateClusterRequest{ + Update: &container.ClusterUpdate{ + UserManagedKeysConfig: expandUserManagedKeysConfig(d.Get("user_managed_keys_config")), + }, + } + updateF := updateFunc(req, "updating GKE cluster user managed keys config.") + if err := transport_tpg.LockedCall(lockKey, updateF); err != nil { + return err + } + + log.Printf("[INFO] GKE cluster %s user managed key config has been updated to %#v", d.Id(), req.Update.UserManagedKeysConfig) + } + if d.HasChange("enable_k8s_beta_apis") { log.Print("[INFO] Enable Kubernetes Beta APIs") if v, ok := d.GetOk("enable_k8s_beta_apis"); ok { @@ -5530,6 +5607,32 @@ func expandFleet(configured interface{}) *container.Fleet { } } +func expandUserManagedKeysConfig(configured interface{}) *container.UserManagedKeysConfig { + l := configured.([]interface{}) + if len(l) == 0 || l[0] == nil { + return nil + } + + config := l[0].(map[string]interface{}) + umkc := &container.UserManagedKeysConfig{ + ClusterCa: config["cluster_ca"].(string), + EtcdApiCa: config["etcd_api_ca"].(string), + EtcdPeerCa: config["etcd_peer_ca"].(string), + AggregationCa: config["aggregation_ca"].(string), + ControlPlaneDiskEncryptionKey: config["control_plane_disk_encryption_key"].(string), + GkeopsEtcdBackupEncryptionKey: config["gkeops_etcd_backup_encryption_key"].(string), + } + if v, ok := config["service_account_signing_keys"]; ok { + sk := v.(*schema.Set) + umkc.ServiceAccountSigningKeys = tpgresource.ConvertStringSet(sk) + } + if v, ok := config["service_account_verification_keys"]; ok { + vk := v.(*schema.Set) + umkc.ServiceAccountVerificationKeys = tpgresource.ConvertStringSet(vk) + } + return umkc +} + func expandEnableK8sBetaApis(configured interface{}, enabledAPIs []string) *container.K8sBetaAPIConfig { l := configured.([]interface{}) if len(l) == 0 || l[0] == nil { @@ -6446,6 +6549,27 @@ func flattenFleet(c *container.Fleet) []map[string]interface{} { } } +func flattenUserManagedKeysConfig(c *container.UserManagedKeysConfig) []map[string]interface{} { + if c == nil { + return nil + } + f := map[string]interface{}{ + "cluster_ca": c.ClusterCa, + "etcd_api_ca": c.EtcdApiCa, + "etcd_peer_ca": c.EtcdPeerCa, + "aggregation_ca": c.AggregationCa, + "control_plane_disk_encryption_key": c.ControlPlaneDiskEncryptionKey, + "gkeops_etcd_backup_encryption_key": c.GkeopsEtcdBackupEncryptionKey, + } + if len(c.ServiceAccountSigningKeys) != 0 { + f["service_account_signing_keys"] = schema.NewSet(schema.HashString, tpgresource.ConvertStringArrToInterface(c.ServiceAccountSigningKeys)) + } + if len(c.ServiceAccountVerificationKeys) != 0 { + f["service_account_verification_keys"] = schema.NewSet(schema.HashString, tpgresource.ConvertStringArrToInterface(c.ServiceAccountVerificationKeys)) + } + return []map[string]interface{}{f} +} + func flattenEnableK8sBetaApis(c *container.K8sBetaAPIConfig) []map[string]interface{} { if c == nil { return nil diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl index 7f7d5980e598..023af48758d7 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/terraform-provider-google/google/acctest" "github.com/hashicorp/terraform-provider-google/google/envvar" "github.com/hashicorp/terraform-provider-google/google/services/container" + cloudkms "google.golang.org/api/cloudkms/v1" ) func TestAccContainerCluster_basic(t *testing.T) { @@ -5451,6 +5452,310 @@ resource "google_container_cluster" "with_security_posture_config" { `, resource_name, networkName, subnetworkName) } +func TestAccContainerCluster_WithCPAFeatures(t *testing.T) { + t.Parallel() + + suffix := acctest.RandString(t, 10) + clusterName := fmt.Sprintf("tf-test-cluster-%s", suffix) + networkName := acctest.BootstrapSharedTestNetwork(t, "gke-cluster") + subnetworkName := acctest.BootstrapSubnet(t, "gke-cluster", networkName) + + // Bootstrap KMS keys and needed IAM role. + diskKey := acctest.BootstrapKMSKeyWithPurposeInLocationAndName(t, "ENCRYPT_DECRYPT", "us-central1", "control-plane-disk-encryption") + signingKey := acctest.BootstrapKMSKeyWithPurposeInLocationAndName(t, "ASYMMETRIC_SIGN", "us-central1", "rs256-service-account-signing") + backupKey := acctest.BootstrapKMSKeyWithPurposeInLocationAndName(t, "ENCRYPT_DECRYPT", "us-central1", "etcd-backups") + + // Here, we are granting the container engine service agent permissions on + // *ALL* Cloud KMS keys in the project. A more realistic usage would be to + // grant the service agent the necessary roles only on the individual keys + // we have created. + roles := []string{ + "roles/container.cloudKmsKeyUser", + "roles/privateca.certificateManager", + "roles/cloudkms.cryptoKeyEncrypterDecrypter", + } + if acctest.BootstrapPSARoles(t, "service-", "container-engine-robot", roles) { + t.Fatal("Stopping the test because a role was added to the policy.") + } + + // Find an active cryptoKeyVersion on the signing key. + var signingCryptoKeyVersion *cloudkms.CryptoKeyVersion + for _, ckv := range signingKey.CryptoKeyVersions { + if ckv.State == "ENABLED" && ckv.Algorithm == "RSA_SIGN_PKCS1_4096_SHA256" { + signingCryptoKeyVersion = ckv + } + } + if signingCryptoKeyVersion == nil { + t.Fatal("Didn't find an appropriate cryptoKeyVersion to use as the service account signing key") + } + + context := map[string]interface{}{ + "resource_name": clusterName, + "networkName": networkName, + "subnetworkName": subnetworkName, + "disk_key": diskKey.CryptoKey.Name, + "backup_key": backupKey.CryptoKey.Name, + "signing_cryptokeyversion": signingCryptoKeyVersion.Name, + "random_suffix": suffix, + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + // We are only supporting CPA features on create for now. + Config: testAccContainerCluster_EnableCPAFeatures(context), + }, + { + ResourceName: "google_container_cluster.with_cpa_features", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"deletion_protection"}, + }, + }, + }) +} + +func testAccContainerCluster_EnableCPAFeatures(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_privateca_ca_pool" "cluster_ca" { + name = "tf-test-cluster-ca-%{random_suffix}" + location = "us-central1" + tier = "DEVOPS" +} + +resource "google_privateca_ca_pool" "etcd_api_ca" { + name = "tf-test-etcd-api-ca-%{random_suffix}" + location = "us-central1" + tier = "DEVOPS" +} + +resource "google_privateca_ca_pool" "etcd_peer_ca" { + name = "tf-test-etcd-peer-%{random_suffix}" + location = "us-central1" + tier = "DEVOPS" +} + +resource "google_privateca_ca_pool" "aggregation_ca" { + name = "tf-test-aggregation-ca-%{random_suffix}" + location = "us-central1" + tier = "DEVOPS" +} + +resource "google_privateca_certificate_authority" "cluster_ca" { + certificate_authority_id = "my-authority" + location = "us-central1" + pool = google_privateca_ca_pool.cluster_ca.name + type = "SELF_SIGNED" + key_spec { + algorithm = "RSA_PKCS1_4096_SHA256" + } + + config { + subject_config { + subject { + country_code = "us" + organization = "google" + organizational_unit = "enterprise" + locality = "mountain view" + province = "california" + street_address = "1600 amphitheatre parkway" + postal_code = "94109" + common_name = "my-certificate-authority" + } + } + x509_config { + ca_options { + is_ca = true + } + key_usage { + base_key_usage { + cert_sign = true + crl_sign = true + } + extended_key_usage { + server_auth = true + client_auth = true + } + } + } + } + + // Disable CA deletion related safe checks for easier cleanup. + deletion_protection = false + skip_grace_period = true + ignore_active_certificates_on_deletion = true +} + +resource "google_privateca_certificate_authority" "etcd_api_ca" { + certificate_authority_id = "my-authority" + location = "us-central1" + pool = google_privateca_ca_pool.etcd_api_ca.name + type = "SELF_SIGNED" + key_spec { + algorithm = "RSA_PKCS1_4096_SHA256" + } + + config { + subject_config { + subject { + country_code = "us" + organization = "google" + organizational_unit = "enterprise" + locality = "mountain view" + province = "california" + street_address = "1600 amphitheatre parkway" + postal_code = "94109" + common_name = "my-certificate-authority" + } + } + x509_config { + ca_options { + is_ca = true + } + key_usage { + base_key_usage { + cert_sign = true + crl_sign = true + } + extended_key_usage { + server_auth = true + client_auth = true + } + } + } + } + // Disable CA deletion related safe checks for easier cleanup. + deletion_protection = false + skip_grace_period = true + ignore_active_certificates_on_deletion = true +} + +resource "google_privateca_certificate_authority" "etcd_peer_ca" { + certificate_authority_id = "my-authority" + location = "us-central1" + pool = google_privateca_ca_pool.etcd_peer_ca.name + type = "SELF_SIGNED" + key_spec { + algorithm = "RSA_PKCS1_4096_SHA256" + } + + config { + subject_config { + subject { + country_code = "us" + organization = "google" + organizational_unit = "enterprise" + locality = "mountain view" + province = "california" + street_address = "1600 amphitheatre parkway" + postal_code = "94109" + common_name = "my-certificate-authority" + } + } + x509_config { + ca_options { + is_ca = true + } + key_usage { + base_key_usage { + cert_sign = true + crl_sign = true + } + extended_key_usage { + server_auth = true + client_auth = true + } + } + } + } + // Disable CA deletion related safe checks for easier cleanup. + deletion_protection = false + skip_grace_period = true + ignore_active_certificates_on_deletion = true +} + +resource "google_privateca_certificate_authority" "aggregation_ca" { + certificate_authority_id = "my-authority" + location = "us-central1" + pool = google_privateca_ca_pool.aggregation_ca.name + type = "SELF_SIGNED" + key_spec { + algorithm = "RSA_PKCS1_4096_SHA256" + } + config { + subject_config { + subject { + country_code = "us" + organization = "google" + organizational_unit = "enterprise" + locality = "mountain view" + province = "california" + street_address = "1600 amphitheatre parkway" + postal_code = "94109" + common_name = "my-certificate-authority" + } + } + x509_config { + ca_options { + is_ca = true + } + key_usage { + base_key_usage { + cert_sign = true + crl_sign = true + } + extended_key_usage { + server_auth = true + client_auth = true + } + } + } + } + + // Disable CA deletion related safe checks for easier cleanup. + deletion_protection = false + skip_grace_period = true + ignore_active_certificates_on_deletion = true +} + +resource "google_container_cluster" "with_cpa_features" { + name = "%{resource_name}" + location = "us-central1-a" + initial_node_count = 1 + release_channel { + channel = "RAPID" + } + user_managed_keys_config { + cluster_ca = google_privateca_ca_pool.cluster_ca.id + etcd_api_ca = google_privateca_ca_pool.etcd_api_ca.id + etcd_peer_ca = google_privateca_ca_pool.etcd_peer_ca.id + aggregation_ca = google_privateca_ca_pool.aggregation_ca.id + control_plane_disk_encryption_key = "%{disk_key}" + gkeops_etcd_backup_encryption_key = "%{backup_key}" + + service_account_signing_keys = [ + "%{signing_cryptokeyversion}", + ] + service_account_verification_keys = [ + "%{signing_cryptokeyversion}", + ] + } + deletion_protection = false + network = "%{networkName}" + subnetwork = "%{subnetworkName}" + depends_on = [ + google_privateca_ca_pool.cluster_ca, + google_privateca_ca_pool.etcd_api_ca, + google_privateca_ca_pool.etcd_peer_ca, + google_privateca_ca_pool.aggregation_ca, + ] +} +`, context) +} + func TestAccContainerCluster_autopilot_minimal(t *testing.T) { t.Parallel()