Skip to content

Commit

Permalink
Data Warehouse acceptance tests for catalog, hive (#173)
Browse files Browse the repository at this point in the history
* DWX-18779 DW acceptance tests for catalog, hive

Cover the dw environment, db catalog and hive vw creations and
deletions as part of acceptance testing.

* Add role and policy creation to dw acceptance test

Use the default role to create an environment and DW cluster. The AWS
policy is queried via CLI.
  • Loading branch information
tevesz authored Nov 7, 2024
1 parent b299e93 commit 388f577
Show file tree
Hide file tree
Showing 2 changed files with 198 additions and 14 deletions.
131 changes: 127 additions & 4 deletions cdpacctest/acctest.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
package cdpacctest

import (
"encoding/base64"
"fmt"
"math/rand"
"os"
Expand All @@ -22,8 +23,11 @@ import (
"github.com/hashicorp/terraform-plugin-go/tfprotov6"
"github.com/hashicorp/terraform-plugin-testing/helper/resource"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"

"github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/cdp"
environmentoperations "github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/gen/environments/client/operations"
environmentsmodels "github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/gen/environments/models"
"github.com/cloudera/terraform-provider-cdp/provider"
)

Expand Down Expand Up @@ -51,6 +55,11 @@ var (
VersionConstraint: "~> 3.4",
},
}
TimeExternalProvider = map[string]resource.ExternalProvider{
"time": {
Source: "hashicorp/time",
},
}

cdpClientOnce sync.Once
cdpClient *cdp.Client
Expand Down Expand Up @@ -96,11 +105,25 @@ provider "cdp" {
`
}

func TestAccAwsProviderConfig() string {
return `
provider "aws" {
type AwsProvider struct {
profile string
region string
}
`

func NewAwsProvider(profile, region string) *AwsProvider {
return &AwsProvider{
profile: profile,
region: region,
}
}

func TestAccAwsProviderConfig(p *AwsProvider) string {
return fmt.Sprintf(`
provider "aws" {
profile = %[1]q
region = %[2]q
}
`, p.profile, p.region)
}

// CheckCrn Checks whether the value is set and is a properly formatted CRN
Expand All @@ -127,3 +150,103 @@ func GetCdpClientForAccTest() *cdp.Client {
})
return cdpClient
}

type AwsAccountCredentials struct {
name string
accountID string
externalID string
defaultPolicy string
}

func NewAwsAccountCredentials(name string) *AwsAccountCredentials {
return &AwsAccountCredentials{
name: name,
}
}

func getEnvironmentPrerequisites(t *testing.T, cloudPlatform string) *environmentsmodels.GetCredentialPrerequisitesResponse {
client := GetCdpClientForAccTest()
response, err := client.Environments.
Operations.
GetCredentialPrerequisites(
environmentoperations.NewGetCredentialPrerequisitesParams().
WithInput(&environmentsmodels.GetCredentialPrerequisitesRequest{
CloudPlatform: &cloudPlatform,
}),
)
assert.Nil(t, err)
payload := response.GetPayload()
assert.NotNil(t, payload)
return payload
}

func (a *AwsAccountCredentials) WithPolicy(t *testing.T) *AwsAccountCredentials {
payload := getEnvironmentPrerequisites(t, "AWS")
assert.NotNil(t, payload)
decodedBytes, err := base64.StdEncoding.DecodeString(*payload.Aws.PolicyJSON)
assert.Nil(t, err)
a.defaultPolicy = string(decodedBytes)
return a
}

func (a *AwsAccountCredentials) WithExternalID(t *testing.T) *AwsAccountCredentials {
payload := getEnvironmentPrerequisites(t, "AWS")
assert.NotNil(t, payload)
a.externalID = *payload.Aws.ExternalID
return a
}

func (a *AwsAccountCredentials) WithAccountID(t *testing.T) *AwsAccountCredentials {
payload := getEnvironmentPrerequisites(t, "AWS")
assert.NotNil(t, payload)
a.accountID = payload.AccountID
return a
}

func CreateDefaultRoleAndPolicy(p *AwsAccountCredentials) string {
return fmt.Sprintf(`
resource "aws_iam_role" "cdp_test_role" {
name = "%[1]s-role"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Statement1",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::%[2]s:root"
},
"Action": "sts:AssumeRole",
"Condition": {
"StringEquals": {
"sts:ExternalId": %[3]q
}
}
}
]
}
EOF
tags = {
owner = "[email protected]"
}
}
resource "aws_iam_policy" "cdp_test_policy" {
name = "%[1]s-policy"
description = "DefaultCBPolicy for CDP, replace the static file with a CLI call"
policy = <<EOF
%[4]s
EOF
}
resource "aws_iam_policy_attachment" "test-attach" {
name = "test_attachment"
roles = [aws_iam_role.cdp_test_role.name]
policy_arn = aws_iam_policy.cdp_test_policy.arn
}
`, p.name, p.accountID, p.externalID, p.defaultPolicy)
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
// OF ANY KIND, either express or implied. Refer to the License for the specific
// permissions and limitations governing your use of the file.

package aws_test
package dw

import (
"context"
Expand All @@ -21,13 +21,15 @@ import (
"github.com/hashicorp/terraform-plugin-testing/helper/resource"
"github.com/hashicorp/terraform-plugin-testing/terraform"

"github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/cdp"
"github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/gen/dw/client/operations"
"github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/gen/dw/models"
"github.com/cloudera/terraform-provider-cdp/cdpacctest"
"github.com/cloudera/terraform-provider-cdp/utils"
)

const (
AwsProfile = "ACCEPTANCETEST_AWS_PROFILE"
AwsXAccRoleArn = "ACCEPTANCETEST_AWS_X_ACC_ROLE_ARN"
AwsRegion = "ACCEPTANCETEST_AWS_REGION"
AwsPublicKeyId = "ACCEPTANCETEST_AWS_PUBLIC_KEY_ID"
Expand Down Expand Up @@ -62,6 +64,9 @@ type awsDataLakeTestParameters struct {

func AwsDataLakePreCheck(t *testing.T) {
errMsg := "AWS CDW Terraform acceptance testing requires environment variable %s to be set"
if _, ok := os.LookupEnv(AwsProfile); !ok {
t.Fatalf(errMsg, AwsProfile)
}
if _, ok := os.LookupEnv(AwsXAccRoleArn); !ok {
t.Fatalf(errMsg, AwsXAccRoleArn)
}
Expand Down Expand Up @@ -97,8 +102,30 @@ func AwsDataLakePreCheck(t *testing.T) {
}
}

func TestAccCluster_basic(t *testing.T) {
func PreCheck(t *testing.T) {
if _, ok := os.LookupEnv(AwsProfile); !ok {
t.Skipf("Terraform acceptance testing requires environment variable %s to be set", AwsProfile)
}

if os.Getenv(cdp.CdpProfileEnvVar) == "" && os.Getenv(cdp.CdpAccessKeyIdEnvVar) == "" {
t.Skipf("Terraform acceptance testing requires either %s or %s environment variables to be set", cdp.CdpProfileEnvVar, cdp.CdpAccessKeyIdEnvVar)
}

if os.Getenv(cdp.CdpAccessKeyIdEnvVar) != "" {
if _, ok := os.LookupEnv(cdp.CdpPrivateKeyEnvVar); !ok {
t.Skipf("Environment variable %s should be set together with %s", cdp.CdpPrivateKeyEnvVar, cdp.CdpAccessKeyIdEnvVar)
}
}
}

func TestAccDwCluster_Basic(t *testing.T) {
PreCheck(t)
credName := acctest.RandomWithPrefix(cdpacctest.ResourcePrefix)
awsProvider := cdpacctest.NewAwsProvider(os.Getenv(AwsProfile), os.Getenv(AwsRegion))
accountParams := cdpacctest.NewAwsAccountCredentials(cdpacctest.RandomShortWithPrefix(cdpacctest.ResourcePrefix)).
WithAccountID(t).
WithExternalID(t).
WithPolicy(t)
envParams := awsEnvironmentTestParameters{
Name: cdpacctest.RandomShortWithPrefix(cdpacctest.ResourcePrefix),
Region: os.Getenv(AwsRegion),
Expand All @@ -123,16 +150,23 @@ func TestAccCluster_basic(t *testing.T) {
AwsDataLakePreCheck(t)
},
ProtoV6ProviderFactories: cdpacctest.TestAccProtoV6ProviderFactories,
CheckDestroy: testCheckClusterDestroy,
ExternalProviders: cdpacctest.ConcatExternalProviders(
cdpacctest.AwsExternalProvider,
cdpacctest.TimeExternalProvider,
),
CheckDestroy: testCheckClusterDestroy,
Steps: []resource.TestStep{
// Create and Read testing
{
Config: utils.Concat(
cdpacctest.TestAccCdpProviderConfig(),
testAccAwsCredentialBasicConfig(credName, os.Getenv(AwsXAccRoleArn)),
cdpacctest.CreateDefaultRoleAndPolicy(accountParams),
cdpacctest.TestAccAwsProviderConfig(awsProvider),
testAccAwsCredentialBasicConfig(credName),
testAccAwsEnvironmentConfig(&envParams),
testAccAwsDataLakeConfig(&dlParams),
testAccAwsClusterBasicConfig(&envParams)),
testAccAwsClusterBasicConfig(&envParams),
testAccDwCatalog(),
testAccHiveVirtualWarehouse(cdpacctest.RandomShortWithPrefix("tf-hive"))),
Check: resource.ComposeAggregateTestCheckFunc(
resource.TestCheckResourceAttr(resourceName, "name", envParams.Name),
resource.TestCheckResourceAttr(resourceName, "status", "Accepted"),
Expand All @@ -143,12 +177,21 @@ func TestAccCluster_basic(t *testing.T) {
})
}

func testAccAwsCredentialBasicConfig(name string, roleArn string) string {
func testAccAwsCredentialBasicConfig(name string) string {
// Wait for the IAM policy attachment to be created before creating the credential, after a couple of seconds,
// the CDP credential creation fails, the privileges are not yet available.
return fmt.Sprintf(`
resource "time_sleep" "wait_10_seconds" {
depends_on = [aws_iam_policy_attachment.test-attach]
create_duration = "10s"
}
resource "cdp_environments_aws_credential" "test_cred" {
credential_name = %[1]q
role_arn = %[2]q
}`, name, roleArn)
credential_name = "%[1]s-cred"
role_arn = aws_iam_role.cdp_test_role.arn
depends_on = [time_sleep.wait_10_seconds]
}
`, name)
}

func testAccAwsEnvironmentConfig(envParams *awsEnvironmentTestParameters) string {
Expand Down Expand Up @@ -226,6 +269,24 @@ func testAccAwsClusterBasicConfig(params *awsEnvironmentTestParameters) string {
`, params.SubnetIds)
}

func testAccDwCatalog() string {
return `
resource "cdp_dw_database_catalog" "test_catalog" {
cluster_id = cdp_dw_aws_cluster.test_data_warehouse_aws.cluster_id
}
`
}

func testAccHiveVirtualWarehouse(name string) string {
return fmt.Sprintf(`
resource "cdp_vw_hive" "test_hive" {
cluster_id = cdp_dw_aws_cluster.test_data_warehouse_aws.cluster_id
database_catalog_id = cdp_dw_database_catalog.test_catalog.id
name = %[1]q
}
`, name)
}

func testCheckClusterDestroy(s *terraform.State) error {
for _, rs := range s.RootModule().Resources {
if rs.Type != "cdp_dw_aws_cluster" {
Expand Down

0 comments on commit 388f577

Please sign in to comment.