update AWS RDS db e2e tests (#40065)

* update AWS RDS db e2e tests

* update GHA workflow env vars to be RDS specific
* add auto db user provisioning tests
* provision any db users needed in test code as setup, instead of
  relying on the cloud-terraform repo provisioning them

* fix data race

* rename myDBName to mysqlDBName

* fix flakiness on first run

* remove db_users for auto user roles

* test role assignments in postgres

* reuse rand string util

* test role assignments in mysql

* rearrange subtests to clearly outline dependencies

* verify AWS CA when provisioning

* fix imports

.github/workflows/aws-e2e-tests-non-root.yaml

@@ -14,9 +14,8 @@ env:
   KUBERNETES_SERVICE_ASSUME_ROLE: arn:aws:iam::307493967395:role/tf-eks-discovery-ci-cluster-kubernetes-service-access-role
   KUBE_DISCOVERY_SERVICE_ASSUME_ROLE: arn:aws:iam::307493967395:role/tf-eks-discovery-ci-cluster-discovery-service-access-role
   EKS_CLUSTER_NAME: gha-discovery-ci-eks-us-west-2-307493967395
-  DATABASE_USER: teleport-ci-e2e-test
-  DATABASE_SERVICE_ASSUME_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-database-svc
-  DATABASE_DISCOVERY_SERVICE_ASSUME_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-discovery-svc
+  RDS_ACCESS_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-rds-access
+  RDS_DISCOVERY_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-rds-discovery
   RDS_POSTGRES_INSTANCE_NAME: ci-database-e2e-tests-rds-postgres-instance-us-west-2-307493967395
   RDS_MYSQL_INSTANCE_NAME: ci-database-e2e-tests-rds-mysql-instance-us-west-2-307493967395
   DISCOVERY_MATCHER_LABELS: "*=*"

e2e/aws/eks_test.go

@@ -128,7 +128,7 @@ func awsEKSDiscoveryMatchedCluster(t *testing.T) {
 	// the permissions are correct.
 	kubeClient, _, err := kube.ProxyClient(kube.ProxyConfig{
 		T:           teleport,
-		Username:    username,
+		Username:    hostUser,
 		KubeUsers:   kubeUsers,
 		KubeGroups:  kubeGroups,
 		KubeCluster: clusters[0].GetName(),
@@ -182,7 +182,7 @@ func awsEKSDiscoveryUnmatchedCluster(t *testing.T) {
 // clusters.
 func withFullKubeAccessUserRole(t *testing.T) testOptionsFunc {
 	// Create a new role with full access to all kube clusters.
-	return withUserRole(t, "kubemaster", types.RoleSpecV6{
+	return withUserRole(t, hostUser, "kubemaster", types.RoleSpecV6{
 		Allow: types.RoleConditions{
 			KubeGroups: kubeGroups,
 			KubeUsers:  kubeUsers,

e2e/aws/fixtures_test.go

@@ -19,11 +19,15 @@
 package e2e
 
 import (
+	"crypto/x509"
+	"io"
+	"net/http"
 	"os"
 	"os/user"
 	"testing"
 	"time"
 
+	"github.com/gravitational/trace"
 	"github.com/stretchr/testify/require"
 
 	"github.com/gravitational/teleport/api/types"
@@ -35,15 +39,52 @@ import (
 	"github.com/gravitational/teleport/lib/utils"
 )
 
-// username is the name of the host user used for tests.
-var username string
+// hostUser is the name of the host user used for tests.
+var hostUser string
+
+// awsCertPool is an x509 cert pool containing the AWS global cert bundle.
+var awsCertPool *x509.CertPool
 
 func init() {
 	me, err := user.Current()
 	if err != nil {
 		panic(err)
 	}
-	username = me.Username
+	hostUser = me.Username
+
+	pool, err := getAWSGlobalCertBundlePool()
+	if err != nil {
+		panic(err)
+	}
+	awsCertPool = pool
+}
+
+func getAWSGlobalCertBundlePool() (*x509.CertPool, error) {
+	// AWS global certificate bundle
+	const certBundleURL = "https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem"
+
+	resp, err := http.Get(certBundleURL)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	defer resp.Body.Close()
+
+	if http.StatusOK != resp.StatusCode {
+		return nil, trace.Errorf("got non-ok response from AWS: %d", resp.StatusCode)
+	}
+
+	certBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	certPool := x509.NewCertPool()
+	ok := certPool.AppendCertsFromPEM(certBytes)
+	if !ok {
+		return nil, trace.Errorf("error parsing AWS cert bundle")
+	}
+
+	return certPool, nil
 }
 
 // mustGetEnv is a test helper that fetches an env variable or fails with an
@@ -67,6 +108,13 @@ func mustGetDiscoveryMatcherLabels(t *testing.T) types.Labels {
 	return out
 }
 
+func mustGetDBAdmin(t *testing.T, db types.Database) types.DatabaseAdminUser {
+	t.Helper()
+	adminUser := db.GetAdminUser()
+	require.NotEmpty(t, adminUser.Name, "unknown db auto user provisioning admin, should have been imported using the %q label", types.DatabaseAdminLabel)
+	return adminUser
+}
+
 // testOptionsFunc is a test option configuration func.
 type testOptionsFunc func(*testOptions)
 
@@ -78,15 +126,16 @@ type testOptions struct {
 	// serviceConfigFuncs are a list of functions that configure the Teleport
 	// cluster before it starts.
 	serviceConfigFuncs []func(*servicecfg.Config)
-	// userRoles are roles that will be bootstrapped and added to the Teleport
-	// user under test.
-	userRoles []types.Role
+	// userRoles is a map from username to that user's roles that will be
+	// bootstrapped and added to the Teleport test cluster.
+	userRoles map[string][]types.Role
 }
 
 // createTeleportCluster sets up a Teleport cluster for tests.
 func createTeleportCluster(t *testing.T, opts ...testOptionsFunc) *helpers.TeleInstance {
 	t.Helper()
 	var options testOptions
+	options.userRoles = make(map[string][]types.Role)
 	for _, opt := range opts {
 		opt(&options)
 	}
@@ -98,7 +147,9 @@ func createTeleportCluster(t *testing.T, opts ...testOptionsFunc) *helpers.TeleI
 	teleport := helpers.NewInstance(t, cfg)
 
 	// Create a new user with the role created above.
-	teleport.AddUserWithRole(username, options.userRoles...)
+	for name, roles := range options.userRoles {
+		teleport.AddUserWithRole(name, roles...)
+	}
 
 	tconf := newTeleportConfig(t)
 	for _, optFn := range options.serviceConfigFuncs {
@@ -149,13 +200,13 @@ func newTeleportConfig(t *testing.T) *servicecfg.Config {
 
 // withUserRole creates a new role that will be bootstraped and then granted to
 // the Teleport user under test.
-func withUserRole(t *testing.T, name string, spec types.RoleSpecV6) testOptionsFunc {
+func withUserRole(t *testing.T, user, name string, spec types.RoleSpecV6) testOptionsFunc {
 	t.Helper()
 	// Create a new role with full access to all databases.
 	role, err := types.NewRole(name, spec)
 	require.NoError(t, err)
 	return func(options *testOptions) {
-		options.userRoles = append(options.userRoles, role)
+		options.userRoles[user] = append(options.userRoles[user], role)
 	}
 }
 
@@ -208,11 +259,32 @@ func withDatabaseService(t *testing.T, matchers ...services.ResourceMatcher) tes
 func withFullDatabaseAccessUserRole(t *testing.T) testOptionsFunc {
 	t.Helper()
 	// Create a new role with full access to all databases.
-	return withUserRole(t, "db-access", types.RoleSpecV6{
+	return withUserRole(t, hostUser, "db-access", allowDatabaseAccessRoleSpec)
+}
+
+var allowDatabaseAccessRoleSpec = types.RoleSpecV6{
+	Allow: types.RoleConditions{
+		DatabaseLabels: types.Labels{types.Wildcard: []string{types.Wildcard}},
+		DatabaseUsers:  []string{types.Wildcard},
+		DatabaseNames:  []string{types.Wildcard},
+	},
+}
+
+func makeAutoUserKeepRoleSpec(roles ...string) types.RoleSpecV6 {
+	return types.RoleSpecV6{
 		Allow: types.RoleConditions{
 			DatabaseLabels: types.Labels{types.Wildcard: []string{types.Wildcard}},
-			DatabaseUsers:  []string{types.Wildcard},
 			DatabaseNames:  []string{types.Wildcard},
+			DatabaseRoles:  roles,
 		},
-	})
+		Options: types.RoleOptions{
+			CreateDatabaseUserMode: types.CreateDatabaseUserMode_DB_USER_MODE_KEEP,
+		},
+	}
+}
+
+func makeAutoUserDropRoleSpec(roles ...string) types.RoleSpecV6 {
+	spec := makeAutoUserKeepRoleSpec(roles...)
+	spec.Options.CreateDatabaseUserMode = types.CreateDatabaseUserMode_DB_USER_MODE_BEST_EFFORT_DROP
+	return spec
 }

e2e/aws/main_test.go

@@ -32,18 +32,15 @@ const (
 	// discoveryMatcherLabelsEnv is the env variable that specifies the matcher
 	// labels to use in test discovery services.
 	discoveryMatcherLabelsEnv = "DISCOVERY_MATCHER_LABELS"
-	// dbSvcRoleARNEnv is the environment variable that specifies the IAM role
-	// that Teleport Database Service will assume to access databases.
-	// check modules/databases-ci/ from cloud-terraform repo for more details.
-	dbSvcRoleARNEnv = "DATABASE_SERVICE_ASSUME_ROLE"
-	// dbDiscoverySvcRoleARNEnv is the environment variable that specifies the
+	// rdsAccessRoleEnv is the environment variable that specifies the IAM role
+	// that Teleport Database Service will assume to access RDS databases.
+	// See modules/databases-ci/ from cloud-terraform repo for more details.
+	rdsAccessRoleEnv = "RDS_ACCESS_ROLE"
+	// rdsDiscoveryRoleEnv is the environment variable that specifies the
 	// IAM role that Teleport Discovery Service will assume to discover
-	// databases.
-	// check modules/databases-ci/ from cloud-terraform repo for more details.
-	dbDiscoverySvcRoleARNEnv = "DATABASE_DISCOVERY_SERVICE_ASSUME_ROLE"
-	// dbUserEnv is the database user configured in databases for access via
-	// Teleport.
-	dbUserEnv = "DATABASE_USER"
+	// RDS databases.
+	// See modules/databases-ci/ from cloud-terraform repo for more details.
+	rdsDiscoveryRoleEnv = "RDS_DISCOVERY_ROLE"
 	// rdsPostgresInstanceNameEnv is the environment variable that specifies the
 	// name of the RDS Postgres DB instance that will be created by the Teleport
 	// Discovery Service.