[v14] GCP MySQL IAM Auth support

lib/cloud/gcp/sql.go

@@ -36,6 +36,8 @@ import (
 
 // SQLAdminClient defines an interface providing access to the GCP Cloud SQL API.
 type SQLAdminClient interface {
+	// GetUser retrieves a resource containing information about a user.
+	GetUser(ctx context.Context, db types.Database, dbUser string) (*sqladmin.User, error)
 	// UpdateUser updates an existing user for the project/instance configured in a session.
 	UpdateUser(ctx context.Context, db types.Database, dbUser string, user *sqladmin.User) error
 	// GetDatabaseInstance returns database instance details for the project/instance
@@ -61,6 +63,16 @@ type gcpSQLAdminClient struct {
 	service *sqladmin.Service
 }
 
+// GetUser retrieves a resource containing information about a user.
+func (g *gcpSQLAdminClient) GetUser(ctx context.Context, db types.Database, dbUser string) (*sqladmin.User, error) {
+	user, err := g.service.Users.Get(
+		db.GetGCP().ProjectID,
+		db.GetGCP().InstanceID,
+		dbUser,
+	).Host("%").Context(ctx).Do()
+	return user, trace.Wrap(convertAPIError(err))
+}
+
 // UpdateUser updates an existing user in a Cloud SQL for the project/instance
 // configured in a session.
 func (g *gcpSQLAdminClient) UpdateUser(ctx context.Context, db types.Database, dbUser string, user *sqladmin.User) error {
@@ -69,7 +81,7 @@ func (g *gcpSQLAdminClient) UpdateUser(ctx context.Context, db types.Database, d
 		db.GetGCP().InstanceID,
 		user).Name(dbUser).Host("%").Context(ctx).Do()
 	if err != nil {
-		return trace.Wrap(err)
+		return trace.Wrap(convertAPIError(err))
 	}
 	return nil
 }
@@ -80,7 +92,7 @@ func (g *gcpSQLAdminClient) GetDatabaseInstance(ctx context.Context, db types.Da
 	gcp := db.GetGCP()
 	dbi, err := g.service.Instances.Get(gcp.ProjectID, gcp.InstanceID).Context(ctx).Do()
 	if err != nil {
-		return nil, trace.Wrap(err)
+		return nil, trace.Wrap(convertAPIError(err))
 	}
 	return dbi, nil
 }
@@ -110,7 +122,7 @@ func (g *gcpSQLAdminClient) GenerateEphemeralCert(ctx context.Context, db types.
 	})
 	resp, err := req.Context(ctx).Do()
 	if err != nil {
-		return nil, trace.Wrap(err)
+		return nil, trace.Wrap(convertAPIError(err))
 	}
 
 	// Create TLS certificate from returned ephemeral certificate and private key.

lib/cloud/mocks/gcp.go

@@ -39,6 +39,15 @@ type GCPSQLAdminClientMock struct {
 	DatabaseInstance *sqladmin.DatabaseInstance
 	// EphemeralCert is returned from GenerateEphemeralCert.
 	EphemeralCert *tls.Certificate
+	// DatabaseUser is returned from GetUser.
+	DatabaseUser *sqladmin.User
+}
+
+func (g *GCPSQLAdminClientMock) GetUser(ctx context.Context, db types.Database, dbUser string) (*sqladmin.User, error) {
+	if g.DatabaseUser == nil {
+		return nil, trace.AccessDenied("unauthorized")
+	}
+	return g.DatabaseUser, nil
 }
 
 func (g *GCPSQLAdminClientMock) UpdateUser(ctx context.Context, db types.Database, dbUser string, user *sqladmin.User) error {

lib/srv/db/common/auth.go

@@ -317,14 +317,19 @@ func (a *dbAuth) GetCloudSQLAuthToken(ctx context.Context, sessionCtx *Session)
 		return "", trace.Wrap(err)
 	}
 	a.cfg.Log.Debugf("Generating GCP auth token for %s.", sessionCtx)
+
+	serviceAccountName := sessionCtx.DatabaseUser
+	if !strings.HasSuffix(serviceAccountName, ".gserviceaccount.com") {
+		serviceAccountName = serviceAccountName + ".gserviceaccount.com"
+	}
 	resp, err := gcpIAM.GenerateAccessToken(ctx,
 		&gcpcredentialspb.GenerateAccessTokenRequest{
 			// From GenerateAccessToken docs:
 			//
 			// The resource name of the service account for which the credentials
 			// are requested, in the following format:
 			//   projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}
-			Name: fmt.Sprintf("projects/-/serviceAccounts/%v.gserviceaccount.com", sessionCtx.DatabaseUser),
+			Name: fmt.Sprintf("projects/-/serviceAccounts/%v", serviceAccountName),
 			// From GenerateAccessToken docs:
 			//
 			// Code to identify the scopes to be included in the OAuth 2.0 access
@@ -390,12 +395,19 @@ func (a *dbAuth) GetCloudSQLPassword(ctx context.Context, sessionCtx *Session) (
 func (a *dbAuth) updateCloudSQLUser(ctx context.Context, sessionCtx *Session, gcpCloudSQL gcp.SQLAdminClient, user *sqladmin.User) error {
 	err := gcpCloudSQL.UpdateUser(ctx, sessionCtx.Database, sessionCtx.DatabaseUser, user)
 	if err != nil {
+		// Note that mysql client has a 1024 char limit for displaying errors
+		// so we need to keep the message short when possible. This message
+		// does get cut off when sessionCtx.DatabaseUser or err is long.
 		return trace.AccessDenied(`Could not update Cloud SQL user %q password:
 
   %v
 
-Make sure Teleport db service has "Cloud SQL Admin" GCP IAM role, or
-"cloudsql.users.update" IAM permission.
+If the db user uses IAM authentication, please use the full service account email
+ID as "--db-user", or grant the Teleport Database Service the
+"cloudsql.users.get" IAM permission so it can discover the user type.
+
+If the db user uses passwords, make sure Teleport Database Service has "Cloud
+SQL Admin" GCP IAM role, or "cloudsql.users.update" IAM permission.
 `, sessionCtx.DatabaseUser, err)
 	}
 	return nil

lib/srv/db/mysql/engine.go

@@ -33,7 +33,6 @@ import (
 
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/api/utils/retryutils"
-	"github.com/gravitational/teleport/lib/defaults"
 	"github.com/gravitational/teleport/lib/services"
 	"github.com/gravitational/teleport/lib/srv/db/cloud"
 	"github.com/gravitational/teleport/lib/srv/db/common"
@@ -220,34 +219,17 @@ func (e *Engine) connect(ctx context.Context, sessionCtx *common.Session) (*clie
 			return nil, trace.Wrap(err)
 		}
 	case sessionCtx.Database.IsCloudSQL():
-		// For Cloud SQL MySQL there is no IAM auth, so we use one-time passwords
-		// by resetting the database user password for each connection. Thus,
-		// acquire a lock to make sure all connection attempts to the same
-		// database and user are serialized.
-		retryCtx, cancel := context.WithTimeout(ctx, defaults.DatabaseConnectTimeout)
-		defer cancel()
-		lease, err := services.AcquireSemaphoreWithRetry(retryCtx, e.makeAcquireSemaphoreConfig(sessionCtx))
-		if err != nil {
-			return nil, trace.Wrap(err)
-		}
-		// Only release the semaphore after the connection has been established
-		// below. If the semaphore fails to release for some reason, it will
-		// expire in a minute on its own.
-		defer func() {
-			err := e.AuthClient.CancelSemaphoreLease(ctx, *lease)
-			if err != nil {
-				e.Log.WithError(err).Errorf("Failed to cancel lease: %v.", lease)
-			}
-		}()
-		password, err = e.Auth.GetCloudSQLPassword(ctx, sessionCtx)
+		// Get the client once for subsequent calls (it acquires a read lock).
+		gcpClient, err := e.CloudClients.GetGCPSQLAdminClient(ctx)
 		if err != nil {
 			return nil, trace.Wrap(err)
 		}
-		// Get the client once for subsequent calls (it acquires a read lock).
-		gcpClient, err := e.CloudClients.GetGCPSQLAdminClient(ctx)
+
+		user, password, err = e.getGCPUserAndPassword(ctx, sessionCtx, gcpClient)
 		if err != nil {
 			return nil, trace.Wrap(err)
 		}
+
 		// Detect whether the instance is set to require SSL.
 		// Fallback to not requiring SSL for access denied errors.
 		requireSSL, err := cloud.GetGCPRequireSSL(ctx, sessionCtx, gcpClient)

lib/srv/db/mysql/gcp.go

@@ -0,0 +1,175 @@
+/*
+ * Teleport
+ * Copyright (C) 2024  Gravitational, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package mysql
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/gravitational/trace"
+
+	"github.com/gravitational/teleport/lib/cloud/gcp"
+	"github.com/gravitational/teleport/lib/defaults"
+	"github.com/gravitational/teleport/lib/services"
+	"github.com/gravitational/teleport/lib/srv/db/common"
+)
+
+func isDBUserFullGCPServerAccountID(dbUser string) bool {
+	// Example: [email protected]
+	return strings.Contains(dbUser, "@") &&
+		strings.HasSuffix(dbUser, ".iam.gserviceaccount.com")
+}
+
+func isDBUserShortGCPServiceAccountID(dbUser string) bool {
+	// Example: [email protected]
+	return strings.Contains(dbUser, "@") &&
+		strings.HasSuffix(dbUser, ".iam")
+}
+
+func gcpServiceAccountToDatabaseUser(serviceAccountName string) string {
+	user, _, _ := strings.Cut(serviceAccountName, "@")
+	return user
+}
+
+func databaseUserToGCPServiceAccount(sessionCtx *common.Session) string {
+	return fmt.Sprintf("%s@%s.iam.gserviceaccount.com", sessionCtx.DatabaseUser, sessionCtx.Database.GetGCP().ProjectID)
+}
+
+func (e *Engine) getGCPUserAndPassword(ctx context.Context, sessionCtx *common.Session, gcpClient gcp.SQLAdminClient) (string, string, error) {
+	// If `--db-user` is the full service account email ID, use IAM Auth.
+	if isDBUserFullGCPServerAccountID(sessionCtx.DatabaseUser) {
+		user := gcpServiceAccountToDatabaseUser(sessionCtx.DatabaseUser)
+		password, err := e.getGCPIAMAuthToken(ctx, sessionCtx)
+		if err != nil {
+			return "", "", trace.Wrap(err)
+		}
+		return user, password, nil
+	}
+
+	// Note that GCP Postgres' format "[email protected]" is not accepted
+	// for GCP MySQL. For GCP Postgres, "[email protected]" is the actual
+	// mapped in-database username. However, the mapped in-database username
+	// for GCP MySQL does not have the "@my-project-id.iam" part.
+	if isDBUserShortGCPServiceAccountID(sessionCtx.DatabaseUser) {
+		return "", "", trace.BadParameter("username %q is not accepted for GCP MySQL. Please use the in-database username or the full service account Email ID.", sessionCtx.DatabaseUser)
+	}
+
+	// Get user info to decide how to authenticate.
+	user := sessionCtx.DatabaseUser
+	dbUserInfo, err := gcpClient.GetUser(ctx, sessionCtx.Database, sessionCtx.DatabaseUser)
+	switch {
+	// GetUser permission is new for IAM auth. If no permission, assume legacy password user.
+	case trace.IsAccessDenied(err):
+		e.Log.WithField("user", sessionCtx.DatabaseUser).Debug("Access denied to get GCP MySQL database user info. Continue with password auth.")
+		password, err := e.getGCPOneTimePassword(ctx, sessionCtx)
+		if err != nil {
+			return "", "", trace.Wrap(err)
+		}
+		return user, password, nil
+
+	// Make the original error message "object not found" more readable. Note
+	// that catching not found here also prevents Google creating a new
+	// database user during OTP generation.
+	case trace.IsNotFound(err):
+		return "", "", trace.NotFound("database user %q does not exist in database %q", sessionCtx.DatabaseUser, sessionCtx.Database.GetName())
+
+	// Report any other error.
+	case err != nil:
+		return "", "", trace.Wrap(err)
+	}
+
+	// The user type constants are documented in their SDK. However, in
+	// practice, type can also be empty for built-in user.
+	switch dbUserInfo.Type {
+	case "",
+		gcpMySQLDBUserTypeBuiltIn:
+		password, err := e.getGCPOneTimePassword(ctx, sessionCtx)
+		if err != nil {
+			return "", "", trace.Wrap(err)
+		}
+		return user, password, nil
+
+	case gcpMySQLDBUserTypeServiceAccount,
+		gcpMySQLDBUserTypeGroupServiceAccount:
+		serviceAccountName := databaseUserToGCPServiceAccount(sessionCtx)
+		password, err := e.getGCPIAMAuthToken(ctx, sessionCtx.WithUser(serviceAccountName))
+		if err != nil {
+			return "", "", trace.Wrap(err)
+		}
+		return user, password, nil
+
+	case gcpMySQLDBUserTypeUser,
+		gcpMySQLDBUserTypeGroupUser:
+		return "", "", trace.BadParameter("GCP MySQL user type %q not supported", dbUserInfo.Type)
+
+	default:
+		return "", "", trace.BadParameter("unknown GCP MySQL user type %q", dbUserInfo.Type)
+	}
+}
+
+func (e *Engine) getGCPIAMAuthToken(ctx context.Context, sessionCtx *common.Session) (string, error) {
+	e.Log.WithField("session", sessionCtx).Debug("Authenticating GCP MySQL with IAM auth.")
+
+	// Note that sessionCtx.DatabaseUser is the service account.
+	password, err := e.Auth.GetCloudSQLAuthToken(ctx, sessionCtx)
+	return password, trace.Wrap(err)
+}
+
+func (e *Engine) getGCPOneTimePassword(ctx context.Context, sessionCtx *common.Session) (string, error) {
+	e.Log.WithField("session", sessionCtx).Debug("Authenticating GCP MySQL with password auth.")
+
+	// For Cloud SQL MySQL legacy auth, we use one-time passwords by resetting
+	// the database user password for each connection. Thus, acquire a lock to
+	// make sure all connection attempts to the same database and user are
+	// serialized.
+	retryCtx, cancel := context.WithTimeout(ctx, defaults.DatabaseConnectTimeout)
+	defer cancel()
+	lease, err := services.AcquireSemaphoreWithRetry(retryCtx, e.makeAcquireSemaphoreConfig(sessionCtx))
+	if err != nil {
+		return "", trace.Wrap(err)
+	}
+	// Only release the semaphore after the connection has been established
+	// below. If the semaphore fails to release for some reason, it will
+	// expire in a minute on its own.
+	defer func() {
+		err := e.AuthClient.CancelSemaphoreLease(ctx, *lease)
+		if err != nil {
+			e.Log.WithError(err).Errorf("Failed to cancel lease: %v.", lease)
+		}
+	}()
+	password, err := e.Auth.GetCloudSQLPassword(ctx, sessionCtx)
+	if err != nil {
+		return "", trace.Wrap(err)
+	}
+	return password, nil
+}
+
+const (
+	// gcpMySQLDBUserTypeBuiltIn indicates the database's built-in user type.
+	gcpMySQLDBUserTypeBuiltIn = "BUILT_IN"
+	// gcpMySQLDBUserTypeServiceAccount indicates a Cloud IAM service account.
+	gcpMySQLDBUserTypeServiceAccount = "CLOUD_IAM_SERVICE_ACCOUNT"
+	//  gcpMySQLDBUserTypeGroupServiceAccount indicates a Cloud IAM group service account.
+	gcpMySQLDBUserTypeGroupServiceAccount = "CLOUD_IAM_GROUP_SERVICE_ACCOUNT"
+	// gcpMySQLDBUserTypeUser indicates a Cloud IAM user.
+	gcpMySQLDBUserTypeUser = "CLOUD_IAM_USER"
+	// gcpMySQLDBUserTypeGroupUser indicates a Cloud IAM group login user.
+	gcpMySQLDBUserTypeGroupUser = "CLOUD_IAM_GROUP_USER"
+)