From 576e6ac1012d9f4b86742b3ca159bba1fbf9707e Mon Sep 17 00:00:00 2001 From: rosstimothy <39066650+rosstimothy@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:53:33 +0000 Subject: [PATCH] Adds support to Firestore backends for custom databases (#47540) Firestore has supported multiple databases in a project for a while (see https://cloud.google.com/blog/products/databases/manage-multiple-firestore-databases-in-a-project), however, Teleport only allowed using the default database in the project. The DatabaseID is now exposed in the file config, and if provided both the state and events backends will use the appropriate database. Closes https://github.com/gravitational/teleport/issues/37227. --- docs/pages/reference/backends.mdx | 6 +++- lib/backend/firestore/firestorebk.go | 29 ++++++++++++++----- lib/events/firestoreevents/firestoreevents.go | 6 +++- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/docs/pages/reference/backends.mdx b/docs/pages/reference/backends.mdx index 2bd9aad5f3aca..8b0b73ecb3465 100644 --- a/docs/pages/reference/backends.mdx +++ b/docs/pages/reference/backends.mdx @@ -1270,6 +1270,10 @@ teleport: # Name of the Firestore table. collection_name: Example_TELEPORT_FIRESTORE_TABLE_NAME + # An optional database id to use. If not provided the default + # database for the project is used. + database_id: Example_TELEPORT_FIRESTORE_DATABASE_ID + credentials_path: /var/lib/teleport/gcs_creds # This setting configures Teleport to send the audit events to three places: @@ -1278,7 +1282,7 @@ teleport: # database table, so attempting to use the same table for both will result in errors. # When using highly available storage like Firestore, you should make sure that the list always specifies # the High Availability storage method first, as this is what the Teleport web UI uses as its source of events to display. - audit_events_uri: ['firestore://Example_TELEPORT_FIRESTORE_EVENTS_TABLE_NAME', 'file:///var/lib/teleport/audit/events', 'stdout://'] + audit_events_uri: ['firestore://Example_TELEPORT_FIRESTORE_EVENTS_TABLE_NAME?projectID=$PROJECT_ID&credentialsPath=$CREDENTIALS_PATH&databaseID=$DATABASE_ID', 'file:///var/lib/teleport/audit/events', 'stdout://'] # This setting configures Teleport to save the recorded sessions in GCP storage: audit_sessions_uri: gs://Example_TELEPORT_GCS_BUCKET/records diff --git a/lib/backend/firestore/firestorebk.go b/lib/backend/firestore/firestorebk.go index 3056b245c6ae9..c7483aede60f8 100644 --- a/lib/backend/firestore/firestorebk.go +++ b/lib/backend/firestore/firestorebk.go @@ -16,6 +16,7 @@ package firestore import ( "bytes" + "cmp" "context" "encoding/base64" "errors" @@ -57,6 +58,9 @@ type Config struct { DisableExpiredDocumentPurge bool `json:"disable_expired_document_purge,omitempty"` // EndPoint is used to point the Firestore clients at emulated Firestore storage. EndPoint string `json:"endpoint,omitempty"` + // DatabaseID is the identifier of a specific Firestore database to use. If not specified, the + // default database for the ProjectID is used. + DatabaseID string `json:"database_id,omitempty"` } type backendConfig struct { @@ -265,14 +269,14 @@ func (t ownerCredentials) GetRequestMetadata(context.Context, ...string) (map[st func (t ownerCredentials) RequireTransportSecurity() bool { return false } // CreateFirestoreClients creates a firestore admin and normal client given the supplied parameters -func CreateFirestoreClients(ctx context.Context, projectID string, endPoint string, credentialsFile string) (*apiv1.FirestoreAdminClient, *firestore.Client, error) { +func CreateFirestoreClients(ctx context.Context, projectID, database string, endpoint string, credentialsFile string) (*apiv1.FirestoreAdminClient, *firestore.Client, error) { var args []option.ClientOption - if endPoint != "" { + if endpoint != "" { args = append(args, option.WithTelemetryDisabled(), option.WithoutAuthentication(), - option.WithEndpoint(endPoint), + option.WithEndpoint(endpoint), option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), option.WithGRPCDialOption(grpc.WithPerRPCCredentials(ownerCredentials{})), ) @@ -280,11 +284,21 @@ func CreateFirestoreClients(ctx context.Context, projectID string, endPoint stri args = append(args, option.WithCredentialsFile(credentialsFile)) } - firestoreClient, err := firestore.NewClient(ctx, projectID, args...) + firestoreAdminClient, err := apiv1.NewFirestoreAdminClient(ctx, args...) if err != nil { return nil, nil, ConvertGRPCError(err) } - firestoreAdminClient, err := apiv1.NewFirestoreAdminClient(ctx, args...) + + if database == "" { + firestoreClient, err := firestore.NewClient(ctx, projectID, args...) + if err != nil { + return nil, nil, ConvertGRPCError(err) + } + + return firestoreAdminClient, firestoreClient, nil + } + + firestoreClient, err := firestore.NewClientWithDatabase(ctx, projectID, database, args...) if err != nil { return nil, nil, ConvertGRPCError(err) } @@ -328,7 +342,7 @@ func New(ctx context.Context, params backend.Params, options Options) (*Backend, } closeCtx, cancel := context.WithCancel(ctx) - firestoreAdminClient, firestoreClient, err := CreateFirestoreClients(closeCtx, cfg.ProjectID, cfg.EndPoint, cfg.CredentialsPath) + firestoreAdminClient, firestoreClient, err := CreateFirestoreClients(closeCtx, cfg.ProjectID, cfg.DatabaseID, cfg.EndPoint, cfg.CredentialsPath) if err != nil { cancel() return nil, trace.Wrap(err) @@ -886,7 +900,8 @@ func ConvertGRPCError(err error, args ...interface{}) error { } func (b *Backend) getIndexParent() string { - return "projects/" + b.ProjectID + "/databases/(default)/collectionGroups/" + b.CollectionName + database := cmp.Or(b.backendConfig.Config.DatabaseID, "(default)") + return "projects/" + b.ProjectID + "/databases/" + database + "/collectionGroups/" + b.CollectionName } func (b *Backend) ensureIndexes(adminSvc *apiv1.FirestoreAdminClient) error { diff --git a/lib/events/firestoreevents/firestoreevents.go b/lib/events/firestoreevents/firestoreevents.go index 8caea7ac46e73..fd131555e4582 100644 --- a/lib/events/firestoreevents/firestoreevents.go +++ b/lib/events/firestoreevents/firestoreevents.go @@ -15,6 +15,7 @@ package firestoreevents import ( + "cmp" "context" "encoding/json" "errors" @@ -198,6 +199,8 @@ func (cfg *EventsConfig) SetFromURL(url *url.URL) error { } cfg.ProjectID = projectIDParamString + cfg.DatabaseID = url.Query().Get("databaseID") + eventRetentionPeriodParamString := url.Query().Get(eventRetentionPeriodPropertyKey) if eventRetentionPeriodParamString == "" { cfg.RetentionPeriod = defaultEventRetentionPeriod @@ -572,7 +575,8 @@ type searchEventsFilter struct { } func (l *Log) getIndexParent() string { - return "projects/" + l.ProjectID + "/databases/(default)/collectionGroups/" + l.CollectionName + database := cmp.Or(l.Config.DatabaseID, "(default)") + return "projects/" + l.ProjectID + "/databases/" + database + "/collectionGroups/" + l.CollectionName } func (l *Log) ensureIndexes(adminSvc *apiv1.FirestoreAdminClient) error {