Merge branch 'master' into joerger/sso-mfa-ceremony

.github/ISSUE_TEMPLATE/testplan.md

@@ -979,10 +979,14 @@ manualy testing.
   - [ ] Self-hosted MariaDB.
   - [ ] Self-hosted MongoDB.
   - [ ] Self-hosted CockroachDB.
-  - [ ] Self-hosted Redis.
+  - [ ] Self-hosted Redis/Valkey.
   - [ ] Self-hosted Redis Cluster.
   - [ ] Self-hosted MSSQL.
   - [ ] Self-hosted MSSQL with PKINIT authentication.
+  - [ ] Self-hosted Elasticsearch.
+  - [ ] Self-hosted Cassandra/ScyllaDB.
+  - [ ] Self-hosted Oracle.
+  - [ ] Self-hosted ClickHouse.
   - [ ] AWS Aurora Postgres.
   - [ ] AWS Aurora MySQL.
     - [ ] MySQL server version reported by Teleport is correct.
@@ -992,53 +996,57 @@ manualy testing.
     - [ ] Verify connection to external AWS account works with `assume_role_arn: ""` and `external_id: "<id>"`
   - [ ] AWS ElastiCache.
   - [ ] AWS MemoryDB.
+  - [ ] AWS OpenSearch.
+  - [ ] AWS Dynamodb.
+    - [ ] Verify connection to external AWS account works with `assume_role_arn: ""` and `external_id: "<id>"`
+  - [ ] AWS DocumentDB
+  - [ ] AWS Keyspaces
+    - [ ] Verify connection to external AWS account works with `assume_role_arn: ""` and `external_id: "<id>"`
   - [ ] GCP Cloud SQL Postgres.
   - [ ] GCP Cloud SQL MySQL.
   - [ ] GCP Cloud Spanner.
-  - [ ] Snowflake.
   - [ ] Azure Cache for Redis.
   - [x] Azure single-server MySQL and Postgres (EOL Sep 2024 and Mar 2025, skip)
-  - [ ] Azure flexible-server MySQL and Postgres
-  - [ ] Elasticsearch.
-  - [ ] OpenSearch.
-  - [ ] Cassandra/ScyllaDB.
-    - [ ] Verify connection to external AWS account works with `assume_role_arn: ""` and `external_id: "<id>"`
-  - [ ] Dynamodb.
-    - [ ] Verify connection to external AWS account works with `assume_role_arn: ""` and `external_id: "<id>"`
+  - [ ] Azure flexible-server MySQL
+  - [ ] Azure flexible-server Postgres
   - [ ] Azure SQL Server.
-  - [ ] Oracle.
-  - [ ] ClickHouse.
+  - [ ] Snowflake.
+  - [ ] MongoDB Atlas.
 - [ ] Connect to a database within a remote cluster via a trusted cluster.
   - [ ] Self-hosted Postgres.
   - [ ] Self-hosted MySQL.
   - [ ] Self-hosted MariaDB.
   - [ ] Self-hosted MongoDB.
   - [ ] Self-hosted CockroachDB.
-  - [ ] Self-hosted Redis.
+  - [ ] Self-hosted Redis/Valkey.
   - [ ] Self-hosted Redis Cluster.
   - [ ] Self-hosted MSSQL.
   - [ ] Self-hosted MSSQL with PKINIT authentication.
+  - [ ] Self-hosted Elasticsearch.
+  - [ ] Self-hosted Cassandra/ScyllaDB.
+  - [ ] Self-hosted Oracle.
+  - [ ] Self-hosted ClickHouse.
   - [ ] AWS Aurora Postgres.
   - [ ] AWS Aurora MySQL.
   - [ ] AWS RDS Proxy (MySQL, Postgres, MariaDB, or SQL Server)
   - [ ] AWS Redshift.
   - [ ] AWS Redshift Serverless.
   - [ ] AWS ElastiCache.
   - [ ] AWS MemoryDB.
+  - [ ] AWS OpenSearch.
+  - [ ] AWS Dynamodb.
+  - [ ] AWS DocumentDB
+  - [ ] AWS Keyspaces
   - [ ] GCP Cloud SQL Postgres.
   - [ ] GCP Cloud SQL MySQL.
   - [ ] GCP Cloud Spanner.
-  - [ ] Snowflake.
   - [ ] Azure Cache for Redis.
   - [x] Azure single-server MySQL and Postgres (EOL Sep 2024 and Mar 2025, skip)
-  - [ ] Azure flexible-server MySQL and Postgres
-  - [ ] Elasticsearch.
-  - [ ] OpenSearch.
-  - [ ] Cassandra/ScyllaDB.
-  - [ ] Dynamodb.
+  - [ ] Azure flexible-server MySQL
+  - [ ] Azure flexible-server Postgres
   - [ ] Azure SQL Server.
-  - [ ] Oracle.
-  - [ ] ClickHouse.
+  - [ ] Snowflake.
+  - [ ] MongoDB Atlas.
 - [ ] Verify auto user provisioning.
   Verify all supported modes: `keep`, `best_effort_drop`
   - [ ] Self-hosted Postgres.
@@ -1084,6 +1092,7 @@ manualy testing.
       - [ ] Can detect and register ElastiCache Redis clusters.
       - [ ] Can detect and register MemoryDB clusters.
       - [ ] Can detect and register OpenSearch domains.
+      - [ ] Can detect and register DocumentDB clusters.
     - [ ] Azure
       - [ ] Can detect and register MySQL and Postgres single-server instances.
       - [ ] Can detect and register MySQL and Postgres flexible-server instances.
@@ -1098,6 +1107,11 @@ manualy testing.
   - [ ] Verify searching for all columns in the search bar works
   - [ ] Verify you can sort by all columns except `labels`
 - [ ] `tsh bench` load tests (instructions on Notion -> Database Access -> Load test)
+- [ ] Verify database session player
+  - [ ] Web UI
+    - [ ] Postgres
+  - [ ] `tsh play`
+    - [ ] Postgres
 
 ## TLS Routing
 
@@ -1574,13 +1588,21 @@ Docs: [IP Pinning](https://goteleport.com/docs/access-controls/guides/ip-pinning
   - [ ] Verify that users can run custom audit queries.
   - [ ] Verify that the Privileged Access Report is generated and periodically refreshed.
 
-- [ ] Access List
+- [ ] Access Lists
   - [ ] Verify Access List membership/ownership/expiration date.
-    - [ ] Verify permissions granted by Access List membership.
-    - [ ] Verify permissions granted by Access List ownership.
-    - [ ] Verify Access List Review.
-    - [ ] verify Access LIst Promotion.
-    - [ ] Verify that owners can only add/remove members and not change other properties.
+  - [ ] Verify permissions granted by Access List membership.
+  - [ ] Verify permissions granted by Access List ownership.
+  - [ ] Verify Access List Review.
+  - [ ] verify Access LIst Promotion.
+  - [ ] Verify that owners can only add/remove members and not change other properties.
+  - [ ] Nested Access Lists
+    - [ ] Verify that Access Lists can be added as members or owners of other Access Lists.
+    - [ ] Verify that member grants from ancestor lists are inherited by members of nested Access Lists added as members.
+    - [ ] Verify that owner grants from ancestor lists are inherited by members of nested Access Lists added as owners.
+    - [ ] Verify that Access List Review and Promotion work with nested Access Lists.
+    - [ ] Verify that manually deleting a nested Access List used as a member or owner does not break UserLoginState generation or listing Access Lists.
+    - [ ] Verify that an Access List can be added as a member or owner of another Access List using `tctl`.
+    - [ ] Verify that Access Lists added as members or owners of other Access Lists using `tctl` are validated (no circular references, no nesting > 10 levels).
 
 - [ ] Verify Okta Sync Service
   - [ ] Verify Okta Plugin configuration.
@@ -1590,6 +1612,7 @@ Docs: [IP Pinning](https://goteleport.com/docs/access-controls/guides/ip-pinning
     - [ ] Verify that users/apps/groups are synced from Okta to Teleport.
     - [ ] Verify the custom `okta_import_rule` rule configuration.
     - [ ] Verify that users/apps/groups are displayed in the Teleport Web UI.
+    - [ ] Verify that users/groups are flattened on import, and are not duplicated on sync when their membership is inherited via nested Access Lists.
   - [ ] Verify that a user is locked/removed from Teleport when the user is Suspended/Deactivated in Okta.
   - [ ] Verify access to Okta apps granted by access_list/access_request.
 

.github/ISSUE_TEMPLATE/webtestplan.md

@@ -574,6 +574,45 @@ With the previous role you created from `Strategy Reason`, change `request_acces
 
 - [ ] Verify after login, dashboard is rendered as normal
 
+## Access Lists
+
+Not available for OSS
+
+- Creating new Access List:
+  - [ ] Verify that traits/roles are not be required in order to create
+  - [ ] Verify that one can be created with members and owners
+  - [ ] Verify the web cache is updated (new list should appear under "Access Lists" page without reloading)
+- Deleting existing Access List:
+  - [ ] Verify the web cache is updated (deleted list should disappear from "Access Lists" page without reloading)
+  - [ ] Verify that an Access List used as a member or owner in other lists cannot be deleted (should show a warning)
+- Reviewing Access List:
+  - [ ] Verify that after reviewing, the web cache is updated (list cards should show any member/role changes)
+- Updating (renaming, removing members, adding members):
+  - [ ] Verify the web cache is updated (changes to name/members appear under "Access Lists" page without reloading)
+- [ ] Verify Access List search is preserved between sub-route navigation (clicking into specific List and navigating back)
+- Can manage members/owners for an existing Access List:
+  - [ ] Verify that existing Users:
+    - [ ] Can be enrolled as members and owners
+    - [ ] Enrolled as members or owners can be removed
+  - [ ] Verify that existing Access Lists:
+    - [ ] Can be enrolled as members and owners
+    - [ ] Enrolled as members or owners can be removed
+  - [ ] Verify that an Access List cannot be added as a member or owner:
+    - [ ] If it is already a member or owner
+    - [ ] If it would result in a circular reference (ACL A -> ACL B -> ACL A)
+    - [ ] If the depth of the inheritance would exceed 10 levels
+    - [ ] If it includes yourself (and you lack RBAC)
+  - [ ] Verify that non-existing Members and Owners can be enrolled in an existing List (e.g., SSO users)
+- Inherited grants are properly calculated and displayed:
+  - [ ] Verify that members of a nested Access List:
+    - [ ] Added as a member to another Access List inherit its Member grants
+    - [ ] Added as an owner to another Access List inherit its Owner grants
+    - [ ] That do not meet Membership Requirements in a Nested List do not inherit any Grants from Parent Lists
+    - [ ] That do not meet the Parent List's Membership/Ownership Requirements do not inherit its Member/Owner Grants
+  - [ ] Verify that owners of Access Lists added as Members/Owners to other Access Lists do *not* inherit any Grants
+  - [ ] Verify that inherited grants are updated on reload or navigating away from / back to Access List View/Edit route
+  - [ ] Verify that 'View More' exists and can be clicked under the 'Inherited Member Grants' section if inherited grants overflows the container
+
 ## Web Terminal (aka console)
 
 - [ ] Verify that top nav has a user menu (Main and Logout)

docs/pages/usage-billing.mdx

@@ -64,25 +64,6 @@ Set the `TELEPORT_REPORTING_HTTPS_PROXY` and `TELEPORT_REPORTING_HTTP_PROXY`
 environment variables to your proxy address. That will apply as the HTTP connect
 proxy setting overriding `HTTPS_PROXY` and `HTTP_PROXY` just for outbound usage reporting.
 
-### Validating usage reports
-
-The system that Teleport uses for submitting usage reports is independent of the
-system that Teleport uses for submitting audit events. 
-
-Teleport processes submit audit events to the Teleport Auth Service, which
-stores them on its audit event backend for retrieval by Teleport API clients. In
-contrast, usage reports are aggregated on a submission service that runs either
-on self-hosted Teleport infrastructure or Teleport Cloud, depending on the
-user's plan. The submission service persists usage reports in the case of a
-submission failure. After a successful submission, the submission service
-deletes the reports.
-
-It is not possible for Teleport users to independently validate usage event
-data, as there is no way to set up a third-party usage event destination or
-retrieve usage events from a Teleport backend. Reach out to
-[email protected] if you have questions about usage reporting on your
-Teleport account.
-
 ## Billing metrics
 
 Teleport uses the anonymized usage data described in the previous section to
@@ -144,6 +125,11 @@ to compute a daily TPR. Then we average the daily TPR over a monthly period,
 which starts on the subscription start date and ends on each monthly anniversary
 thereafter.
 
+If you recreate a single resource more than once an hour, this will affect the
+hourly average. For example, if were to create then delete 10 servers three
+times in one hour, Teleport would display 10 servers at any given time. However,
+for the entire hour, Teleport would report 30 protected servers.
+
 ## Usage measurement for billing
 
 We aggregate all counts of the billing metrics on a monthly basis starting on
@@ -155,3 +141,50 @@ Subscription, also known as a high water mark calculation.
 
 Reach out to [email protected] if you have questions about the
 commercial editions of Teleport.
+
+## Troubleshooting usage and billing
+
+Teleport aggregates usage reports on a submission service that runs either on
+self-hosted Teleport infrastructure or Teleport Cloud, depending on the user's
+plan. The submission service persists usage reports in the case of a submission
+failure, and deletes the reports after a successful submission. It is not
+possible to set up a third-party destination for usage events to independently
+verify usage event data. 
+
+If you are using Teleport Enterprise (Cloud), your usage data is accurate as
+long as Teleport-managed reporting infrastructure works as expected (check the
+[status page](https://status.teleport.sh/) for any incidents). On self-hosted
+Teleport Enterprise clusters, some conditions can interfere with data reporting.
+This section describes some scenarios that can lead to inaccurate data on
+self-hosted clusters.
+
+If you suspect that any of these scenarios describe your Teleport cluster, or
+your usage data appears inaccurate, reach out to [email protected].
+
+### Multiple Teleport clusters
+
+In versions older than v14.3.1, Teleport does not de-duplicate users as expected
+across multiple Teleport clusters that belong to the same account. If you are
+running multiple Teleport clusters with affected versions, the count of active
+users may be higher than expected.
+
+### Unexpected license differences
+
+When distributing copies of your Teleport Enterprise (Self-Hosted) license
+across Auth Service instances, you must not download a license multiple times
+from your Teleport account. Instead, you must download a license once and copy
+that license across Auth Service instances. Otherwise, the Teleport usage
+reporting infrastructure will identify multiple licenses and misrepresent your
+usage numbers.
+
+### SSO users
+
+In Teleport, single sign-on (SSO) users are
+[ephemeral](reference/user-types.mdx#temporary-users). Teleport deletes an SSO user
+when its session expires. To count the number of SSO users in your cluster, you
+can examine Teleport audit events for unique SSO users that have authenticated
+to Teleport during a given time period. The Teleport documentation includes
+[how-to guides](./admin-guides/management/export-audit-events/export-audit-events.mdx) for
+exporting audit events to common log management solutions so you can identify
+users that have authenticated using an SSO provider.
+

lib/kube/proxy/server.go

@@ -45,6 +45,7 @@ import (
 	"github.com/gravitational/teleport/lib/multiplexer"
 	"github.com/gravitational/teleport/lib/reversetunnel"
 	"github.com/gravitational/teleport/lib/services"
+	"github.com/gravitational/teleport/lib/services/readonly"
 	"github.com/gravitational/teleport/lib/srv"
 	"github.com/gravitational/teleport/lib/srv/ingress"
 )
@@ -98,7 +99,7 @@ type TLSServerConfig struct {
 	// kubernetes cluster name. Proxy uses this map to route requests to the correct
 	// kubernetes_service. The servers are kept in memory to avoid making unnecessary
 	// unmarshal calls followed by filtering and to improve memory usage.
-	KubernetesServersWatcher *services.KubeServerWatcher
+	KubernetesServersWatcher *services.GenericWatcher[types.KubeServer, readonly.KubeServer]
 	// PROXYProtocolMode controls behavior related to unsigned PROXY protocol headers.
 	PROXYProtocolMode multiplexer.PROXYProtocolMode
 	// InventoryHandle is used to send kube server heartbeats via the inventory control stream.
@@ -170,7 +171,7 @@ type TLSServer struct {
 	closeContext context.Context
 	closeFunc    context.CancelFunc
 	// kubeClusterWatcher monitors changes to kube cluster resources.
-	kubeClusterWatcher *services.KubeClusterWatcher
+	kubeClusterWatcher *services.GenericWatcher[types.KubeCluster, readonly.KubeCluster]
 	// reconciler reconciles proxied kube clusters with kube_clusters resources.
 	reconciler *services.Reconciler[types.KubeCluster]
 	// monitoredKubeClusters contains all kube clusters the proxied kube_clusters are
@@ -620,7 +621,9 @@ func (t *TLSServer) getKubernetesServersForKubeClusterFunc() (getKubeServersByNa
 		}, nil
 	case ProxyService:
 		return func(ctx context.Context, name string) ([]types.KubeServer, error) {
-			servers, err := t.KubernetesServersWatcher.GetKubeServersByClusterName(ctx, name)
+			servers, err := t.KubernetesServersWatcher.CurrentResourcesWithFilter(ctx, func(ks readonly.KubeServer) bool {
+				return ks.GetCluster().GetName() == name
+			})
 			return servers, trace.Wrap(err)
 		}, nil
 	case LegacyProxyService:
@@ -630,7 +633,9 @@ func (t *TLSServer) getKubernetesServersForKubeClusterFunc() (getKubeServersByNa
 			// and forward the request to the next proxy.
 			kube, err := t.getKubeClusterWithServiceLabels(name)
 			if err != nil {
-				servers, err := t.KubernetesServersWatcher.GetKubeServersByClusterName(ctx, name)
+				servers, err := t.KubernetesServersWatcher.CurrentResourcesWithFilter(ctx, func(ks readonly.KubeServer) bool {
+					return ks.GetCluster().GetName() == name
+				})
 				return servers, trace.Wrap(err)
 			}
 			srv, err := types.NewKubernetesServerV3FromCluster(kube, "", t.HostID)

lib/kube/proxy/utils_testing.go

@@ -294,6 +294,7 @@ func SetupTestContext(ctx context.Context, t *testing.T, cfg TestConfig) *TestCo
 				Component: teleport.ComponentKube,
 				Client:    client,
 			},
+			KubernetesServerGetter: client,
 		},
 	)
 	require.NoError(t, err)
@@ -387,7 +388,7 @@ func SetupTestContext(ctx context.Context, t *testing.T, cfg TestConfig) *TestCo
 
 	// Ensure watcher has the correct list of clusters.
 	require.Eventually(t, func() bool {
-		kubeServers, err := kubeServersWatcher.GetKubernetesServers(ctx)
+		kubeServers, err := kubeServersWatcher.CurrentResources(ctx)
 		return err == nil && len(kubeServers) == len(cfg.Clusters)
 	}, 3*time.Second, time.Millisecond*100)
 

lib/kube/proxy/watcher.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/lib/services"
+	"github.com/gravitational/teleport/lib/services/readonly"
 	"github.com/gravitational/teleport/lib/utils"
 )
 
@@ -89,7 +90,7 @@ func (s *TLSServer) startReconciler(ctx context.Context) (err error) {
 
 // startKubeClusterResourceWatcher starts watching changes to Kube Clusters resources and
 // registers/unregisters the proxied Kube Cluster accordingly.
-func (s *TLSServer) startKubeClusterResourceWatcher(ctx context.Context) (*services.KubeClusterWatcher, error) {
+func (s *TLSServer) startKubeClusterResourceWatcher(ctx context.Context) (*services.GenericWatcher[types.KubeCluster, readonly.KubeCluster], error) {
 	if len(s.ResourceMatchers) == 0 || s.KubeServiceType != KubeService {
 		s.log.Debug("Not initializing Kube Cluster resource watcher.")
 		return nil, nil
@@ -102,6 +103,7 @@ func (s *TLSServer) startKubeClusterResourceWatcher(ctx context.Context) (*servi
 			// Logger:       s.log,
 			Client: s.AccessPoint,
 		},
+		KubernetesClusterGetter: s.AccessPoint,
 	})
 	if err != nil {
 		return nil, trace.Wrap(err)
@@ -110,7 +112,7 @@ func (s *TLSServer) startKubeClusterResourceWatcher(ctx context.Context) (*servi
 		defer watcher.Close()
 		for {
 			select {
-			case clusters := <-watcher.KubeClustersC:
+			case clusters := <-watcher.ResourcesC:
 				s.monitoredKubeClusters.setResources(clusters)
 				select {
 				case s.reconcileCh <- struct{}{}: