From 410207b64941c9c9d31b27c01e8485d99e1952dc Mon Sep 17 00:00:00 2001
From: rosstimothy <39066650+rosstimothy@users.noreply.github.com>
Date: Wed, 21 Aug 2024 11:10:04 -0400
Subject: [PATCH 1/7] Update resolving SSH hosts to use unified API (#45644)

A new GetAllUnifiedResources function was added to replace
GetAllResources so that ssh host resolution during tsh ssh used
the more performant API. Additionally, the fallback clientside
filtering in GetSSHTargets that used GetAllResources was removed
as all supported auth instances should implement the GetSSHTargets
RPC.
---
 api/client/client.go   | 56 ++++++++++++++++++++----------------------
 lib/client/api.go      | 18 +++++++++++---
 lib/client/api_test.go |  9 +++++++
 3 files changed, 50 insertions(+), 33 deletions(-)

diff --git a/api/client/client.go b/api/client/client.go
index 0dca361ed8f7c..3e2d6b34584ee 100644
--- a/api/client/client.go
+++ b/api/client/client.go
@@ -3788,6 +3788,31 @@ func GetUnifiedResourcePage(ctx context.Context, clt ListUnifiedResourcesClient,
 	}
 }
 
+// GetAllUnifiedResources is a helper for getting all existing resources that match the provided request. In addition to
+// iterating pages, it also correctly handles downsizing pages when LimitExceeded errors are encountered.
+func GetAllUnifiedResources(ctx context.Context, clt ListUnifiedResourcesClient, req *proto.ListUnifiedResourcesRequest) ([]*types.EnrichedResource, error) {
+	var out []*types.EnrichedResource
+
+	// Set the limit to the default size.
+	req.Limit = int32(defaults.DefaultChunkSize)
+	for {
+		resources, nextKey, err := GetUnifiedResourcePage(ctx, clt, req)
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
+
+		out = append(out, resources...)
+
+		if nextKey == "" || len(resources) == 0 {
+			break
+		}
+
+		req.StartKey = nextKey
+	}
+
+	return out, nil
+}
+
 // GetEnrichedResourcePage is a helper for getting a single page of enriched resources.
 func GetEnrichedResourcePage(ctx context.Context, clt GetResourcesClient, req *proto.ListResourcesRequest) (ResourcePage[*types.EnrichedResource], error) {
 	var out ResourcePage[*types.EnrichedResource]
@@ -4064,36 +4089,7 @@ func GetKubernetesResourcesWithFilters(ctx context.Context, clt kubeproto.KubeSe
 // but may result in confusing behavior if it is used outside of those contexts.
 func (c *Client) GetSSHTargets(ctx context.Context, req *proto.GetSSHTargetsRequest) (*proto.GetSSHTargetsResponse, error) {
 	rsp, err := c.grpc.GetSSHTargets(ctx, req)
-	if err := trace.Wrap(err); !trace.IsNotImplemented(err) {
-		return rsp, err
-	}
-
-	// if we got a not implemented error, fallback to client-side filtering
-	servers, err := GetAllResources[*types.ServerV2](ctx, c, &proto.ListResourcesRequest{
-		ResourceType:     types.KindNode,
-		UseSearchAsRoles: true,
-	})
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-
-	// we only get here if we hit a NotImplementedError from GetSSHTargets, which means
-	// we should be performing client-side filtering with default parameters instead.
-	routeMatcher := utils.NewSSHRouteMatcher(req.Host, req.Port, false)
-
-	// do client-side filtering
-	filtered := servers[:0]
-	for _, srv := range servers {
-		if !routeMatcher.RouteToServer(srv) {
-			continue
-		}
-
-		filtered = append(filtered, srv)
-	}
-
-	return &proto.GetSSHTargetsResponse{
-		Servers: filtered,
-	}, nil
+	return rsp, trace.Wrap(err)
 }
 
 // CreateSessionTracker creates a tracker resource for an active session.
diff --git a/lib/client/api.go b/lib/client/api.go
index 4f1086827dd9c..b02f177a35c76 100644
--- a/lib/client/api.go
+++ b/lib/client/api.go
@@ -1374,7 +1374,7 @@ type targetNode struct {
 
 // getTargetNodes returns a list of node addresses this SSH command needs to
 // operate on.
-func (tc *TeleportClient) getTargetNodes(ctx context.Context, clt client.GetResourcesClient, options SSHOptions) ([]targetNode, error) {
+func (tc *TeleportClient) getTargetNodes(ctx context.Context, clt client.ListUnifiedResourcesClient, options SSHOptions) ([]targetNode, error) {
 	ctx, span := tc.Tracer.Start(
 		ctx,
 		"teleportClient/getTargetNodes",
@@ -1393,16 +1393,28 @@ func (tc *TeleportClient) getTargetNodes(ctx context.Context, clt client.GetReso
 
 	// Query for nodes if labels, fuzzy search, or predicate expressions were provided.
 	if len(tc.Labels) > 0 || len(tc.SearchKeywords) > 0 || tc.PredicateExpression != "" {
-		nodes, err := client.GetAllResources[types.Server](ctx, clt, tc.ResourceFilter(types.KindNode))
+		nodes, err := client.GetAllUnifiedResources(ctx, clt, &proto.ListUnifiedResourcesRequest{
+			Kinds:               []string{types.KindNode},
+			SortBy:              types.SortBy{Field: types.ResourceMetadataName},
+			Labels:              tc.Labels,
+			SearchKeywords:      tc.SearchKeywords,
+			PredicateExpression: tc.PredicateExpression,
+			UseSearchAsRoles:    tc.UseSearchAsRoles,
+		})
 		if err != nil {
 			return nil, trace.Wrap(err)
 		}
 
 		retval := make([]targetNode, 0, len(nodes))
 		for _, resource := range nodes {
+			server, ok := resource.ResourceWithLabels.(types.Server)
+			if !ok {
+				continue
+			}
+
 			// always dial nodes by UUID
 			retval = append(retval, targetNode{
-				hostname: resource.GetHostname(),
+				hostname: server.GetHostname(),
 				addr:     fmt.Sprintf("%s:0", resource.GetName()),
 			})
 		}
diff --git a/lib/client/api_test.go b/lib/client/api_test.go
index 9b1e18e7d2f1d..3dcf18b2aa15b 100644
--- a/lib/client/api_test.go
+++ b/lib/client/api_test.go
@@ -1270,6 +1270,15 @@ func (f fakeResourceClient) GetResources(ctx context.Context, req *proto.ListRes
 	return &proto.ListResourcesResponse{Resources: out}, nil
 }
 
+func (f fakeResourceClient) ListUnifiedResources(ctx context.Context, req *proto.ListUnifiedResourcesRequest) (*proto.ListUnifiedResourcesResponse, error) {
+	out := make([]*proto.PaginatedResource, 0, len(f.nodes))
+	for _, n := range f.nodes {
+		out = append(out, &proto.PaginatedResource{Resource: &proto.PaginatedResource_Node{Node: n}})
+	}
+
+	return &proto.ListUnifiedResourcesResponse{Resources: out}, nil
+}
+
 func TestGetTargetNodes(t *testing.T) {
 	tests := []struct {
 		name      string

From ea0460581596226077ffc34afc1e4daeac099235 Mon Sep 17 00:00:00 2001
From: Anton Miniailo <anton@goteleport.com>
Date: Wed, 21 Aug 2024 11:30:20 -0400
Subject: [PATCH 2/7] Unify getting agent version for EKS enrollment during
 manual and auto-discovery flows (#45661)

---
 lib/kube/utils/utils.go                       | 28 ++++++++
 lib/kube/utils/utils_test.go                  | 72 +++++++++++++++++++
 lib/srv/discovery/kube_integration_watcher.go | 21 +-----
 .../kube_integration_watcher_test.go          | 70 ------------------
 lib/web/integrations_awsoidc.go               | 13 ++--
 5 files changed, 106 insertions(+), 98 deletions(-)

diff --git a/lib/kube/utils/utils.go b/lib/kube/utils/utils.go
index 5b62621bc1147..f52aad3166301 100644
--- a/lib/kube/utils/utils.go
+++ b/lib/kube/utils/utils.go
@@ -23,6 +23,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"slices"
+	"strings"
 
 	"github.com/gravitational/trace"
 	"k8s.io/client-go/kubernetes"
@@ -32,6 +33,8 @@ import (
 	"github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/types"
+	"github.com/gravitational/teleport/lib/automaticupgrades"
+	"github.com/gravitational/teleport/lib/automaticupgrades/version"
 )
 
 // GetKubeClient returns instance of client to the kubernetes cluster
@@ -218,6 +221,31 @@ func extractAndSortKubeClusters(kss []types.KubeServer) []types.KubeCluster {
 	return []types.KubeCluster(sorted)
 }
 
+type Pinger interface {
+	Ping(context.Context) (proto.PingResponse, error)
+}
+
+// GetKubeAgentVersion returns a version of the Kube agent appropriate for this Teleport cluster. Used for example when deciding version
+// for enrolling EKS clusters.
+func GetKubeAgentVersion(ctx context.Context, pinger Pinger, clusterFeatures proto.Features, releaseChannels automaticupgrades.Channels) (string, error) {
+	pingResponse, err := pinger.Ping(ctx)
+	if err != nil {
+		return "", trace.Wrap(err)
+	}
+	agentVersion := pingResponse.ServerVersion
+
+	if clusterFeatures.GetAutomaticUpgrades() && clusterFeatures.GetCloud() {
+		defaultVersion, err := releaseChannels.DefaultVersion(ctx)
+		if err == nil {
+			agentVersion = defaultVersion
+		} else if !errors.Is(err, &version.NoNewVersionError{}) {
+			return "", trace.Wrap(err)
+		}
+	}
+
+	return strings.TrimPrefix(agentVersion, "v"), nil
+}
+
 // CheckKubeCluster validates kubeClusterName is registered with this Teleport cluster.
 func CheckKubeCluster(ctx context.Context, p KubeServicesPresence, kubeClusterName string) error {
 	if kubeClusterName == "" {
diff --git a/lib/kube/utils/utils_test.go b/lib/kube/utils/utils_test.go
index 1e32c07d3241d..fb90ebd0a3194 100644
--- a/lib/kube/utils/utils_test.go
+++ b/lib/kube/utils/utils_test.go
@@ -22,9 +22,12 @@ import (
 	"context"
 	"testing"
 
+	"github.com/gravitational/trace"
 	"github.com/stretchr/testify/require"
 
+	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/types"
+	"github.com/gravitational/teleport/lib/automaticupgrades"
 )
 
 func TestCheckKubeCluster(t *testing.T) {
@@ -91,6 +94,75 @@ func kubeServer(t *testing.T, kubeCluster, hostname, hostID string) types.KubeSe
 	return server
 }
 
+func TestGetAgentVersion(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	testCases := []struct {
+		desc            string
+		ping            func(ctx context.Context) (proto.PingResponse, error)
+		clusterFeatures proto.Features
+		channelVersion  string
+		expectedVersion string
+		errorAssert     require.ErrorAssertionFunc
+	}{
+		{
+			desc: "ping error",
+			ping: func(ctx context.Context) (proto.PingResponse, error) {
+				return proto.PingResponse{}, trace.BadParameter("ping error")
+			},
+			expectedVersion: "",
+			errorAssert:     require.Error,
+		},
+		{
+			desc: "no automatic upgrades",
+			ping: func(ctx context.Context) (proto.PingResponse, error) {
+				return proto.PingResponse{ServerVersion: "1.2.3"}, nil
+			},
+			expectedVersion: "1.2.3",
+			errorAssert:     require.NoError,
+		},
+		{
+			desc: "automatic upgrades",
+			ping: func(ctx context.Context) (proto.PingResponse, error) {
+				return proto.PingResponse{ServerVersion: "10"}, nil
+			},
+			clusterFeatures: proto.Features{AutomaticUpgrades: true, Cloud: true},
+			channelVersion:  "v1.2.3",
+			expectedVersion: "1.2.3",
+			errorAssert:     require.NoError,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			p := &pinger{pingFn: tt.ping}
+
+			var channel *automaticupgrades.Channel
+			if tt.channelVersion != "" {
+				channel = &automaticupgrades.Channel{StaticVersion: tt.channelVersion}
+				err := channel.CheckAndSetDefaults()
+				require.NoError(t, err)
+			}
+			releaseChannels := automaticupgrades.Channels{automaticupgrades.DefaultChannelName: channel}
+
+			version, err := GetKubeAgentVersion(ctx, p, tt.clusterFeatures, releaseChannels)
+
+			tt.errorAssert(t, err)
+			require.Equal(t, tt.expectedVersion, version)
+		})
+	}
+}
+
+type pinger struct {
+	pingFn func(ctx context.Context) (proto.PingResponse, error)
+}
+
+func (p *pinger) Ping(ctx context.Context) (proto.PingResponse, error) {
+	return p.pingFn(ctx)
+}
+
 func TestExtractAndSortKubeClusterNames(t *testing.T) {
 	t.Parallel()
 
diff --git a/lib/srv/discovery/kube_integration_watcher.go b/lib/srv/discovery/kube_integration_watcher.go
index fa593e8a668ae..58c7228b4f031 100644
--- a/lib/srv/discovery/kube_integration_watcher.go
+++ b/lib/srv/discovery/kube_integration_watcher.go
@@ -20,7 +20,6 @@ package discovery
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"slices"
 	"strings"
@@ -32,7 +31,7 @@ import (
 	integrationv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/integration/v1"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/lib/automaticupgrades"
-	"github.com/gravitational/teleport/lib/automaticupgrades/version"
+	kubeutils "github.com/gravitational/teleport/lib/kube/utils"
 	"github.com/gravitational/teleport/lib/srv/discovery/common"
 )
 
@@ -215,23 +214,7 @@ func (s *Server) enrollEKSClusters(region, integration string, clusters []types.
 }
 
 func (s *Server) getKubeAgentVersion(releaseChannels automaticupgrades.Channels) (string, error) {
-	pingResponse, err := s.AccessPoint.Ping(s.ctx)
-	if err != nil {
-		return "", trace.Wrap(err)
-	}
-	agentVersion := pingResponse.ServerVersion
-
-	clusterFeatures := s.ClusterFeatures()
-	if clusterFeatures.GetAutomaticUpgrades() && clusterFeatures.GetCloud() {
-		defaultVersion, err := releaseChannels.DefaultVersion(s.ctx)
-		if err == nil {
-			agentVersion = defaultVersion
-		} else if !errors.Is(err, &version.NoNewVersionError{}) {
-			return "", trace.Wrap(err)
-		}
-	}
-
-	return strings.TrimPrefix(agentVersion, "v"), nil
+	return kubeutils.GetKubeAgentVersion(s.ctx, s.AccessPoint, s.ClusterFeatures(), releaseChannels)
 }
 
 type IntegrationFetcher interface {
diff --git a/lib/srv/discovery/kube_integration_watcher_test.go b/lib/srv/discovery/kube_integration_watcher_test.go
index 5075da163d392..f6cab69c9ec46 100644
--- a/lib/srv/discovery/kube_integration_watcher_test.go
+++ b/lib/srv/discovery/kube_integration_watcher_test.go
@@ -46,7 +46,6 @@ import (
 	"github.com/gravitational/teleport/lib/auth"
 	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/authz"
-	"github.com/gravitational/teleport/lib/automaticupgrades"
 	"github.com/gravitational/teleport/lib/cloud"
 	"github.com/gravitational/teleport/lib/cloud/mocks"
 	"github.com/gravitational/teleport/lib/integrations/awsoidc"
@@ -55,75 +54,6 @@ import (
 	"github.com/gravitational/teleport/lib/srv/discovery/fetchers"
 )
 
-func TestGetAgentVersion(t *testing.T) {
-	t.Parallel()
-
-	ctx := context.Background()
-
-	testCases := []struct {
-		desc            string
-		ping            func(ctx context.Context) (proto.PingResponse, error)
-		clusterFeatures proto.Features
-		channelVersion  string
-		expectedVersion string
-		errorAssert     require.ErrorAssertionFunc
-	}{
-		{
-			desc: "ping error",
-			ping: func(ctx context.Context) (proto.PingResponse, error) {
-				return proto.PingResponse{}, trace.BadParameter("ping error")
-			},
-			expectedVersion: "",
-			errorAssert:     require.Error,
-		},
-		{
-			desc: "no automatic upgrades",
-			ping: func(ctx context.Context) (proto.PingResponse, error) {
-				return proto.PingResponse{ServerVersion: "1.2.3"}, nil
-			},
-			expectedVersion: "1.2.3",
-			errorAssert:     require.NoError,
-		},
-		{
-			desc: "automatic upgrades",
-			ping: func(ctx context.Context) (proto.PingResponse, error) {
-				return proto.PingResponse{ServerVersion: "10"}, nil
-			},
-			clusterFeatures: proto.Features{AutomaticUpgrades: true, Cloud: true},
-			channelVersion:  "v1.2.3",
-			expectedVersion: "1.2.3",
-			errorAssert:     require.NoError,
-		},
-	}
-
-	for _, tt := range testCases {
-		t.Run(tt.desc, func(t *testing.T) {
-			server := Server{
-				ctx: ctx,
-				Config: &Config{
-					AccessPoint: &fakeAccessPoint{ping: tt.ping},
-					ClusterFeatures: func() proto.Features {
-						return tt.clusterFeatures
-					},
-				},
-			}
-
-			var channel *automaticupgrades.Channel
-			if tt.channelVersion != "" {
-				channel = &automaticupgrades.Channel{StaticVersion: tt.channelVersion}
-				err := channel.CheckAndSetDefaults()
-				require.NoError(t, err)
-			}
-			releaseChannels := automaticupgrades.Channels{automaticupgrades.DefaultChannelName: channel}
-
-			version, err := server.getKubeAgentVersion(releaseChannels)
-
-			tt.errorAssert(t, err)
-			require.Equal(t, tt.expectedVersion, version)
-		})
-	}
-}
-
 func TestServer_getKubeFetchers(t *testing.T) {
 	eks1, err := fetchers.NewEKSFetcher(fetchers.EKSFetcherConfig{
 		ClientGetter: &cloud.TestCloudClients{STS: &mocks.STSMock{}},
diff --git a/lib/web/integrations_awsoidc.go b/lib/web/integrations_awsoidc.go
index 9c0ca267130d1..888882579f8dc 100644
--- a/lib/web/integrations_awsoidc.go
+++ b/lib/web/integrations_awsoidc.go
@@ -45,6 +45,7 @@ import (
 	"github.com/gravitational/teleport/lib/httplib"
 	"github.com/gravitational/teleport/lib/integrations/awsoidc"
 	"github.com/gravitational/teleport/lib/integrations/awsoidc/deployserviceconfig"
+	kubeutils "github.com/gravitational/teleport/lib/kube/utils"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
 	"github.com/gravitational/teleport/lib/services"
 	libutils "github.com/gravitational/teleport/lib/utils"
@@ -480,15 +481,9 @@ func (h *Handler) awsOIDCEnrollEKSClusters(w http.ResponseWriter, r *http.Reques
 		return nil, trace.BadParameter("an integration name is required")
 	}
 
-	// todo(anton): get auth server version and use it instead of this proxy teleport.version.
-	agentVersion := teleport.Version
-	if h.ClusterFeatures.GetAutomaticUpgrades() {
-		upgradesVersion, err := h.cfg.AutomaticUpgradesChannels.DefaultVersion(ctx)
-		if err != nil {
-			return "", trace.Wrap(err)
-		}
-
-		agentVersion = strings.TrimPrefix(upgradesVersion, "v")
+	agentVersion, err := kubeutils.GetKubeAgentVersion(ctx, h.cfg.ProxyClient, h.ClusterFeatures, h.cfg.AutomaticUpgradesChannels)
+	if err != nil {
+		return nil, trace.Wrap(err)
 	}
 
 	response, err := clt.IntegrationAWSOIDCClient().EnrollEKSClusters(ctx, &integrationv1.EnrollEKSClustersRequest{

From 4558f28ca2682e05c6e75f89f771e0601e14ad90 Mon Sep 17 00:00:00 2001
From: Noah Stride <noah.stride@goteleport.com>
Date: Wed, 21 Aug 2024 18:03:21 +0100
Subject: [PATCH 3/7] Machine ID: Document Kubernetes Workload Attestation
 (#45369) (#45584)

* Add KWA features into the machine id config reference

* Add example manifests for tbot service for workload attestation

* Update docs/pages/enroll-resources/machine-id/workload-identity/workload-attestation.mdx



* Add more detail to explanation of workload attestation

* Add table

* Update cspell

* Fix spelling of connect

* Fix spelling of attested

---------

Co-authored-by: Paul Gottschling <paul.gottschling@goteleport.com>
---
 docs/cspell.json                              |   2 +
 .../machine-id/workload-identity.mdx          |   1 +
 .../workload-attestation.mdx                  | 239 ++++++++++++++++++
 .../reference/machine-id/configuration.mdx    |  77 ++++--
 4 files changed, 300 insertions(+), 19 deletions(-)
 create mode 100644 docs/pages/enroll-resources/machine-id/workload-identity/workload-attestation.mdx

diff --git a/docs/cspell.json b/docs/cspell.json
index 9d5fb1b66b94c..6efcb0d625837 100644
--- a/docs/cspell.json
+++ b/docs/cspell.json
@@ -121,6 +121,7 @@
     "Keycloak",
     "Keyspaces",
     "knownhosts",
+    "Kubelet",
     "Kubernetes",
     "Kubes",
     "LDAPS",
@@ -292,6 +293,7 @@
     "argoproj",
     "armv",
     "atburke",
+    "attested",
     "attrname",
     "attobj",
     "auditctl",
diff --git a/docs/pages/enroll-resources/machine-id/workload-identity.mdx b/docs/pages/enroll-resources/machine-id/workload-identity.mdx
index d8f78125ec175..af56023caad1f 100644
--- a/docs/pages/enroll-resources/machine-id/workload-identity.mdx
+++ b/docs/pages/enroll-resources/machine-id/workload-identity.mdx
@@ -148,6 +148,7 @@ Teleport's Workload Identity currently only supports issuing X.509-SVIDs.
 
 - [Getting Started](./workload-identity/getting-started.mdx): How to configure Teleport for Workload Identity.
 - [Best Practices](./workload-identity/best-practices.mdx): Best practices for using Workload Identity in Production.
+- [Workload Attestation](./workload-identity/workload-attestation.mdx): Learn about using Workload Attestation to securely issue SVIDs to specific workloads.
 - [TSH Support](./workload-identity/tsh.mdx): How to use `tsh` with Workload Identity to issue SVIDs to users.
 - [AWS Roles Anywhere](./workload-identity/aws-roles-anywhere.mdx): Configuring AWS to accept Workload ID certificates as authentication using AWS Roles Anywhere.
 
diff --git a/docs/pages/enroll-resources/machine-id/workload-identity/workload-attestation.mdx b/docs/pages/enroll-resources/machine-id/workload-identity/workload-attestation.mdx
new file mode 100644
index 0000000000000..8bc4846432628
--- /dev/null
+++ b/docs/pages/enroll-resources/machine-id/workload-identity/workload-attestation.mdx
@@ -0,0 +1,239 @@
+---
+title: Workload Attestation
+description: An overview of the Teleport Workload Identity Workload Attestation feature.
+---
+
+<Admonition type="tip" title="Preview">
+Teleport Workload Identity is currently in Preview. This means that some
+features may be missing. We're actively looking for design partners to help us
+shape the future of Workload Identity and would love to
+[hear your feedback](mailto:product@goteleport.com).
+</Admonition>
+
+Workload Attestation is the process completed by `tbot` to assert the identity
+of a workload that has connected to the Workload API and requested certificates.
+The information gathered during attestation is used to decide which, if any,
+SPIFFE IDs should be encoded into an SVID and issued to the workload.
+
+Workload Attestors are the individual components that perform this attestation.
+They use the process ID of the workload to gather information about the workload
+from platform-specific APIs. For example, the Kubernetes Workload Attestor
+queries the local Kubelet API to determine which Kubernetes pod the process
+belongs to.
+
+The result of the attestation process is known as attestation metadata. This
+attestation metadata is referred to by the rules you configured for `tbot`'s
+Workload API service. For example, you may state that only workloads running in
+a specific Kubernetes namespace should be issued a specific SPIFFE ID.
+
+Additionally, this metadata is included in the log messages output by `tbot`
+when it issues an SVID. This allows you to audit the issuance of SVIDs and
+understand why a specific SPIFFE ID was issued to a workload.
+
+## Unix
+
+The Unix Workload Attestor is the most basic attestor and allows you to restrict
+the issuance of SVIDs to specific Unix processes based on a range of criteria.
+
+### Attestation Metadata
+
+The following metadata is produced by the Unix Workload Attestor and is
+available to be used when configuring rules for `tbot`'s Workload API service:
+
+| Field             | Description                                                                  |
+|-------------------|------------------------------------------------------------------------------|
+| `unix.attested` | Indicates that the workload has been attested by the Unix Workload Attestor. |
+| `unix.pid`        | The process ID of the attested workload.                                     |
+| `unix.uid`        | The effective user ID of the attested workload.                              |
+| `unix.gid`        | The effective primary group ID of the attested workload.                     |
+
+## Kubernetes
+
+The Kubernetes Workload Attestor allows you to restrict the issuance of SVIDs
+to specific Kubernetes workloads based on a range of criteria.
+
+It works by first determining the pod ID for a given process ID and then by
+querying the local kubelet API for details about that pod.
+
+### Attestation Metadata
+
+The following metadata is produced by the Kubernetes Workload Attestor and is
+available to be used when configuring rules for `tbot`'s Workload API service:
+
+| Field                        | Description                                                                        |
+|------------------------------|------------------------------------------------------------------------------------|
+| `kubernetes.attested`      | Indicates that the workload has been attested by the Kubernetes Workload Attestor. |
+| `kubernetes.namespace`       | The namespace of the Kubernetes Pod.                                               |
+| `kubernetes.service_account` | The service account of the Kubernetes Pod.                                         |
+| `kubernetes.pod_name`        | The name of the Kubernetes Pod.                                                    |
+
+### Deployment Guidance
+
+To use Kubernetes Workload Attestation, `tbot` must be deployed as a daemon
+set. This is because the unix domain socket can only be accessed by pods on the
+same node as the agent. Additionally, the daemon set must have the `hostPID`
+property set to `true` to allow the agent to access information about
+processes within other containers.
+
+The daemon set must also have a service account assigned that allows it to query
+the Kubelet API. This is an example role with the required RBAC:
+
+```yaml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: tbot
+rules:
+  - resources: ["pods","nodes","nodes/proxy"]
+    apiGroups: [""]
+    verbs: ["get"]
+```
+
+Mapping the Workload API Unix domain socket into the containers of workloads
+can be done in two ways:
+
+- Directly configuring a hostPath volume for the `tbot` daemonset and workloads
+  which will need to connect to it.
+- Using [spiffe-csi-driver](https://github.com/spiffe/spiffe-csi).
+
+Example manifests for required Kubernetes resources:
+
+```yaml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: tbot
+rules:
+  - resources: ["pods","nodes","nodes/proxy"]
+    apiGroups: [""]
+    verbs: ["get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: tbot
+subjects:
+  - kind: ServiceAccount
+    name: tbot
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: tbot
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: tbot
+  namespace: default
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tbot-config
+  namespace: default
+data:
+  tbot.yaml: |
+    version: v2
+    onboarding:
+      join_method: kubernetes
+      # replace with the name of a join token you have created.
+      token: example-token
+    storage:
+      type: memory
+    # ensure this is configured to the address of your Teleport Proxy Service.
+    proxy_server: example.teleport.sh:443
+    services:
+      - type: spiffe-workload-api
+        listen: unix:///run/tbot/sockets/workload.sock
+        attestor:
+          kubernetes:
+            enabled: true
+            kubelet:
+              # skip verification of the Kubelet API certificate as this is not
+              # usually issued by the cluster CA.
+              skip_verify: true
+        # replace the svid entries with the SPIFFE IDs that you wish to issue,
+        # using the `rules` blocks to restrict these to specific Kubernetes
+        # workloads.
+        svids:
+          - path: /my-service
+            rules:
+              - kubernetes:
+                  namespace: default
+                  service_account: example-sa
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: tbot
+spec:
+  selector:
+      matchLabels:
+        app: tbot
+  template:
+    metadata:
+      labels:
+        app: tbot
+    spec:
+      securityContext:
+        runAsUser: 0
+        runAsGroup: 0
+      hostPID: true
+      containers:
+        - name: tbot
+          image: public.ecr.aws/gravitational/tbot-distroless:(=teleport.version=)
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            privileged: true
+          args:
+            - start
+            - -c
+            - /config/tbot.yaml
+            - --log-format
+            - json
+          volumeMounts:
+            - mountPath: /config
+              name: config
+            - mountPath: /var/run/secrets/tokens
+              name: join-sa-token
+            - name: tbot-sockets
+              mountPath: /run/tbot/sockets
+              readOnly: false
+          env:
+            - name: TELEPORT_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: KUBERNETES_TOKEN_PATH
+              value: /var/run/secrets/tokens/join-sa-token
+      serviceAccountName: tbot
+      volumes:
+        - name: tbot-sockets
+          hostPath:
+            path: /run/tbot/sockets
+            type: DirectoryOrCreate
+        - name: config
+          configMap:
+            name: tbot-config
+        - name: join-sa-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                  path: join-sa-token
+                  # 600 seconds is the minimum that Kubernetes supports. We
+                  # recommend this value is used.
+                  expirationSeconds: 600
+                  # `example.teleport.sh` must be replaced with the name of
+                  # your Teleport cluster.
+                  audience: example.teleport.sh
+```
+
+## Next steps
+
+- [Workload Identity Overview](../workload-identity.mdx): Overview of Teleport
+Workload Identity.
+- [Best Practices](./best-practices.mdx): Best practices for using Workload
+Identity in Production.
+- Read the [configuration reference](../../../reference/machine-id/configuration.mdx) to explore
+all the available configuration options.
diff --git a/docs/pages/reference/machine-id/configuration.mdx b/docs/pages/reference/machine-id/configuration.mdx
index 3ba87bf005797..f17189443858a 100644
--- a/docs/pages/reference/machine-id/configuration.mdx
+++ b/docs/pages/reference/machine-id/configuration.mdx
@@ -21,8 +21,6 @@ artifacts include configuration files, certificates, and cryptographic key
 material. Usually, artifacts are files, but this term is explicitly avoided
 because a destination isn't required to be a filesystem.
 
-## Configuration v2
-
 From Teleport 14, `tbot` supports the v2 configuration version.
 
 ```yaml
@@ -174,7 +172,7 @@ outputs:
       path: ./tbot-user
 ```
 
-### Outputs
+## Outputs
 
 Outputs define what actions `tbot` should take when it runs. They describe
 the format of the certificates to be generated, the roles used to generate the certificates, and the
@@ -183,7 +181,7 @@ destination where they should be written.
 There are multiple types of output. Select the one that is most appropriate for
 your intended use-case.
 
-#### `identity`
+### `identity`
 
 The `identity` output can be used to authenticate:
 
@@ -224,7 +222,7 @@ app_name: grafana
 (!docs/pages/includes/machine-id/common-output-config.yaml!)
 ```
 
-#### `database`
+### `database`
 
 The `database` output is used to generate credentials that can be used to
 access databases that have been configured with Teleport.
@@ -255,7 +253,7 @@ format: tls
 (!docs/pages/includes/machine-id/common-output-config.yaml!)
 ```
 
-##### Supported formats
+#### Supported formats
 
 You can provide the following values to the `format` configuration field in
 the `database` output type:
@@ -267,7 +265,7 @@ the `database` output type:
 | `cockroach`   | Provides `cockroach/node.key`, `cockroach/node.crt`, and `cockroach/ca.crt`. This is designed to be used with CockroachDB clients. |
 | `tls`         | Provides `tls.key`, `tls.crt`, and `tls.cas`. This is for generic clients that require the specific file extensions. |
 
-#### `kubernetes`
+### `kubernetes`
 
 The `kubernetes` output is used to generate credentials that can be used to
 access Kubernetes clusters that have been configured with Teleport.
@@ -290,7 +288,7 @@ kubernetes_cluster: my-cluster
 (!docs/pages/includes/machine-id/common-output-config.yaml!)
 ```
 
-#### `ssh_host`
+### `ssh_host`
 
 The `ssh_host` output is used to generate the artifacts required to configure
 an OpenSSH server with Teleport in order to allow Teleport users to connect to
@@ -313,7 +311,7 @@ principals:
 (!docs/pages/includes/machine-id/common-output-config.yaml!)
 ```
 
-#### `spiffe-svid`
+### `spiffe-svid`
 
 The `spiffe-svid` output is used to generate a SPIFFE X509 SVID and write this
 to a configured destination.
@@ -346,14 +344,14 @@ svid:
 (!docs/pages/includes/machine-id/common-output-config.yaml!)
 ```
 
-### Services
+## Services
 
 Services are configurable long-lived components that run within `tbot`. Unlike
 Outputs, they may not necessarily generate artifacts. Typically, services
 provide supporting functionality for machine to machine access, for example,
 opening tunnels or providing APIs.
 
-#### `spiffe-workload-api`
+### `spiffe-workload-api`
 
 The `spiffe-workload-api` service opens a listener for a service that implements
 the SPIFFE Workload API. This service is used to provide SPIFFE SVIDs to
@@ -372,6 +370,47 @@ type: spiffe-workload-api
 # - TCP: `tcp://<address>:<port>`
 # - Unix socket: `unix:///<path>`
 listen: unix:///opt/machine-id/workload.sock
+# attestors allows Workload Attestation to be configured for this Workload
+# API.
+attestors:
+  # kubernetes is configuration for the Kubernetes Workload Attestor. See
+  # the Kubernetes Workload Attestor section for more information.
+  kubernetes:
+    # enabled specifies whether the Kubernetes Workload Attestor should be
+    # enabled. If unspecified, this defaults to false.
+    enabled: true
+    # kubelet holds configuration relevant to the Kubernetes Workload Attestors
+    # interaction with the Kubelet API.
+    kubelet:
+      # read_only_port is the port on which the Kubelet API is exposed for
+      # read-only operations. Since Kubernetes 1.16, the read-only port is
+      # typically disabled by default and secure_port should be used instead.
+      read_only_port: 10255
+      # secure_port is the port on which the attestor should connect to the
+      # Kubelet secure API. If unspecified, this defaults to `10250`. This is
+      # mutually exclusive with ReadOnlyPort.
+      secure_port: 10250
+      # token_path is the path to the token file that the Kubelet API client
+      # should use to authenticate with the Kubelet API. If unspecified, this
+      # defaults to `/var/run/secrets/kubernetes.io/serviceaccount/token`.
+      token_path: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      # ca_path is the path to the CA file that the Kubelet API client should
+      # use to validate the Kubelet API server's certificate. If unspecified,
+      # this defaults to `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt`.
+      ca_path: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+      # skip_verify is used to disable verification of the Kubelet API server's
+      # certificate. If unspecified, this defaults to false.
+      #
+      # If specified, the value specified in ca_path is ignored.
+      #
+      # This is useful in cases where the Kubelet API server has not been issued
+      # with a certificate signed by the Kubernetes cluster's CA. This is fairly
+      # common with a number of Kubernetes distributions.
+      skip_verify: true
+      # anonymous is used to disable authentication with the Kubelet API. If
+      # unspecified, this defaults to false. If set, the token_path field is
+      # ignored.
+      anonymous: false
 # svids specifies the SPIFFE SVIDs that the Workload API should provide.
 svids:
     # path specifies what the path element should be requested for the SPIFFE
@@ -421,7 +460,7 @@ svids:
           gid: 50
 ```
 
-#### `database-tunnel`
+### `database-tunnel`
 
 The `database-tunnel` service opens a listener for a service that tunnels
 connections to a database server.
@@ -453,7 +492,7 @@ database: postgres
 username: postgres
 ```
 
-#### `application-tunnel`
+### `application-tunnel`
 
 The `application-tunnel` service opens a listener that tunnels connections to
 an application in Teleport. It supports both HTTP and TCP applications. This is
@@ -481,7 +520,7 @@ listen: tcp://127.0.0.1:8084
 app_name: my-application
 ```
 
-#### `ssh-multiplexer`
+### `ssh-multiplexer`
 
 The `ssh-multiplexer` service opens a listener for a high-performance local
 SSH multiplexer. This is designed for use-cases which create a large number
@@ -546,7 +585,7 @@ destination:
 - `v1.sock`: the Unix socket that the multiplexer listens on.
 - `agent.sock`: the Unix socket that the SSH agent listens on.
 
-##### Using the SSH multiplexer programmatically
+#### Using the SSH multiplexer programmatically
 
 To use the SSH multiplexer programmatically, your SSH client library will need
 to support one of two things:
@@ -690,7 +729,7 @@ func main() {
 ```
 </Details>
 
-### Destinations
+## Destinations
 
 A destination is somewhere that `tbot` can read and write artifacts.
 
@@ -702,7 +741,7 @@ Destinations are used in two places in the `tbot` configuration:
 Destinations come in multiple types. Usually, the `directory` type is the most
 appropriate.
 
-#### `directory`
+### `directory`
 
 The `directory` destination type stores artifacts as files in a specified
 directory.
@@ -739,7 +778,7 @@ symlinks: try-secure
 acls: try
 ```
 
-#### `memory`
+### `memory`
 
 The `memory` destination type stores artifacts in the process memory. When
 the process exits, nothing is persisted. This destination type
@@ -753,7 +792,7 @@ Configuration:
 type: memory
 ```
 
-#### `kubernetes_secret`
+### `kubernetes_secret`
 
 The `kubernetes_secret` destination type stores artifacts in a Kubernetes
 secret. This allows them to be mounted into other containers deployed in

From 4f2b340105fa7a972c75cc9b5db14a67e346cd64 Mon Sep 17 00:00:00 2001
From: "STeve (Xin) Huang" <xin.huang@goteleport.com>
Date: Wed, 21 Aug 2024 13:39:46 -0400
Subject: [PATCH 4/7] [buddy] Truncate AssumeRole session name to API limits
 (#45202) (#45658)

* 44833 Truncate AssumeRole session name to API limits

* Link reference



* Hash the username and add audit log

* review comments

* remove audit

---------

Co-authored-by: Joao Ubaldo <3534309+joaoubaldo@users.noreply.github.com>
---
 lib/srv/app/cloud.go         |  2 +-
 lib/utils/aws/aws.go         | 38 ++++++++++++++++++++++++++++++++++--
 lib/utils/aws/aws_test.go    | 30 ++++++++++++++++++++++++++++
 lib/utils/aws/credentials.go |  2 +-
 4 files changed, 68 insertions(+), 4 deletions(-)

diff --git a/lib/srv/app/cloud.go b/lib/srv/app/cloud.go
index 1757188d3fe36..3675bc14910ed 100644
--- a/lib/srv/app/cloud.go
+++ b/lib/srv/app/cloud.go
@@ -191,7 +191,7 @@ func (c *cloud) getAWSSigninToken(ctx context.Context, req *AWSSigninRequest, en
 	options = append(options, func(creds *stscreds.AssumeRoleProvider) {
 		// Setting role session name to Teleport username will allow to
 		// associate CloudTrail events with the Teleport user.
-		creds.RoleSessionName = req.Identity.Username
+		creds.RoleSessionName = awsutils.MaybeHashRoleSessionName(req.Identity.Username)
 
 		// Setting web console session duration through AssumeRole call for AWS
 		// sessions with temporary credentials.
diff --git a/lib/utils/aws/aws.go b/lib/utils/aws/aws.go
index 950a572946d10..b79afbdd23f0a 100644
--- a/lib/utils/aws/aws.go
+++ b/lib/utils/aws/aws.go
@@ -21,7 +21,10 @@ package aws
 import (
 	"bytes"
 	"context"
+	"crypto/sha1"
+	"encoding/hex"
 	"fmt"
+	"log/slog"
 	"net/http"
 	"net/textproto"
 	"sort"
@@ -33,7 +36,6 @@ import (
 	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
 	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/gravitational/trace"
-	"github.com/sirupsen/logrus"
 
 	apievents "github.com/gravitational/teleport/api/types/events"
 	apiawsutils "github.com/gravitational/teleport/api/utils/aws"
@@ -68,6 +70,11 @@ const (
 	AmzJSON1_0 = "application/x-amz-json-1.0"
 	// AmzJSON1_1 is an AWS Content-Type header that indicates the media type is JSON.
 	AmzJSON1_1 = "application/x-amz-json-1.1"
+
+	// MaxRoleSessionNameLength is the maximum length of the role session name
+	// used by the AssumeRole call.
+	// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html
+	MaxRoleSessionNameLength = 64
 )
 
 // SigV4 contains parsed content of the AWS Authorization header.
@@ -249,7 +256,7 @@ func FilterAWSRoles(arns []string, accountID string) (result Roles) {
 	for _, roleARN := range arns {
 		parsed, err := ParseRoleARN(roleARN)
 		if err != nil {
-			logrus.Warnf("skipping invalid AWS role ARN: %v", err)
+			slog.WarnContext(context.Background(), "Skipping invalid AWS role ARN.", "error", err)
 			continue
 		}
 		if accountID != "" && parsed.AccountID != accountID {
@@ -484,3 +491,30 @@ func iamResourceARN(partition, accountID, resourceType, resourceName string) str
 		Resource:  fmt.Sprintf("%s/%s", resourceType, resourceName),
 	}.String()
 }
+
+// MaybeHashRoleSessionName truncates the role session name and adds a hash
+// when the original role session name is greater than AWS character limit
+// (64).
+func MaybeHashRoleSessionName(roleSessionName string) (ret string) {
+	if len(roleSessionName) <= MaxRoleSessionNameLength {
+		return roleSessionName
+	}
+
+	const hashLen = 16
+	hash := sha1.New()
+	hash.Write([]byte(roleSessionName))
+	hex := hex.EncodeToString(hash.Sum(nil))[:hashLen]
+
+	// "1" for the delimiter.
+	keepPrefixIndex := MaxRoleSessionNameLength - len(hex) - 1
+
+	// Sanity check. This should never happen since hash length and
+	// MaxRoleSessionNameLength are both constant.
+	if keepPrefixIndex < 0 {
+		keepPrefixIndex = 0
+	}
+
+	ret = fmt.Sprintf("%s-%s", roleSessionName[:keepPrefixIndex], hex)
+	slog.DebugContext(context.Background(), "AWS role session name is too long. Using a hash instead.", "hashed", ret, "original", roleSessionName)
+	return ret
+}
diff --git a/lib/utils/aws/aws_test.go b/lib/utils/aws/aws_test.go
index 61ca9dfc48c4f..e42da143252fa 100644
--- a/lib/utils/aws/aws_test.go
+++ b/lib/utils/aws/aws_test.go
@@ -493,3 +493,33 @@ func TestResourceARN(t *testing.T) {
 		})
 	}
 }
+
+func TestMaybeHashRoleSessionName(t *testing.T) {
+	for _, tt := range []struct {
+		name     string
+		role     string
+		expected string
+	}{
+		{
+			name:     "role session name not hashed, less than 64 characters",
+			role:     "MyRole",
+			expected: "MyRole",
+		},
+		{
+			name:     "role session name not hashed, exactly 64 characters",
+			role:     "Role123456789012345678901234567890123456789012345678901234567890",
+			expected: "Role123456789012345678901234567890123456789012345678901234567890",
+		},
+		{
+			name:     "role session name hashed, longer than 64 characters",
+			role:     "remote-raimundo.oliveira@abigcompany.com-teleport.abigcompany.com",
+			expected: "remote-raimundo.oliveira@abigcompany.com-telepo-8fe1f87e599b043e",
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := MaybeHashRoleSessionName(tt.role)
+			require.Equal(t, tt.expected, actual)
+			require.LessOrEqual(t, len(actual), MaxRoleSessionNameLength)
+		})
+	}
+}
diff --git a/lib/utils/aws/credentials.go b/lib/utils/aws/credentials.go
index 4c415c98448f1..257d6606d42a9 100644
--- a/lib/utils/aws/credentials.go
+++ b/lib/utils/aws/credentials.go
@@ -74,7 +74,7 @@ func (g *credentialsGetter) Get(_ context.Context, request GetCredentialsRequest
 	logrus.Debugf("Creating STS session %q for %q.", request.SessionName, request.RoleARN)
 	return stscreds.NewCredentials(request.Provider, request.RoleARN,
 		func(cred *stscreds.AssumeRoleProvider) {
-			cred.RoleSessionName = request.SessionName
+			cred.RoleSessionName = MaybeHashRoleSessionName(request.SessionName)
 			cred.Expiry.SetExpiration(request.Expiry, 0)
 
 			if request.ExternalID != "" {

From 4616c07928e47af8d202730a907f51a614797eb7 Mon Sep 17 00:00:00 2001
From: Maxim Dietz <maxim.dietz@goteleport.com>
Date: Wed, 21 Aug 2024 15:07:21 -0400
Subject: [PATCH 5/7] chore: Bump `e` to `7ecf946` (#45664)

---
 e | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/e b/e
index 17fe288dc44a6..7ecf946f6b564 160000
--- a/e
+++ b/e
@@ -1 +1 @@
-Subproject commit 17fe288dc44a6b3df10ac0ea857e6cdd7b508d29
+Subproject commit 7ecf946f6b5647f4cb194b51c27177d9aa65737f

From c8ed024e167f095ca11b0440b757be891d501a48 Mon Sep 17 00:00:00 2001
From: David Boslee <david@goteleport.com>
Date: Wed, 21 Aug 2024 13:18:19 -0600
Subject: [PATCH 6/7] keystore: add metrics for key creation and signing
 (#45338) (#45634)

* keystore: add metrics for key creation and signing

* register metrics

* remove sshCountSigner

* add store name as label

* add label for key algorithm

* Update lib/auth/keystore/manager.go



* Update lib/auth/keystore/manager.go



* Update lib/auth/keystore/manager.go



* Update lib/auth/keystore/manager.go



* Update lib/auth/keystore/manager.go



---------

Co-authored-by: Nic Klaassen <nic@goteleport.com>
Co-authored-by: rosstimothy <39066650+rosstimothy@users.noreply.github.com>
---
 lib/auth/keystore/aws_kms.go  |   4 ++
 lib/auth/keystore/gcp_kms.go  |   4 ++
 lib/auth/keystore/manager.go  | 117 +++++++++++++++++++++++++++++++++-
 lib/auth/keystore/pkcs11.go   |   4 ++
 lib/auth/keystore/software.go |   4 ++
 5 files changed, 131 insertions(+), 2 deletions(-)

diff --git a/lib/auth/keystore/aws_kms.go b/lib/auth/keystore/aws_kms.go
index 1db86295f09ae..3ca2a6c1db9dc 100644
--- a/lib/auth/keystore/aws_kms.go
+++ b/lib/auth/keystore/aws_kms.go
@@ -102,6 +102,10 @@ func newAWSKMSKeystore(ctx context.Context, cfg *servicecfg.AWSKMSConfig, opts *
 	}, nil
 }
 
+func (a *awsKMSKeystore) name() string {
+	return storeAWS
+}
+
 // keyTypeDescription returns a human-readable description of the types of keys
 // this backend uses.
 func (a *awsKMSKeystore) keyTypeDescription() string {
diff --git a/lib/auth/keystore/gcp_kms.go b/lib/auth/keystore/gcp_kms.go
index cb607913ffb6c..59bed3686ba3a 100644
--- a/lib/auth/keystore/gcp_kms.go
+++ b/lib/auth/keystore/gcp_kms.go
@@ -99,6 +99,10 @@ func newGCPKMSKeyStore(ctx context.Context, cfg *servicecfg.GCPKMSConfig, opts *
 	}, nil
 }
 
+func (a *gcpKMSKeyStore) name() string {
+	return storeGCP
+}
+
 // keyTypeDescription returns a human-readable description of the types of keys
 // this backend uses.
 func (g *gcpKMSKeyStore) keyTypeDescription() string {
diff --git a/lib/auth/keystore/manager.go b/lib/auth/keystore/manager.go
index 654fa0f80dfb5..dce91e2e394cd 100644
--- a/lib/auth/keystore/manager.go
+++ b/lib/auth/keystore/manager.go
@@ -25,11 +25,13 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"io"
 	"log/slog"
 
 	kms "cloud.google.com/go/kms/apiv1"
 	"github.com/gravitational/trace"
 	"github.com/jonboulle/clockwork"
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/exp/maps"
 
@@ -38,10 +40,53 @@ import (
 	"github.com/gravitational/teleport/api/utils/keys"
 	"github.com/gravitational/teleport/lib/auth/keystore/internal/faketime"
 	"github.com/gravitational/teleport/lib/defaults"
+	"github.com/gravitational/teleport/lib/observability/metrics"
 	"github.com/gravitational/teleport/lib/service/servicecfg"
 	"github.com/gravitational/teleport/lib/tlsca"
 )
 
+const (
+	keystoreSubsystem = "keystore"
+
+	labelKeyType = "key_type"
+	keyTypeTLS   = "tls"
+	keyTypeSSH   = "ssh"
+	keyTypeJWT   = "jwt"
+
+	labelStoreType = "store_type"
+	storePKCS11    = "pkcs11"
+	storeGCP       = "gcp_kms"
+	storeAWS       = "aws_kms"
+	storeSoftware  = "software"
+)
+
+var (
+	signCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: teleport.MetricNamespace,
+		Subsystem: keystoreSubsystem,
+		Name:      "sign_requests_total",
+		Help:      "Total number of sign requests",
+	}, []string{labelKeyType, labelStoreType})
+	signErrorCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: teleport.MetricNamespace,
+		Subsystem: keystoreSubsystem,
+		Name:      "sign_requests_error",
+		Help:      "Total number of sign request errors",
+	}, []string{labelKeyType, labelStoreType})
+	createCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: teleport.MetricNamespace,
+		Subsystem: keystoreSubsystem,
+		Name:      "key_create_requests_total",
+		Help:      "Total number of key create requests",
+	}, []string{labelKeyType, labelStoreType})
+	createErrorCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: teleport.MetricNamespace,
+		Subsystem: keystoreSubsystem,
+		Name:      "key_create_requests_error",
+		Help:      "Total number of key create request errors",
+	}, []string{labelKeyType, labelStoreType})
+)
+
 // Manager provides an interface to interact with teleport CA private keys,
 // which may be software keys or held in an HSM or other key manager.
 type Manager struct {
@@ -98,6 +143,9 @@ type backend interface {
 	// keyTypeDescription returns a human-readable description of the types of
 	// keys this backend uses.
 	keyTypeDescription() string
+
+	// name returns the name of the backend.
+	name() string
 }
 
 // Options holds keystore options.
@@ -133,6 +181,14 @@ func (opts *Options) CheckAndSetDefaults() error {
 
 // NewManager returns a new keystore Manager
 func NewManager(ctx context.Context, cfg *servicecfg.KeystoreConfig, opts *Options) (*Manager, error) {
+	if err := metrics.RegisterPrometheusCollectors(
+		signCounter,
+		signErrorCounter,
+		createCounter,
+		createErrorCounter,
+	); err != nil {
+		return nil, trace.Wrap(err)
+	}
 	if err := cfg.CheckAndSetDefaults(); err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -174,6 +230,22 @@ func NewManager(ctx context.Context, cfg *servicecfg.KeystoreConfig, opts *Optio
 	}, nil
 }
 
+type cryptoCountSigner struct {
+	crypto.Signer
+	keyType string
+	store   string
+}
+
+func (s *cryptoCountSigner) Sign(rand io.Reader, digest []byte, opts crypto.SignerOpts) (signature []byte, err error) {
+	signCounter.WithLabelValues(s.keyType, s.store).Inc()
+	sig, err := s.Signer.Sign(rand, digest, opts)
+	if err != nil {
+		signErrorCounter.WithLabelValues(s.keyType, s.store).Inc()
+		return nil, trace.Wrap(err)
+	}
+	return sig, nil
+}
+
 // GetSSHSigner selects a usable SSH keypair from the given CA ActiveKeys and
 // returns an [ssh.Signer].
 func (m *Manager) GetSSHSigner(ctx context.Context, ca types.CertAuthority) (ssh.Signer, error) {
@@ -206,6 +278,7 @@ func (m *Manager) getSSHSigner(ctx context.Context, keySet types.CAKeySet) (ssh.
 			if err != nil {
 				return nil, trace.Wrap(err)
 			}
+			signer = &cryptoCountSigner{Signer: signer, keyType: keyTypeSSH, store: backend.name()}
 			sshSigner, err := ssh.NewSignerFromSigner(signer)
 			if err != nil {
 				return nil, trace.Wrap(err)
@@ -251,6 +324,9 @@ func (s rsaSHA512Signer) Algorithms() []string {
 // and returns the PEM-encoded TLS certificate and a [crypto.Signer].
 func (m *Manager) GetTLSCertAndSigner(ctx context.Context, ca types.CertAuthority) ([]byte, crypto.Signer, error) {
 	cert, signer, err := m.getTLSCertAndSigner(ctx, ca.GetActiveKeys())
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
 	return cert, signer, trace.Wrap(err)
 }
 
@@ -258,6 +334,9 @@ func (m *Manager) GetTLSCertAndSigner(ctx context.Context, ca types.CertAuthorit
 // and returns the PEM-encoded TLS certificate and a [crypto.Signer].
 func (m *Manager) GetAdditionalTrustedTLSCertAndSigner(ctx context.Context, ca types.CertAuthority) ([]byte, crypto.Signer, error) {
 	cert, signer, err := m.getTLSCertAndSigner(ctx, ca.GetAdditionalTrustedKeys())
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
 	return cert, signer, trace.Wrap(err)
 }
 
@@ -279,6 +358,7 @@ func (m *Manager) getTLSCertAndSigner(ctx context.Context, keySet types.CAKeySet
 			if err != nil {
 				return nil, nil, trace.Wrap(err)
 			}
+			signer = &cryptoCountSigner{Signer: signer, keyType: keyTypeTLS, store: backend.name()}
 			return keyPair.Cert, signer, nil
 		}
 	}
@@ -314,7 +394,10 @@ func (m *Manager) GetJWTSigner(ctx context.Context, ca types.CertAuthority) (cry
 				return nil, trace.Wrap(err)
 			}
 			signer, err := backend.getSigner(ctx, keyPair.PrivateKey, pub)
-			return signer, trace.Wrap(err)
+			if err != nil {
+				return nil, trace.Wrap(err)
+			}
+			return &cryptoCountSigner{Signer: signer, keyType: keyTypeJWT, store: backend.name()}, trace.Wrap(err)
 		}
 	}
 	return nil, trace.NotFound("no usable JWT key pairs found")
@@ -322,6 +405,16 @@ func (m *Manager) GetJWTSigner(ctx context.Context, ca types.CertAuthority) (cry
 
 // NewSSHKeyPair generates a new SSH keypair in the keystore backend and returns it.
 func (m *Manager) NewSSHKeyPair(ctx context.Context) (*types.SSHKeyPair, error) {
+	createCounter.WithLabelValues(keyTypeSSH, m.backendForNewKeys.name()).Inc()
+	key, err := m.newSSHKeyPair(ctx)
+	if err != nil {
+		createErrorCounter.WithLabelValues(keyTypeSSH, m.backendForNewKeys.name()).Inc()
+		return nil, trace.Wrap(err)
+	}
+	return key, nil
+}
+
+func (m *Manager) newSSHKeyPair(ctx context.Context) (*types.SSHKeyPair, error) {
 	// The default hash length for SSH signers is 512 bits.
 	sshKey, cryptoSigner, err := m.backendForNewKeys.generateRSA(ctx, withDigestAlgorithm(crypto.SHA512))
 	if err != nil {
@@ -341,12 +434,22 @@ func (m *Manager) NewSSHKeyPair(ctx context.Context) (*types.SSHKeyPair, error)
 
 // NewTLSKeyPair creates a new TLS keypair in the keystore backend and returns it.
 func (m *Manager) NewTLSKeyPair(ctx context.Context, clusterName string) (*types.TLSKeyPair, error) {
+	createCounter.WithLabelValues(keyTypeTLS, m.backendForNewKeys.name()).Inc()
+	key, err := m.newTLSKeyPair(ctx, clusterName)
+	if err != nil {
+		createErrorCounter.WithLabelValues(keyTypeTLS, m.backendForNewKeys.name()).Inc()
+		return nil, trace.Wrap(err)
+	}
+	return key, nil
+}
+
+func (m *Manager) newTLSKeyPair(ctx context.Context, clusterName string) (*types.TLSKeyPair, error) {
 	tlsKey, signer, err := m.backendForNewKeys.generateRSA(ctx)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
 	tlsCert, err := tlsca.GenerateSelfSignedCAWithSigner(
-		signer,
+		&cryptoCountSigner{Signer: signer, keyType: keyTypeTLS, store: m.backendForNewKeys.name()},
 		pkix.Name{
 			CommonName:   clusterName,
 			Organization: []string{clusterName},
@@ -364,6 +467,16 @@ func (m *Manager) NewTLSKeyPair(ctx context.Context, clusterName string) (*types
 // New JWTKeyPair create a new JWT keypair in the keystore backend and returns
 // it.
 func (m *Manager) NewJWTKeyPair(ctx context.Context) (*types.JWTKeyPair, error) {
+	createCounter.WithLabelValues(keyTypeJWT, m.backendForNewKeys.name()).Inc()
+	key, err := m.newJWTKeyPair(ctx)
+	if err != nil {
+		createErrorCounter.WithLabelValues(keyTypeJWT, m.backendForNewKeys.name()).Inc()
+		return nil, trace.Wrap(err)
+	}
+	return key, nil
+}
+
+func (m *Manager) newJWTKeyPair(ctx context.Context) (*types.JWTKeyPair, error) {
 	jwtKey, signer, err := m.backendForNewKeys.generateRSA(ctx)
 	if err != nil {
 		return nil, trace.Wrap(err)
diff --git a/lib/auth/keystore/pkcs11.go b/lib/auth/keystore/pkcs11.go
index a488ff875ed59..ed48e8e789a8d 100644
--- a/lib/auth/keystore/pkcs11.go
+++ b/lib/auth/keystore/pkcs11.go
@@ -76,6 +76,10 @@ func newPKCS11KeyStore(config *servicecfg.PKCS11Config, opts *Options) (*pkcs11K
 	}, nil
 }
 
+func (p *pkcs11KeyStore) name() string {
+	return storePKCS11
+}
+
 // keyTypeDescription returns a human-readable description of the types of keys
 // this backend uses.
 func (p *pkcs11KeyStore) keyTypeDescription() string {
diff --git a/lib/auth/keystore/software.go b/lib/auth/keystore/software.go
index 08ee777cd98b0..996302248c14e 100644
--- a/lib/auth/keystore/software.go
+++ b/lib/auth/keystore/software.go
@@ -53,6 +53,10 @@ func newSoftwareKeyStore(config *softwareConfig) *softwareKeyStore {
 	}
 }
 
+func (p *softwareKeyStore) name() string {
+	return storeSoftware
+}
+
 // keyTypeDescription returns a human-readable description of the types of keys
 // this backend uses.
 func (s *softwareKeyStore) keyTypeDescription() string {

From 76f2a52ce4f35d41a324ee623b4c2e91d1c53bc5 Mon Sep 17 00:00:00 2001
From: rosstimothy <39066650+rosstimothy@users.noreply.github.com>
Date: Wed, 21 Aug 2024 18:15:47 -0400
Subject: [PATCH 7/7] Add backend.NewKey function (#45669)

Refactoring in preparation for creating a concrete backend.Key
type instead of using []byte for keys. The backend.Key function
still exists and was not deprecated because it is still consumed
in e. It will be removed when the e ref is updated after e uses
backend.NewKey.
---
 lib/auth/migration/migration.go               |   2 +-
 lib/auth/storage/storage.go                   |  14 +-
 lib/backend/backend.go                        |   8 +-
 lib/backend/clone.go                          |   2 +-
 lib/backend/clone_test.go                     |   8 +-
 lib/backend/etcdbk/etcd_test.go               |   4 +-
 lib/backend/kubernetes/kubernetes_test.go     |  26 ++--
 lib/cache/cache_test.go                       |   6 +-
 lib/service/service.go                        |   4 +-
 lib/services/local/access.go                  |  22 ++--
 lib/services/local/apps.go                    |   8 +-
 lib/services/local/assertion_replay.go        |   2 +-
 lib/services/local/configuration.go           |  82 ++++++------
 lib/services/local/connection_diagnostic.go   |  10 +-
 lib/services/local/databases.go               |   8 +-
 lib/services/local/databaseservice.go         |   4 +-
 lib/services/local/desktops.go                |  12 +-
 lib/services/local/dynamic_access.go          |   6 +-
 lib/services/local/events.go                  | 122 +++++++++---------
 lib/services/local/externalauditstorage.go    |   4 +-
 lib/services/local/generic/generic.go         |   6 +-
 lib/services/local/headlessauthn.go           |   2 +-
 lib/services/local/headlessauthn_watcher.go   |   2 +-
 lib/services/local/inventory.go               |   4 +-
 lib/services/local/kube.go                    |   8 +-
 lib/services/local/plugin_data.go             |   2 +-
 lib/services/local/plugins.go                 |  14 +-
 lib/services/local/presence.go                |  76 +++++------
 lib/services/local/provisioning.go            |   8 +-
 lib/services/local/resource.go                |  28 ++--
 lib/services/local/restrictions.go            |   6 +-
 lib/services/local/session.go                 |  16 +--
 lib/services/local/sessiontracker.go          |  14 +-
 lib/services/local/status.go                  |  10 +-
 lib/services/local/trust.go                   |  30 ++---
 lib/services/local/unstable.go                |   2 +-
 lib/services/local/userpreferences.go         |   4 +-
 lib/services/local/users.go                   |  94 +++++++-------
 lib/services/local/users_test.go              |   2 +-
 lib/services/local/usertoken.go               |   8 +-
 lib/services/unified_resource.go              |  16 +--
 .../teleport/aggregating/service.go           |   6 +-
 42 files changed, 359 insertions(+), 353 deletions(-)

diff --git a/lib/auth/migration/migration.go b/lib/auth/migration/migration.go
index 625fd277a9dc6..987834f10ac61 100644
--- a/lib/auth/migration/migration.go
+++ b/lib/auth/migration/migration.go
@@ -163,7 +163,7 @@ type migrationStatus struct {
 	Completed time.Time      `json:"completed"`
 }
 
-var key = backend.Key("migrations", "current")
+var key = backend.NewKey("migrations", "current")
 
 func setCurrentMigration(ctx context.Context, b backend.Backend, status migrationStatus) error {
 	value, err := json.Marshal(status)
diff --git a/lib/auth/storage/storage.go b/lib/auth/storage/storage.go
index 8caf0ca93684c..9596f12e1e645 100644
--- a/lib/auth/storage/storage.go
+++ b/lib/auth/storage/storage.go
@@ -88,7 +88,7 @@ func (p *ProcessStorage) Close() error {
 
 // GetState reads rotation state from disk.
 func (p *ProcessStorage) GetState(ctx context.Context, role types.SystemRole) (*state.StateV2, error) {
-	item, err := p.stateStorage.Get(ctx, backend.Key(statesPrefix, strings.ToLower(role.String()), stateName))
+	item, err := p.stateStorage.Get(ctx, backend.NewKey(statesPrefix, strings.ToLower(role.String()), stateName))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -121,7 +121,7 @@ func (p *ProcessStorage) CreateState(role types.SystemRole, state state.StateV2)
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:   backend.Key(statesPrefix, strings.ToLower(role.String()), stateName),
+		Key:   backend.NewKey(statesPrefix, strings.ToLower(role.String()), stateName),
 		Value: value,
 	}
 	_, err = p.stateStorage.Create(context.TODO(), item)
@@ -141,7 +141,7 @@ func (p *ProcessStorage) WriteState(role types.SystemRole, state state.StateV2)
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:   backend.Key(statesPrefix, strings.ToLower(role.String()), stateName),
+		Key:   backend.NewKey(statesPrefix, strings.ToLower(role.String()), stateName),
 		Value: value,
 	}
 	_, err = p.stateStorage.Put(context.TODO(), item)
@@ -156,7 +156,7 @@ func (p *ProcessStorage) ReadIdentity(name string, role types.SystemRole) (*stat
 	if name == "" {
 		return nil, trace.BadParameter("missing parameter name")
 	}
-	item, err := p.stateStorage.Get(context.TODO(), backend.Key(idsPrefix, strings.ToLower(role.String()), name))
+	item, err := p.stateStorage.Get(context.TODO(), backend.NewKey(idsPrefix, strings.ToLower(role.String()), name))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -201,7 +201,7 @@ func (p *ProcessStorage) WriteIdentity(name string, id state.Identity) error {
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:   backend.Key(idsPrefix, strings.ToLower(id.ID.Role.String()), name),
+		Key:   backend.NewKey(idsPrefix, strings.ToLower(id.ID.Role.String()), name),
 		Value: value,
 	}
 	_, err = p.stateStorage.Put(context.TODO(), item)
@@ -210,7 +210,7 @@ func (p *ProcessStorage) WriteIdentity(name string, id state.Identity) error {
 
 // GetTeleportVersion reads the last known Teleport version from storage.
 func (p *ProcessStorage) GetTeleportVersion(ctx context.Context) (*semver.Version, error) {
-	item, err := p.stateStorage.Get(ctx, backend.Key(teleportPrefix, lastKnownVersion))
+	item, err := p.stateStorage.Get(ctx, backend.NewKey(teleportPrefix, lastKnownVersion))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -223,7 +223,7 @@ func (p *ProcessStorage) WriteTeleportVersion(ctx context.Context, version *semv
 		return trace.BadParameter("wrong version parameter")
 	}
 	item := backend.Item{
-		Key:   backend.Key(teleportPrefix, lastKnownVersion),
+		Key:   backend.NewKey(teleportPrefix, lastKnownVersion),
 		Value: []byte(version.String()),
 	}
 	_, err := p.stateStorage.Put(ctx, item)
diff --git a/lib/backend/backend.go b/lib/backend/backend.go
index aa5d5ed4843cb..b5a8bc52ed90c 100644
--- a/lib/backend/backend.go
+++ b/lib/backend/backend.go
@@ -437,6 +437,12 @@ const Separator = '/'
 // Key joins parts into path separated by Separator,
 // makes sure path always starts with Separator ("/")
 func Key(parts ...string) []byte {
+	return NewKey(parts...)
+}
+
+// NewKey joins parts into path separated by Separator,
+// makes sure path always starts with Separator ("/")
+func NewKey(parts ...string) []byte {
 	return internalKey("", parts...)
 }
 
@@ -445,7 +451,7 @@ func Key(parts ...string) []byte {
 // math child paths and not other paths that have the resulting path
 // as a prefix.
 func ExactKey(parts ...string) []byte {
-	return append(Key(parts...), Separator)
+	return append(NewKey(parts...), Separator)
 }
 
 func internalKey(internalPrefix string, parts ...string) []byte {
diff --git a/lib/backend/clone.go b/lib/backend/clone.go
index 9579d2571eebc..6ee3f7e72527d 100644
--- a/lib/backend/clone.go
+++ b/lib/backend/clone.go
@@ -52,7 +52,7 @@ type CloneConfig struct {
 func Clone(ctx context.Context, src, dst Backend, parallel int, force bool) error {
 	log := slog.With(teleport.ComponentKey, "clone")
 	itemC := make(chan Item, bufferSize)
-	start := Key("")
+	start := NewKey("")
 	migrated := &atomic.Int32{}
 
 	if parallel <= 0 {
diff --git a/lib/backend/clone_test.go b/lib/backend/clone_test.go
index c4cc492441049..ab776ee0dcede 100644
--- a/lib/backend/clone_test.go
+++ b/lib/backend/clone_test.go
@@ -44,7 +44,7 @@ func TestClone(t *testing.T) {
 
 	for i := 0; i < itemCount; i++ {
 		item := backend.Item{
-			Key:   backend.Key(fmt.Sprintf("key-%05d", i)),
+			Key:   backend.NewKey(fmt.Sprintf("key-%05d", i)),
 			Value: []byte(fmt.Sprintf("value-%d", i)),
 		}
 		_, err := src.Put(ctx, item)
@@ -55,7 +55,7 @@ func TestClone(t *testing.T) {
 	err = backend.Clone(ctx, src, dst, 10, false)
 	require.NoError(t, err)
 
-	start := backend.Key("")
+	start := backend.NewKey("")
 	result, err := dst.GetRange(ctx, start, backend.RangeEnd(start), 0)
 	require.NoError(t, err)
 
@@ -80,7 +80,7 @@ func TestCloneForce(t *testing.T) {
 
 	for i := 0; i < itemCount; i++ {
 		item := backend.Item{
-			Key:   backend.Key(fmt.Sprintf("key-%05d", i)),
+			Key:   backend.NewKey(fmt.Sprintf("key-%05d", i)),
 			Value: []byte(fmt.Sprintf("value-%d", i)),
 		}
 		_, err := src.Put(ctx, item)
@@ -97,7 +97,7 @@ func TestCloneForce(t *testing.T) {
 	err = backend.Clone(ctx, src, dst, 10, true)
 	require.NoError(t, err)
 
-	start := backend.Key("")
+	start := backend.NewKey("")
 	result, err := dst.GetRange(ctx, start, backend.RangeEnd(start), 0)
 	require.NoError(t, err)
 
diff --git a/lib/backend/etcdbk/etcd_test.go b/lib/backend/etcdbk/etcd_test.go
index 7ec811818065d..7c1ea2260611c 100644
--- a/lib/backend/etcdbk/etcd_test.go
+++ b/lib/backend/etcdbk/etcd_test.go
@@ -219,7 +219,7 @@ func TestLeaseBucketing(t *testing.T) {
 
 	buckets := make(map[int64]struct{})
 	for i := 0; i < count; i++ {
-		key := backend.Key(pfx, fmt.Sprintf("%d", i))
+		key := backend.NewKey(pfx, fmt.Sprintf("%d", i))
 		_, err := bk.Put(ctx, backend.Item{
 			Key:     key,
 			Value:   []byte(fmt.Sprintf("val-%d", i)),
@@ -234,7 +234,7 @@ func TestLeaseBucketing(t *testing.T) {
 		time.Sleep(time.Millisecond * 200)
 	}
 
-	start := backend.Key(pfx, "")
+	start := backend.NewKey(pfx, "")
 
 	rslt, err := bk.GetRange(ctx, start, backend.RangeEnd(start), backend.NoLimit)
 	require.NoError(t, err)
diff --git a/lib/backend/kubernetes/kubernetes_test.go b/lib/backend/kubernetes/kubernetes_test.go
index 09adacb93314e..c7326eebc59ca 100644
--- a/lib/backend/kubernetes/kubernetes_test.go
+++ b/lib/backend/kubernetes/kubernetes_test.go
@@ -160,7 +160,7 @@ func TestBackend_Get(t *testing.T) {
 				replicaName: "agent-0",
 			},
 			args: args{
-				key: backend.Key("ids", "kube", "current"),
+				key: backend.NewKey("ids", "kube", "current"),
 			},
 			want: nil,
 
@@ -169,7 +169,7 @@ func TestBackend_Get(t *testing.T) {
 		{
 			name: "secret exists and key is present",
 			args: args{
-				key: backend.Key("ids", "kube", "current"),
+				key: backend.NewKey("ids", "kube", "current"),
 			},
 			fields: fields{
 				objects: []runtime.Object{
@@ -177,7 +177,7 @@ func TestBackend_Get(t *testing.T) {
 						"agent-0-state",
 						"test",
 						map[string][]byte{
-							backendKeyToSecret(backend.Key("ids", "kube", "current")): payloadTestData,
+							backendKeyToSecret(backend.NewKey("ids", "kube", "current")): payloadTestData,
 						},
 					),
 				},
@@ -189,7 +189,7 @@ func TestBackend_Get(t *testing.T) {
 		{
 			name: "secret exists and key is present but empty",
 			args: args{
-				key: backend.Key("ids", "kube", "current"),
+				key: backend.NewKey("ids", "kube", "current"),
 			},
 			fields: fields{
 				objects: []runtime.Object{
@@ -197,7 +197,7 @@ func TestBackend_Get(t *testing.T) {
 						"agent-0-state",
 						"test",
 						map[string][]byte{
-							backendKeyToSecret(backend.Key("ids", "kube", "current")): payloadEmpty,
+							backendKeyToSecret(backend.NewKey("ids", "kube", "current")): payloadEmpty,
 						},
 					),
 				},
@@ -210,7 +210,7 @@ func TestBackend_Get(t *testing.T) {
 		{
 			name: "secret exists but key not present",
 			args: args{
-				key: backend.Key("ids", "kube", "replacement"),
+				key: backend.NewKey("ids", "kube", "replacement"),
 			},
 			fields: fields{
 				objects: []runtime.Object{
@@ -218,7 +218,7 @@ func TestBackend_Get(t *testing.T) {
 						"agent-0-state",
 						"test",
 						map[string][]byte{
-							backendKeyToSecret(backend.Key("ids", "kube", "current")): payloadTestData,
+							backendKeyToSecret(backend.NewKey("ids", "kube", "current")): payloadTestData,
 						},
 					),
 				},
@@ -283,7 +283,7 @@ func TestBackend_Put(t *testing.T) {
 			},
 			args: args{
 				item: backend.Item{
-					Key:   backend.Key("ids", "kube", "current"),
+					Key:   backend.NewKey("ids", "kube", "current"),
 					Value: payloadTestData,
 				},
 			},
@@ -291,7 +291,7 @@ func TestBackend_Put(t *testing.T) {
 				"agent-0-state",
 				"test",
 				map[string][]byte{
-					backendKeyToSecret(backend.Key("ids", "kube", "current")): payloadTestData,
+					backendKeyToSecret(backend.NewKey("ids", "kube", "current")): payloadTestData,
 				},
 			),
 		},
@@ -299,7 +299,7 @@ func TestBackend_Put(t *testing.T) {
 			name: "secret exists and has keys",
 			args: args{
 				item: backend.Item{
-					Key:   backend.Key("ids", "kube", "current2"),
+					Key:   backend.NewKey("ids", "kube", "current2"),
 					Value: payloadTestData,
 				},
 			},
@@ -309,7 +309,7 @@ func TestBackend_Put(t *testing.T) {
 						"agent-0-state",
 						"test",
 						map[string][]byte{
-							backendKeyToSecret(backend.Key("ids", "kube", "current")): payloadTestData,
+							backendKeyToSecret(backend.NewKey("ids", "kube", "current")): payloadTestData,
 						},
 					),
 				},
@@ -320,8 +320,8 @@ func TestBackend_Put(t *testing.T) {
 				"agent-0-state",
 				"test",
 				map[string][]byte{
-					backendKeyToSecret(backend.Key("ids", "kube", "current")):  payloadTestData,
-					backendKeyToSecret(backend.Key("ids", "kube", "current2")): payloadTestData,
+					backendKeyToSecret(backend.NewKey("ids", "kube", "current")):  payloadTestData,
+					backendKeyToSecret(backend.NewKey("ids", "kube", "current2")): payloadTestData,
 				},
 			),
 		},
diff --git a/lib/cache/cache_test.go b/lib/cache/cache_test.go
index 923d8fe7b63ad..3b676cd57bc48 100644
--- a/lib/cache/cache_test.go
+++ b/lib/cache/cache_test.go
@@ -3384,7 +3384,7 @@ func TestInvalidDatabases(t *testing.T) {
 				value, err := services.MarshalDatabase(db)
 				require.NoError(t, err)
 				_, err = b.Create(ctx, backend.Item{
-					Key:     backend.Key("db", db.GetName()),
+					Key:     backend.NewKey("db", db.GetName()),
 					Value:   value,
 					Expires: db.Expiry(),
 				})
@@ -3406,7 +3406,7 @@ func TestInvalidDatabases(t *testing.T) {
 				marshalledDB, err := services.MarshalDatabase(validDB)
 				require.NoError(t, err)
 				_, err = b.Create(ctx, backend.Item{
-					Key:     backend.Key("db", validDB.GetName()),
+					Key:     backend.NewKey("db", validDB.GetName()),
 					Value:   marshalledDB,
 					Expires: validDB.Expiry(),
 				})
@@ -3426,7 +3426,7 @@ func TestInvalidDatabases(t *testing.T) {
 				value, err := services.MarshalDatabase(invalidDB)
 				require.NoError(t, err)
 				_, err = b.Update(ctx, backend.Item{
-					Key:     backend.Key("db", cacheDB.GetName()),
+					Key:     backend.NewKey("db", cacheDB.GetName()),
 					Value:   value,
 					Expires: invalidDB.Expiry(),
 				})
diff --git a/lib/service/service.go b/lib/service/service.go
index 6de1b7d4ae230..85455e06a44d2 100644
--- a/lib/service/service.go
+++ b/lib/service/service.go
@@ -6322,7 +6322,7 @@ func persistHostIDToStorages(ctx context.Context, cfg *servicecfg.Config, kubeBa
 // loadHostIDFromKubeSecret reads the host_uuid from the Kubernetes secret with
 // the expected key: `/host_uuid`.
 func loadHostIDFromKubeSecret(ctx context.Context, kubeBackend kubernetesBackend) (string, error) {
-	item, err := kubeBackend.Get(ctx, backend.Key(utils.HostUUIDFile))
+	item, err := kubeBackend.Get(ctx, backend.NewKey(utils.HostUUIDFile))
 	if err != nil {
 		return "", trace.Wrap(err)
 	}
@@ -6335,7 +6335,7 @@ func writeHostIDToKubeSecret(ctx context.Context, kubeBackend kubernetesBackend,
 	_, err := kubeBackend.Put(
 		ctx,
 		backend.Item{
-			Key:   backend.Key(utils.HostUUIDFile),
+			Key:   backend.NewKey(utils.HostUUIDFile),
 			Value: []byte(id),
 		},
 	)
diff --git a/lib/services/local/access.go b/lib/services/local/access.go
index 4c748120a76b5..be3ab9ce41018 100644
--- a/lib/services/local/access.go
+++ b/lib/services/local/access.go
@@ -50,7 +50,7 @@ func NewAccessService(backend backend.Backend) *AccessService {
 
 // DeleteAllRoles deletes all roles
 func (s *AccessService) DeleteAllRoles(ctx context.Context) error {
-	startKey := backend.Key(rolesPrefix)
+	startKey := backend.NewKey(rolesPrefix)
 	endKey := backend.RangeEnd(startKey)
 	return s.DeleteRange(ctx, startKey, endKey)
 }
@@ -101,7 +101,7 @@ func (s *AccessService) ListRoles(ctx context.Context, req *proto.ListRolesReque
 
 	startKey := backend.ExactKey(rolesPrefix)
 	if req.StartKey != "" {
-		startKey = backend.Key(rolesPrefix, req.StartKey, paramsPrefix)
+		startKey = backend.NewKey(rolesPrefix, req.StartKey, paramsPrefix)
 	}
 
 	endKey := backend.RangeEnd(backend.ExactKey(rolesPrefix))
@@ -168,7 +168,7 @@ func (s *AccessService) CreateRole(ctx context.Context, role types.Role) (types.
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(rolesPrefix, role.GetName(), paramsPrefix),
+		Key:      backend.NewKey(rolesPrefix, role.GetName(), paramsPrefix),
 		Value:    value,
 		Expires:  role.Expiry(),
 		Revision: rev,
@@ -196,7 +196,7 @@ func (s *AccessService) UpdateRole(ctx context.Context, role types.Role) (types.
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(rolesPrefix, role.GetName(), paramsPrefix),
+		Key:      backend.NewKey(rolesPrefix, role.GetName(), paramsPrefix),
 		Value:    value,
 		Expires:  role.Expiry(),
 		Revision: rev,
@@ -224,7 +224,7 @@ func (s *AccessService) UpsertRole(ctx context.Context, role types.Role) (types.
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(rolesPrefix, role.GetName(), paramsPrefix),
+		Key:      backend.NewKey(rolesPrefix, role.GetName(), paramsPrefix),
 		Value:    value,
 		Expires:  role.Expiry(),
 		Revision: rev,
@@ -243,7 +243,7 @@ func (s *AccessService) GetRole(ctx context.Context, name string) (types.Role, e
 	if name == "" {
 		return nil, trace.BadParameter("missing role name")
 	}
-	item, err := s.Get(ctx, backend.Key(rolesPrefix, name, paramsPrefix))
+	item, err := s.Get(ctx, backend.NewKey(rolesPrefix, name, paramsPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("role %v is not found", name)
@@ -259,7 +259,7 @@ func (s *AccessService) DeleteRole(ctx context.Context, name string) error {
 	if name == "" {
 		return trace.BadParameter("missing role name")
 	}
-	err := s.Delete(ctx, backend.Key(rolesPrefix, name, paramsPrefix))
+	err := s.Delete(ctx, backend.NewKey(rolesPrefix, name, paramsPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("role %q is not found", name)
@@ -273,7 +273,7 @@ func (s *AccessService) GetLock(ctx context.Context, name string) (types.Lock, e
 	if name == "" {
 		return nil, trace.BadParameter("missing lock name")
 	}
-	item, err := s.Get(ctx, backend.Key(locksPrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(locksPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("lock %q is not found", name)
@@ -324,7 +324,7 @@ func (s *AccessService) UpsertLock(ctx context.Context, lock types.Lock) error {
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(locksPrefix, lock.GetName()),
+		Key:      backend.NewKey(locksPrefix, lock.GetName()),
 		Value:    value,
 		Expires:  lock.Expiry(),
 		Revision: rev,
@@ -341,7 +341,7 @@ func (s *AccessService) DeleteLock(ctx context.Context, name string) error {
 	if name == "" {
 		return trace.BadParameter("missing lock name")
 	}
-	err := s.Delete(ctx, backend.Key(locksPrefix, name))
+	err := s.Delete(ctx, backend.NewKey(locksPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("lock %q is not found", name)
@@ -382,7 +382,7 @@ func (s *AccessService) ReplaceRemoteLocks(ctx context.Context, clusterName stri
 				return trace.Wrap(err)
 			}
 			item := backend.Item{
-				Key:      backend.Key(locksPrefix, lock.GetName()),
+				Key:      backend.NewKey(locksPrefix, lock.GetName()),
 				Value:    value,
 				Expires:  lock.Expiry(),
 				Revision: rev,
diff --git a/lib/services/local/apps.go b/lib/services/local/apps.go
index d691edd0141f2..f058d7fbd754f 100644
--- a/lib/services/local/apps.go
+++ b/lib/services/local/apps.go
@@ -59,7 +59,7 @@ func (s *AppService) GetApps(ctx context.Context) ([]types.Application, error) {
 
 // GetApp returns the specified application resource.
 func (s *AppService) GetApp(ctx context.Context, name string) (types.Application, error) {
-	item, err := s.Get(ctx, backend.Key(appPrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(appPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("application %q doesn't exist", name)
@@ -84,7 +84,7 @@ func (s *AppService) CreateApp(ctx context.Context, app types.Application) error
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(appPrefix, app.GetName()),
+		Key:     backend.NewKey(appPrefix, app.GetName()),
 		Value:   value,
 		Expires: app.Expiry(),
 	}
@@ -110,7 +110,7 @@ func (s *AppService) UpdateApp(ctx context.Context, app types.Application) error
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(appPrefix, app.GetName()),
+		Key:      backend.NewKey(appPrefix, app.GetName()),
 		Value:    value,
 		Expires:  app.Expiry(),
 		Revision: rev,
@@ -128,7 +128,7 @@ func (s *AppService) UpdateApp(ctx context.Context, app types.Application) error
 
 // DeleteApp removes the specified application resource.
 func (s *AppService) DeleteApp(ctx context.Context, name string) error {
-	err := s.Delete(ctx, backend.Key(appPrefix, name))
+	err := s.Delete(ctx, backend.NewKey(appPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("application %q doesn't exist", name)
diff --git a/lib/services/local/assertion_replay.go b/lib/services/local/assertion_replay.go
index 76698bd8c5861..d294a24282c42 100644
--- a/lib/services/local/assertion_replay.go
+++ b/lib/services/local/assertion_replay.go
@@ -46,7 +46,7 @@ func NewAssertionReplayService(bk backend.Backend) *AssertionReplayService {
 // `safeAfter` must be either at or after the point in time that a given SSO assertion becomes invalid in order to mitigate replay attacks.
 // This function shouldn't be used if the assertion never verifiably expires.
 func (s *AssertionReplayService) RecognizeSSOAssertion(ctx context.Context, connectorID string, assertionID string, user string, safeAfter time.Time) error {
-	key := backend.Key(assertionReplayPrefix, connectorID, assertionID)
+	key := backend.NewKey(assertionReplayPrefix, connectorID, assertionID)
 	item := backend.Item{Key: key, Value: []byte(user), Expires: safeAfter}
 	_, err := s.bk.Create(ctx, item)
 	switch {
diff --git a/lib/services/local/configuration.go b/lib/services/local/configuration.go
index 38865478326cc..753a341534495 100644
--- a/lib/services/local/configuration.go
+++ b/lib/services/local/configuration.go
@@ -62,7 +62,7 @@ func NewClusterConfigurationService(backend backend.Backend) (*ClusterConfigurat
 
 // GetClusterName gets the name of the cluster from the backend.
 func (s *ClusterConfigurationService) GetClusterName(opts ...services.MarshalOption) (types.ClusterName, error) {
-	item, err := s.Get(context.TODO(), backend.Key(clusterConfigPrefix, namePrefix))
+	item, err := s.Get(context.TODO(), backend.NewKey(clusterConfigPrefix, namePrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			clusterNameNotFound.Inc()
@@ -76,7 +76,7 @@ func (s *ClusterConfigurationService) GetClusterName(opts ...services.MarshalOpt
 
 // DeleteClusterName deletes types.ClusterName from the backend.
 func (s *ClusterConfigurationService) DeleteClusterName() error {
-	err := s.Delete(context.TODO(), backend.Key(clusterConfigPrefix, namePrefix))
+	err := s.Delete(context.TODO(), backend.NewKey(clusterConfigPrefix, namePrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("cluster configuration not found")
@@ -96,7 +96,7 @@ func (s *ClusterConfigurationService) SetClusterName(c types.ClusterName) error
 	}
 
 	_, err = s.Create(context.TODO(), backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, namePrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, namePrefix),
 		Value:    value,
 		Expires:  c.Expiry(),
 		Revision: rev,
@@ -117,7 +117,7 @@ func (s *ClusterConfigurationService) UpsertClusterName(c types.ClusterName) err
 	}
 
 	_, err = s.Put(context.TODO(), backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, namePrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, namePrefix),
 		Value:    value,
 		Expires:  c.Expiry(),
 		Revision: rev,
@@ -131,7 +131,7 @@ func (s *ClusterConfigurationService) UpsertClusterName(c types.ClusterName) err
 
 // GetStaticTokens gets the list of static tokens used to provision nodes.
 func (s *ClusterConfigurationService) GetStaticTokens() (types.StaticTokens, error) {
-	item, err := s.Get(context.TODO(), backend.Key(clusterConfigPrefix, staticTokensPrefix))
+	item, err := s.Get(context.TODO(), backend.NewKey(clusterConfigPrefix, staticTokensPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("static tokens not found")
@@ -150,7 +150,7 @@ func (s *ClusterConfigurationService) SetStaticTokens(c types.StaticTokens) erro
 		return trace.Wrap(err)
 	}
 	_, err = s.Put(context.TODO(), backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, staticTokensPrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, staticTokensPrefix),
 		Value:    value,
 		Expires:  c.Expiry(),
 		Revision: rev,
@@ -164,7 +164,7 @@ func (s *ClusterConfigurationService) SetStaticTokens(c types.StaticTokens) erro
 
 // DeleteStaticTokens deletes static tokens
 func (s *ClusterConfigurationService) DeleteStaticTokens() error {
-	err := s.Delete(context.TODO(), backend.Key(clusterConfigPrefix, staticTokensPrefix))
+	err := s.Delete(context.TODO(), backend.NewKey(clusterConfigPrefix, staticTokensPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("static tokens are not found")
@@ -177,7 +177,7 @@ func (s *ClusterConfigurationService) DeleteStaticTokens() error {
 // GetAuthPreference fetches the cluster authentication preferences
 // from the backend and return them.
 func (s *ClusterConfigurationService) GetAuthPreference(ctx context.Context) (types.AuthPreference, error) {
-	item, err := s.Get(ctx, backend.Key(authPrefix, preferencePrefix, generalPrefix))
+	item, err := s.Get(ctx, backend.NewKey(authPrefix, preferencePrefix, generalPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("authentication preference not found")
@@ -201,7 +201,7 @@ func (s *ClusterConfigurationService) CreateAuthPreference(ctx context.Context,
 	}
 
 	item := backend.Item{
-		Key:   backend.Key(authPrefix, preferencePrefix, generalPrefix),
+		Key:   backend.NewKey(authPrefix, preferencePrefix, generalPrefix),
 		Value: value,
 	}
 
@@ -228,7 +228,7 @@ func (s *ClusterConfigurationService) UpdateAuthPreference(ctx context.Context,
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(authPrefix, preferencePrefix, generalPrefix),
+		Key:      backend.NewKey(authPrefix, preferencePrefix, generalPrefix),
 		Value:    value,
 		Revision: rev,
 	}
@@ -256,7 +256,7 @@ func (s *ClusterConfigurationService) UpsertAuthPreference(ctx context.Context,
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(authPrefix, preferencePrefix, generalPrefix),
+		Key:      backend.NewKey(authPrefix, preferencePrefix, generalPrefix),
 		Value:    value,
 		Revision: rev,
 	}
@@ -272,7 +272,7 @@ func (s *ClusterConfigurationService) UpsertAuthPreference(ctx context.Context,
 
 // DeleteAuthPreference deletes types.AuthPreference from the backend.
 func (s *ClusterConfigurationService) DeleteAuthPreference(ctx context.Context) error {
-	err := s.Delete(ctx, backend.Key(authPrefix, preferencePrefix, generalPrefix))
+	err := s.Delete(ctx, backend.NewKey(authPrefix, preferencePrefix, generalPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("auth preference not found")
@@ -284,7 +284,7 @@ func (s *ClusterConfigurationService) DeleteAuthPreference(ctx context.Context)
 
 // GetClusterAuditConfig gets cluster audit config from the backend.
 func (s *ClusterConfigurationService) GetClusterAuditConfig(ctx context.Context) (types.ClusterAuditConfig, error) {
-	item, err := s.Get(ctx, backend.Key(clusterConfigPrefix, auditPrefix))
+	item, err := s.Get(ctx, backend.NewKey(clusterConfigPrefix, auditPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("cluster audit config not found")
@@ -308,7 +308,7 @@ func (s *ClusterConfigurationService) CreateClusterAuditConfig(ctx context.Conte
 	}
 
 	item := backend.Item{
-		Key:   backend.Key(clusterConfigPrefix, auditPrefix),
+		Key:   backend.NewKey(clusterConfigPrefix, auditPrefix),
 		Value: value,
 	}
 
@@ -330,7 +330,7 @@ func (s *ClusterConfigurationService) UpdateClusterAuditConfig(ctx context.Conte
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, auditPrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, auditPrefix),
 		Value:    value,
 		Revision: rev,
 	}
@@ -351,7 +351,7 @@ func (s *ClusterConfigurationService) UpsertClusterAuditConfig(ctx context.Conte
 	}
 
 	item := backend.Item{
-		Key:   backend.Key(clusterConfigPrefix, auditPrefix),
+		Key:   backend.NewKey(clusterConfigPrefix, auditPrefix),
 		Value: value,
 	}
 
@@ -366,7 +366,7 @@ func (s *ClusterConfigurationService) UpsertClusterAuditConfig(ctx context.Conte
 
 // DeleteClusterAuditConfig deletes ClusterAuditConfig from the backend.
 func (s *ClusterConfigurationService) DeleteClusterAuditConfig(ctx context.Context) error {
-	err := s.Delete(ctx, backend.Key(clusterConfigPrefix, auditPrefix))
+	err := s.Delete(ctx, backend.NewKey(clusterConfigPrefix, auditPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("cluster audit config not found")
@@ -378,7 +378,7 @@ func (s *ClusterConfigurationService) DeleteClusterAuditConfig(ctx context.Conte
 
 // GetClusterNetworkingConfig gets cluster networking config from the backend.
 func (s *ClusterConfigurationService) GetClusterNetworkingConfig(ctx context.Context) (types.ClusterNetworkingConfig, error) {
-	item, err := s.Get(ctx, backend.Key(clusterConfigPrefix, networkingPrefix))
+	item, err := s.Get(ctx, backend.NewKey(clusterConfigPrefix, networkingPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("cluster networking config not found")
@@ -401,7 +401,7 @@ func (s *ClusterConfigurationService) CreateClusterNetworkingConfig(ctx context.
 	}
 
 	item := backend.Item{
-		Key:   backend.Key(clusterConfigPrefix, networkingPrefix),
+		Key:   backend.NewKey(clusterConfigPrefix, networkingPrefix),
 		Value: value,
 	}
 
@@ -428,7 +428,7 @@ func (s *ClusterConfigurationService) UpdateClusterNetworkingConfig(ctx context.
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, networkingPrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, networkingPrefix),
 		Value:    value,
 		Revision: rev,
 	}
@@ -455,7 +455,7 @@ func (s *ClusterConfigurationService) UpsertClusterNetworkingConfig(ctx context.
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, networkingPrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, networkingPrefix),
 		Value:    value,
 		Revision: rev,
 	}
@@ -471,7 +471,7 @@ func (s *ClusterConfigurationService) UpsertClusterNetworkingConfig(ctx context.
 
 // DeleteClusterNetworkingConfig deletes ClusterNetworkingConfig from the backend.
 func (s *ClusterConfigurationService) DeleteClusterNetworkingConfig(ctx context.Context) error {
-	err := s.Delete(ctx, backend.Key(clusterConfigPrefix, networkingPrefix))
+	err := s.Delete(ctx, backend.NewKey(clusterConfigPrefix, networkingPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("cluster networking config not found")
@@ -483,7 +483,7 @@ func (s *ClusterConfigurationService) DeleteClusterNetworkingConfig(ctx context.
 
 // GetSessionRecordingConfig gets session recording config from the backend.
 func (s *ClusterConfigurationService) GetSessionRecordingConfig(ctx context.Context) (types.SessionRecordingConfig, error) {
-	item, err := s.Get(ctx, backend.Key(clusterConfigPrefix, sessionRecordingPrefix))
+	item, err := s.Get(ctx, backend.NewKey(clusterConfigPrefix, sessionRecordingPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("session recording config not found")
@@ -506,7 +506,7 @@ func (s *ClusterConfigurationService) CreateSessionRecordingConfig(ctx context.C
 	}
 
 	item := backend.Item{
-		Key:   backend.Key(clusterConfigPrefix, sessionRecordingPrefix),
+		Key:   backend.NewKey(clusterConfigPrefix, sessionRecordingPrefix),
 		Value: value,
 	}
 
@@ -533,7 +533,7 @@ func (s *ClusterConfigurationService) UpdateSessionRecordingConfig(ctx context.C
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, sessionRecordingPrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, sessionRecordingPrefix),
 		Value:    value,
 		Revision: rev,
 	}
@@ -561,7 +561,7 @@ func (s *ClusterConfigurationService) UpsertSessionRecordingConfig(ctx context.C
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, sessionRecordingPrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, sessionRecordingPrefix),
 		Value:    value,
 		Revision: rev,
 	}
@@ -577,7 +577,7 @@ func (s *ClusterConfigurationService) UpsertSessionRecordingConfig(ctx context.C
 
 // DeleteSessionRecordingConfig deletes SessionRecordingConfig from the backend.
 func (s *ClusterConfigurationService) DeleteSessionRecordingConfig(ctx context.Context) error {
-	err := s.Delete(ctx, backend.Key(clusterConfigPrefix, sessionRecordingPrefix))
+	err := s.Delete(ctx, backend.NewKey(clusterConfigPrefix, sessionRecordingPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("session recording config not found")
@@ -607,7 +607,7 @@ func (s *ClusterConfigurationService) GetInstallers(ctx context.Context) ([]type
 }
 
 func (s *ClusterConfigurationService) GetUIConfig(ctx context.Context) (types.UIConfig, error) {
-	item, err := s.Get(ctx, backend.Key(clusterConfigPrefix, uiPrefix))
+	item, err := s.Get(ctx, backend.NewKey(clusterConfigPrefix, uiPrefix))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -622,7 +622,7 @@ func (s *ClusterConfigurationService) SetUIConfig(ctx context.Context, uic types
 	}
 
 	_, err = s.Put(ctx, backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, uiPrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, uiPrefix),
 		Value:    value,
 		Revision: rev,
 	})
@@ -630,12 +630,12 @@ func (s *ClusterConfigurationService) SetUIConfig(ctx context.Context, uic types
 }
 
 func (s *ClusterConfigurationService) DeleteUIConfig(ctx context.Context) error {
-	return trace.Wrap(s.Delete(ctx, backend.Key(clusterConfigPrefix, uiPrefix)))
+	return trace.Wrap(s.Delete(ctx, backend.NewKey(clusterConfigPrefix, uiPrefix)))
 }
 
 // GetInstaller gets the script of the cluster from the backend.
 func (s *ClusterConfigurationService) GetInstaller(ctx context.Context, name string) (types.Installer, error) {
-	item, err := s.Get(ctx, backend.Key(clusterConfigPrefix, scriptsPrefix, installerPrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(clusterConfigPrefix, scriptsPrefix, installerPrefix, name))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -651,7 +651,7 @@ func (s *ClusterConfigurationService) SetInstaller(ctx context.Context, ins type
 	}
 
 	_, err = s.Put(ctx, backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, scriptsPrefix, installerPrefix, ins.GetName()),
+		Key:      backend.NewKey(clusterConfigPrefix, scriptsPrefix, installerPrefix, ins.GetName()),
 		Value:    value,
 		Expires:  ins.Expiry(),
 		Revision: rev,
@@ -662,7 +662,7 @@ func (s *ClusterConfigurationService) SetInstaller(ctx context.Context, ins type
 // DeleteInstaller sets the installer script to default script in the backend.
 func (s *ClusterConfigurationService) DeleteInstaller(ctx context.Context, name string) error {
 	return trace.Wrap(
-		s.Delete(ctx, backend.Key(clusterConfigPrefix, scriptsPrefix, installerPrefix, name)))
+		s.Delete(ctx, backend.NewKey(clusterConfigPrefix, scriptsPrefix, installerPrefix, name)))
 }
 
 // DeleteAllInstallers removes all installer resources.
@@ -677,7 +677,7 @@ func (s *ClusterConfigurationService) DeleteAllInstallers(ctx context.Context) e
 
 // GetClusterMaintenanceConfig loads the maintenance config singleton resource.
 func (s *ClusterConfigurationService) GetClusterMaintenanceConfig(ctx context.Context) (types.ClusterMaintenanceConfig, error) {
-	item, err := s.Get(ctx, backend.Key(clusterConfigPrefix, maintenancePrefix))
+	item, err := s.Get(ctx, backend.NewKey(clusterConfigPrefix, maintenancePrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("no maintenance config has been created")
@@ -702,7 +702,7 @@ func (s *ClusterConfigurationService) UpdateClusterMaintenanceConfig(ctx context
 	err := generic.FastUpdateNonceProtectedResource(
 		ctx,
 		s.Backend,
-		backend.Key(clusterConfigPrefix, maintenancePrefix),
+		backend.NewKey(clusterConfigPrefix, maintenancePrefix),
 		cmc,
 	)
 
@@ -715,7 +715,7 @@ func (s *ClusterConfigurationService) UpdateClusterMaintenanceConfig(ctx context
 
 // DeleteClusterMaintenanceConfig deletes the maintenance config singleton resource.
 func (s *ClusterConfigurationService) DeleteClusterMaintenanceConfig(ctx context.Context) error {
-	err := s.Delete(ctx, backend.Key(clusterConfigPrefix, maintenancePrefix))
+	err := s.Delete(ctx, backend.NewKey(clusterConfigPrefix, maintenancePrefix))
 	if err != nil {
 		return trace.Wrap(err)
 	}
@@ -724,7 +724,7 @@ func (s *ClusterConfigurationService) DeleteClusterMaintenanceConfig(ctx context
 
 // GetAccessGraphSettings fetches the cluster *clusterconfigpb.AccessGraphSettings from the backend and return them.
 func (s *ClusterConfigurationService) GetAccessGraphSettings(ctx context.Context) (*clusterconfigpb.AccessGraphSettings, error) {
-	item, err := s.Get(ctx, backend.Key(clusterConfigPrefix, accessGraphSettingsPrefix))
+	item, err := s.Get(ctx, backend.NewKey(clusterConfigPrefix, accessGraphSettingsPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("AccessGraphSettings preference not found")
@@ -743,7 +743,7 @@ func (s *ClusterConfigurationService) CreateAccessGraphSettings(ctx context.Cont
 	}
 
 	item := backend.Item{
-		Key:   backend.Key(clusterConfigPrefix, accessGraphSettingsPrefix),
+		Key:   backend.NewKey(clusterConfigPrefix, accessGraphSettingsPrefix),
 		Value: value,
 	}
 
@@ -766,7 +766,7 @@ func (s *ClusterConfigurationService) UpdateAccessGraphSettings(ctx context.Cont
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, accessGraphSettingsPrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, accessGraphSettingsPrefix),
 		Value:    value,
 		Revision: rev,
 	}
@@ -789,7 +789,7 @@ func (s *ClusterConfigurationService) UpsertAccessGraphSettings(ctx context.Cont
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, accessGraphSettingsPrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, accessGraphSettingsPrefix),
 		Value:    value,
 		Revision: rev,
 	}
@@ -805,7 +805,7 @@ func (s *ClusterConfigurationService) UpsertAccessGraphSettings(ctx context.Cont
 
 // DeleteAccessGraphSettings deletes *clusterconfigpb.AccessGraphSettings from the backend.
 func (s *ClusterConfigurationService) DeleteAccessGraphSettings(ctx context.Context) error {
-	err := s.Delete(ctx, backend.Key(clusterConfigPrefix, accessGraphSettingsPrefix))
+	err := s.Delete(ctx, backend.NewKey(clusterConfigPrefix, accessGraphSettingsPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("access graph settings not found")
diff --git a/lib/services/local/connection_diagnostic.go b/lib/services/local/connection_diagnostic.go
index 7157f31a331c7..645318152b4c3 100644
--- a/lib/services/local/connection_diagnostic.go
+++ b/lib/services/local/connection_diagnostic.go
@@ -54,7 +54,7 @@ func (s *ConnectionDiagnosticService) CreateConnectionDiagnostic(ctx context.Con
 	}
 
 	item := backend.Item{
-		Key:     backend.Key(connectionDiagnosticPrefix, connectionDiagnostic.GetName()),
+		Key:     backend.NewKey(connectionDiagnosticPrefix, connectionDiagnostic.GetName()),
 		Value:   value,
 		Expires: connectionDiagnostic.Expiry(),
 	}
@@ -74,7 +74,7 @@ func (s *ConnectionDiagnosticService) UpdateConnectionDiagnostic(ctx context.Con
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(connectionDiagnosticPrefix, connectionDiagnostic.GetName()),
+		Key:      backend.NewKey(connectionDiagnosticPrefix, connectionDiagnostic.GetName()),
 		Value:    value,
 		Expires:  connectionDiagnostic.Expiry(),
 		Revision: rev,
@@ -87,7 +87,7 @@ func (s *ConnectionDiagnosticService) UpdateConnectionDiagnostic(ctx context.Con
 // AppendDiagnosticTrace adds a Trace into the ConnectionDiagnostics.
 // It does a CompareAndSwap to ensure atomicity.
 func (s *ConnectionDiagnosticService) AppendDiagnosticTrace(ctx context.Context, name string, t *types.ConnectionDiagnosticTrace) (types.ConnectionDiagnostic, error) {
-	existing, err := s.Get(ctx, backend.Key(connectionDiagnosticPrefix, name))
+	existing, err := s.Get(ctx, backend.NewKey(connectionDiagnosticPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("connection diagnostic %q doesn't exist", name)
@@ -109,7 +109,7 @@ func (s *ConnectionDiagnosticService) AppendDiagnosticTrace(ctx context.Context,
 	}
 
 	newItem := backend.Item{
-		Key:      backend.Key(connectionDiagnosticPrefix, connectionDiagnostic.GetName()),
+		Key:      backend.NewKey(connectionDiagnosticPrefix, connectionDiagnostic.GetName()),
 		Value:    value,
 		Expires:  connectionDiagnostic.Expiry(),
 		Revision: existing.Revision,
@@ -127,7 +127,7 @@ func (s *ConnectionDiagnosticService) AppendDiagnosticTrace(ctx context.Context,
 //
 // If not found, a `trace.NotFound` error is returned
 func (s *ConnectionDiagnosticService) GetConnectionDiagnostic(ctx context.Context, name string) (types.ConnectionDiagnostic, error) {
-	item, err := s.Get(ctx, backend.Key(connectionDiagnosticPrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(connectionDiagnosticPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("connection diagnostic %q doesn't exist", name)
diff --git a/lib/services/local/databases.go b/lib/services/local/databases.go
index cb1f1c2022341..8248d34951162 100644
--- a/lib/services/local/databases.go
+++ b/lib/services/local/databases.go
@@ -59,7 +59,7 @@ func (s *DatabaseService) GetDatabases(ctx context.Context) ([]types.Database, e
 
 // GetDatabase returns the specified database resource.
 func (s *DatabaseService) GetDatabase(ctx context.Context, name string) (types.Database, error) {
-	item, err := s.Get(ctx, backend.Key(databasesPrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(databasesPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("database %q doesn't exist", name)
@@ -84,7 +84,7 @@ func (s *DatabaseService) CreateDatabase(ctx context.Context, database types.Dat
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(databasesPrefix, database.GetName()),
+		Key:     backend.NewKey(databasesPrefix, database.GetName()),
 		Value:   value,
 		Expires: database.Expiry(),
 	}
@@ -110,7 +110,7 @@ func (s *DatabaseService) UpdateDatabase(ctx context.Context, database types.Dat
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(databasesPrefix, database.GetName()),
+		Key:      backend.NewKey(databasesPrefix, database.GetName()),
 		Value:    value,
 		Expires:  database.Expiry(),
 		Revision: rev,
@@ -128,7 +128,7 @@ func (s *DatabaseService) UpdateDatabase(ctx context.Context, database types.Dat
 
 // DeleteDatabase removes the specified database resource.
 func (s *DatabaseService) DeleteDatabase(ctx context.Context, name string) error {
-	err := s.Delete(ctx, backend.Key(databasesPrefix, name))
+	err := s.Delete(ctx, backend.NewKey(databasesPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("database %q doesn't exist", name)
diff --git a/lib/services/local/databaseservice.go b/lib/services/local/databaseservice.go
index 135c9c3170022..ad7f064e35b68 100644
--- a/lib/services/local/databaseservice.go
+++ b/lib/services/local/databaseservice.go
@@ -50,7 +50,7 @@ func (s *DatabaseServicesService) UpsertDatabaseService(ctx context.Context, ser
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(databaseServicePrefix, service.GetName()),
+		Key:      backend.NewKey(databaseServicePrefix, service.GetName()),
 		Value:    value,
 		Expires:  service.Expiry(),
 		Revision: rev,
@@ -73,7 +73,7 @@ func (s *DatabaseServicesService) UpsertDatabaseService(ctx context.Context, ser
 
 // DeleteDatabaseService removes the specified DatabaseService resource.
 func (s *DatabaseServicesService) DeleteDatabaseService(ctx context.Context, name string) error {
-	err := s.Delete(ctx, backend.Key(databaseServicePrefix, name))
+	err := s.Delete(ctx, backend.NewKey(databaseServicePrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("databaseService %q doesn't exist", name)
diff --git a/lib/services/local/desktops.go b/lib/services/local/desktops.go
index 59aba247c7d13..2fd0afbe8d1b4 100644
--- a/lib/services/local/desktops.go
+++ b/lib/services/local/desktops.go
@@ -76,7 +76,7 @@ func (s *WindowsDesktopService) CreateWindowsDesktop(ctx context.Context, deskto
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(windowsDesktopsPrefix, desktop.GetHostID(), desktop.GetName()),
+		Key:     backend.NewKey(windowsDesktopsPrefix, desktop.GetHostID(), desktop.GetName()),
 		Value:   value,
 		Expires: desktop.Expiry(),
 	}
@@ -102,7 +102,7 @@ func (s *WindowsDesktopService) UpdateWindowsDesktop(ctx context.Context, deskto
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(windowsDesktopsPrefix, desktop.GetHostID(), desktop.GetName()),
+		Key:      backend.NewKey(windowsDesktopsPrefix, desktop.GetHostID(), desktop.GetName()),
 		Value:    value,
 		Expires:  desktop.Expiry(),
 		Revision: rev,
@@ -129,7 +129,7 @@ func (s *WindowsDesktopService) UpsertWindowsDesktop(ctx context.Context, deskto
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(windowsDesktopsPrefix, desktop.GetHostID(), desktop.GetName()),
+		Key:      backend.NewKey(windowsDesktopsPrefix, desktop.GetHostID(), desktop.GetName()),
 		Value:    value,
 		Expires:  desktop.Expiry(),
 		Revision: rev,
@@ -147,7 +147,7 @@ func (s *WindowsDesktopService) DeleteWindowsDesktop(ctx context.Context, hostID
 		return trace.Errorf("name must not be empty")
 	}
 
-	key := backend.Key(windowsDesktopsPrefix, hostID, name)
+	key := backend.NewKey(windowsDesktopsPrefix, hostID, name)
 
 	err := s.Delete(ctx, key)
 	if err != nil {
@@ -176,7 +176,7 @@ func (s *WindowsDesktopService) ListWindowsDesktops(ctx context.Context, req typ
 		return nil, trace.BadParameter("nonpositive parameter limit")
 	}
 
-	rangeStart := backend.Key(windowsDesktopsPrefix, req.StartKey)
+	rangeStart := backend.NewKey(windowsDesktopsPrefix, req.StartKey)
 	rangeEnd := backend.RangeEnd(backend.ExactKey(windowsDesktopsPrefix))
 	filter := services.MatchResourceFilter{
 		ResourceKind:   types.KindWindowsDesktop,
@@ -248,7 +248,7 @@ func (s *WindowsDesktopService) ListWindowsDesktopServices(ctx context.Context,
 		return nil, trace.BadParameter("nonpositive parameter limit")
 	}
 
-	rangeStart := backend.Key(windowsDesktopServicesPrefix, req.StartKey)
+	rangeStart := backend.NewKey(windowsDesktopServicesPrefix, req.StartKey)
 	rangeEnd := backend.RangeEnd(backend.ExactKey(windowsDesktopServicesPrefix))
 	filter := services.MatchResourceFilter{
 		ResourceKind:   types.KindWindowsDesktopService,
diff --git a/lib/services/local/dynamic_access.go b/lib/services/local/dynamic_access.go
index 7c21ff945fe27..89424bc50cae9 100644
--- a/lib/services/local/dynamic_access.go
+++ b/lib/services/local/dynamic_access.go
@@ -331,7 +331,7 @@ func (s *DynamicAccessService) ListAccessRequests(ctx context.Context, req *prot
 
 	startKey := backend.ExactKey(accessRequestsPrefix)
 	if req.StartKey != "" {
-		startKey = backend.Key(accessRequestsPrefix, req.StartKey)
+		startKey = backend.NewKey(accessRequestsPrefix, req.StartKey)
 	}
 	endKey := backend.RangeEnd(backend.ExactKey(accessRequestsPrefix))
 
@@ -486,11 +486,11 @@ func itemToAccessRequest(item backend.Item, opts ...services.MarshalOption) (*ty
 }
 
 func accessRequestKey(name string) []byte {
-	return backend.Key(accessRequestsPrefix, name, paramsPrefix)
+	return backend.NewKey(accessRequestsPrefix, name, paramsPrefix)
 }
 
 func AccessRequestAllowedPromotionKey(name string) []byte {
-	return backend.Key(accessRequestPromotionPrefix, name, paramsPrefix)
+	return backend.NewKey(accessRequestPromotionPrefix, name, paramsPrefix)
 }
 
 const (
diff --git a/lib/services/local/events.go b/lib/services/local/events.go
index d0908209da0a8..18ee515318256 100644
--- a/lib/services/local/events.go
+++ b/lib/services/local/events.go
@@ -398,7 +398,7 @@ func newCertAuthorityParser(loadSecrets bool, filter map[string]string) *certAut
 	caFilter.FromMap(filter)
 	return &certAuthorityParser{
 		loadSecrets: loadSecrets,
-		baseParser:  newBaseParser(backend.Key(authoritiesPrefix)),
+		baseParser:  newBaseParser(backend.NewKey(authoritiesPrefix)),
 		filter:      caFilter,
 	}
 }
@@ -445,7 +445,7 @@ func (p *certAuthorityParser) parse(event backend.Event) (types.Resource, error)
 
 func newProvisionTokenParser() *provisionTokenParser {
 	return &provisionTokenParser{
-		baseParser: newBaseParser(backend.Key(tokensPrefix)),
+		baseParser: newBaseParser(backend.NewKey(tokensPrefix)),
 	}
 }
 
@@ -473,7 +473,7 @@ func (p *provisionTokenParser) parse(event backend.Event) (types.Resource, error
 
 func newStaticTokensParser() *staticTokensParser {
 	return &staticTokensParser{
-		baseParser: newBaseParser(backend.Key(clusterConfigPrefix, staticTokensPrefix)),
+		baseParser: newBaseParser(backend.NewKey(clusterConfigPrefix, staticTokensPrefix)),
 	}
 }
 
@@ -506,7 +506,7 @@ func (p *staticTokensParser) parse(event backend.Event) (types.Resource, error)
 
 func newClusterAuditConfigParser() *clusterAuditConfigParser {
 	return &clusterAuditConfigParser{
-		baseParser: newBaseParser(backend.Key(clusterConfigPrefix, auditPrefix)),
+		baseParser: newBaseParser(backend.NewKey(clusterConfigPrefix, auditPrefix)),
 	}
 }
 
@@ -540,7 +540,7 @@ func (p *clusterAuditConfigParser) parse(event backend.Event) (types.Resource, e
 
 func newClusterNetworkingConfigParser() *clusterNetworkingConfigParser {
 	return &clusterNetworkingConfigParser{
-		baseParser: newBaseParser(backend.Key(clusterConfigPrefix, networkingPrefix)),
+		baseParser: newBaseParser(backend.NewKey(clusterConfigPrefix, networkingPrefix)),
 	}
 }
 
@@ -574,7 +574,7 @@ func (p *clusterNetworkingConfigParser) parse(event backend.Event) (types.Resour
 
 func newAuthPreferenceParser() *authPreferenceParser {
 	return &authPreferenceParser{
-		baseParser: newBaseParser(backend.Key(authPrefix, preferencePrefix, generalPrefix)),
+		baseParser: newBaseParser(backend.NewKey(authPrefix, preferencePrefix, generalPrefix)),
 	}
 }
 
@@ -608,7 +608,7 @@ func (p *authPreferenceParser) parse(event backend.Event) (types.Resource, error
 
 func newUIConfigParser() *uiConfigParser {
 	return &uiConfigParser{
-		baseParser: newBaseParser(backend.Key(clusterConfigPrefix, uiPrefix)),
+		baseParser: newBaseParser(backend.NewKey(clusterConfigPrefix, uiPrefix)),
 	}
 }
 
@@ -642,7 +642,7 @@ func (p *uiConfigParser) parse(event backend.Event) (types.Resource, error) {
 
 func newSessionRecordingConfigParser() *sessionRecordingConfigParser {
 	return &sessionRecordingConfigParser{
-		baseParser: newBaseParser(backend.Key(clusterConfigPrefix, sessionRecordingPrefix)),
+		baseParser: newBaseParser(backend.NewKey(clusterConfigPrefix, sessionRecordingPrefix)),
 	}
 }
 
@@ -676,7 +676,7 @@ func (p *sessionRecordingConfigParser) parse(event backend.Event) (types.Resourc
 
 func newClusterNameParser() *clusterNameParser {
 	return &clusterNameParser{
-		baseParser: newBaseParser(backend.Key(clusterConfigPrefix, namePrefix)),
+		baseParser: newBaseParser(backend.NewKey(clusterConfigPrefix, namePrefix)),
 	}
 }
 
@@ -708,9 +708,9 @@ func (p *clusterNameParser) parse(event backend.Event) (types.Resource, error) {
 }
 
 func newNamespaceParser(name string) *namespaceParser {
-	prefix := backend.Key(namespacesPrefix)
+	prefix := backend.NewKey(namespacesPrefix)
 	if name != "" {
-		prefix = backend.Key(namespacesPrefix, name, paramsPrefix)
+		prefix = backend.NewKey(namespacesPrefix, name, paramsPrefix)
 	}
 	return &namespaceParser{
 		baseParser: newBaseParser(prefix),
@@ -749,7 +749,7 @@ func (p *namespaceParser) parse(event backend.Event) (types.Resource, error) {
 
 func newRoleParser() *roleParser {
 	return &roleParser{
-		baseParser: newBaseParser(backend.Key(rolesPrefix)),
+		baseParser: newBaseParser(backend.NewKey(rolesPrefix)),
 	}
 }
 
@@ -782,8 +782,8 @@ func newAccessRequestParser(m map[string]string) (*accessRequestParser, error) {
 	}
 	return &accessRequestParser{
 		filter:      filter,
-		matchPrefix: backend.Key(accessRequestsPrefix),
-		matchSuffix: backend.Key(paramsPrefix),
+		matchPrefix: backend.NewKey(accessRequestsPrefix),
+		matchSuffix: backend.NewKey(paramsPrefix),
 	}, nil
 }
 
@@ -827,7 +827,7 @@ func (p *accessRequestParser) parse(event backend.Event) (types.Resource, error)
 
 func newUserParser() *userParser {
 	return &userParser{
-		baseParser: newBaseParser(backend.Key(webPrefix, usersPrefix)),
+		baseParser: newBaseParser(backend.NewKey(webPrefix, usersPrefix)),
 	}
 }
 
@@ -863,7 +863,7 @@ func (p *userParser) parse(event backend.Event) (types.Resource, error) {
 
 func newNodeParser() *nodeParser {
 	return &nodeParser{
-		baseParser: newBaseParser(backend.Key(nodesPrefix, apidefaults.Namespace)),
+		baseParser: newBaseParser(backend.NewKey(nodesPrefix, apidefaults.Namespace)),
 	}
 }
 
@@ -877,7 +877,7 @@ func (p *nodeParser) parse(event backend.Event) (types.Resource, error) {
 
 func newProxyParser() *proxyParser {
 	return &proxyParser{
-		baseParser: newBaseParser(backend.Key(proxiesPrefix)),
+		baseParser: newBaseParser(backend.NewKey(proxiesPrefix)),
 	}
 }
 
@@ -891,7 +891,7 @@ func (p *proxyParser) parse(event backend.Event) (types.Resource, error) {
 
 func newAuthServerParser() *authServerParser {
 	return &authServerParser{
-		baseParser: newBaseParser(backend.Key(authServersPrefix)),
+		baseParser: newBaseParser(backend.NewKey(authServersPrefix)),
 	}
 }
 
@@ -905,7 +905,7 @@ func (p *authServerParser) parse(event backend.Event) (types.Resource, error) {
 
 func newTunnelConnectionParser() *tunnelConnectionParser {
 	return &tunnelConnectionParser{
-		baseParser: newBaseParser(backend.Key(tunnelConnectionsPrefix)),
+		baseParser: newBaseParser(backend.NewKey(tunnelConnectionsPrefix)),
 	}
 }
 
@@ -945,7 +945,7 @@ func (p *tunnelConnectionParser) parse(event backend.Event) (types.Resource, err
 
 func newReverseTunnelParser() *reverseTunnelParser {
 	return &reverseTunnelParser{
-		baseParser: newBaseParser(backend.Key(reverseTunnelsPrefix)),
+		baseParser: newBaseParser(backend.NewKey(reverseTunnelsPrefix)),
 	}
 }
 
@@ -973,7 +973,7 @@ func (p *reverseTunnelParser) parse(event backend.Event) (types.Resource, error)
 
 func newAppServerV3Parser() *appServerV3Parser {
 	return &appServerV3Parser{
-		baseParser: newBaseParser(backend.Key(appServersPrefix, apidefaults.Namespace)),
+		baseParser: newBaseParser(backend.NewKey(appServersPrefix, apidefaults.Namespace)),
 	}
 }
 
@@ -1009,7 +1009,7 @@ func (p *appServerV3Parser) parse(event backend.Event) (types.Resource, error) {
 
 func newSAMLIdPSessionParser(loadSecrets bool) *webSessionParser {
 	return &webSessionParser{
-		baseParser:  newBaseParser(backend.Key(samlIdPPrefix, sessionsPrefix)),
+		baseParser:  newBaseParser(backend.NewKey(samlIdPPrefix, sessionsPrefix)),
 		loadSecrets: loadSecrets,
 		hdr: types.ResourceHeader{
 			Kind:    types.KindWebSession,
@@ -1021,7 +1021,7 @@ func newSAMLIdPSessionParser(loadSecrets bool) *webSessionParser {
 
 func newSnowflakeSessionParser(loadSecrets bool) *webSessionParser {
 	return &webSessionParser{
-		baseParser:  newBaseParser(backend.Key(snowflakePrefix, sessionsPrefix)),
+		baseParser:  newBaseParser(backend.NewKey(snowflakePrefix, sessionsPrefix)),
 		loadSecrets: loadSecrets,
 		hdr: types.ResourceHeader{
 			Kind:    types.KindWebSession,
@@ -1033,7 +1033,7 @@ func newSnowflakeSessionParser(loadSecrets bool) *webSessionParser {
 
 func newAppSessionParser(loadSecrets bool) *webSessionParser {
 	return &webSessionParser{
-		baseParser:  newBaseParser(backend.Key(appsPrefix, sessionsPrefix)),
+		baseParser:  newBaseParser(backend.NewKey(appsPrefix, sessionsPrefix)),
 		loadSecrets: loadSecrets,
 		hdr: types.ResourceHeader{
 			Kind:    types.KindWebSession,
@@ -1045,7 +1045,7 @@ func newAppSessionParser(loadSecrets bool) *webSessionParser {
 
 func newWebSessionParser(loadSecrets bool) *webSessionParser {
 	return &webSessionParser{
-		baseParser:  newBaseParser(backend.Key(webPrefix, sessionsPrefix)),
+		baseParser:  newBaseParser(backend.NewKey(webPrefix, sessionsPrefix)),
 		loadSecrets: loadSecrets,
 		hdr: types.ResourceHeader{
 			Kind:    types.KindWebSession,
@@ -1084,7 +1084,7 @@ func (p *webSessionParser) parse(event backend.Event) (types.Resource, error) {
 
 func newWebTokenParser() *webTokenParser {
 	return &webTokenParser{
-		baseParser: newBaseParser(backend.Key(webPrefix, tokensPrefix)),
+		baseParser: newBaseParser(backend.NewKey(webPrefix, tokensPrefix)),
 	}
 }
 
@@ -1112,7 +1112,7 @@ func (p *webTokenParser) parse(event backend.Event) (types.Resource, error) {
 
 func newKubeServerParser() *kubeServerParser {
 	return &kubeServerParser{
-		baseParser: newBaseParser(backend.Key(kubeServersPrefix)),
+		baseParser: newBaseParser(backend.NewKey(kubeServersPrefix)),
 	}
 }
 
@@ -1149,7 +1149,7 @@ func (p *kubeServerParser) parse(event backend.Event) (types.Resource, error) {
 
 func newDatabaseServerParser() *databaseServerParser {
 	return &databaseServerParser{
-		baseParser: newBaseParser(backend.Key(dbServersPrefix, apidefaults.Namespace)),
+		baseParser: newBaseParser(backend.NewKey(dbServersPrefix, apidefaults.Namespace)),
 	}
 }
 
@@ -1186,7 +1186,7 @@ func (p *databaseServerParser) parse(event backend.Event) (types.Resource, error
 
 func newDatabaseServiceParser() *databaseServiceParser {
 	return &databaseServiceParser{
-		baseParser: newBaseParser(backend.Key(databaseServicePrefix)),
+		baseParser: newBaseParser(backend.NewKey(databaseServicePrefix)),
 	}
 }
 
@@ -1211,7 +1211,7 @@ func (p *databaseServiceParser) parse(event backend.Event) (types.Resource, erro
 
 func newKubeClusterParser() *kubeClusterParser {
 	return &kubeClusterParser{
-		baseParser: newBaseParser(backend.Key(kubernetesPrefix)),
+		baseParser: newBaseParser(backend.NewKey(kubernetesPrefix)),
 	}
 }
 
@@ -1235,7 +1235,7 @@ func (p *kubeClusterParser) parse(event backend.Event) (types.Resource, error) {
 
 func newCrownJewelParser() *crownJewelParser {
 	return &crownJewelParser{
-		baseParser: newBaseParser(backend.Key(crownJewelsKey)),
+		baseParser: newBaseParser(backend.NewKey(crownJewelsKey)),
 	}
 }
 
@@ -1263,7 +1263,7 @@ func (p *crownJewelParser) parse(event backend.Event) (types.Resource, error) {
 
 func newAppParser() *appParser {
 	return &appParser{
-		baseParser: newBaseParser(backend.Key(appPrefix)),
+		baseParser: newBaseParser(backend.NewKey(appPrefix)),
 	}
 }
 
@@ -1287,7 +1287,7 @@ func (p *appParser) parse(event backend.Event) (types.Resource, error) {
 
 func newDatabaseParser() *databaseParser {
 	return &databaseParser{
-		baseParser: newBaseParser(backend.Key(databasesPrefix)),
+		baseParser: newBaseParser(backend.NewKey(databasesPrefix)),
 	}
 }
 
@@ -1311,7 +1311,7 @@ func (p *databaseParser) parse(event backend.Event) (types.Resource, error) {
 
 func newDatabaseObjectParser() *databaseObjectParser {
 	return &databaseObjectParser{
-		baseParser: newBaseParser(backend.Key(databaseObjectPrefix)),
+		baseParser: newBaseParser(backend.NewKey(databaseObjectPrefix)),
 	}
 }
 
@@ -1359,7 +1359,7 @@ func parseServer(event backend.Event, kind string) (types.Resource, error) {
 
 func newRemoteClusterParser() *remoteClusterParser {
 	return &remoteClusterParser{
-		matchPrefix: backend.Key(remoteClustersPrefix),
+		matchPrefix: backend.NewKey(remoteClustersPrefix),
 	}
 }
 
@@ -1395,7 +1395,7 @@ func (p *remoteClusterParser) parse(event backend.Event) (types.Resource, error)
 
 func newLockParser() *lockParser {
 	return &lockParser{
-		baseParser: newBaseParser(backend.Key(locksPrefix)),
+		baseParser: newBaseParser(backend.NewKey(locksPrefix)),
 	}
 }
 
@@ -1420,7 +1420,7 @@ func (p *lockParser) parse(event backend.Event) (types.Resource, error) {
 
 func newNetworkRestrictionsParser() *networkRestrictionsParser {
 	return &networkRestrictionsParser{
-		matchPrefix: backend.Key(restrictionsPrefix, network),
+		matchPrefix: backend.NewKey(restrictionsPrefix, network),
 	}
 }
 
@@ -1456,7 +1456,7 @@ func (p *networkRestrictionsParser) parse(event backend.Event) (types.Resource,
 
 func newWindowsDesktopServicesParser() *windowsDesktopServicesParser {
 	return &windowsDesktopServicesParser{
-		baseParser: newBaseParser(backend.Key(windowsDesktopServicesPrefix, "")),
+		baseParser: newBaseParser(backend.NewKey(windowsDesktopServicesPrefix, "")),
 	}
 }
 
@@ -1481,7 +1481,7 @@ func (p *windowsDesktopServicesParser) parse(event backend.Event) (types.Resourc
 
 func newWindowsDesktopsParser() *windowsDesktopsParser {
 	return &windowsDesktopsParser{
-		baseParser: newBaseParser(backend.Key(windowsDesktopsPrefix, "")),
+		baseParser: newBaseParser(backend.NewKey(windowsDesktopsPrefix, "")),
 	}
 }
 
@@ -1522,7 +1522,7 @@ type installerParser struct {
 
 func newInstallerParser() *installerParser {
 	return &installerParser{
-		baseParser: newBaseParser(backend.Key(clusterConfigPrefix, scriptsPrefix, installerPrefix)),
+		baseParser: newBaseParser(backend.NewKey(clusterConfigPrefix, scriptsPrefix, installerPrefix)),
 	}
 }
 
@@ -1555,7 +1555,7 @@ type pluginParser struct {
 
 func newPluginParser(loadSecrets bool) *pluginParser {
 	return &pluginParser{
-		baseParser:  newBaseParser(backend.Key(pluginsPrefix)),
+		baseParser:  newBaseParser(backend.NewKey(pluginsPrefix)),
 		loadSecrets: loadSecrets,
 	}
 }
@@ -1584,7 +1584,7 @@ func (p *pluginParser) parse(event backend.Event) (types.Resource, error) {
 
 func newSAMLIDPServiceProviderParser() *samlIDPServiceProviderParser {
 	return &samlIDPServiceProviderParser{
-		baseParser: newBaseParser(backend.Key(samlIDPServiceProviderPrefix)),
+		baseParser: newBaseParser(backend.NewKey(samlIDPServiceProviderPrefix)),
 	}
 }
 
@@ -1608,7 +1608,7 @@ func (p *samlIDPServiceProviderParser) parse(event backend.Event) (types.Resourc
 
 func newUserGroupParser() *userGroupParser {
 	return &userGroupParser{
-		baseParser: newBaseParser(backend.Key(userGroupPrefix)),
+		baseParser: newBaseParser(backend.NewKey(userGroupPrefix)),
 	}
 }
 
@@ -1632,7 +1632,7 @@ func (p *userGroupParser) parse(event backend.Event) (types.Resource, error) {
 
 func newOktaImportRuleParser() *oktaImportRuleParser {
 	return &oktaImportRuleParser{
-		baseParser: newBaseParser(backend.Key(oktaImportRulePrefix)),
+		baseParser: newBaseParser(backend.NewKey(oktaImportRulePrefix)),
 	}
 }
 
@@ -1656,7 +1656,7 @@ func (p *oktaImportRuleParser) parse(event backend.Event) (types.Resource, error
 
 func newOktaAssignmentParser() *oktaAssignmentParser {
 	return &oktaAssignmentParser{
-		baseParser: newBaseParser(backend.Key(oktaAssignmentPrefix)),
+		baseParser: newBaseParser(backend.NewKey(oktaAssignmentPrefix)),
 	}
 }
 
@@ -1680,7 +1680,7 @@ func (p *oktaAssignmentParser) parse(event backend.Event) (types.Resource, error
 
 func newIntegrationParser() *integrationParser {
 	return &integrationParser{
-		baseParser: newBaseParser(backend.Key(integrationsPrefix)),
+		baseParser: newBaseParser(backend.NewKey(integrationsPrefix)),
 	}
 }
 
@@ -1704,7 +1704,7 @@ func (p *integrationParser) parse(event backend.Event) (types.Resource, error) {
 
 func newDiscoveryConfigParser() *discoveryConfigParser {
 	return &discoveryConfigParser{
-		baseParser: newBaseParser(backend.Key(discoveryConfigPrefix)),
+		baseParser: newBaseParser(backend.NewKey(discoveryConfigPrefix)),
 	}
 }
 
@@ -1733,7 +1733,7 @@ func newHeadlessAuthenticationParser(m map[string]string) (*headlessAuthenticati
 	}
 
 	return &headlessAuthenticationParser{
-		baseParser: newBaseParser(backend.Key(headlessAuthenticationPrefix)),
+		baseParser: newBaseParser(backend.NewKey(headlessAuthenticationPrefix)),
 		filter:     filter,
 	}, nil
 }
@@ -1787,7 +1787,7 @@ func (p *accessListParser) parse(event backend.Event) (types.Resource, error) {
 
 func newAuditQueryParser() *auditQueryParser {
 	return &auditQueryParser{
-		baseParser: newBaseParser(backend.Key(AuditQueryPrefix)),
+		baseParser: newBaseParser(backend.NewKey(AuditQueryPrefix)),
 	}
 }
 
@@ -1810,7 +1810,7 @@ func (p *auditQueryParser) parse(event backend.Event) (types.Resource, error) {
 
 func newSecurityReportParser() *securityReportParser {
 	return &securityReportParser{
-		baseParser: newBaseParser(backend.Key(SecurityReportPrefix)),
+		baseParser: newBaseParser(backend.NewKey(SecurityReportPrefix)),
 	}
 }
 
@@ -1833,7 +1833,7 @@ func (p *securityReportParser) parse(event backend.Event) (types.Resource, error
 
 func newSecurityReportStateParser() *securityReportStateParser {
 	return &securityReportStateParser{
-		baseParser: newBaseParser(backend.Key(SecurityReportStatePrefix)),
+		baseParser: newBaseParser(backend.NewKey(SecurityReportStatePrefix)),
 	}
 }
 
@@ -1856,7 +1856,7 @@ func (p *securityReportStateParser) parse(event backend.Event) (types.Resource,
 
 func newUserLoginStateParser() *userLoginStateParser {
 	return &userLoginStateParser{
-		baseParser: newBaseParser(backend.Key(userLoginStatePrefix)),
+		baseParser: newBaseParser(backend.NewKey(userLoginStatePrefix)),
 	}
 }
 
@@ -1952,7 +1952,7 @@ func (p *accessListReviewParser) parse(event backend.Event) (types.Resource, err
 
 func newKubeWaitingContainerParser() *kubeWaitingContainerParser {
 	return &kubeWaitingContainerParser{
-		baseParser: newBaseParser(backend.Key(kubeWaitingContPrefix)),
+		baseParser: newBaseParser(backend.NewKey(kubeWaitingContPrefix)),
 	}
 }
 
@@ -2035,7 +2035,7 @@ func (p *AccessMonitoringRuleParser) parse(event backend.Event) (types.Resource,
 
 func newUserNotificationParser() *userNotificationParser {
 	return &userNotificationParser{
-		baseParser: newBaseParser(backend.Key(notificationsUserSpecificPrefix)),
+		baseParser: newBaseParser(backend.NewKey(notificationsUserSpecificPrefix)),
 	}
 }
 
@@ -2083,7 +2083,7 @@ func (p *userNotificationParser) parse(event backend.Event) (types.Resource, err
 
 func newGlobalNotificationParser() *globalNotificationParser {
 	return &globalNotificationParser{
-		baseParser: newBaseParser(backend.Key(notificationsGlobalPrefix)),
+		baseParser: newBaseParser(backend.NewKey(notificationsGlobalPrefix)),
 	}
 }
 
@@ -2134,7 +2134,7 @@ func (p *globalNotificationParser) parse(event backend.Event) (types.Resource, e
 
 func newInstanceParser() *instanceParser {
 	return &instanceParser{
-		baseParser: newBaseParser(backend.Key(instancePrefix)),
+		baseParser: newBaseParser(backend.NewKey(instancePrefix)),
 	}
 }
 
@@ -2226,7 +2226,7 @@ func WaitForEvent(ctx context.Context, watcher types.Watcher, m EventMatcher, cl
 
 func newDeviceParser() *deviceParser {
 	return &deviceParser{
-		baseParser: newBaseParser(backend.Key(devicetrust.DevicesIDPrefix...)),
+		baseParser: newBaseParser(backend.NewKey(devicetrust.DevicesIDPrefix...)),
 	}
 }
 
@@ -2270,7 +2270,7 @@ func (p *deviceParser) parse(event backend.Event) (types.Resource, error) {
 
 func newAccessGraphSecretPrivateKeyParser() *accessGraphSecretPrivateKeyParser {
 	return &accessGraphSecretPrivateKeyParser{
-		baseParser: newBaseParser(backend.Key(privateKeysPrefix)),
+		baseParser: newBaseParser(backend.NewKey(privateKeysPrefix)),
 	}
 }
 
@@ -2314,7 +2314,7 @@ func (p *accessGraphSecretPrivateKeyParser) parse(event backend.Event) (types.Re
 
 func newAccessGraphSecretAuthorizedKeyParser() *accessGraphSecretAuthorizedKeyParser {
 	return &accessGraphSecretAuthorizedKeyParser{
-		baseParser: newBaseParser(backend.Key(authorizedKeysPrefix)),
+		baseParser: newBaseParser(backend.NewKey(authorizedKeysPrefix)),
 	}
 }
 
@@ -2394,7 +2394,7 @@ func baseTwoKeys(key []byte) (string, string, error) {
 
 func newSPIFFEFederationParser() *spiffeFederationParser {
 	return &spiffeFederationParser{
-		baseParser: newBaseParser(backend.Key(spiffeFederationPrefix)),
+		baseParser: newBaseParser(backend.NewKey(spiffeFederationPrefix)),
 	}
 }
 
@@ -2422,7 +2422,7 @@ func (p *spiffeFederationParser) parse(event backend.Event) (types.Resource, err
 
 func newAccessGraphSettingsParser() *accessGraphSettingsParser {
 	return &accessGraphSettingsParser{
-		baseParser: newBaseParser(backend.Key(clusterConfigPrefix, accessGraphSettingsPrefix)),
+		baseParser: newBaseParser(backend.NewKey(clusterConfigPrefix, accessGraphSettingsPrefix)),
 	}
 }
 
diff --git a/lib/services/local/externalauditstorage.go b/lib/services/local/externalauditstorage.go
index 981ac764c999e..668db8f1385de 100644
--- a/lib/services/local/externalauditstorage.go
+++ b/lib/services/local/externalauditstorage.go
@@ -38,8 +38,8 @@ const (
 )
 
 var (
-	draftExternalAuditStorageBackendKey   = backend.Key(externalAuditStoragePrefix, externalAuditStorageDraftName)
-	clusterExternalAuditStorageBackendKey = backend.Key(externalAuditStoragePrefix, externalAuditStorageClusterName)
+	draftExternalAuditStorageBackendKey   = backend.NewKey(externalAuditStoragePrefix, externalAuditStorageDraftName)
+	clusterExternalAuditStorageBackendKey = backend.NewKey(externalAuditStoragePrefix, externalAuditStorageClusterName)
 )
 
 // ExternalAuditStorageService manages External Audit Storage resources in the Backend.
diff --git a/lib/services/local/generic/generic.go b/lib/services/local/generic/generic.go
index 123c1bfa34840..2c2c89ff18589 100644
--- a/lib/services/local/generic/generic.go
+++ b/lib/services/local/generic/generic.go
@@ -177,7 +177,7 @@ func (s *Service[T]) ListResourcesReturnNextResource(ctx context.Context, pageSi
 	return resources, next, trace.Wrap(err)
 }
 func (s *Service[T]) listResourcesReturnNextResourceWithKey(ctx context.Context, pageSize int, pageToken string) ([]T, *T, string, error) {
-	rangeStart := backend.Key(s.backendPrefix, pageToken)
+	rangeStart := backend.NewKey(s.backendPrefix, pageToken)
 	rangeEnd := backend.RangeEnd(backend.ExactKey(s.backendPrefix))
 
 	// Adjust page size, so it can't be too large.
@@ -220,7 +220,7 @@ func (s *Service[T]) listResourcesReturnNextResourceWithKey(ctx context.Context,
 
 // ListResourcesWithFilter returns a paginated list of resources that match the given filter.
 func (s *Service[T]) ListResourcesWithFilter(ctx context.Context, pageSize int, pageToken string, matcher func(T) bool) ([]T, string, error) {
-	rangeStart := backend.Key(s.backendPrefix, pageToken)
+	rangeStart := backend.NewKey(s.backendPrefix, pageToken)
 	rangeEnd := backend.RangeEnd(backend.ExactKey(s.backendPrefix))
 
 	// Adjust page size, so it can't be too large.
@@ -444,7 +444,7 @@ func (s *Service[T]) MakeBackendItem(resource T, name string) (backend.Item, err
 
 // MakeKey will make a key for the service given a name.
 func (s *Service[T]) MakeKey(name string) []byte {
-	return backend.Key(s.backendPrefix, name)
+	return backend.NewKey(s.backendPrefix, name)
 }
 
 // RunWhileLocked will run the given function in a backend lock. This is a wrapper around the backend.RunWhileLocked function.
diff --git a/lib/services/local/headlessauthn.go b/lib/services/local/headlessauthn.go
index 36f102c338115..2c2cf51e4e3e0 100644
--- a/lib/services/local/headlessauthn.go
+++ b/lib/services/local/headlessauthn.go
@@ -158,5 +158,5 @@ func unmarshalHeadlessAuthentication(data []byte) (*types.HeadlessAuthentication
 const headlessAuthenticationPrefix = "headless_authentication"
 
 func headlessAuthenticationKey(username, name string) []byte {
-	return backend.Key(headlessAuthenticationPrefix, usersPrefix, username, name)
+	return backend.NewKey(headlessAuthenticationPrefix, usersPrefix, username, name)
 }
diff --git a/lib/services/local/headlessauthn_watcher.go b/lib/services/local/headlessauthn_watcher.go
index e87995f8bfb1d..3cb060576a237 100644
--- a/lib/services/local/headlessauthn_watcher.go
+++ b/lib/services/local/headlessauthn_watcher.go
@@ -205,7 +205,7 @@ func (h *HeadlessAuthenticationWatcher) newWatcher(ctx context.Context) (backend
 	watcher, err := h.identityService.NewWatcher(ctx, backend.Watch{
 		Name:            types.KindHeadlessAuthentication,
 		MetricComponent: types.KindHeadlessAuthentication,
-		Prefixes:        [][]byte{backend.Key(headlessAuthenticationPrefix)},
+		Prefixes:        [][]byte{backend.NewKey(headlessAuthenticationPrefix)},
 	})
 	if err != nil {
 		return nil, trace.Wrap(err)
diff --git a/lib/services/local/inventory.go b/lib/services/local/inventory.go
index 8d906ec502de4..2b488e49e58aa 100644
--- a/lib/services/local/inventory.go
+++ b/lib/services/local/inventory.go
@@ -70,7 +70,7 @@ func (s *PresenceService) GetInstances(ctx context.Context, req types.InstanceFi
 
 // getInstance gets an instance resource by server ID.
 func (s *PresenceService) getInstance(ctx context.Context, serverID string) (types.Instance, error) {
-	item, err := s.Get(ctx, backend.Key(instancePrefix, serverID))
+	item, err := s.Get(ctx, backend.NewKey(instancePrefix, serverID))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -105,7 +105,7 @@ func (s *PresenceService) UpsertInstance(ctx context.Context, instance types.Ins
 		return trace.BadParameter("unexpected type %T, expected %T", instance, v1)
 	}
 
-	item, err := generic.FastMarshal(backend.Key(instancePrefix, instance.GetName()), v1)
+	item, err := generic.FastMarshal(backend.NewKey(instancePrefix, instance.GetName()), v1)
 	if err != nil {
 		return trace.Errorf("failed to marshal Instance: %v", err)
 	}
diff --git a/lib/services/local/kube.go b/lib/services/local/kube.go
index f21b6442ac9d1..335f8ca3952ca 100644
--- a/lib/services/local/kube.go
+++ b/lib/services/local/kube.go
@@ -59,7 +59,7 @@ func (s *KubernetesService) GetKubernetesClusters(ctx context.Context) ([]types.
 
 // GetKubernetesCluster returns the specified kubernetes cluster resource.
 func (s *KubernetesService) GetKubernetesCluster(ctx context.Context, name string) (types.KubeCluster, error) {
-	item, err := s.Get(ctx, backend.Key(kubernetesPrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(kubernetesPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("kubernetes cluster %q doesn't exist", name)
@@ -84,7 +84,7 @@ func (s *KubernetesService) CreateKubernetesCluster(ctx context.Context, cluster
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(kubernetesPrefix, cluster.GetName()),
+		Key:     backend.NewKey(kubernetesPrefix, cluster.GetName()),
 		Value:   value,
 		Expires: cluster.Expiry(),
 	}
@@ -110,7 +110,7 @@ func (s *KubernetesService) UpdateKubernetesCluster(ctx context.Context, cluster
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(kubernetesPrefix, cluster.GetName()),
+		Key:      backend.NewKey(kubernetesPrefix, cluster.GetName()),
 		Value:    value,
 		Expires:  cluster.Expiry(),
 		Revision: rev,
@@ -127,7 +127,7 @@ func (s *KubernetesService) UpdateKubernetesCluster(ctx context.Context, cluster
 
 // DeleteKubernetesCluster removes the specified kubernetes cluster resource.
 func (s *KubernetesService) DeleteKubernetesCluster(ctx context.Context, name string) error {
-	err := s.Delete(ctx, backend.Key(kubernetesPrefix, name))
+	err := s.Delete(ctx, backend.NewKey(kubernetesPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("kubernetes cluster %q doesn't exist", name)
diff --git a/lib/services/local/plugin_data.go b/lib/services/local/plugin_data.go
index 721994567feb5..03b5b898e66ba 100644
--- a/lib/services/local/plugin_data.go
+++ b/lib/services/local/plugin_data.go
@@ -249,7 +249,7 @@ func itemToPluginData(item backend.Item) (types.PluginData, error) {
 }
 
 func pluginDataKey(kind string, name string) []byte {
-	return backend.Key(pluginDataPrefix, kind, name, paramsPrefix)
+	return backend.NewKey(pluginDataPrefix, kind, name, paramsPrefix)
 }
 
 const (
diff --git a/lib/services/local/plugins.go b/lib/services/local/plugins.go
index ae5a40937c5ee..4f2bcfe7a1fa2 100644
--- a/lib/services/local/plugins.go
+++ b/lib/services/local/plugins.go
@@ -48,7 +48,7 @@ func (s *PluginsService) CreatePlugin(ctx context.Context, plugin types.Plugin)
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(pluginsPrefix, plugin.GetName()),
+		Key:     backend.NewKey(pluginsPrefix, plugin.GetName()),
 		Value:   value,
 		Expires: plugin.Expiry(),
 	}
@@ -62,7 +62,7 @@ func (s *PluginsService) CreatePlugin(ctx context.Context, plugin types.Plugin)
 
 // DeletePlugin implements service.Plugins
 func (s *PluginsService) DeletePlugin(ctx context.Context, name string) error {
-	err := s.backend.Delete(ctx, backend.Key(pluginsPrefix, name))
+	err := s.backend.Delete(ctx, backend.NewKey(pluginsPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("plugin %q doesn't exist", name)
@@ -82,7 +82,7 @@ func (s *PluginsService) UpdatePlugin(ctx context.Context, plugin types.Plugin)
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(pluginsPrefix, plugin.GetName()),
+		Key:      backend.NewKey(pluginsPrefix, plugin.GetName()),
 		Value:    value,
 		Expires:  plugin.Expiry(),
 		Revision: plugin.GetRevision(),
@@ -109,7 +109,7 @@ func (s *PluginsService) DeleteAllPlugins(ctx context.Context) error {
 
 // GetPlugin implements services.Plugins
 func (s *PluginsService) GetPlugin(ctx context.Context, name string, withSecrets bool) (types.Plugin, error) {
-	item, err := s.backend.Get(ctx, backend.Key(pluginsPrefix, name))
+	item, err := s.backend.Get(ctx, backend.NewKey(pluginsPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("plugin %q doesn't exist", name)
@@ -159,7 +159,7 @@ func (s *PluginsService) ListPlugins(ctx context.Context, limit int, startKey st
 	// Get at most limit+1 results to determine if there will be a next key.
 	maxLimit := limit + 1
 
-	startKeyBytes := backend.Key(pluginsPrefix, startKey)
+	startKeyBytes := backend.NewKey(pluginsPrefix, startKey)
 	endKey := backend.RangeEnd(backend.ExactKey(pluginsPrefix))
 	result, err := s.backend.GetRange(ctx, startKeyBytes, endKey, maxLimit)
 	if err != nil {
@@ -218,7 +218,7 @@ func (s *PluginsService) SetPluginStatus(ctx context.Context, name string, statu
 }
 
 func (s *PluginsService) updateAndSwap(ctx context.Context, name string, modify func(types.Plugin) error) error {
-	key := backend.Key(pluginsPrefix, name)
+	key := backend.NewKey(pluginsPrefix, name)
 	item, err := s.backend.Get(ctx, key)
 	if err != nil {
 		if trace.IsNotFound(err) {
@@ -247,7 +247,7 @@ func (s *PluginsService) updateAndSwap(ctx context.Context, name string, modify
 	}
 
 	_, err = s.backend.CompareAndSwap(ctx, *item, backend.Item{
-		Key:      backend.Key(pluginsPrefix, plugin.GetName()),
+		Key:      backend.NewKey(pluginsPrefix, plugin.GetName()),
 		Value:    value,
 		Expires:  plugin.Expiry(),
 		Revision: rev,
diff --git a/lib/services/local/presence.go b/lib/services/local/presence.go
index 27c30776b4cd7..7108812744ea9 100644
--- a/lib/services/local/presence.go
+++ b/lib/services/local/presence.go
@@ -104,7 +104,7 @@ func (s *PresenceService) UpsertNamespace(n types.Namespace) error {
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(namespacesPrefix, n.Metadata.Name, paramsPrefix),
+		Key:      backend.NewKey(namespacesPrefix, n.Metadata.Name, paramsPrefix),
 		Value:    value,
 		Expires:  n.Metadata.Expiry(),
 		Revision: rev,
@@ -122,7 +122,7 @@ func (s *PresenceService) GetNamespace(name string) (*types.Namespace, error) {
 	if name == "" {
 		return nil, trace.BadParameter("missing namespace name")
 	}
-	item, err := s.Get(context.TODO(), backend.Key(namespacesPrefix, name, paramsPrefix))
+	item, err := s.Get(context.TODO(), backend.NewKey(namespacesPrefix, name, paramsPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("namespace %q is not found", name)
@@ -138,7 +138,7 @@ func (s *PresenceService) DeleteNamespace(namespace string) error {
 	if namespace == "" {
 		return trace.BadParameter("missing namespace name")
 	}
-	err := s.Delete(context.TODO(), backend.Key(namespacesPrefix, namespace, paramsPrefix))
+	err := s.Delete(context.TODO(), backend.NewKey(namespacesPrefix, namespace, paramsPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("namespace %q is not found", namespace)
@@ -229,9 +229,9 @@ func (s *PresenceService) DeleteServerInfo(ctx context.Context, name string) err
 func serverInfoKey(subkind, name string) []byte {
 	switch subkind {
 	case types.SubKindCloudInfo:
-		return backend.Key(serverInfoPrefix, cloudLabelsPrefix, name)
+		return backend.NewKey(serverInfoPrefix, cloudLabelsPrefix, name)
 	default:
-		return backend.Key(serverInfoPrefix, name)
+		return backend.NewKey(serverInfoPrefix, name)
 	}
 }
 
@@ -266,7 +266,7 @@ func (s *PresenceService) upsertServer(ctx context.Context, prefix string, serve
 		return trace.Wrap(err)
 	}
 	_, err = s.Put(ctx, backend.Item{
-		Key:      backend.Key(prefix, server.GetName()),
+		Key:      backend.NewKey(prefix, server.GetName()),
 		Value:    value,
 		Expires:  server.Expiry(),
 		Revision: rev,
@@ -282,7 +282,7 @@ func (s *PresenceService) DeleteAllNodes(ctx context.Context, namespace string)
 
 // DeleteNode deletes node
 func (s *PresenceService) DeleteNode(ctx context.Context, namespace string, name string) error {
-	key := backend.Key(nodesPrefix, namespace, name)
+	key := backend.NewKey(nodesPrefix, namespace, name)
 	return s.Delete(ctx, key)
 }
 
@@ -294,7 +294,7 @@ func (s *PresenceService) GetNode(ctx context.Context, namespace, name string) (
 	if name == "" {
 		return nil, trace.BadParameter("missing parameter name")
 	}
-	item, err := s.Get(ctx, backend.Key(nodesPrefix, namespace, name))
+	item, err := s.Get(ctx, backend.NewKey(nodesPrefix, namespace, name))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -354,7 +354,7 @@ func (s *PresenceService) UpsertNode(ctx context.Context, server types.Server) (
 		return nil, trace.Wrap(err)
 	}
 	_, err = s.Put(ctx, backend.Item{
-		Key:      backend.Key(nodesPrefix, server.GetNamespace(), server.GetName()),
+		Key:      backend.NewKey(nodesPrefix, server.GetNamespace(), server.GetName()),
 		Value:    value,
 		Expires:  server.Expiry(),
 		Revision: rev,
@@ -390,7 +390,7 @@ func (s *PresenceService) DeleteAllAuthServers() error {
 
 // DeleteAuthServer deletes auth server by name
 func (s *PresenceService) DeleteAuthServer(name string) error {
-	key := backend.Key(authServersPrefix, name)
+	key := backend.NewKey(authServersPrefix, name)
 	return s.Delete(context.TODO(), key)
 }
 
@@ -413,7 +413,7 @@ func (s *PresenceService) DeleteAllProxies() error {
 
 // DeleteProxy deletes proxy
 func (s *PresenceService) DeleteProxy(ctx context.Context, name string) error {
-	key := backend.Key(proxiesPrefix, name)
+	key := backend.NewKey(proxiesPrefix, name)
 	return s.Delete(ctx, key)
 }
 
@@ -434,7 +434,7 @@ func (s *PresenceService) UpsertReverseTunnel(tunnel types.ReverseTunnel) error
 		return trace.Wrap(err)
 	}
 	_, err = s.Put(context.TODO(), backend.Item{
-		Key:      backend.Key(reverseTunnelsPrefix, tunnel.GetName()),
+		Key:      backend.NewKey(reverseTunnelsPrefix, tunnel.GetName()),
 		Value:    value,
 		Expires:  tunnel.Expiry(),
 		Revision: rev,
@@ -444,7 +444,7 @@ func (s *PresenceService) UpsertReverseTunnel(tunnel types.ReverseTunnel) error
 
 // GetReverseTunnel returns reverse tunnel by name
 func (s *PresenceService) GetReverseTunnel(name string, opts ...services.MarshalOption) (types.ReverseTunnel, error) {
-	item, err := s.Get(context.TODO(), backend.Key(reverseTunnelsPrefix, name))
+	item, err := s.Get(context.TODO(), backend.NewKey(reverseTunnelsPrefix, name))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -478,7 +478,7 @@ func (s *PresenceService) GetReverseTunnels(ctx context.Context, opts ...service
 
 // DeleteReverseTunnel deletes reverse tunnel by it's cluster name
 func (s *PresenceService) DeleteReverseTunnel(clusterName string) error {
-	err := s.Delete(context.TODO(), backend.Key(reverseTunnelsPrefix, clusterName))
+	err := s.Delete(context.TODO(), backend.NewKey(reverseTunnelsPrefix, clusterName))
 	return trace.Wrap(err)
 }
 
@@ -509,7 +509,7 @@ func (s *PresenceService) AcquireSemaphore(ctx context.Context, req types.Acquir
 	leaseID := uuid.New().String()
 
 	// key is not modified, so allocate it once
-	key := backend.Key(semaphoresPrefix, req.SemaphoreKind, req.SemaphoreName)
+	key := backend.NewKey(semaphoresPrefix, req.SemaphoreKind, req.SemaphoreName)
 
 Acquire:
 	for i := int64(0); i < leaseRetryAttempts; i++ {
@@ -636,7 +636,7 @@ func (s *PresenceService) KeepAliveSemaphoreLease(ctx context.Context, lease typ
 		return trace.BadParameter("lease %v has expired at %v", lease.LeaseID, lease.Expires)
 	}
 
-	key := backend.Key(semaphoresPrefix, lease.SemaphoreKind, lease.SemaphoreName)
+	key := backend.NewKey(semaphoresPrefix, lease.SemaphoreKind, lease.SemaphoreName)
 	item, err := s.Get(ctx, key)
 	if err != nil {
 		if trace.IsNotFound(err) {
@@ -703,7 +703,7 @@ func (s *PresenceService) CancelSemaphoreLease(ctx context.Context, lease types.
 			}
 		}
 
-		key := backend.Key(semaphoresPrefix, lease.SemaphoreKind, lease.SemaphoreName)
+		key := backend.NewKey(semaphoresPrefix, lease.SemaphoreKind, lease.SemaphoreName)
 		item, err := s.Get(ctx, key)
 		if err != nil {
 			return trace.Wrap(err)
@@ -751,7 +751,7 @@ func (s *PresenceService) GetSemaphores(ctx context.Context, filter types.Semaph
 	var items []backend.Item
 	if filter.SemaphoreKind != "" && filter.SemaphoreName != "" {
 		// special case: filter corresponds to a single semaphore
-		item, err := s.Get(ctx, backend.Key(semaphoresPrefix, filter.SemaphoreKind, filter.SemaphoreName))
+		item, err := s.Get(ctx, backend.NewKey(semaphoresPrefix, filter.SemaphoreKind, filter.SemaphoreName))
 		if err != nil {
 			if trace.IsNotFound(err) {
 				return nil, nil
@@ -796,7 +796,7 @@ func (s *PresenceService) DeleteSemaphore(ctx context.Context, filter types.Sema
 	if filter.SemaphoreKind == "" || filter.SemaphoreName == "" {
 		return trace.BadParameter("semaphore kind and name must be specified for deletion")
 	}
-	return trace.Wrap(s.Delete(ctx, backend.Key(semaphoresPrefix, filter.SemaphoreKind, filter.SemaphoreName)))
+	return trace.Wrap(s.Delete(ctx, backend.NewKey(semaphoresPrefix, filter.SemaphoreKind, filter.SemaphoreName)))
 }
 
 // UpsertKubernetesServer registers an kubernetes server.
@@ -814,7 +814,7 @@ func (s *PresenceService) UpsertKubernetesServer(ctx context.Context, server typ
 	// the following path in the backend:
 	//   /kubeServers/<host-uuid>/<name>
 	_, err = s.Put(ctx, backend.Item{
-		Key: backend.Key(kubeServersPrefix,
+		Key: backend.NewKey(kubeServersPrefix,
 			server.GetHostID(),
 			server.GetName()),
 		Value:    value,
@@ -844,7 +844,7 @@ func (s *PresenceService) DeleteKubernetesServer(ctx context.Context, hostID, na
 	if hostID == "" {
 		return trace.BadParameter("no hostID specified for kubernetes server deletion")
 	}
-	key := backend.Key(kubeServersPrefix, hostID, name)
+	key := backend.NewKey(kubeServersPrefix, hostID, name)
 	return s.Delete(ctx, key)
 }
 
@@ -919,7 +919,7 @@ func (s *PresenceService) UpsertDatabaseServer(ctx context.Context, server types
 	// they are stored under the following path in the backend:
 	//   /databaseServers/<namespace>/<host-uuid>/<name>
 	_, err = s.Put(ctx, backend.Item{
-		Key: backend.Key(dbServersPrefix,
+		Key: backend.NewKey(dbServersPrefix,
 			server.GetNamespace(),
 			server.GetHostID(),
 			server.GetName()),
@@ -953,7 +953,7 @@ func (s *PresenceService) DeleteDatabaseServer(ctx context.Context, namespace, h
 	if name == "" {
 		return trace.BadParameter("missing database server name")
 	}
-	key := backend.Key(dbServersPrefix, namespace, hostID, name)
+	key := backend.NewKey(dbServersPrefix, namespace, hostID, name)
 	return s.Delete(ctx, key)
 }
 
@@ -1013,7 +1013,7 @@ func (s *PresenceService) UpsertApplicationServer(ctx context.Context, server ty
 	// the following path in the backend:
 	//   /appServers/<namespace>/<host-uuid>/<name>
 	_, err = s.Put(ctx, backend.Item{
-		Key: backend.Key(appServersPrefix,
+		Key: backend.NewKey(appServersPrefix,
 			server.GetNamespace(),
 			server.GetHostID(),
 			server.GetName()),
@@ -1038,7 +1038,7 @@ func (s *PresenceService) UpsertApplicationServer(ctx context.Context, server ty
 
 // DeleteApplicationServer removes specified application server.
 func (s *PresenceService) DeleteApplicationServer(ctx context.Context, namespace, hostID, name string) error {
-	key := backend.Key(appServersPrefix, namespace, hostID, name)
+	key := backend.NewKey(appServersPrefix, namespace, hostID, name)
 	return s.Delete(ctx, key)
 }
 
@@ -1058,21 +1058,21 @@ func (s *PresenceService) KeepAliveServer(ctx context.Context, h types.KeepAlive
 	var key []byte
 	switch h.GetType() {
 	case constants.KeepAliveNode:
-		key = backend.Key(nodesPrefix, h.Namespace, h.Name)
+		key = backend.NewKey(nodesPrefix, h.Namespace, h.Name)
 	case constants.KeepAliveApp:
 		if h.HostID != "" {
-			key = backend.Key(appServersPrefix, h.Namespace, h.HostID, h.Name)
+			key = backend.NewKey(appServersPrefix, h.Namespace, h.HostID, h.Name)
 		} else { // DELETE IN 9.0. Legacy app server is heartbeating back.
-			key = backend.Key(appsPrefix, serversPrefix, h.Namespace, h.Name)
+			key = backend.NewKey(appsPrefix, serversPrefix, h.Namespace, h.Name)
 		}
 	case constants.KeepAliveDatabase:
-		key = backend.Key(dbServersPrefix, h.Namespace, h.HostID, h.Name)
+		key = backend.NewKey(dbServersPrefix, h.Namespace, h.HostID, h.Name)
 	case constants.KeepAliveWindowsDesktopService:
-		key = backend.Key(windowsDesktopServicesPrefix, h.Name)
+		key = backend.NewKey(windowsDesktopServicesPrefix, h.Name)
 	case constants.KeepAliveKube:
-		key = backend.Key(kubeServersPrefix, h.HostID, h.Name)
+		key = backend.NewKey(kubeServersPrefix, h.HostID, h.Name)
 	case constants.KeepAliveDatabaseService:
-		key = backend.Key(databaseServicePrefix, h.Name)
+		key = backend.NewKey(databaseServicePrefix, h.Name)
 	default:
 		return trace.BadParameter("unknown keep-alive type %q", h.GetType())
 	}
@@ -1106,7 +1106,7 @@ func (s *PresenceService) GetWindowsDesktopServices(ctx context.Context) ([]type
 }
 
 func (s *PresenceService) GetWindowsDesktopService(ctx context.Context, name string) (types.WindowsDesktopService, error) {
-	result, err := s.Get(ctx, backend.Key(windowsDesktopServicesPrefix, name))
+	result, err := s.Get(ctx, backend.NewKey(windowsDesktopServicesPrefix, name))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -1132,7 +1132,7 @@ func (s *PresenceService) UpsertWindowsDesktopService(ctx context.Context, srv t
 		return nil, trace.Wrap(err)
 	}
 	_, err = s.Put(ctx, backend.Item{
-		Key:      backend.Key(windowsDesktopServicesPrefix, srv.GetName()),
+		Key:      backend.NewKey(windowsDesktopServicesPrefix, srv.GetName()),
 		Value:    value,
 		Expires:  srv.Expiry(),
 		Revision: rev,
@@ -1156,7 +1156,7 @@ func (s *PresenceService) DeleteWindowsDesktopService(ctx context.Context, name
 	if name == "" {
 		return trace.BadParameter("missing windows desktop service name")
 	}
-	key := backend.Key(windowsDesktopServicesPrefix, name)
+	key := backend.NewKey(windowsDesktopServicesPrefix, name)
 	return s.Delete(ctx, key)
 }
 
@@ -1173,7 +1173,7 @@ func (s *PresenceService) UpsertHostUserInteractionTime(ctx context.Context, nam
 		return err
 	}
 	_, err = s.Put(ctx, backend.Item{
-		Key:   backend.Key(loginTimePrefix, name),
+		Key:   backend.NewKey(loginTimePrefix, name),
 		Value: val,
 	})
 	return trace.Wrap(err)
@@ -1181,7 +1181,7 @@ func (s *PresenceService) UpsertHostUserInteractionTime(ctx context.Context, nam
 
 // GetHostUserInteractionTime retrieves a unix user's interaction time
 func (s *PresenceService) GetHostUserInteractionTime(ctx context.Context, name string) (time.Time, error) {
-	item, err := s.Get(ctx, backend.Key(loginTimePrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(loginTimePrefix, name))
 	if err != nil {
 		return time.Time{}, trace.Wrap(err)
 	}
@@ -1263,7 +1263,7 @@ func (s *PresenceService) listResources(ctx context.Context, req proto.ListResou
 		return nil, trace.NotImplemented("%s not implemented at ListResources", req.ResourceType)
 	}
 
-	rangeStart := backend.Key(append(keyPrefix, req.StartKey)...)
+	rangeStart := backend.NewKey(append(keyPrefix, req.StartKey)...)
 	rangeEnd := backend.RangeEnd(backend.ExactKey(keyPrefix...))
 	filter := services.MatchResourceFilter{
 		ResourceKind:   req.ResourceType,
diff --git a/lib/services/local/provisioning.go b/lib/services/local/provisioning.go
index 6ee087762b6f3..4d9eeac954a41 100644
--- a/lib/services/local/provisioning.go
+++ b/lib/services/local/provisioning.go
@@ -77,7 +77,7 @@ func (s *ProvisioningService) tokenToItem(p types.ProvisionToken) (*backend.Item
 		return nil, trace.Wrap(err)
 	}
 	item := &backend.Item{
-		Key:      backend.Key(tokensPrefix, p.GetName()),
+		Key:      backend.NewKey(tokensPrefix, p.GetName()),
 		Value:    data,
 		Expires:  p.Expiry(),
 		Revision: rev,
@@ -87,7 +87,7 @@ func (s *ProvisioningService) tokenToItem(p types.ProvisionToken) (*backend.Item
 
 // DeleteAllTokens deletes all provisioning tokens
 func (s *ProvisioningService) DeleteAllTokens() error {
-	startKey := backend.Key(tokensPrefix)
+	startKey := backend.NewKey(tokensPrefix)
 	return s.DeleteRange(context.TODO(), startKey, backend.RangeEnd(startKey))
 }
 
@@ -96,7 +96,7 @@ func (s *ProvisioningService) GetToken(ctx context.Context, token string) (types
 	if token == "" {
 		return nil, trace.BadParameter("missing parameter token")
 	}
-	item, err := s.Get(ctx, backend.Key(tokensPrefix, token))
+	item, err := s.Get(ctx, backend.NewKey(tokensPrefix, token))
 	if trace.IsNotFound(err) {
 		return nil, trace.NotFound("provisioning token(%s) not found", backend.MaskKeyName(token))
 	} else if err != nil {
@@ -111,7 +111,7 @@ func (s *ProvisioningService) DeleteToken(ctx context.Context, token string) err
 	if token == "" {
 		return trace.BadParameter("missing parameter token")
 	}
-	err := s.Delete(ctx, backend.Key(tokensPrefix, token))
+	err := s.Delete(ctx, backend.NewKey(tokensPrefix, token))
 	if trace.IsNotFound(err) {
 		return trace.NotFound("provisioning token(%s) not found", backend.MaskKeyName(token))
 	}
diff --git a/lib/services/local/resource.go b/lib/services/local/resource.go
index ffa2cd99a2d8e..f820d603efb7e 100644
--- a/lib/services/local/resource.go
+++ b/lib/services/local/resource.go
@@ -133,7 +133,7 @@ func itemFromClusterNetworkingConfig(cnc types.ClusterNetworkingConfig) (*backen
 	}
 
 	item := &backend.Item{
-		Key:      backend.Key(clusterConfigPrefix, networkingPrefix),
+		Key:      backend.NewKey(clusterConfigPrefix, networkingPrefix),
 		Value:    value,
 		Revision: cnc.GetRevision(),
 	}
@@ -152,7 +152,7 @@ func itemFromAuthPreference(ap types.AuthPreference) (*backend.Item, error) {
 	}
 
 	item := &backend.Item{
-		Key:      backend.Key(authPrefix, preferencePrefix, generalPrefix),
+		Key:      backend.NewKey(authPrefix, preferencePrefix, generalPrefix),
 		Value:    value,
 		Revision: ap.GetRevision(),
 	}
@@ -172,7 +172,7 @@ func itemFromUser(user types.User) (*backend.Item, error) {
 		return nil, trace.Wrap(err)
 	}
 	item := &backend.Item{
-		Key:      backend.Key(webPrefix, usersPrefix, user.GetName(), paramsPrefix),
+		Key:      backend.NewKey(webPrefix, usersPrefix, user.GetName(), paramsPrefix),
 		Value:    value,
 		Expires:  user.Expiry(),
 		Revision: rev,
@@ -209,7 +209,7 @@ func itemFromCertAuthority(ca types.CertAuthority) (*backend.Item, error) {
 		return nil, trace.Wrap(err)
 	}
 	item := &backend.Item{
-		Key:      backend.Key(authoritiesPrefix, string(ca.GetType()), ca.GetName()),
+		Key:      backend.NewKey(authoritiesPrefix, string(ca.GetType()), ca.GetName()),
 		Value:    value,
 		Expires:  ca.Expiry(),
 		Revision: rev,
@@ -229,7 +229,7 @@ func itemFromProvisionToken(p types.ProvisionToken) (*backend.Item, error) {
 		return nil, trace.Wrap(err)
 	}
 	item := &backend.Item{
-		Key:      backend.Key(tokensPrefix, p.GetName()),
+		Key:      backend.NewKey(tokensPrefix, p.GetName()),
 		Value:    value,
 		Expires:  p.Expiry(),
 		Revision: rev,
@@ -249,7 +249,7 @@ func itemFromTrustedCluster(tc types.TrustedCluster) (*backend.Item, error) {
 		return nil, trace.Wrap(err)
 	}
 	item := &backend.Item{
-		Key:      backend.Key(trustedClustersPrefix, tc.GetName()),
+		Key:      backend.NewKey(trustedClustersPrefix, tc.GetName()),
 		Value:    value,
 		Expires:  tc.Expiry(),
 		Revision: rev,
@@ -269,7 +269,7 @@ func itemFromGithubConnector(gc types.GithubConnector) (*backend.Item, error) {
 		return nil, trace.Wrap(err)
 	}
 	item := &backend.Item{
-		Key:      backend.Key(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, gc.GetName()),
+		Key:      backend.NewKey(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, gc.GetName()),
 		Value:    value,
 		Expires:  gc.Expiry(),
 		Revision: rev,
@@ -287,7 +287,7 @@ func itemFromRole(role types.Role) (*backend.Item, error) {
 	}
 
 	item := &backend.Item{
-		Key:      backend.Key(rolesPrefix, role.GetName(), paramsPrefix),
+		Key:      backend.NewKey(rolesPrefix, role.GetName(), paramsPrefix),
 		Value:    value,
 		Expires:  role.Expiry(),
 		Revision: rev,
@@ -304,7 +304,7 @@ func itemFromOIDCConnector(connector types.OIDCConnector) (*backend.Item, error)
 		return nil, trace.Wrap(err)
 	}
 	item := &backend.Item{
-		Key:      backend.Key(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, connector.GetName()),
+		Key:      backend.NewKey(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, connector.GetName()),
 		Value:    value,
 		Expires:  connector.Expiry(),
 		Revision: rev,
@@ -324,7 +324,7 @@ func itemFromSAMLConnector(connector types.SAMLConnector) (*backend.Item, error)
 		return nil, trace.Wrap(err)
 	}
 	item := &backend.Item{
-		Key:      backend.Key(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, connector.GetName()),
+		Key:      backend.NewKey(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, connector.GetName()),
 		Value:    value,
 		Expires:  connector.Expiry(),
 		Revision: rev,
@@ -387,7 +387,7 @@ func itemsFromLocalAuthSecrets(user string, auth types.LocalAuthSecrets) ([]back
 	}
 	if len(auth.PasswordHash) > 0 {
 		item := backend.Item{
-			Key:   backend.Key(webPrefix, usersPrefix, user, pwdPrefix),
+			Key:   backend.NewKey(webPrefix, usersPrefix, user, pwdPrefix),
 			Value: auth.PasswordHash,
 		}
 		items = append(items, item)
@@ -398,7 +398,7 @@ func itemsFromLocalAuthSecrets(user string, auth types.LocalAuthSecrets) ([]back
 			return nil, trace.Wrap(err)
 		}
 		items = append(items, backend.Item{
-			Key:   backend.Key(webPrefix, usersPrefix, user, mfaDevicePrefix, mfa.Id),
+			Key:   backend.NewKey(webPrefix, usersPrefix, user, mfaDevicePrefix, mfa.Id),
 			Value: value,
 		})
 	}
@@ -417,7 +417,7 @@ func itemFromLock(l types.Lock) (*backend.Item, error) {
 		return nil, trace.Wrap(err)
 	}
 	return &backend.Item{
-		Key:      backend.Key(locksPrefix, l.GetName()),
+		Key:      backend.NewKey(locksPrefix, l.GetName()),
 		Value:    value,
 		Expires:  l.Expiry(),
 		Revision: rev,
@@ -428,7 +428,7 @@ func itemFromLock(l types.Lock) (*backend.Item, error) {
 // has order N cost.
 
 // fullUsersPrefix is the entire string preceding the name of a user in a key
-var fullUsersPrefix = string(backend.Key(webPrefix, usersPrefix)) + "/"
+var fullUsersPrefix = string(backend.NewKey(webPrefix, usersPrefix)) + "/"
 
 // splitUsernameAndSuffix is a helper for extracting usernames and suffixes from
 // backend key values.
diff --git a/lib/services/local/restrictions.go b/lib/services/local/restrictions.go
index c5408f87185ab..57333b4c41897 100644
--- a/lib/services/local/restrictions.go
+++ b/lib/services/local/restrictions.go
@@ -50,7 +50,7 @@ func (s *RestrictionsService) SetNetworkRestrictions(ctx context.Context, nr typ
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(restrictionsPrefix, network),
+		Key:      backend.NewKey(restrictionsPrefix, network),
 		Value:    value,
 		Expires:  nr.Expiry(),
 		Revision: rev,
@@ -64,7 +64,7 @@ func (s *RestrictionsService) SetNetworkRestrictions(ctx context.Context, nr typ
 }
 
 func (s *RestrictionsService) GetNetworkRestrictions(ctx context.Context) (types.NetworkRestrictions, error) {
-	item, err := s.Get(context.TODO(), backend.Key(restrictionsPrefix, network))
+	item, err := s.Get(context.TODO(), backend.NewKey(restrictionsPrefix, network))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -74,7 +74,7 @@ func (s *RestrictionsService) GetNetworkRestrictions(ctx context.Context) (types
 
 // SetNetworkRestrictions upserts NetworkRestrictions
 func (s *RestrictionsService) DeleteNetworkRestrictions(ctx context.Context) error {
-	return trace.Wrap(s.Delete(ctx, backend.Key(restrictionsPrefix, network)))
+	return trace.Wrap(s.Delete(ctx, backend.NewKey(restrictionsPrefix, network)))
 }
 
 const (
diff --git a/lib/services/local/session.go b/lib/services/local/session.go
index 02b597d6c1940..43ce714107963 100644
--- a/lib/services/local/session.go
+++ b/lib/services/local/session.go
@@ -59,7 +59,7 @@ func (s *IdentityService) GetSAMLIdPSession(ctx context.Context, req types.GetSA
 }
 
 func (s *IdentityService) getSession(ctx context.Context, keyParts ...string) (types.WebSession, error) {
-	item, err := s.Get(ctx, backend.Key(keyParts...))
+	item, err := s.Get(ctx, backend.NewKey(keyParts...))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -106,7 +106,7 @@ func (s *IdentityService) ListSAMLIdPSessions(ctx context.Context, pageSize int,
 
 // listSessions gets a paginated list of sessions.
 func (s *IdentityService) listSessions(ctx context.Context, pageSize int, pageToken, user string, keyPrefix ...string) ([]types.WebSession, string, error) {
-	rangeStart := backend.Key(append(keyPrefix, pageToken)...)
+	rangeStart := backend.NewKey(append(keyPrefix, pageToken)...)
 	rangeEnd := backend.RangeEnd(backend.ExactKey(keyPrefix...))
 
 	// Adjust page size, so it can't be too large.
@@ -192,7 +192,7 @@ func (s *IdentityService) upsertSession(ctx context.Context, session types.WebSe
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(append(keyPrefix, session.GetName())...),
+		Key:      backend.NewKey(append(keyPrefix, session.GetName())...),
 		Value:    value,
 		Expires:  session.GetExpiryTime(),
 		Revision: rev,
@@ -206,7 +206,7 @@ func (s *IdentityService) upsertSession(ctx context.Context, session types.WebSe
 
 // DeleteAppSession removes an application web session.
 func (s *IdentityService) DeleteAppSession(ctx context.Context, req types.DeleteAppSessionRequest) error {
-	if err := s.Delete(ctx, backend.Key(appsPrefix, sessionsPrefix, req.SessionID)); err != nil {
+	if err := s.Delete(ctx, backend.NewKey(appsPrefix, sessionsPrefix, req.SessionID)); err != nil {
 		return trace.Wrap(err)
 	}
 	return nil
@@ -214,7 +214,7 @@ func (s *IdentityService) DeleteAppSession(ctx context.Context, req types.Delete
 
 // DeleteSnowflakeSession removes a Snowflake web session.
 func (s *IdentityService) DeleteSnowflakeSession(ctx context.Context, req types.DeleteSnowflakeSessionRequest) error {
-	if err := s.Delete(ctx, backend.Key(snowflakePrefix, sessionsPrefix, req.SessionID)); err != nil {
+	if err := s.Delete(ctx, backend.NewKey(snowflakePrefix, sessionsPrefix, req.SessionID)); err != nil {
 		return trace.Wrap(err)
 	}
 	return nil
@@ -223,7 +223,7 @@ func (s *IdentityService) DeleteSnowflakeSession(ctx context.Context, req types.
 // DeleteSAMLIdPSession removes a SAML IdP session.
 // TODO(Joerger): DELETE IN v18.0.0
 func (s *IdentityService) DeleteSAMLIdPSession(ctx context.Context, req types.DeleteSAMLIdPSessionRequest) error {
-	if err := s.Delete(ctx, backend.Key(samlIdPPrefix, sessionsPrefix, req.SessionID)); err != nil {
+	if err := s.Delete(ctx, backend.NewKey(samlIdPPrefix, sessionsPrefix, req.SessionID)); err != nil {
 		return trace.Wrap(err)
 	}
 	return nil
@@ -510,9 +510,9 @@ type webTokens struct {
 }
 
 func webSessionKey(sessionID string) (key []byte) {
-	return backend.Key(webPrefix, sessionsPrefix, sessionID)
+	return backend.NewKey(webPrefix, sessionsPrefix, sessionID)
 }
 
 func webTokenKey(token string) (key []byte) {
-	return backend.Key(webPrefix, tokensPrefix, token)
+	return backend.NewKey(webPrefix, tokensPrefix, token)
 }
diff --git a/lib/services/local/sessiontracker.go b/lib/services/local/sessiontracker.go
index 780344b208013..b8906b02d04ad 100644
--- a/lib/services/local/sessiontracker.go
+++ b/lib/services/local/sessiontracker.go
@@ -48,7 +48,7 @@ func NewSessionTrackerService(bk backend.Backend) (services.SessionTrackerServic
 }
 
 func (s *sessionTracker) loadSession(ctx context.Context, sessionID string) (types.SessionTracker, error) {
-	sessionJSON, err := s.bk.Get(ctx, backend.Key(sessionPrefix, sessionID))
+	sessionJSON, err := s.bk.Get(ctx, backend.NewKey(sessionPrefix, sessionID))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -64,7 +64,7 @@ func (s *sessionTracker) loadSession(ctx context.Context, sessionID string) (typ
 // UpdatePresence updates the presence status of a user in a session.
 func (s *sessionTracker) UpdatePresence(ctx context.Context, sessionID, user string) error {
 	for i := 0; i < casRetryLimit; i++ {
-		sessionItem, err := s.bk.Get(ctx, backend.Key(sessionPrefix, sessionID))
+		sessionItem, err := s.bk.Get(ctx, backend.NewKey(sessionPrefix, sessionID))
 		if err != nil {
 			return trace.Wrap(err)
 		}
@@ -84,7 +84,7 @@ func (s *sessionTracker) UpdatePresence(ctx context.Context, sessionID, user str
 		}
 
 		item := backend.Item{
-			Key:      backend.Key(sessionPrefix, sessionID),
+			Key:      backend.NewKey(sessionPrefix, sessionID),
 			Value:    sessionJSON,
 			Expires:  session.Expiry(),
 			Revision: sessionItem.Revision,
@@ -188,7 +188,7 @@ func (s *sessionTracker) CreateSessionTracker(ctx context.Context, tracker types
 	}
 
 	item := backend.Item{
-		Key:     backend.Key(sessionPrefix, tracker.GetSessionID()),
+		Key:     backend.NewKey(sessionPrefix, tracker.GetSessionID()),
 		Value:   json,
 		Expires: tracker.Expiry(),
 	}
@@ -203,7 +203,7 @@ func (s *sessionTracker) CreateSessionTracker(ctx context.Context, tracker types
 // UpdateSessionTracker updates a tracker resource for an active session.
 func (s *sessionTracker) UpdateSessionTracker(ctx context.Context, req *proto.UpdateSessionTrackerRequest) error {
 	for i := 0; i < casRetryLimit; i++ {
-		sessionItem, err := s.bk.Get(ctx, backend.Key(sessionPrefix, req.SessionID))
+		sessionItem, err := s.bk.Get(ctx, backend.NewKey(sessionPrefix, req.SessionID))
 		if err != nil {
 			return trace.Wrap(err)
 		}
@@ -263,7 +263,7 @@ func (s *sessionTracker) UpdateSessionTracker(ctx context.Context, req *proto.Up
 		}
 
 		item := backend.Item{
-			Key:      backend.Key(sessionPrefix, req.SessionID),
+			Key:      backend.NewKey(sessionPrefix, req.SessionID),
 			Value:    sessionJSON,
 			Expires:  expiry,
 			Revision: sessionItem.Revision,
@@ -286,5 +286,5 @@ func (s *sessionTracker) UpdateSessionTracker(ctx context.Context, req *proto.Up
 
 // RemoveSessionTracker removes a tracker resource for an active session.
 func (s *sessionTracker) RemoveSessionTracker(ctx context.Context, sessionID string) error {
-	return trace.Wrap(s.bk.Delete(ctx, backend.Key(sessionPrefix, sessionID)))
+	return trace.Wrap(s.bk.Delete(ctx, backend.NewKey(sessionPrefix, sessionID)))
 }
diff --git a/lib/services/local/status.go b/lib/services/local/status.go
index a8fc9d6ff5581..d9855a00fce01 100644
--- a/lib/services/local/status.go
+++ b/lib/services/local/status.go
@@ -102,7 +102,7 @@ func (s *StatusService) getAllClusterAlerts(ctx context.Context) ([]types.Cluste
 }
 
 func (s *StatusService) getClusterAlert(ctx context.Context, alertID string) (types.ClusterAlert, error) {
-	key := backend.Key(clusterAlertPrefix, alertID)
+	key := backend.NewKey(clusterAlertPrefix, alertID)
 	item, err := s.Backend.Get(ctx, key)
 	if err != nil {
 		return types.ClusterAlert{}, trace.Wrap(err)
@@ -136,7 +136,7 @@ func (s *StatusService) UpsertClusterAlert(ctx context.Context, alert types.Clus
 	}
 
 	_, err = s.Backend.Put(ctx, backend.Item{
-		Key:      backend.Key(clusterAlertPrefix, alert.Metadata.Name),
+		Key:      backend.NewKey(clusterAlertPrefix, alert.Metadata.Name),
 		Value:    val,
 		Expires:  alert.Metadata.Expiry(),
 		Revision: rev,
@@ -145,7 +145,7 @@ func (s *StatusService) UpsertClusterAlert(ctx context.Context, alert types.Clus
 }
 
 func (s *StatusService) DeleteClusterAlert(ctx context.Context, alertID string) error {
-	err := s.Backend.Delete(ctx, backend.Key(clusterAlertPrefix, alertID))
+	err := s.Backend.Delete(ctx, backend.NewKey(clusterAlertPrefix, alertID))
 	if trace.IsNotFound(err) {
 		return trace.NotFound("cluster alert %q not found", alertID)
 	}
@@ -164,7 +164,7 @@ func (s *StatusService) CreateAlertAck(ctx context.Context, ack types.AlertAckno
 	}
 
 	_, err = s.Backend.Create(ctx, backend.Item{
-		Key:     backend.Key(alertAckPrefix, ack.AlertID),
+		Key:     backend.NewKey(alertAckPrefix, ack.AlertID),
 		Value:   val,
 		Expires: ack.Expires,
 	})
@@ -205,7 +205,7 @@ func (s *StatusService) ClearAlertAcks(ctx context.Context, req proto.ClearAlert
 		return trace.Wrap(s.Backend.DeleteRange(ctx, startKey, backend.RangeEnd(startKey)))
 	}
 
-	err := s.Backend.Delete(ctx, backend.Key(alertAckPrefix, req.AlertID))
+	err := s.Backend.Delete(ctx, backend.NewKey(alertAckPrefix, req.AlertID))
 	if trace.IsNotFound(err) {
 		return nil
 	}
diff --git a/lib/services/local/trust.go b/lib/services/local/trust.go
index 47ff0d807fb8a..33fb5f3342001 100644
--- a/lib/services/local/trust.go
+++ b/lib/services/local/trust.go
@@ -161,7 +161,7 @@ func (s *CA) CompareAndSwapCertAuthority(new, expected types.CertAuthority) erro
 		return trace.Wrap(err)
 	}
 
-	key := backend.Key(authoritiesPrefix, string(new.GetType()), new.GetName())
+	key := backend.NewKey(authoritiesPrefix, string(new.GetType()), new.GetName())
 
 	actualItem, err := s.Get(context.TODO(), key)
 	if err != nil {
@@ -436,7 +436,7 @@ func (s *CA) UpsertTrustedCluster(ctx context.Context, trustedCluster types.Trus
 		return nil, trace.Wrap(err)
 	}
 	_, err = s.Put(ctx, backend.Item{
-		Key:      backend.Key(trustedClustersPrefix, trustedCluster.GetName()),
+		Key:      backend.NewKey(trustedClustersPrefix, trustedCluster.GetName()),
 		Value:    value,
 		Expires:  trustedCluster.Expiry(),
 		Revision: rev,
@@ -452,7 +452,7 @@ func (s *CA) GetTrustedCluster(ctx context.Context, name string) (types.TrustedC
 	if name == "" {
 		return nil, trace.BadParameter("missing trusted cluster name")
 	}
-	item, err := s.Get(ctx, backend.Key(trustedClustersPrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(trustedClustersPrefix, name))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -485,7 +485,7 @@ func (s *CA) DeleteTrustedCluster(ctx context.Context, name string) error {
 	if name == "" {
 		return trace.BadParameter("missing trusted cluster name")
 	}
-	err := s.Delete(ctx, backend.Key(trustedClustersPrefix, name))
+	err := s.Delete(ctx, backend.NewKey(trustedClustersPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return trace.NotFound("trusted cluster %q is not found", name)
@@ -506,7 +506,7 @@ func (s *CA) UpsertTunnelConnection(conn types.TunnelConnection) error {
 		return trace.Wrap(err)
 	}
 	_, err = s.Put(context.TODO(), backend.Item{
-		Key:      backend.Key(tunnelConnectionsPrefix, conn.GetClusterName(), conn.GetName()),
+		Key:      backend.NewKey(tunnelConnectionsPrefix, conn.GetClusterName(), conn.GetName()),
 		Value:    value,
 		Expires:  conn.Expiry(),
 		Revision: rev,
@@ -519,7 +519,7 @@ func (s *CA) UpsertTunnelConnection(conn types.TunnelConnection) error {
 
 // GetTunnelConnection returns connection by cluster name and connection name
 func (s *CA) GetTunnelConnection(clusterName, connectionName string, opts ...services.MarshalOption) (types.TunnelConnection, error) {
-	item, err := s.Get(context.TODO(), backend.Key(tunnelConnectionsPrefix, clusterName, connectionName))
+	item, err := s.Get(context.TODO(), backend.NewKey(tunnelConnectionsPrefix, clusterName, connectionName))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("trusted cluster connection %q is not found", connectionName)
@@ -588,7 +588,7 @@ func (s *CA) DeleteTunnelConnection(clusterName, connectionName string) error {
 	if connectionName == "" {
 		return trace.BadParameter("missing connection name")
 	}
-	return s.Delete(context.TODO(), backend.Key(tunnelConnectionsPrefix, clusterName, connectionName))
+	return s.Delete(context.TODO(), backend.NewKey(tunnelConnectionsPrefix, clusterName, connectionName))
 }
 
 // DeleteTunnelConnections deletes all tunnel connections for cluster
@@ -617,7 +617,7 @@ func (s *CA) CreateRemoteCluster(
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(remoteClustersPrefix, rc.GetName()),
+		Key:     backend.NewKey(remoteClustersPrefix, rc.GetName()),
 		Value:   value,
 		Expires: rc.Expiry(),
 	}
@@ -658,7 +658,7 @@ func (s *CA) UpdateRemoteCluster(ctx context.Context, rc types.RemoteCluster) (t
 		}
 
 		lease, err := s.ConditionalUpdate(ctx, backend.Item{
-			Key:      backend.Key(remoteClustersPrefix, existing.GetName()),
+			Key:      backend.NewKey(remoteClustersPrefix, existing.GetName()),
 			Value:    updateValue,
 			Expires:  existing.Expiry(),
 			Revision: existing.GetRevision(),
@@ -713,7 +713,7 @@ func (s *CA) PatchRemoteCluster(
 		}
 
 		lease, err := s.ConditionalUpdate(ctx, backend.Item{
-			Key:      backend.Key(remoteClustersPrefix, name),
+			Key:      backend.NewKey(remoteClustersPrefix, name),
 			Value:    updatedValue,
 			Expires:  updated.Expiry(),
 			Revision: updated.GetRevision(),
@@ -759,7 +759,7 @@ func (s *CA) GetRemoteClusters(ctx context.Context) ([]types.RemoteCluster, erro
 func (s *CA) ListRemoteClusters(
 	ctx context.Context, pageSize int, pageToken string,
 ) ([]types.RemoteCluster, string, error) {
-	rangeStart := backend.Key(remoteClustersPrefix, pageToken)
+	rangeStart := backend.NewKey(remoteClustersPrefix, pageToken)
 	rangeEnd := backend.RangeEnd(backend.ExactKey(remoteClustersPrefix))
 
 	// Adjust page size, so it can't be too large.
@@ -804,7 +804,7 @@ func (s *CA) GetRemoteCluster(
 	if clusterName == "" {
 		return nil, trace.BadParameter("missing parameter cluster name")
 	}
-	item, err := s.Get(ctx, backend.Key(remoteClustersPrefix, clusterName))
+	item, err := s.Get(ctx, backend.NewKey(remoteClustersPrefix, clusterName))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("remote cluster %q is not found", clusterName)
@@ -828,7 +828,7 @@ func (s *CA) DeleteRemoteCluster(
 	if clusterName == "" {
 		return trace.BadParameter("missing parameter cluster name")
 	}
-	return s.Delete(ctx, backend.Key(remoteClustersPrefix, clusterName))
+	return s.Delete(ctx, backend.NewKey(remoteClustersPrefix, clusterName))
 }
 
 // DeleteAllRemoteClusters deletes all remote clusters
@@ -855,12 +855,12 @@ func caToItem(key []byte, ca types.CertAuthority) (backend.Item, error) {
 
 // activeCAKey builds the active key variant for the supplied ca id.
 func activeCAKey(id types.CertAuthID) []byte {
-	return backend.Key(authoritiesPrefix, string(id.Type), id.DomainName)
+	return backend.NewKey(authoritiesPrefix, string(id.Type), id.DomainName)
 }
 
 // inactiveCAKey builds the inactive key variant for the supplied ca id.
 func inactiveCAKey(id types.CertAuthID) []byte {
-	return backend.Key(authoritiesPrefix, deactivatedPrefix, string(id.Type), id.DomainName)
+	return backend.NewKey(authoritiesPrefix, deactivatedPrefix, string(id.Type), id.DomainName)
 }
 
 const (
diff --git a/lib/services/local/unstable.go b/lib/services/local/unstable.go
index 3b3194ffca847..cbce002b39217 100644
--- a/lib/services/local/unstable.go
+++ b/lib/services/local/unstable.go
@@ -111,7 +111,7 @@ func (s UnstableService) GetSystemRoleAssertions(ctx context.Context, serverID s
 }
 
 func systemRoleAssertionsKey(serverID string, assertionID string) []byte {
-	return backend.Key(systemRoleAssertionsPrefix, serverID, assertionID)
+	return backend.NewKey(systemRoleAssertionsPrefix, serverID, assertionID)
 }
 
 const (
diff --git a/lib/services/local/userpreferences.go b/lib/services/local/userpreferences.go
index a5518b80a99de..76b1bbe4aadf0 100644
--- a/lib/services/local/userpreferences.go
+++ b/lib/services/local/userpreferences.go
@@ -118,7 +118,7 @@ func (u *UserPreferencesService) getUserPreferences(ctx context.Context, usernam
 
 // backendKey returns the backend key for the user preferences for the given username.
 func backendKey(username string) []byte {
-	return backend.Key(userPreferencesPrefix, username)
+	return backend.NewKey(userPreferencesPrefix, username)
 }
 
 // validatePreferences validates the given preferences.
@@ -132,7 +132,7 @@ func validatePreferences(preferences *userpreferencesv1.UserPreferences) error {
 
 // createBackendItem creates a backend.Item for the given username and user preferences.
 func createBackendItem(username string, preferences *userpreferencesv1.UserPreferences) (backend.Item, error) {
-	settingsKey := backend.Key(userPreferencesPrefix, username)
+	settingsKey := backend.NewKey(userPreferencesPrefix, username)
 
 	payload, err := json.Marshal(preferences)
 	if err != nil {
diff --git a/lib/services/local/users.go b/lib/services/local/users.go
index 505e1abdb2034..7d39351233c2d 100644
--- a/lib/services/local/users.go
+++ b/lib/services/local/users.go
@@ -97,7 +97,7 @@ func (s *IdentityService) DeleteAllUsers(ctx context.Context) error {
 
 // ListUsers returns a page of users.
 func (s *IdentityService) ListUsers(ctx context.Context, req *userspb.ListUsersRequest) (*userspb.ListUsersResponse, error) {
-	rangeStart := backend.Key(webPrefix, usersPrefix, req.PageToken)
+	rangeStart := backend.NewKey(webPrefix, usersPrefix, req.PageToken)
 	rangeEnd := backend.RangeEnd(backend.ExactKey(webPrefix, usersPrefix))
 	pageSize := req.PageSize
 
@@ -319,7 +319,7 @@ func (s *IdentityService) CreateUser(ctx context.Context, user types.User) (type
 	}
 
 	item := backend.Item{
-		Key:     backend.Key(webPrefix, usersPrefix, user.GetName(), paramsPrefix),
+		Key:     backend.NewKey(webPrefix, usersPrefix, user.GetName(), paramsPrefix),
 		Value:   value,
 		Expires: user.Expiry(),
 	}
@@ -356,7 +356,7 @@ func (s *IdentityService) LegacyUpdateUser(ctx context.Context, user types.User)
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, usersPrefix, user.GetName(), paramsPrefix),
+		Key:      backend.NewKey(webPrefix, usersPrefix, user.GetName(), paramsPrefix),
 		Value:    value,
 		Expires:  user.Expiry(),
 		Revision: rev,
@@ -386,7 +386,7 @@ func (s *IdentityService) UpdateUser(ctx context.Context, user types.User) (type
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, usersPrefix, user.GetName(), paramsPrefix),
+		Key:      backend.NewKey(webPrefix, usersPrefix, user.GetName(), paramsPrefix),
 		Value:    value,
 		Expires:  user.Expiry(),
 		Revision: rev,
@@ -450,7 +450,7 @@ func (s *IdentityService) UpsertUser(ctx context.Context, user types.User) (type
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, usersPrefix, user.GetName(), paramsPrefix),
+		Key:      backend.NewKey(webPrefix, usersPrefix, user.GetName(), paramsPrefix),
 		Value:    value,
 		Expires:  user.Expiry(),
 		Revision: rev,
@@ -490,7 +490,7 @@ func (s *IdentityService) CompareAndSwapUser(ctx context.Context, new, existing
 	}
 
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, usersPrefix, new.GetName(), paramsPrefix),
+		Key:      backend.NewKey(webPrefix, usersPrefix, new.GetName(), paramsPrefix),
 		Value:    nil, // avoid marshaling new until we pass one comparison
 		Expires:  new.Expiry(),
 		Revision: "",
@@ -557,7 +557,7 @@ func (s *IdentityService) getUser(ctx context.Context, user string, withSecrets
 		return u, items, trace.Wrap(err)
 	}
 
-	item, err := s.Get(ctx, backend.Key(webPrefix, usersPrefix, user, paramsPrefix))
+	item, err := s.Get(ctx, backend.NewKey(webPrefix, usersPrefix, user, paramsPrefix))
 	if err != nil {
 		return nil, nil, trace.NotFound("user %q not found", user)
 	}
@@ -686,7 +686,7 @@ func (s *IdentityService) upsertPasswordHash(username string, hash []byte) error
 		}
 	}
 	item := backend.Item{
-		Key:   backend.Key(webPrefix, usersPrefix, username, pwdPrefix),
+		Key:   backend.NewKey(webPrefix, usersPrefix, username, pwdPrefix),
 		Value: hash,
 	}
 	_, err = s.Put(context.TODO(), item)
@@ -701,7 +701,7 @@ func (s *IdentityService) GetPasswordHash(user string) ([]byte, error) {
 	if user == "" {
 		return nil, trace.BadParameter("missing user name")
 	}
-	item, err := s.Get(context.TODO(), backend.Key(webPrefix, usersPrefix, user, pwdPrefix))
+	item, err := s.Get(context.TODO(), backend.NewKey(webPrefix, usersPrefix, user, pwdPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("user %q is not found", user)
@@ -718,7 +718,7 @@ func (s *IdentityService) UpsertUsedTOTPToken(user string, otpToken string) erro
 		return trace.BadParameter("missing user name")
 	}
 	item := backend.Item{
-		Key:     backend.Key(webPrefix, usersPrefix, user, usedTOTPPrefix),
+		Key:     backend.NewKey(webPrefix, usersPrefix, user, usedTOTPPrefix),
 		Value:   []byte(otpToken),
 		Expires: s.Clock().Now().UTC().Add(usedTOTPTTL),
 	}
@@ -734,7 +734,7 @@ func (s *IdentityService) GetUsedTOTPToken(user string) (string, error) {
 	if user == "" {
 		return "", trace.BadParameter("missing user name")
 	}
-	item, err := s.Get(context.TODO(), backend.Key(webPrefix, usersPrefix, user, usedTOTPPrefix))
+	item, err := s.Get(context.TODO(), backend.NewKey(webPrefix, usersPrefix, user, usedTOTPPrefix))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return "0", nil
@@ -751,7 +751,7 @@ func (s *IdentityService) DeleteUsedTOTPToken(user string) error {
 	if user == "" {
 		return trace.BadParameter("missing user name")
 	}
-	return s.Delete(context.TODO(), backend.Key(webPrefix, usersPrefix, user, usedTOTPPrefix))
+	return s.Delete(context.TODO(), backend.NewKey(webPrefix, usersPrefix, user, usedTOTPPrefix))
 }
 
 // AddUserLoginAttempt logs user login attempt
@@ -764,7 +764,7 @@ func (s *IdentityService) AddUserLoginAttempt(user string, attempt services.Logi
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(webPrefix, usersPrefix, user, attemptsPrefix, uuid.New().String()),
+		Key:     backend.NewKey(webPrefix, usersPrefix, user, attemptsPrefix, uuid.New().String()),
 		Value:   value,
 		Expires: backend.Expiry(s.Clock(), ttl),
 	}
@@ -851,7 +851,7 @@ func (s *IdentityService) DeletePassword(ctx context.Context, user string) error
 		return trace.BadParameter("missing username")
 	}
 
-	delErr := s.Delete(ctx, backend.Key(webPrefix, usersPrefix, user, pwdPrefix))
+	delErr := s.Delete(ctx, backend.NewKey(webPrefix, usersPrefix, user, pwdPrefix))
 	// Don't bail out just yet if the error is "not found"; the password state
 	// flag may still be unspecified, and we want to make it UNSET.
 	if delErr != nil && !trace.IsNotFound(delErr) {
@@ -966,12 +966,12 @@ type webauthnUser struct {
 }
 
 func webauthnLocalAuthKey(user string) []byte {
-	return backend.Key(webPrefix, usersPrefix, user, webauthnLocalAuthPrefix)
+	return backend.NewKey(webPrefix, usersPrefix, user, webauthnLocalAuthPrefix)
 }
 
 func webauthnUserKey(id []byte) []byte {
 	key := base64.RawURLEncoding.EncodeToString(id)
-	return backend.Key(webauthnPrefix, usersPrefix, key)
+	return backend.NewKey(webauthnPrefix, usersPrefix, key)
 }
 
 func (s *IdentityService) UpsertWebauthnSessionData(ctx context.Context, user, sessionID string, sd *wantypes.SessionData) error {
@@ -1024,7 +1024,7 @@ func (s *IdentityService) DeleteWebauthnSessionData(ctx context.Context, user, s
 }
 
 func sessionDataKey(user, sessionID string) []byte {
-	return backend.Key(webPrefix, usersPrefix, user, webauthnSessionData, sessionID)
+	return backend.NewKey(webPrefix, usersPrefix, user, webauthnSessionData, sessionID)
 }
 
 // globalSessionDataLimiter keeps a count of in-flight session data challenges
@@ -1135,7 +1135,7 @@ func (s *IdentityService) DeleteGlobalWebauthnSessionData(ctx context.Context, s
 }
 
 func globalSessionDataKey(scope, id string) []byte {
-	return backend.Key(webauthnPrefix, webauthnGlobalSessionData, scope, id)
+	return backend.NewKey(webauthnPrefix, webauthnGlobalSessionData, scope, id)
 }
 
 func (s *IdentityService) UpsertMFADevice(ctx context.Context, user string, d *types.MFADevice) error {
@@ -1183,7 +1183,7 @@ func (s *IdentityService) UpsertMFADevice(ctx context.Context, user string, d *t
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, usersPrefix, user, mfaDevicePrefix, d.Id),
+		Key:      backend.NewKey(webPrefix, usersPrefix, user, mfaDevicePrefix, d.Id),
 		Value:    value,
 		Revision: rev,
 	}
@@ -1212,7 +1212,7 @@ func (s *IdentityService) DeleteMFADevice(ctx context.Context, user, id string)
 		return trace.BadParameter("missing parameter id")
 	}
 
-	err := s.Delete(ctx, backend.Key(webPrefix, usersPrefix, user, mfaDevicePrefix, id))
+	err := s.Delete(ctx, backend.NewKey(webPrefix, usersPrefix, user, mfaDevicePrefix, id))
 	return trace.Wrap(err)
 }
 
@@ -1255,7 +1255,7 @@ func (s *IdentityService) UpsertOIDCConnector(ctx context.Context, connector typ
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, connector.GetName()),
+		Key:      backend.NewKey(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, connector.GetName()),
 		Value:    value,
 		Expires:  connector.Expiry(),
 		Revision: rev,
@@ -1278,7 +1278,7 @@ func (s *IdentityService) CreateOIDCConnector(ctx context.Context, connector typ
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, connector.GetName()),
+		Key:     backend.NewKey(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, connector.GetName()),
 		Value:   value,
 		Expires: connector.Expiry(),
 	}
@@ -1300,7 +1300,7 @@ func (s *IdentityService) UpdateOIDCConnector(ctx context.Context, connector typ
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, connector.GetName()),
+		Key:      backend.NewKey(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, connector.GetName()),
 		Value:    value,
 		Expires:  connector.Expiry(),
 		Revision: connector.GetRevision(),
@@ -1318,7 +1318,7 @@ func (s *IdentityService) DeleteOIDCConnector(ctx context.Context, name string)
 	if name == "" {
 		return trace.BadParameter("missing parameter name")
 	}
-	err := s.Delete(ctx, backend.Key(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, name))
+	err := s.Delete(ctx, backend.NewKey(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, name))
 	return trace.Wrap(err)
 }
 
@@ -1328,7 +1328,7 @@ func (s *IdentityService) GetOIDCConnector(ctx context.Context, name string, wit
 	if name == "" {
 		return nil, trace.BadParameter("missing parameter name")
 	}
-	item, err := s.Get(ctx, backend.Key(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(webPrefix, connectorsPrefix, oidcPrefix, connectorsPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("OpenID connector '%v' is not configured", name)
@@ -1382,7 +1382,7 @@ func (s *IdentityService) CreateOIDCAuthRequest(ctx context.Context, req types.O
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(webPrefix, connectorsPrefix, oidcPrefix, requestsPrefix, req.StateToken),
+		Key:     backend.NewKey(webPrefix, connectorsPrefix, oidcPrefix, requestsPrefix, req.StateToken),
 		Value:   buf.Bytes(),
 		Expires: backend.Expiry(s.Clock(), ttl),
 	}
@@ -1397,7 +1397,7 @@ func (s *IdentityService) GetOIDCAuthRequest(ctx context.Context, stateToken str
 	if stateToken == "" {
 		return nil, trace.BadParameter("missing parameter stateToken")
 	}
-	item, err := s.Get(ctx, backend.Key(webPrefix, connectorsPrefix, oidcPrefix, requestsPrefix, stateToken))
+	item, err := s.Get(ctx, backend.NewKey(webPrefix, connectorsPrefix, oidcPrefix, requestsPrefix, stateToken))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -1419,7 +1419,7 @@ func (s *IdentityService) UpsertSAMLConnector(ctx context.Context, connector typ
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, connector.GetName()),
+		Key:      backend.NewKey(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, connector.GetName()),
 		Value:    value,
 		Expires:  connector.Expiry(),
 		Revision: rev,
@@ -1442,7 +1442,7 @@ func (s *IdentityService) UpdateSAMLConnector(ctx context.Context, connector typ
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, connector.GetName()),
+		Key:      backend.NewKey(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, connector.GetName()),
 		Value:    value,
 		Expires:  connector.Expiry(),
 		Revision: connector.GetRevision(),
@@ -1465,7 +1465,7 @@ func (s *IdentityService) CreateSAMLConnector(ctx context.Context, connector typ
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, connector.GetName()),
+		Key:     backend.NewKey(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, connector.GetName()),
 		Value:   value,
 		Expires: connector.Expiry(),
 	}
@@ -1482,7 +1482,7 @@ func (s *IdentityService) DeleteSAMLConnector(ctx context.Context, name string)
 	if name == "" {
 		return trace.BadParameter("missing parameter name")
 	}
-	err := s.Delete(ctx, backend.Key(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, name))
+	err := s.Delete(ctx, backend.NewKey(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, name))
 	return trace.Wrap(err)
 }
 
@@ -1492,7 +1492,7 @@ func (s *IdentityService) GetSAMLConnector(ctx context.Context, name string, wit
 	if name == "" {
 		return nil, trace.BadParameter("missing parameter name")
 	}
-	item, err := s.Get(ctx, backend.Key(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(webPrefix, connectorsPrefix, samlPrefix, connectorsPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("SAML connector %q is not configured", name)
@@ -1553,7 +1553,7 @@ func (s *IdentityService) CreateSAMLAuthRequest(ctx context.Context, req types.S
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(webPrefix, connectorsPrefix, samlPrefix, requestsPrefix, req.ID),
+		Key:     backend.NewKey(webPrefix, connectorsPrefix, samlPrefix, requestsPrefix, req.ID),
 		Value:   buf.Bytes(),
 		Expires: backend.Expiry(s.Clock(), ttl),
 	}
@@ -1568,7 +1568,7 @@ func (s *IdentityService) GetSAMLAuthRequest(ctx context.Context, id string) (*t
 	if id == "" {
 		return nil, trace.BadParameter("missing parameter id")
 	}
-	item, err := s.Get(ctx, backend.Key(webPrefix, connectorsPrefix, samlPrefix, requestsPrefix, id))
+	item, err := s.Get(ctx, backend.NewKey(webPrefix, connectorsPrefix, samlPrefix, requestsPrefix, id))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -1598,7 +1598,7 @@ func (s *IdentityService) CreateSSODiagnosticInfo(ctx context.Context, authKind
 	}
 
 	item := backend.Item{
-		Key:     backend.Key(webPrefix, connectorsPrefix, authKind, requestsTracePrefix, authRequestID),
+		Key:     backend.NewKey(webPrefix, connectorsPrefix, authKind, requestsTracePrefix, authRequestID),
 		Value:   jsonValue,
 		Expires: backend.Expiry(s.Clock(), time.Minute*15),
 	}
@@ -1622,7 +1622,7 @@ func (s *IdentityService) GetSSODiagnosticInfo(ctx context.Context, authKind str
 		return nil, trace.BadParameter("unsupported authKind %q", authKind)
 	}
 
-	item, err := s.Get(ctx, backend.Key(webPrefix, connectorsPrefix, authKind, requestsTracePrefix, authRequestID))
+	item, err := s.Get(ctx, backend.NewKey(webPrefix, connectorsPrefix, authKind, requestsTracePrefix, authRequestID))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -1646,7 +1646,7 @@ func (s *IdentityService) UpsertGithubConnector(ctx context.Context, connector t
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, connector.GetName()),
+		Key:      backend.NewKey(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, connector.GetName()),
 		Value:    value,
 		Expires:  connector.Expiry(),
 		Revision: rev,
@@ -1669,7 +1669,7 @@ func (s *IdentityService) UpdateGithubConnector(ctx context.Context, connector t
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:      backend.Key(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, connector.GetName()),
+		Key:      backend.NewKey(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, connector.GetName()),
 		Value:    value,
 		Expires:  connector.Expiry(),
 		Revision: connector.GetRevision(),
@@ -1692,7 +1692,7 @@ func (s *IdentityService) CreateGithubConnector(ctx context.Context, connector t
 		return nil, trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, connector.GetName()),
+		Key:     backend.NewKey(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, connector.GetName()),
 		Value:   value,
 		Expires: connector.Expiry(),
 	}
@@ -1734,7 +1734,7 @@ func (s *IdentityService) GetGithubConnector(ctx context.Context, name string, w
 	if name == "" {
 		return nil, trace.BadParameter("missing parameter name")
 	}
-	item, err := s.Get(ctx, backend.Key(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, name))
+	item, err := s.Get(ctx, backend.NewKey(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, name))
 	if err != nil {
 		if trace.IsNotFound(err) {
 			return nil, trace.NotFound("github connector %q is not configured", name)
@@ -1756,7 +1756,7 @@ func (s *IdentityService) DeleteGithubConnector(ctx context.Context, name string
 	if name == "" {
 		return trace.BadParameter("missing parameter name")
 	}
-	return trace.Wrap(s.Delete(ctx, backend.Key(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, name)))
+	return trace.Wrap(s.Delete(ctx, backend.NewKey(webPrefix, connectorsPrefix, githubPrefix, connectorsPrefix, name)))
 }
 
 // CreateGithubAuthRequest creates a new auth request for Github OAuth2 flow
@@ -1769,7 +1769,7 @@ func (s *IdentityService) CreateGithubAuthRequest(ctx context.Context, req types
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(webPrefix, connectorsPrefix, githubPrefix, requestsPrefix, req.StateToken),
+		Key:     backend.NewKey(webPrefix, connectorsPrefix, githubPrefix, requestsPrefix, req.StateToken),
 		Value:   buf.Bytes(),
 		Expires: req.Expiry(),
 	}
@@ -1784,7 +1784,7 @@ func (s *IdentityService) GetGithubAuthRequest(ctx context.Context, stateToken s
 	if stateToken == "" {
 		return nil, trace.BadParameter("missing parameter stateToken")
 	}
-	item, err := s.Get(ctx, backend.Key(webPrefix, connectorsPrefix, githubPrefix, requestsPrefix, stateToken))
+	item, err := s.Get(ctx, backend.NewKey(webPrefix, connectorsPrefix, githubPrefix, requestsPrefix, stateToken))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -1801,7 +1801,7 @@ func (s *IdentityService) GetRecoveryCodes(ctx context.Context, user string, wit
 		return nil, trace.BadParameter("missing parameter user")
 	}
 
-	item, err := s.Get(ctx, backend.Key(webPrefix, usersPrefix, user, recoveryCodesPrefix))
+	item, err := s.Get(ctx, backend.NewKey(webPrefix, usersPrefix, user, recoveryCodesPrefix))
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -1835,7 +1835,7 @@ func (s *IdentityService) UpsertRecoveryCodes(ctx context.Context, user string,
 	}
 
 	item := backend.Item{
-		Key:   backend.Key(webPrefix, usersPrefix, user, recoveryCodesPrefix),
+		Key:   backend.NewKey(webPrefix, usersPrefix, user, recoveryCodesPrefix),
 		Value: value,
 	}
 
@@ -1857,7 +1857,7 @@ func (s *IdentityService) UpsertKeyAttestationData(ctx context.Context, attestat
 
 	key := keyAttestationDataFingerprint(attestationData.PublicKeyDER)
 	item := backend.Item{
-		Key:     backend.Key(attestationsPrefix, key),
+		Key:     backend.NewKey(attestationsPrefix, key),
 		Value:   value,
 		Expires: s.Clock().Now().UTC().Add(ttl),
 	}
@@ -1875,7 +1875,7 @@ func (s *IdentityService) GetKeyAttestationData(ctx context.Context, pubDER []by
 	}
 
 	key := keyAttestationDataFingerprint(pubDER)
-	item, err := s.Get(ctx, backend.Key(attestationsPrefix, key))
+	item, err := s.Get(ctx, backend.NewKey(attestationsPrefix, key))
 
 	if trace.IsNotFound(err) {
 		return nil, trace.NotFound("hardware key attestation not found")
diff --git a/lib/services/local/users_test.go b/lib/services/local/users_test.go
index 20b790139342c..7474b58c0801a 100644
--- a/lib/services/local/users_test.go
+++ b/lib/services/local/users_test.go
@@ -1305,7 +1305,7 @@ func TestCompareAndSwapUser(t *testing.T) {
 	require.NoError(err)
 	require.True(services.UsersEquals(currentBob, bob2))
 
-	item, err := identity.Backend.Get(ctx, backend.Key(local.WebPrefix, local.UsersPrefix, "bob", local.ParamsPrefix))
+	item, err := identity.Backend.Get(ctx, backend.NewKey(local.WebPrefix, local.UsersPrefix, "bob", local.ParamsPrefix))
 	require.NoError(err)
 	var m map[string]any
 	require.NoError(json.Unmarshal(item.Value, &m))
diff --git a/lib/services/local/usertoken.go b/lib/services/local/usertoken.go
index e6ce141b82f6d..5a56adb739636 100644
--- a/lib/services/local/usertoken.go
+++ b/lib/services/local/usertoken.go
@@ -71,7 +71,7 @@ func (s *IdentityService) DeleteUserToken(ctx context.Context, tokenID string) e
 
 // GetUserToken returns a token by its ID.
 func (s *IdentityService) GetUserToken(ctx context.Context, tokenID string) (types.UserToken, error) {
-	item, err := s.Get(ctx, backend.Key(userTokenPrefix, tokenID, paramsPrefix))
+	item, err := s.Get(ctx, backend.NewKey(userTokenPrefix, tokenID, paramsPrefix))
 	switch {
 	case trace.IsNotFound(err):
 		return nil, trace.NotFound("user token(%s) not found", backend.MaskKeyName(tokenID))
@@ -99,7 +99,7 @@ func (s *IdentityService) CreateUserToken(ctx context.Context, token types.UserT
 	}
 
 	item := backend.Item{
-		Key:     backend.Key(userTokenPrefix, token.GetName(), paramsPrefix),
+		Key:     backend.NewKey(userTokenPrefix, token.GetName(), paramsPrefix),
 		Value:   value,
 		Expires: token.Expiry(),
 	}
@@ -113,7 +113,7 @@ func (s *IdentityService) CreateUserToken(ctx context.Context, token types.UserT
 
 // GetUserTokenSecrets returns token secrets.
 func (s *IdentityService) GetUserTokenSecrets(ctx context.Context, tokenID string) (types.UserTokenSecrets, error) {
-	item, err := s.Get(ctx, backend.Key(userTokenPrefix, tokenID, secretsPrefix))
+	item, err := s.Get(ctx, backend.NewKey(userTokenPrefix, tokenID, secretsPrefix))
 	switch {
 	case trace.IsNotFound(err):
 		return nil, trace.NotFound("user token(%s) secrets not found", backend.MaskKeyName(tokenID))
@@ -140,7 +140,7 @@ func (s *IdentityService) UpsertUserTokenSecrets(ctx context.Context, secrets ty
 		return trace.Wrap(err)
 	}
 	item := backend.Item{
-		Key:     backend.Key(userTokenPrefix, secrets.GetName(), secretsPrefix),
+		Key:     backend.NewKey(userTokenPrefix, secrets.GetName(), secretsPrefix),
 		Value:   value,
 		Expires: secrets.Expiry(),
 	}
diff --git a/lib/services/unified_resource.go b/lib/services/unified_resource.go
index 629de623cc48d..3e0921487ed57 100644
--- a/lib/services/unified_resource.go
+++ b/lib/services/unified_resource.go
@@ -220,10 +220,10 @@ func (c *UnifiedResourceCache) getRange(ctx context.Context, startKey []byte, ma
 		var endKey []byte
 		if req.SortBy.IsDesc {
 			iterateRange = tree.DescendRange
-			endKey = backend.Key(prefix)
+			endKey = backend.NewKey(prefix)
 		} else {
 			iterateRange = tree.AscendRange
-			endKey = backend.RangeEnd(backend.Key(prefix))
+			endKey = backend.RangeEnd(backend.NewKey(prefix))
 		}
 		var iteratorErr error
 		iterateRange(&item{Key: startKey}, &item{Key: endKey}, func(item *item) bool {
@@ -276,10 +276,10 @@ func getStartKey(req *proto.ListUnifiedResourcesRequest) []byte {
 	// if startkey doesnt exist, we check the sort direction.
 	// If sort is descending, startkey is end of the list
 	if req.SortBy.IsDesc {
-		return backend.RangeEnd(backend.Key(prefix))
+		return backend.RangeEnd(backend.NewKey(prefix))
 	}
 	// return start of the list
-	return backend.Key(prefix)
+	return backend.NewKey(prefix)
 }
 
 func (c *UnifiedResourceCache) IterateUnifiedResources(ctx context.Context, matchFn func(types.ResourceWithLabels) (bool, error), req *proto.ListUnifiedResourcesRequest) ([]types.ResourceWithLabels, string, error) {
@@ -300,7 +300,7 @@ func (c *UnifiedResourceCache) IterateUnifiedResources(ctx context.Context, matc
 // GetUnifiedResources returns a list of all resources stored in the current unifiedResourceCollector tree in ascending order
 func (c *UnifiedResourceCache) GetUnifiedResources(ctx context.Context) ([]types.ResourceWithLabels, error) {
 	req := &proto.ListUnifiedResourcesRequest{Limit: backend.NoLimit, SortBy: types.SortBy{IsDesc: false, Field: sortByName}}
-	result, _, err := c.getRange(ctx, backend.Key(prefix), func(rwl types.ResourceWithLabels) (bool, error) { return true, nil }, req)
+	result, _, err := c.getRange(ctx, backend.NewKey(prefix), func(rwl types.ResourceWithLabels) (bool, error) { return true, nil }, req)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -319,7 +319,7 @@ func (c *UnifiedResourceCache) GetUnifiedResourcesByIDs(ctx context.Context, ids
 
 	err := c.read(ctx, func(cache *UnifiedResourceCache) error {
 		for _, id := range ids {
-			key := backend.Key(prefix, id)
+			key := backend.NewKey(prefix, id)
 			res, found := cache.nameTree.Get(&item{Key: key})
 			if !found || res == nil {
 				continue
@@ -418,8 +418,8 @@ func makeResourceSortKey(resource types.Resource) resourceSortKey {
 	return resourceSortKey{
 		// names should be stored as lowercase to keep items sorted as
 		// expected, regardless of case
-		byName: backend.Key(prefix, strings.ToLower(name), kind),
-		byType: backend.Key(prefix, kind, strings.ToLower(name)),
+		byName: backend.NewKey(prefix, strings.ToLower(name), kind),
+		byType: backend.NewKey(prefix, kind, strings.ToLower(name)),
 	}
 }
 
diff --git a/lib/usagereporter/teleport/aggregating/service.go b/lib/usagereporter/teleport/aggregating/service.go
index 3a74fad9de773..180b4815a1830 100644
--- a/lib/usagereporter/teleport/aggregating/service.go
+++ b/lib/usagereporter/teleport/aggregating/service.go
@@ -48,7 +48,7 @@ const (
 // a given UUID and start time, such that reports with an earlier start time
 // will appear earlier in lexicographic ordering.
 func userActivityReportKey(reportUUID uuid.UUID, startTime time.Time) []byte {
-	return backend.Key(userActivityReportsPrefix, startTime.Format(time.RFC3339), reportUUID.String())
+	return backend.NewKey(userActivityReportsPrefix, startTime.Format(time.RFC3339), reportUUID.String())
 }
 
 func prepareUserActivityReports(
@@ -84,7 +84,7 @@ func prepareUserActivityReports(
 // a given UUID and start time, such that reports with an earlier start time
 // will appear earlier in lexicographic ordering.
 func resourcePresenceReportKey(reportUUID uuid.UUID, startTime time.Time) []byte {
-	return backend.Key(ResourcePresenceReportsPrefix, startTime.Format(time.RFC3339), reportUUID.String())
+	return backend.NewKey(ResourcePresenceReportsPrefix, startTime.Format(time.RFC3339), reportUUID.String())
 }
 
 // prepareResourcePresenceReport prepares a resource presence report for storage.
@@ -240,7 +240,7 @@ func (r reportService) createUserActivityReportsLock(ctx context.Context, ttl ti
 	if len(payload) == 0 {
 		payload = []byte("null")
 	}
-	lockKey := backend.Key(userActivityReportsLock)
+	lockKey := backend.NewKey(userActivityReportsLock)
 	// HACK(espadolini): dynamodbbk doesn't let you Create over an expired item
 	// but it will explicitly delete expired items on a Get; in addition, reads
 	// are cheaper than writes in most backends, so we do a Get here first