split user keys for k8s access

api/utils/keypaths/keypaths.go

@@ -111,13 +111,16 @@ const (
 //    │   ├── foo-kube                 --> Kubernetes certs for user "foo"
 //    │   |    ├── root                 --> Kubernetes certs for Teleport cluster "root"
 //    │   |    │   ├── kubeA-kubeconfig --> standalone kubeconfig for Kubernetes cluster "kubeA"
-//    │   |    │   ├── kubeA-x509.pem   --> TLS cert for Kubernetes cluster "kubeA"
+//    │   |    │   ├── kubeA.crt        --> TLS cert for Kubernetes cluster "kubeA"
+//    │   |    │   ├── kubeA.key        --> private key for Kubernetes cluster "kubeA"
 //    │   |    │   ├── kubeB-kubeconfig --> standalone kubeconfig for Kubernetes cluster "kubeB"
-//    │   |    │   ├── kubeB-x509.pem   --> TLS cert for Kubernetes cluster "kubeB"
+//    │   |    │   ├── kubeB.crt        --> TLS cert for Kubernetes cluster "kubeB"
+//    │   |    │   ├── kubeB.key        --> private key for Kubernetes cluster "kubeB"
 //    │   |    │   └── localca.pem      --> Self-signed localhost CA cert for Teleport cluster "root"
 //    │   |    └── leaf                 --> Kubernetes certs for Teleport cluster "leaf"
 //    │   |        ├── kubeC-kubeconfig --> standalone kubeconfig for Kubernetes cluster "kubeC"
-//    │   |        └── kubeC-x509.pem   --> TLS cert for Kubernetes cluster "kubeC"
+//    │   |        └── kubeC.crt        --> TLS cert for Kubernetes cluster "kubeC"
+//    │   |        └── kubeC.key        --> private key for Kubernetes cluster "kubeC"
 //    |   └── cas                       --> Trusted clusters certificates
 //    |        ├── root.pem             --> TLS CA for teleport cluster "root"
 //    |        ├── leaf1.pem            --> TLS CA for teleport cluster "leaf1"
@@ -316,28 +319,36 @@ func KubeDir(baseDir, proxy, username string) string {
 	return filepath.Join(ProxyKeyDir(baseDir, proxy), username+kubeDirSuffix)
 }
 
-// KubeCertDir returns the path to the user's kube cert directory
+// KubeCredentialDir returns the path to the user's kube credential directory
 // for the given proxy and cluster.
 //
 // <baseDir>/keys/<proxy>/<username>-kube/<cluster>
-func KubeCertDir(baseDir, proxy, username, cluster string) string {
+func KubeCredentialDir(baseDir, proxy, username, cluster string) string {
 	return filepath.Join(KubeDir(baseDir, proxy, username), cluster)
 }
 
 // KubeCertPath returns the path to the user's TLS certificate
 // for the given proxy, cluster, and kube cluster.
 //
-// <baseDir>/keys/<proxy>/<username>-kube/<cluster>/<kubename>-x509.pem
+// <baseDir>/keys/<proxy>/<username>-kube/<cluster>/<kubename>.crt
 func KubeCertPath(baseDir, proxy, username, cluster, kubename string) string {
-	return filepath.Join(KubeCertDir(baseDir, proxy, username, cluster), kubename+fileExtTLSCertLegacy)
+	return filepath.Join(KubeCredentialDir(baseDir, proxy, username, cluster), kubename+fileExtTLSCert)
+}
+
+// KubeKeyPath returns the path to the user's TLS private key
+// for the given proxy, cluster, and kube cluster.
+//
+// <baseDir>/keys/<proxy>/<username>-kube/<cluster>/<kubename>.key
+func KubeKeyPath(baseDir, proxy, username, cluster, kubename string) string {
+	return filepath.Join(KubeCredentialDir(baseDir, proxy, username, cluster), kubename+fileExtTLSKey)
 }
 
 // KubeConfigPath returns the path to the user's standalone kubeconfig
 // for the given proxy, cluster, and kube cluster.
 //
 // <baseDir>/keys/<proxy>/<username>-kube/<cluster>/<kubename>-kubeconfig
 func KubeConfigPath(baseDir, proxy, username, cluster, kubename string) string {
-	return filepath.Join(KubeCertDir(baseDir, proxy, username, cluster), kubename+kubeConfigSuffix)
+	return filepath.Join(KubeCredentialDir(baseDir, proxy, username, cluster), kubename+kubeConfigSuffix)
 }
 
 // KubeCredLockfilePath returns the kube credentials lock file for given proxy

lib/benchmark/kube.go

@@ -99,9 +99,9 @@ func getKubeTLSClientConfig(ctx context.Context, tc *client.TeleportClient) (res
 		return rest.TLSClientConfig{}, trace.Wrap(err)
 	}
 
-	certPem := k.KubeTLSCerts[tc.KubernetesCluster]
+	cred := k.KubeTLSCredentials[tc.KubernetesCluster]
 
-	keyPEM, err := k.PrivateKey.SoftwarePrivateKeyPEM()
+	keyPEM, err := cred.PrivateKey.SoftwarePrivateKeyPEM()
 	if err != nil {
 		return rest.TLSClientConfig{}, trace.Wrap(err)
 	}
@@ -132,7 +132,7 @@ func getKubeTLSClientConfig(ctx context.Context, tc *client.TeleportClient) (res
 
 	return rest.TLSClientConfig{
 		CAData:     bytes.Join(clusterCAs, []byte("\n")),
-		CertData:   certPem,
+		CertData:   cred.Cert,
 		KeyData:    keyPEM,
 		ServerName: tlsServerName,
 	}, nil

lib/client/api.go

@@ -3716,7 +3716,10 @@ func (tc *TeleportClient) SSHLogin(ctx context.Context, sshLoginFunc SSHLoginFun
 	keyRing.ProxyHost = tc.WebProxyHost()
 
 	if tc.KubernetesCluster != "" {
-		keyRing.KubeTLSCerts[tc.KubernetesCluster] = response.TLSCert
+		keyRing.KubeTLSCredentials[tc.KubernetesCluster] = TLSCredential{
+			Cert:       response.TLSCert,
+			PrivateKey: priv,
+		}
 	}
 	if tc.DatabaseService != "" {
 		keyRing.DBTLSCredentials[tc.DatabaseService] = TLSCredential{

lib/client/client_store.go

@@ -260,8 +260,8 @@ func (s *Store) FullProfileStatus() (*ProfileStatus, []*ProfileStatus, error) {
 // parallel.
 // Although this function speeds up the process since it removes all transversals,
 // it still has to read 2 different files:
-// - $TSH_HOME/keys/$PROXY/$USER-kube/$TELEPORT_CLUSTER/$KUBE_CLUSTER-x509.pem
-// - $TSH_HOME/keys/$PROXY/$USER
+// - $TSH_HOME/keys/$PROXY/$USER-kube/$TELEPORT_CLUSTER/$KUBE_CLUSTER.crt
+// - $TSH_HOME/keys/$PROXY/$USER-kube/$TELEPORT_CLUSTER/$KUBE_CLUSTER.key
 func LoadKeysToKubeFromStore(profile *profile.Profile, dirPath, teleportCluster, kubeCluster string) ([]byte, []byte, error) {
 	fsKeyStore := NewFSKeyStore(dirPath)
 
@@ -271,7 +271,7 @@ func LoadKeysToKubeFromStore(profile *profile.Profile, dirPath, teleportCluster,
 		return nil, nil, trace.Wrap(err)
 	}
 
-	privKeyPath := fsKeyStore.userKeyPath(KeyRingIndex{ProxyHost: profile.SiteName, Username: profile.Username})
+	privKeyPath := fsKeyStore.kubeKeyPath(KeyRingIndex{ProxyHost: profile.SiteName, ClusterName: teleportCluster, Username: profile.Username}, kubeCluster)
 	privKey, err := os.ReadFile(privKeyPath)
 	if err != nil {
 		return nil, nil, trace.Wrap(err)

lib/client/cluster_client.go

@@ -241,7 +241,10 @@ func (c *ClusterClient) generateUserCerts(ctx context.Context, cachePolicy CertC
 			PrivateKey: privKey,
 		}
 	case proto.UserCertsRequest_Kubernetes:
-		keyRing.KubeTLSCerts[params.KubernetesCluster] = certs.TLS
+		keyRing.KubeTLSCredentials[params.KubernetesCluster] = TLSCredential{
+			PrivateKey: privKey,
+			Cert:       certs.TLS,
+		}
 	case proto.UserCertsRequest_WindowsDesktop:
 		keyRing.WindowsDesktopCerts[params.RouteToWindowsDesktop.WindowsDesktop] = certs.TLS
 	}
@@ -353,7 +356,7 @@ func (c *ClusterClient) prepareUserCertsRequest(ctx context.Context, params Reis
 	var sshPublicKey, tlsPublicKey []byte
 	var privateKey *keys.PrivateKey
 	switch params.usage() {
-	case proto.UserCertsRequest_App:
+	case proto.UserCertsRequest_App, proto.UserCertsRequest_Kubernetes:
 		privateKey, err = keyRing.GenerateKey(ctx, c.tc, cryptosuites.UserTLS)
 		if err != nil {
 			return nil, nil, trace.Wrap(err)
@@ -670,10 +673,13 @@ func PerformMFACeremony(ctx context.Context, params PerformMFACeremonyParams) (*
 	case len(newCerts.TLS) > 0:
 		switch certsReq.Usage {
 		case proto.UserCertsRequest_Kubernetes:
-			if keyRing.KubeTLSCerts == nil {
-				keyRing.KubeTLSCerts = make(map[string][]byte)
+			if keyRing.KubeTLSCredentials == nil {
+				keyRing.KubeTLSCredentials = make(map[string]TLSCredential)
+			}
+			keyRing.KubeTLSCredentials[certsReq.KubernetesCluster] = TLSCredential{
+				Cert:       newCerts.TLS,
+				PrivateKey: params.PrivateKey,
 			}
-			keyRing.KubeTLSCerts[certsReq.KubernetesCluster] = newCerts.TLS
 
 		case proto.UserCertsRequest_Database:
 			dbCert, err := makeDatabaseClientPEM(certsReq.RouteToDatabase.Protocol, newCerts.TLS, params.PrivateKey)

lib/client/identityfile/identity.go

@@ -787,7 +787,12 @@ func KeyRingFromIdentityFile(identityPath, proxyHost, clusterName string) (*clie
 		// If this identity file has any kubernetes certs, copy it into the
 		// KubeTLSCerts map.
 		if parsedIdent.KubernetesCluster != "" {
-			keyRing.KubeTLSCerts[parsedIdent.KubernetesCluster] = ident.Certs.TLS
+			keyRing.KubeTLSCredentials[parsedIdent.KubernetesCluster] = client.TLSCredential{
+				// Identity files only have room for one private key, it must
+				// match the kube cert.
+				PrivateKey: priv,
+				Cert:       ident.Certs.TLS,
+			}
 		}
 	} else {
 		keyRing.Username, err = keyRing.CertUsername()

lib/client/identityfile/identity_test.go

@@ -416,8 +416,8 @@ func TestKeyFromIdentityFile(t *testing.T) {
 		require.NoError(t, err)
 		parsedKeyRing, err := KeyRingFromIdentityFile(identityFilePath, proxyHost, cluster)
 		require.NoError(t, err)
-		require.NotNil(t, parsedKeyRing.KubeTLSCerts[k8sCluster])
-		require.Equal(t, keyRing.TLSCert, parsedKeyRing.KubeTLSCerts[k8sCluster])
+		require.NotNil(t, parsedKeyRing.KubeTLSCredentials[k8sCluster].PrivateKey)
+		require.Equal(t, keyRing.TLSCert, parsedKeyRing.KubeTLSCredentials[k8sCluster].Cert)
 	})
 }
 

lib/client/interfaces.go

@@ -109,9 +109,9 @@ type KeyRing struct {
 	// TLSCert is a PEM encoded client TLS x509 certificate.
 	// It's used to authenticate to the Teleport APIs.
 	TLSCert []byte `json:"TLSCert,omitempty"`
-	// KubeTLSCerts are TLS certificates (PEM-encoded) for individual
+	// KubeTLSCredentials are TLS credentials for individual
 	// kubernetes clusters. Map key is a kubernetes cluster name.
-	KubeTLSCerts map[string][]byte `json:"KubeCerts,omitempty"`
+	KubeTLSCredentials map[string]TLSCredential
 	// DBTLSCredentials are TLS credentials for database access.
 	// Map key is the database service name.
 	DBTLSCredentials map[string]TLSCredential
@@ -176,7 +176,7 @@ func GenerateRSAKeyRing() (*KeyRing, error) {
 func NewKeyRing(priv *keys.PrivateKey) *KeyRing {
 	return &KeyRing{
 		PrivateKey:          priv,
-		KubeTLSCerts:        make(map[string][]byte),
+		KubeTLSCredentials:  make(map[string]TLSCredential),
 		DBTLSCredentials:    make(map[string]TLSCredential),
 		AppTLSCredentials:   make(map[string]TLSCredential),
 		WindowsDesktopCerts: make(map[string][]byte),
@@ -220,12 +220,12 @@ func (k *KeyRing) KubeClientTLSConfig(cipherSuites []uint16, kubeClusterName str
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
-	tlsCert, ok := k.KubeTLSCerts[kubeClusterName]
+	cred, ok := k.KubeTLSCredentials[kubeClusterName]
 	if !ok {
 		return nil, trace.NotFound("TLS certificate for kubernetes cluster %q not found", kubeClusterName)
 	}
 
-	tlsConfig, err := k.clientTLSConfig(cipherSuites, tlsCert, []string{rootCluster})
+	tlsConfig, err := k.clientTLSConfig(cipherSuites, cred, []string{rootCluster})
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -274,11 +274,14 @@ func (k *KeyRing) TeleportClientTLSConfig(cipherSuites []uint16, clusters []stri
 	if len(k.TLSCert) == 0 {
 		return nil, trace.NotFound("TLS certificate not found")
 	}
-	return k.clientTLSConfig(cipherSuites, k.TLSCert, clusters)
+	return k.clientTLSConfig(cipherSuites, TLSCredential{
+		PrivateKey: k.PrivateKey,
+		Cert:       k.TLSCert,
+	}, clusters)
 }
 
-func (k *KeyRing) clientTLSConfig(cipherSuites []uint16, tlsCertRaw []byte, clusters []string) (*tls.Config, error) {
-	tlsCert, err := k.PrivateKey.TLSCertificate(tlsCertRaw)
+func (k *KeyRing) clientTLSConfig(cipherSuites []uint16, cred TLSCredential, clusters []string) (*tls.Config, error) {
+	tlsCert, err := cred.TLSCertificate()
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -459,21 +462,21 @@ func (k *KeyRing) TeleportTLSCertificate() (*x509.Certificate, error) {
 // KubeX509Cert returns the parsed x509 certificate for authentication against
 // a named kubernetes cluster.
 func (k *KeyRing) KubeX509Cert(kubeClusterName string) (*x509.Certificate, error) {
-	tlsCert, ok := k.KubeTLSCerts[kubeClusterName]
+	cred, ok := k.KubeTLSCredentials[kubeClusterName]
 	if !ok {
-		return nil, trace.NotFound("TLS certificate for kubernetes cluster %q not found", kubeClusterName)
+		return nil, trace.NotFound("TLS credential for kubernetes cluster %q not found", kubeClusterName)
 	}
-	return tlsca.ParseCertificatePEM(tlsCert)
+	return tlsca.ParseCertificatePEM(cred.Cert)
 }
 
 // KubeTLSCert returns the tls.Certificate for authentication against a named
 // kubernetes cluster.
 func (k *KeyRing) KubeTLSCert(kubeClusterName string) (tls.Certificate, error) {
-	certPem, ok := k.KubeTLSCerts[kubeClusterName]
+	cred, ok := k.KubeTLSCredentials[kubeClusterName]
 	if !ok {
 		return tls.Certificate{}, trace.NotFound("TLS certificate for kubernetes cluster %q not found", kubeClusterName)
 	}
-	tlsCert, err := k.PrivateKey.TLSCertificate(certPem)
+	tlsCert, err := cred.TLSCertificate()
 	if err != nil {
 		return tls.Certificate{}, trace.Wrap(err)
 	}

lib/client/keyagent.go

@@ -515,7 +515,7 @@ func (a *LocalKeyAgent) AddDatabaseKeyRing(keyRing *KeyRing) error {
 // AddKubeKeyRing activates a new signed Kubernetes key by adding it into the keystore.
 // key must contain at least one Kubernetes cert. ssh cert is not required.
 func (a *LocalKeyAgent) AddKubeKeyRing(keyRing *KeyRing) error {
-	if len(keyRing.KubeTLSCerts) == 0 {
+	if len(keyRing.KubeTLSCredentials) == 0 {
 		return trace.BadParameter("key ring must contain at least one Kubernetes access certificate")
 	}
 	return a.addKeyRing(keyRing)

lib/client/keystore.go

@@ -165,6 +165,11 @@ func (fs *FSKeyStore) kubeCertPath(idx KeyRingIndex, kubename string) string {
 	return keypaths.KubeCertPath(fs.KeyDir, idx.ProxyHost, idx.Username, idx.ClusterName, kubename)
 }
 
+// kubeKeyPath returns the private key path for the given KeyRingIndex and kube cluster name.
+func (fs *FSKeyStore) kubeKeyPath(idx KeyRingIndex, kubename string) string {
+	return keypaths.KubeKeyPath(fs.KeyDir, idx.ProxyHost, idx.Username, idx.ClusterName, kubename)
+}
+
 // AddKeyRing adds the given key ring to the store.
 func (fs *FSKeyStore) AddKeyRing(keyRing *KeyRing) error {
 	if err := keyRing.KeyRingIndex.Check(); err != nil {
@@ -205,17 +210,20 @@ func (fs *FSKeyStore) AddKeyRing(keyRing *KeyRing) error {
 		}
 	}
 
-	// TODO(awly): unit test this.
-	for kubeCluster, cert := range keyRing.KubeTLSCerts {
+	for kubeCluster, cred := range keyRing.KubeTLSCredentials {
 		// Prevent directory traversal via a crafted kubernetes cluster name.
 		//
 		// This will confuse cluster cert loading (GetKeyRing will return
 		// kubernetes cluster names different from the ones stored here), but I
 		// don't expect any well-meaning user to create bad names.
 		kubeCluster = filepath.Clean(kubeCluster)
 
-		path := fs.kubeCertPath(keyRing.KeyRingIndex, kubeCluster)
-		if err := fs.writeBytes(cert, path); err != nil {
+		certPath := fs.kubeCertPath(keyRing.KeyRingIndex, kubeCluster)
+		if err := fs.writeBytes(cred.Cert, certPath); err != nil {
+			return trace.Wrap(err)
+		}
+		keyPath := fs.kubeKeyPath(keyRing.KeyRingIndex, kubeCluster)
+		if err := fs.writeBytes(cred.PrivateKey.PrivateKeyPEM(), keyPath); err != nil {
 			return trace.Wrap(err)
 		}
 	}
@@ -401,25 +409,6 @@ func (fs *FSKeyStore) GetSSHCertificates(proxyHost, username string) ([]*ssh.Cer
 	return sshCerts, nil
 }
 
-func getCertsByName(certDir string) (map[string][]byte, error) {
-	certsByName := make(map[string][]byte)
-	certFiles, err := os.ReadDir(certDir)
-	if err != nil {
-		return nil, trace.ConvertSystemError(err)
-	}
-	for _, certFile := range certFiles {
-		name := keypaths.TrimCertPathSuffix(certFile.Name())
-		if isCert := name != certFile.Name(); isCert {
-			data, err := os.ReadFile(filepath.Join(certDir, certFile.Name()))
-			if err != nil {
-				return nil, trace.ConvertSystemError(err)
-			}
-			certsByName[name] = data
-		}
-	}
-	return certsByName, nil
-}
-
 func getCredentialsByName(credentialDir string) (map[string]TLSCredential, error) {
 	credsByName := make(map[string]TLSCredential)
 	files, err := os.ReadDir(credentialDir)
@@ -504,24 +493,24 @@ func (o WithSSHCerts) deleteFromKeyRing(keyRing *KeyRing) {
 type WithKubeCerts struct{}
 
 func (o WithKubeCerts) updateKeyRing(keyDir string, idx KeyRingIndex, keyRing *KeyRing) error {
-	certDir := keypaths.KubeCertDir(keyDir, idx.ProxyHost, idx.Username, idx.ClusterName)
-	certsByName, err := getCertsByName(certDir)
+	credentialDir := keypaths.KubeCredentialDir(keyDir, idx.ProxyHost, idx.Username, idx.ClusterName)
+	credsByName, err := getCredentialsByName(credentialDir)
 	if err != nil {
 		return trace.Wrap(err)
 	}
-	keyRing.KubeTLSCerts = certsByName
+	keyRing.KubeTLSCredentials = credsByName
 	return nil
 }
 
 func (o WithKubeCerts) pathsToDelete(keyDir string, idx KeyRingIndex) []string {
 	if idx.ClusterName == "" {
 		return []string{keypaths.KubeDir(keyDir, idx.ProxyHost, idx.Username)}
 	}
-	return []string{keypaths.KubeCertDir(keyDir, idx.ProxyHost, idx.Username, idx.ClusterName)}
+	return []string{keypaths.KubeCredentialDir(keyDir, idx.ProxyHost, idx.Username, idx.ClusterName)}
 }
 
 func (o WithKubeCerts) deleteFromKeyRing(keyRing *KeyRing) {
-	keyRing.KubeTLSCerts = make(map[string][]byte)
+	keyRing.KubeTLSCredentials = make(map[string]TLSCredential)
 }
 
 // WithDBCerts is a CertOption for handling database access certificates.
@@ -661,7 +650,7 @@ func (ms *MemKeyStore) GetKeyRing(idx KeyRingIndex, opts ...CertOption) (*KeyRin
 		case WithSSHCerts:
 			retKeyRing.Cert = keyRing.Cert
 		case WithKubeCerts:
-			retKeyRing.KubeTLSCerts = keyRing.KubeTLSCerts
+			retKeyRing.KubeTLSCredentials = keyRing.KubeTLSCredentials
 		case WithDBCerts:
 			retKeyRing.DBTLSCredentials = keyRing.DBTLSCredentials
 		case WithAppCerts:

lib/client/kube/kube.go

@@ -42,12 +42,12 @@ func CheckIfCertsAreAllowedToAccessCluster(k *client.KeyRing, rootCluster, telep
 	if rootCluster != teleportCluster {
 		return nil
 	}
-	for k8sCluster, cert := range k.KubeTLSCerts {
+	for k8sCluster, cred := range k.KubeTLSCredentials {
 		if k8sCluster != kubeCluster {
 			continue
 		}
 		log.Debugf("Got TLS cert for Kubernetes cluster %q", k8sCluster)
-		exist, err := checkIfCertHasKubeGroupsAndUsers(cert)
+		exist, err := checkIfCertHasKubeGroupsAndUsers(cred.Cert)
 		if err != nil {
 			return trace.Wrap(err)
 		} else if exist {