From 60b91c47a4dedc8168834b7415254ea471fd3123 Mon Sep 17 00:00:00 2001 From: "Joseph, Abhilash" Date: Thu, 2 Nov 2023 17:19:47 -0400 Subject: [PATCH 1/6] Use registry authentication for azure sp credentials, when authType is azureCloudConfig --- cmd/azure-keyvault-secrets-webhook/main.go | 2 +- pkg/docker/registry/registry.go | 82 +++++++++++++++++----- pkg/docker/registry/registry_secret.go | 46 ++++++++++++ 3 files changed, 112 insertions(+), 18 deletions(-) create mode 100644 pkg/docker/registry/registry_secret.go diff --git a/cmd/azure-keyvault-secrets-webhook/main.go b/cmd/azure-keyvault-secrets-webhook/main.go index f501b21b..c75a2309 100644 --- a/cmd/azure-keyvault-secrets-webhook/main.go +++ b/cmd/azure-keyvault-secrets-webhook/main.go @@ -319,7 +319,7 @@ func main() { wg := new(sync.WaitGroup) wg.Add(2) - config.registry = registry.NewRegistry(config.cloudConfig) + config.registry = registry.NewRegistry(config.authType, config.credentialProvider) createHTTPEndpoint(wg, config.httpPort, config.useAuthService, config.authService) createMTLSEndpoint(wg, config.mtlsPort, config.useAuthService, config.authService) diff --git a/pkg/docker/registry/registry.go b/pkg/docker/registry/registry.go index 8647159e..c10afe61 100644 --- a/pkg/docker/registry/registry.go +++ b/pkg/docker/registry/registry.go @@ -23,6 +23,8 @@ import ( "fmt" "net/http" + "github.com/SparebankenVest/azure-key-vault-to-kubernetes/pkg/azure/credentialprovider" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/authn/k8schain" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -50,13 +52,17 @@ type ImageRegistryOptions struct { // Registry impl type Registry struct { - imageCache *cache.Cache + authType string + imageCache *cache.Cache + credentialProvider credentialprovider.CredentialProvider } // NewRegistry creates and initializes registry -func NewRegistry(cloudConfigPath string) ImageRegistry { +func NewRegistry(authType string, credentialProvider credentialprovider.CredentialProvider) ImageRegistry { //, credentialProvider credentialprovider.CredentialProvider return &Registry{ - imageCache: cache.New(cache.NoExpiration, cache.NoExpiration), + authType: authType, + imageCache: cache.New(cache.NoExpiration, cache.NoExpiration), + credentialProvider: credentialProvider, } } @@ -100,7 +106,12 @@ func (r *Registry) GetImageConfig( containerInfo.ImagePullSecrets = append(containerInfo.ImagePullSecrets, imagePullSecret.Name) } - imageConfig, err := getImageConfig(ctx, client, containerInfo, opt) + remoteOptions, err := getContainerRegistryRemoteOptions(ctx, client, containerInfo, r.authType, opt, r.credentialProvider) + if err != nil { + return nil, fmt.Errorf("failed to get remote options: %w", err) + } + + imageConfig, err := getImageConfig(containerInfo, remoteOptions) if imageConfig != nil && allowToCache { r.imageCache.Set(container.Image, imageConfig, cache.DefaultExpiration) } @@ -108,23 +119,56 @@ func (r *Registry) GetImageConfig( return imageConfig, err } -// getImageConfig download image blob from registry -func getImageConfig(ctx context.Context, client kubernetes.Interface, container containerInfo, opt ImageRegistryOptions) (*v1.Config, error) { - authChain, err := k8schain.New( - ctx, - client, - k8schain.Options{ - Namespace: container.Namespace, - ServiceAccountName: container.ServiceAccountName, - ImagePullSecrets: container.ImagePullSecrets, - }, - ) +// getContainerRegistryRemoteOptions get container registry remote option +func getContainerRegistryRemoteOptions(ctx context.Context, client kubernetes.Interface, container containerInfo, authType string, opt ImageRegistryOptions, r credentialprovider.CredentialProvider) ([]remote.Option, error) { + ref, err := name.ParseReference(container.Image) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to parse image reference: %w", err) + } + registry := ref.Context().Registry.Name() + + klog.InfoS("using registry", "imageRegistry", registry) + + authChain := new(authn.Keychain) + switch authType { + case "azureCloudConfig": + klog.InfoS("using cloudConfig for registry authentication", "config.authType", authType) + dockerConfigEntry, err := r.GetAcrCredentials(container.Image) + if err != nil { + return nil, fmt.Errorf("cannot fetch acr credentials: %w", err) + } + + sec := []corev1.Secret{ //{ + *dockerCfgSecretType.Create(container.Namespace, "secret", registry, authn.AuthConfig{ + Username: dockerConfigEntry.Username, Password: dockerConfigEntry.Password, + }), + } + *authChain, err = k8schain.NewFromPullSecrets( + ctx, + sec, + ) + if err != nil { + return nil, err + } + + default: + klog.InfoS("using imagePullSecrets for registry authentication", "config.authType", authType) + *authChain, err = k8schain.New( + ctx, + client, + k8schain.Options{ + Namespace: container.Namespace, + ServiceAccountName: container.ServiceAccountName, + ImagePullSecrets: container.ImagePullSecrets, + }, + ) + if err != nil { + return nil, err + } } options := []remote.Option{ - remote.WithAuthFromKeychain(authChain), + remote.WithAuthFromKeychain(*authChain), } if opt.SkipVerify { @@ -133,7 +177,11 @@ func getImageConfig(ctx context.Context, client kubernetes.Interface, container } options = append(options, remote.WithTransport(tr)) } + return options, err +} +// getImageConfig download image blob from registry +func getImageConfig(container containerInfo, options []remote.Option) (*v1.Config, error) { ref, err := name.ParseReference(container.Image) if err != nil { return nil, fmt.Errorf("failed to parse image reference: %w", err) diff --git a/pkg/docker/registry/registry_secret.go b/pkg/docker/registry/registry_secret.go new file mode 100644 index 00000000..e38e23e8 --- /dev/null +++ b/pkg/docker/registry/registry_secret.go @@ -0,0 +1,46 @@ +package registry + +import ( + "encoding/json" + "fmt" + + "github.com/google/go-containerregistry/pkg/authn" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type secretType struct { + name corev1.SecretType + key string + marshal func(registry string, auth authn.AuthConfig) []byte +} + +func (s *secretType) Create(namespace, name string, registry string, auth authn.AuthConfig) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Type: s.name, + Data: map[string][]byte{ + s.key: s.marshal(registry, auth), + }, + } +} + +var dockerCfgSecretType = secretType{ + name: corev1.SecretTypeDockercfg, + key: corev1.DockerConfigKey, + marshal: func(target string, auth authn.AuthConfig) []byte { + return toJSON(map[string]authn.AuthConfig{target: auth}) + }, +} + +func toJSON(obj any) []byte { + bites, err := json.Marshal(obj) + + if err != nil { + fmt.Errorf("unable to json marshal: %w", err) + } + return bites +} From 8a3131ad49aada2d80a1a2811350da14a978ad74 Mon Sep 17 00:00:00 2001 From: "Joseph, Abhilash" Date: Thu, 2 Nov 2023 17:35:41 -0400 Subject: [PATCH 2/6] cleanup --- pkg/docker/registry/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/docker/registry/registry.go b/pkg/docker/registry/registry.go index c10afe61..b745ace0 100644 --- a/pkg/docker/registry/registry.go +++ b/pkg/docker/registry/registry.go @@ -58,7 +58,7 @@ type Registry struct { } // NewRegistry creates and initializes registry -func NewRegistry(authType string, credentialProvider credentialprovider.CredentialProvider) ImageRegistry { //, credentialProvider credentialprovider.CredentialProvider +func NewRegistry(authType string, credentialProvider credentialprovider.CredentialProvider) ImageRegistry { return &Registry{ authType: authType, imageCache: cache.New(cache.NoExpiration, cache.NoExpiration), From 2bee192a2dc0deba54de19e86c00d25e8031706f Mon Sep 17 00:00:00 2001 From: "Joseph, Abhilash" Date: Tue, 7 Nov 2023 14:26:42 -0500 Subject: [PATCH 3/6] Use logging option default streams --- cmd/azure-keyvault-controller/main.go | 5 ++++- cmd/azure-keyvault-env/main.go | 5 ++++- cmd/azure-keyvault-secrets-webhook/main.go | 6 +++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cmd/azure-keyvault-controller/main.go b/cmd/azure-keyvault-controller/main.go index 71f2d0c9..e195bca4 100644 --- a/cmd/azure-keyvault-controller/main.go +++ b/cmd/azure-keyvault-controller/main.go @@ -102,7 +102,10 @@ func main() { if logFormat == "json" { loggerFactory := jsonlogs.Factory{} - logger, _ := loggerFactory.Create(logConfig.LoggingConfiguration{}, logConfig.LoggingOptions{}) + logger, _ := loggerFactory.Create(*logConfig.NewLoggingConfiguration(), logConfig.LoggingOptions{ + ErrorStream: os.Stderr, + InfoStream: os.Stdout, + }) klog.SetLogger(logger) } klog.InfoS("log settings", "format", logFormat, "level", flag.Lookup("v").Value) diff --git a/cmd/azure-keyvault-env/main.go b/cmd/azure-keyvault-env/main.go index e8c4ab12..7d80263a 100644 --- a/cmd/azure-keyvault-env/main.go +++ b/cmd/azure-keyvault-env/main.go @@ -157,7 +157,10 @@ func main() { if logFormat == "json" { loggerFactory := jsonlogs.Factory{} - logger, _ := loggerFactory.Create(logConfig.LoggingConfiguration{}, logConfig.LoggingOptions{}) + logger, _ := loggerFactory.Create(*logConfig.NewLoggingConfiguration(), logConfig.LoggingOptions{ + ErrorStream: os.Stderr, + InfoStream: os.Stdout, + }) klog.SetLogger(logger) } diff --git a/cmd/azure-keyvault-secrets-webhook/main.go b/cmd/azure-keyvault-secrets-webhook/main.go index c75a2309..56bca941 100644 --- a/cmd/azure-keyvault-secrets-webhook/main.go +++ b/cmd/azure-keyvault-secrets-webhook/main.go @@ -231,7 +231,11 @@ func main() { if params.logFormat == "json" { loggerFactory := jsonlogs.Factory{} - logger, _ := loggerFactory.Create(logConfig.LoggingConfiguration{}, logConfig.LoggingOptions{}) + logger, _ := loggerFactory.Create(*logConfig.NewLoggingConfiguration(), logConfig.LoggingOptions{ + ErrorStream: os.Stderr, + InfoStream: os.Stdout, + }) + klog.SetLogger(logger) } From 10765d92dbaac6629ac4eada6a5143426c2092c9 Mon Sep 17 00:00:00 2001 From: "Joseph, Abhilash" Date: Thu, 9 Nov 2023 00:34:27 -0500 Subject: [PATCH 4/6] upd regex for akvs name to match rfc1123 dns label names --- cmd/azure-keyvault-env/environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/azure-keyvault-env/environment.go b/cmd/azure-keyvault-env/environment.go index aa5f674f..3799dd9e 100644 --- a/cmd/azure-keyvault-env/environment.go +++ b/cmd/azure-keyvault-env/environment.go @@ -9,7 +9,7 @@ import ( ) const ( - envLookupRegex = `^([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)@azurekeyvault(\?([a-zA-Z_][a-zA-Z0-9_\.]*)?)?$` + envLookupRegex = `^([a-zA-Z0-9]([a-zA-Z0-9-]{0,239}[a-zA-Z0-9])?)@azurekeyvault(\?([a-zA-Z_][a-zA-Z0-9_\.]*)?)?$` ) type EnvSecret struct { From 49cc1bf815690b6e57edce086bd28db5938d24cd Mon Sep 17 00:00:00 2001 From: "Joseph, Abhilash" Date: Thu, 9 Nov 2023 13:13:36 -0500 Subject: [PATCH 5/6] Use 253 chars for env name regex --- cmd/azure-keyvault-env/environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/azure-keyvault-env/environment.go b/cmd/azure-keyvault-env/environment.go index 3799dd9e..8124c606 100644 --- a/cmd/azure-keyvault-env/environment.go +++ b/cmd/azure-keyvault-env/environment.go @@ -9,7 +9,7 @@ import ( ) const ( - envLookupRegex = `^([a-zA-Z0-9]([a-zA-Z0-9-]{0,239}[a-zA-Z0-9])?)@azurekeyvault(\?([a-zA-Z_][a-zA-Z0-9_\.]*)?)?$` + envLookupRegex = `^([a-zA-Z0-9]([a-zA-Z0-9-]{0,253}[a-zA-Z0-9])?)@azurekeyvault(\?([a-zA-Z_][a-zA-Z0-9_\.]*)?)?$` ) type EnvSecret struct { From 5a546e00b22df4f92d08951a220afb4b9a0d77e2 Mon Sep 17 00:00:00 2001 From: "Joseph, Abhilash" Date: Fri, 10 Nov 2023 14:11:37 -0500 Subject: [PATCH 6/6] use auth for acrs only --- pkg/azure/credentialprovider/acr.go | 5 ++++ pkg/docker/registry/registry.go | 36 ++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/pkg/azure/credentialprovider/acr.go b/pkg/azure/credentialprovider/acr.go index 2743eaf0..6a7e4a49 100644 --- a/pkg/azure/credentialprovider/acr.go +++ b/pkg/azure/credentialprovider/acr.go @@ -69,6 +69,11 @@ func (c CloudConfigCredentialProvider) GetAcrCredentials(image string) (k8sCrede Password: "", } + if !c.IsAcrRegistry(image) { + klog.V(4).Info("image not from acr, returning empty credentials") + return cred, nil + } + if c.config.UseManagedIdentityExtension { klog.V(4).Info("using managed identity for acr credentials") loginServer := parseACRLoginServerFromImage(image, c.environment) diff --git a/pkg/docker/registry/registry.go b/pkg/docker/registry/registry.go index b745ace0..7479a018 100644 --- a/pkg/docker/registry/registry.go +++ b/pkg/docker/registry/registry.go @@ -138,17 +138,31 @@ func getContainerRegistryRemoteOptions(ctx context.Context, client kubernetes.In return nil, fmt.Errorf("cannot fetch acr credentials: %w", err) } - sec := []corev1.Secret{ //{ - *dockerCfgSecretType.Create(container.Namespace, "secret", registry, authn.AuthConfig{ - Username: dockerConfigEntry.Username, Password: dockerConfigEntry.Password, - }), - } - *authChain, err = k8schain.NewFromPullSecrets( - ctx, - sec, - ) - if err != nil { - return nil, err + if dockerConfigEntry.Username != "" { + + sec := []corev1.Secret{ //{ + *dockerCfgSecretType.Create(container.Namespace, "secret", registry, authn.AuthConfig{ + Username: dockerConfigEntry.Username, Password: dockerConfigEntry.Password, + }), + } + *authChain, err = k8schain.NewFromPullSecrets( + ctx, + sec, + ) + if err != nil { + return nil, err + } + } else { + *authChain, err = k8schain.New( + ctx, + client, + k8schain.Options{ + Namespace: container.Namespace, + ServiceAccountName: container.ServiceAccountName}, + ) + if err != nil { + return nil, err + } } default: