Skip to content

Commit

Permalink
Adds support for TLS and token reviews
Browse files Browse the repository at this point in the history
Signed-off-by: Bipul Adhikari <[email protected]>
  • Loading branch information
bipuladh committed Nov 13, 2024
1 parent 8b610a1 commit 9ee2d7a
Show file tree
Hide file tree
Showing 10 changed files with 212 additions and 29 deletions.
16 changes: 8 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@

# Image URL to use all building/pushing image targets
CONTROLLER_IMG ?= quay.io/csiaddons/k8s-controller
SIDECAR_IMG ?= quay.io/csiaddons/k8s-sidecar
BUNDLE_IMG ?= quay.io/csiaddons/k8s-bundle
TOOLS_IMG ?= quay.io/csiaddons/tools
CONTROLLER_IMG ?= quay.io/badhikar/k8s-controller
SIDECAR_IMG ?= quay.io/badhikar/k8s-sidecar
BUNDLE_IMG ?= quay.io/badhikar/k8s-bundle
TOOLS_IMG ?= quay.io/badhikar/tools

# set TAG to a release for consumption in the bundle
TAG ?= latest
TAG ?= v4

# In case the *_IMG variables can contain a full qualified container-image
# resource (includes a ":"), the container-images should not use the TAG
Expand Down Expand Up @@ -175,16 +175,16 @@ run: manifests generate fmt vet ## Run a controller from your host.
go run ./cmd/manager/main.go

.PHONY: docker-build
docker-build: container-cmd test ## Build docker image with the manager.
$(CONTAINER_CMD) build -t ${CONTROLLER_IMG} .
docker-build: container-cmd ## Build docker image with the manager.
$(CONTAINER_CMD) build -t ${CONTROLLER_IMG} . --platform=linux/amd64

.PHONY: docker-push
docker-push: container-cmd ## Push docker image with the manager.
$(CONTAINER_CMD) push ${CONTROLLER_IMG}

.PHONY: docker-build-sidecar
docker-build-sidecar: container-cmd
$(CONTAINER_CMD) build -f ./build/Containerfile.sidecar -t ${SIDECAR_IMG} .
$(CONTAINER_CMD) build -f ./build/Containerfile.sidecar -t ${SIDECAR_IMG} . --platform=linux/amd64

.PHONY: docker-push-sidecar
docker-push-sidecar: container-cmd
Expand Down
4 changes: 4 additions & 0 deletions api/csiaddons/v1alpha1/csiaddonsnode_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ type CSIAddonsNodeDriver struct {
// side-car listens to.
EndPoint string `json:"endpoint"`

// The name of the service that has the sidecar
// side-car listens to.
SidecarService string `json:"sidecarService"`

// NodeID is the ID of the node to identify on which node the side-car
// is running.
// +kubebuilder:validation:Required
Expand Down
13 changes: 9 additions & 4 deletions cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ func main() {
enableAdmissionWebhooks bool
ctx = context.Background()
cfg = util.NewConfig()
enableTLS bool
skipInsecureVerify bool
)
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
Expand All @@ -92,6 +94,8 @@ func main() {
flag.BoolVar(&enableAdmissionWebhooks, "enable-admission-webhooks", false, "[DEPRECATED] Enable the admission webhooks")
flag.BoolVar(&showVersion, "version", false, "Print Version details")
flag.StringVar(&cfg.SchedulePrecedence, "schedule-precedence", "", "The order of precedence in which schedule of reclaimspace and keyrotation is considered. Possible values are sc-only")
flag.BoolVar(&enableTLS, "enable-tls", false, "Enable TLS(disabled by default)")
flag.BoolVar(&skipInsecureVerify, "insecure-skip-tls-verify", false, "skip server certificate verification")
opts := zap.Options{
Development: true,
TimeEncoder: zapcore.ISO8601TimeEncoder,
Expand Down Expand Up @@ -145,16 +149,17 @@ func main() {
setupLog.Error(err, "unable to start manager")
os.Exit(1)
}

connPool := connection.NewConnectionPool()

ctrlOptions := controller.Options{
MaxConcurrentReconciles: cfg.MaxConcurrentReconciles,
}
if err = (&controllers.CSIAddonsNodeReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ConnPool: connPool,
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ConnPool: connPool,
EnableTLS: enableTLS,
SkipInsecureVerify: skipInsecureVerify,
}).SetupWithManager(mgr, ctrlOptions); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "CSIAddonsNode")
os.Exit(1)
Expand Down
4 changes: 2 additions & 2 deletions config/manager/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
images:
- name: controller
newName: quay.io/csiaddons/k8s-controller
newTag: latest
newName: quay.io/badhikar/k8s-controller
newTag: v2
- name: rbac-proxy
newName: quay.io/brancz/kube-rbac-proxy
newTag: v0.18.0
Expand Down
2 changes: 1 addition & 1 deletion deploy/controller/setup-controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: quay.io/csiaddons/k8s-controller:latest
image: quay.io/badhikar/k8s-controller:v2
livenessProbe:
httpGet:
path: /healthz
Expand Down
30 changes: 27 additions & 3 deletions internal/connection/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,17 @@ package connection

import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"time"

"github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token"

"github.com/csi-addons/spec/lib/go/identity"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/credentials"
)

// Connection struct consists of to NodeID, DriverName, Capabilities for the controller
Expand All @@ -39,11 +45,29 @@ type Connection struct {

// NewConnection establishes connection with sidecar, fetches capability and returns Connection object
// filled with required information.
func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string) (*Connection, error) {
func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string, enableTLS, skipInsecureVerify bool) (*Connection, error) {
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithIdleTimeout(time.Duration(0)),
}

if enableTLS {
opts = append(opts, token.WithServiceAccountToken())

caFile, caError := token.GetCACert()
if caError != nil {
return nil, fmt.Errorf("failed to get server cert %w", caError)
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM([]byte(caFile)) {
return nil, errors.New("failed to append CA cert")
}
tlsConfig := &tls.Config{
RootCAs: caCertPool, // The CA certificates to verify the server
InsecureSkipVerify: skipInsecureVerify,
}
creds := credentials.NewTLS(tlsConfig)
opts = append(opts, grpc.WithTransportCredentials(creds))
}
cc, err := grpc.NewClient(endpoint, opts...)
if err != nil {
return nil, err
Expand Down
12 changes: 9 additions & 3 deletions internal/controller/csiaddons/csiaddonsnode_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,10 @@ var (
// CSIAddonsNodeReconciler reconciles a CSIAddonsNode object
type CSIAddonsNodeReconciler struct {
client.Client
Scheme *runtime.Scheme
ConnPool *connection.ConnectionPool
Scheme *runtime.Scheme
ConnPool *connection.ConnectionPool
EnableTLS bool
SkipInsecureVerify bool
}

//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
Expand Down Expand Up @@ -120,7 +122,7 @@ func (r *CSIAddonsNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reques
}

logger.Info("Connecting to sidecar")
newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name)
newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name, r.EnableTLS, r.SkipInsecureVerify)
if err != nil {
logger.Error(err, "Failed to establish connection with sidecar")

Expand Down Expand Up @@ -207,6 +209,10 @@ func (r *CSIAddonsNodeReconciler) removeFinalizer(
// by GRPC to connect to the sidecar.
func (r *CSIAddonsNodeReconciler) resolveEndpoint(ctx context.Context, rawURL string) (string, error) {
namespace, podname, port, err := parseEndpoint(rawURL)
if r.EnableTLS {
// We need to use this name to accept certificates signed for pods
return podname + "." + namespace + ".pod" + ":" + port, nil
}
if err != nil && errors.Is(err, errLegacyEndpoint) {
return rawURL, nil
} else if err != nil {
Expand Down
123 changes: 123 additions & 0 deletions internal/kubernetes/token/grpc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
Copyright 2024 The Kubernetes-CSI-Addons Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package token

import (
"context"
"fmt"
"io"
"os"
"strings"

"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
authv1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

const bearerPrefix = "Bearer "

func WithServiceAccountToken() grpc.DialOption {
return grpc.WithUnaryInterceptor(addAuthorizationHeader)
}

func addAuthorizationHeader(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
token, err := getToken()
if err != nil {
return err
}

authCtx := metadata.AppendToOutgoingContext(ctx, "Authorization", "Bearer "+token)
return invoker(authCtx, method, req, reply, cc, opts...)
}

func getToken() (string, error) {

const tokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"
return readFile(tokenPath)
}

func AuthorizationInterceptor(kubeclient kubernetes.Clientset) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
if err := authorizeConnection(ctx, kubeclient); err != nil {
return nil, err
}
return handler(ctx, req)
}
}

func authorizeConnection(ctx context.Context, kubeclient kubernetes.Clientset) error {

md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return status.Error(codes.Unauthenticated, "missing metadata")
}

authHeader, ok := md["authorization"]
if !ok || len(authHeader) == 0 {
return status.Error(codes.Unauthenticated, "missing authorization token")
}

token := authHeader[0]
isValidated, err := validateBearerToken(ctx, token, kubeclient)
if !isValidated || (err != nil) {
return status.Error(codes.Unauthenticated, fmt.Sprint("invalid token: %w", err))
}
return nil
}

func parseToken(authHeader string) string {
return strings.TrimPrefix(authHeader, bearerPrefix)
}

func validateBearerToken(ctx context.Context, token string, kubeclient kubernetes.Clientset) (bool, error) {
tokenReview := &authv1.TokenReview{
Spec: authv1.TokenReviewSpec{
Token: parseToken(token),
},
}
result, err := kubeclient.AuthenticationV1().TokenReviews().Create(ctx, tokenReview, metav1.CreateOptions{})
if err != nil {
return false, fmt.Errorf("failed to review token %w", err)
}

if result.Status.Authenticated {
return true, nil
}
return false, nil
}

func readFile(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()

data, err := io.ReadAll(file)
if err != nil {
return "", err
}
return string(data), nil
}

func GetCACert() (string, error) {
caCertFile := "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
return readFile(caCertFile)
}
34 changes: 27 additions & 7 deletions sidecar/internal/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,13 @@ package server

import (
"errors"
"fmt"
"net"

"github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
k8s "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
)

Expand All @@ -38,24 +42,31 @@ type SidecarServer struct {
// URL components to listen on the tcp port
scheme string
endpoint string
client *k8s.Clientset

server *grpc.Server
services []SidecarService
server *grpc.Server
services []SidecarService
enableTLS bool
tlsCertFile string
tlsKeyFile string
}

// NewSidecarServer create a new SidecarServer on the given IP-address and
// port. If the IP-address is an empty string, the server will listen on all
// available IP-addresses. Only tcp ports are supported.
func NewSidecarServer(ip, port string) *SidecarServer {
func NewSidecarServer(ip, port string, client *k8s.Clientset, enableTLS bool, tlsCertFile, tlsKeyFile string) *SidecarServer {
ss := &SidecarServer{}

if ss.services == nil {
ss.services = make([]SidecarService, 0)
}

ss.scheme = "tcp"
ss.endpoint = ip + ":" + port

ss.endpoint = ip + ":" + fmt.Sprint(port)
ss.client = client
ss.enableTLS = enableTLS
ss.tlsCertFile = tlsCertFile
ss.tlsKeyFile = tlsKeyFile
return ss
}

Expand All @@ -69,8 +80,17 @@ func (ss *SidecarServer) RegisterService(svc SidecarService) {
// Init creates the internal gRPC server, and registers the SidecarServices.
// and starts gRPC server.
func (ss *SidecarServer) Start() {
// create the gRPC server and register services
ss.server = grpc.NewServer()
if ss.enableTLS {
creds, err := credentials.NewServerTLSFromFile(ss.tlsCertFile, ss.tlsKeyFile)
if err != nil {
klog.Fatalf("failed to load TLS certificate and key: %v", err)
}
// create the gRPC server and register services
ss.server = grpc.NewServer(grpc.UnaryInterceptor(token.AuthorizationInterceptor(*ss.client)), grpc.Creds(creds))
}
if !ss.enableTLS {
ss.server = grpc.NewServer()
}

for _, svc := range ss.services {
svc.RegisterService(ss.server)
Expand Down
Loading

0 comments on commit 9ee2d7a

Please sign in to comment.