From 7e1aa64088a0164b8fe0cec84f47717b11e4f9c9 Mon Sep 17 00:00:00 2001 From: moonek Date: Wed, 20 Nov 2024 07:54:59 +0000 Subject: [PATCH] [manila-csi-plugin] support muilple share rules Signed-off-by: moonek --- pkg/csi/manila/controllerserver.go | 12 +- pkg/csi/manila/shareadapters/cephfs.go | 112 +++++++++---------- pkg/csi/manila/shareadapters/nfs.go | 49 +++++--- pkg/csi/manila/shareadapters/shareadapter.go | 2 +- 4 files changed, 101 insertions(+), 74 deletions(-) diff --git a/pkg/csi/manila/controllerserver.go b/pkg/csi/manila/controllerserver.go index ea891f926e..a7f81484ff 100644 --- a/pkg/csi/manila/controllerserver.go +++ b/pkg/csi/manila/controllerserver.go @@ -193,18 +193,24 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol ad := getShareAdapter(shareOpts.Protocol) - accessRight, err := ad.GetOrGrantAccess(&shareadapters.GrantAccessArgs{Share: share, ManilaClient: manilaClient, Options: shareOpts}) + accessRights, err := ad.GetOrGrantAccesses(&shareadapters.GrantAccessArgs{Share: share, ManilaClient: manilaClient, Options: shareOpts}) if err != nil { if wait.Interrupted(err) { - return nil, status.Errorf(codes.DeadlineExceeded, "deadline exceeded while waiting for access rule %s for volume %s to become available", accessRight.ID, share.Name) + return nil, status.Errorf(codes.DeadlineExceeded, "deadline exceeded while waiting for access rules for volume %s to become available", share.Name) } return nil, status.Errorf(codes.Internal, "failed to grant access to volume %s: %v", share.Name, err) } + var accessRightIDs []string + for _, ar := range accessRights { + accessRightIDs = append(accessRightIDs, ar.ID) + } + shareAccessIDs := strings.Join(accessRightIDs, ",") + volCtx := filterParametersForVolumeContext(params, options.NodeVolumeContextFields()) volCtx = util.SetMapIfNotEmpty(volCtx, "shareID", share.ID) - volCtx = util.SetMapIfNotEmpty(volCtx, "shareAccessID", accessRight.ID) + volCtx = util.SetMapIfNotEmpty(volCtx, "shareAccessIDs", shareAccessIDs) volCtx = util.SetMapIfNotEmpty(volCtx, "groupID", share.ShareGroupID) volCtx = util.SetMapIfNotEmpty(volCtx, "affinity", shareOpts.Affinity) volCtx = util.SetMapIfNotEmpty(volCtx, "antiAffinity", shareOpts.AntiAffinity) diff --git a/pkg/csi/manila/shareadapters/cephfs.go b/pkg/csi/manila/shareadapters/cephfs.go index 38b0e7119c..4c729c3f34 100644 --- a/pkg/csi/manila/shareadapters/cephfs.go +++ b/pkg/csi/manila/shareadapters/cephfs.go @@ -18,6 +18,7 @@ package shareadapters import ( "fmt" + "strings" "time" "github.com/gophercloud/gophercloud/v2" @@ -31,82 +32,81 @@ type Cephfs struct{} var _ ShareAdapter = &Cephfs{} -func (Cephfs) GetOrGrantAccess(args *GrantAccessArgs) (accessRight *shares.AccessRight, err error) { +func (Cephfs) GetOrGrantAccesses(args *GrantAccessArgs) ([]shares.AccessRight, error) { // First, check if the access right exists or needs to be created - var rights []shares.AccessRight - - accessTo := args.Options.CephfsClientID - if accessTo == "" { - accessTo = args.Share.Name - } - - rights, err = args.ManilaClient.GetAccessRights(args.Share.ID) + rights, err := args.ManilaClient.GetAccessRights(args.Share.ID) if err != nil { if _, ok := err.(gophercloud.ErrResourceNotFound); !ok { return nil, fmt.Errorf("failed to list access rights: %v", err) } - } else { - // Try to find the access right + } + + accessToList := []string{args.Share.Name} + if args.Options.CephfsClientID != "" { + accessToList = strings.Split(args.Options.CephfsClientID, ",") + } + created := false + for _, at := range accessToList { + // Try to find the access right + found := false for _, r := range rights { - if r.AccessTo == accessTo && r.AccessType == "cephx" && r.AccessLevel == "rw" { + if r.AccessTo == at && r.AccessType == "cephx" && r.AccessLevel == "rw" { klog.V(4).Infof("cephx access right for share %s already exists", args.Share.Name) - - accessRight = &r + found = true break } } - } - if accessRight == nil { // Not found, create it - - accessRight, err = args.ManilaClient.GrantAccess(args.Share.ID, shares.GrantAccessOpts{ - AccessType: "cephx", - AccessLevel: "rw", - AccessTo: accessTo, - }) - - if err != nil { - return + if !found { + result, err := args.ManilaClient.GrantAccess(args.Share.ID, shares.GrantAccessOpts{ + AccessType: "cephx", + AccessLevel: "rw", + AccessTo: at, + }) + if err != nil { + return nil, fmt.Errorf("failed to grant access right: %v", err) + } + if result.AccessKey == "" { + // Wait till a ceph key is assigned to the access right + backoff := wait.Backoff{ + Duration: time.Second * 5, + Factor: 1.2, + Steps: 10, + } + wait.ExponentialBackoff(backoff, func() (bool, error) { + rights, err := args.ManilaClient.GetAccessRights(args.Share.ID) + if err != nil { + return false, fmt.Errorf("error get access rights for share %s: %v", args.Share.ID, err) + } + if len(rights) == 0 { + return false, fmt.Errorf("cannot find the access right we've just created") + } + for _, r := range rights { + if r.AccessTo == at && r.AccessKey != "" { + return true, nil + } + } + klog.V(4).Infof("Access key for %s is not set yet, retrying...", at) + return false, nil + }) + } + created = true } } - if accessRight.AccessKey != "" { - // The access right is ready - return - } - - // Wait till a ceph key is assigned to the access right - - backoff := wait.Backoff{ - Duration: time.Second * 5, - Factor: 1.2, - Steps: 10, - } - - return accessRight, wait.ExponentialBackoff(backoff, func() (bool, error) { - rights, err := args.ManilaClient.GetAccessRights(args.Share.ID) + // Search again because access rights have changed + if created { + rights, err = args.ManilaClient.GetAccessRights(args.Share.ID) if err != nil { - return false, err - } - - var accessRight *shares.AccessRight - - for i := range rights { - if rights[i].AccessTo == accessTo { - accessRight = &rights[i] - break + if _, ok := err.(gophercloud.ErrResourceNotFound); !ok { + return nil, fmt.Errorf("failed to list access rights: %v", err) } } - - if accessRight == nil { - return false, fmt.Errorf("cannot find the access right we've just created") - } - - return accessRight.AccessKey != "", nil - }) + } + return rights, nil } func (Cephfs) BuildVolumeContext(args *VolumeContextArgs) (volumeContext map[string]string, err error) { diff --git a/pkg/csi/manila/shareadapters/nfs.go b/pkg/csi/manila/shareadapters/nfs.go index ab24320741..75accda33b 100644 --- a/pkg/csi/manila/shareadapters/nfs.go +++ b/pkg/csi/manila/shareadapters/nfs.go @@ -32,7 +32,7 @@ type NFS struct{} var _ ShareAdapter = &NFS{} -func (NFS) GetOrGrantAccess(args *GrantAccessArgs) (*shares.AccessRight, error) { +func (NFS) GetOrGrantAccesses(args *GrantAccessArgs) ([]shares.AccessRight, error) { // First, check if the access right exists or needs to be created rights, err := args.ManilaClient.GetAccessRights(args.Share.ID) @@ -42,22 +42,43 @@ func (NFS) GetOrGrantAccess(args *GrantAccessArgs) (*shares.AccessRight, error) } } - // Try to find the access right - - for _, r := range rights { - if r.AccessTo == args.Options.NFSShareClient && r.AccessType == "ip" && r.AccessLevel == "rw" { - klog.V(4).Infof("IP access right for share %s already exists", args.Share.Name) - return &r, nil + accessToList := strings.Split(args.Options.NFSShareClient, ",") + + created := false + for _, at := range accessToList { + // Try to find the access right + found := false + for _, r := range rights { + if r.AccessTo == at && r.AccessType == "ip" && r.AccessLevel == "rw" { + klog.V(4).Infof("IP access right %s for share %s already exists", at, args.Share.Name) + found = true + break + } + } + // Not found, create it + if !found { + _, err = args.ManilaClient.GrantAccess(args.Share.ID, shares.GrantAccessOpts{ + AccessType: "ip", + AccessLevel: "rw", + AccessTo: at, + }) + if err != nil { + return nil, fmt.Errorf("failed to grant access right: %v", err) + } + created = true } } - // Not found, create it - - return args.ManilaClient.GrantAccess(args.Share.ID, shares.GrantAccessOpts{ - AccessType: "ip", - AccessLevel: "rw", - AccessTo: args.Options.NFSShareClient, - }) + // Search again because access rights have changed + if created { + rights, err = args.ManilaClient.GetAccessRights(args.Share.ID) + if err != nil { + if _, ok := err.(gophercloud.ErrResourceNotFound); !ok { + return nil, fmt.Errorf("failed to list access rights: %v", err) + } + } + } + return rights, nil } func (NFS) BuildVolumeContext(args *VolumeContextArgs) (volumeContext map[string]string, err error) { diff --git a/pkg/csi/manila/shareadapters/shareadapter.go b/pkg/csi/manila/shareadapters/shareadapter.go index d1e1d6fe0e..cfb9063b21 100644 --- a/pkg/csi/manila/shareadapters/shareadapter.go +++ b/pkg/csi/manila/shareadapters/shareadapter.go @@ -44,7 +44,7 @@ type ShareAdapter interface { // GetOrGrantAccess first tries to retrieve an access right for args.Share. // An access right is created for the share in case it doesn't exist yet. // Returns an existing or new access right for args.Share. - GetOrGrantAccess(args *GrantAccessArgs) (accessRight *shares.AccessRight, err error) + GetOrGrantAccesses(args *GrantAccessArgs) (accessRights []shares.AccessRight, err error) // BuildVolumeContext builds a volume context map that's passed to NodeStageVolumeRequest and NodePublishVolumeRequest BuildVolumeContext(args *VolumeContextArgs) (volumeContext map[string]string, err error)