From d28309fc6a46963060ebf0f1a0e54e169cd0d490 Mon Sep 17 00:00:00 2001 From: lidongsheng Date: Thu, 18 Apr 2024 21:00:07 +0800 Subject: [PATCH 1/8] [occm] add tag on floating ip create --- pkg/openstack/loadbalancer.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/openstack/loadbalancer.go b/pkg/openstack/loadbalancer.go index b9708c8920..e9ffb6209f 100644 --- a/pkg/openstack/loadbalancer.go +++ b/pkg/openstack/loadbalancer.go @@ -31,6 +31,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" v2monitors "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors" v2pools "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" corev1 "k8s.io/api/core/v1" @@ -563,7 +564,7 @@ func (lbaas *LbaasV2) deleteOctaviaListeners(lbID string, listenerList []listene return nil } -func (lbaas *LbaasV2) createFloatingIP(msg string, floatIPOpts floatingips.CreateOpts) (*floatingips.FloatingIP, error) { +func (lbaas *LbaasV2) createFloatingIP(msg string, floatIPOpts floatingips.CreateOpts, tag *string) (*floatingips.FloatingIP, error) { klog.V(4).Infof("%s floating ip with opts %+v", msg, floatIPOpts) mc := metrics.NewMetricContext("floating_ip", "create") floatIP, err := floatingips.Create(lbaas.network, floatIPOpts).Extract() @@ -571,6 +572,16 @@ func (lbaas *LbaasV2) createFloatingIP(msg string, floatIPOpts floatingips.Creat if mc.ObserveRequest(err) != nil { return floatIP, fmt.Errorf("error creating LB floatingip: %v", err) } + + if tag != nil { + err = attributestags.Add(lbaas.network, "floatingips", floatIP.ID, *tag).ExtractErr() + err = PreserveGopherError(err) + if mc.ObserveRequest(err) != nil { + floatingips.Delete(lbaas.network, floatIP.ID) + return floatIP, fmt.Errorf("error update LB floatingip tag: %v", err) + } + } + return floatIP, err } @@ -704,7 +715,7 @@ func (lbaas *LbaasV2) ensureFloatingIP(clusterName string, service *corev1.Servi svcConf.lbPublicSubnetSpec, svcConf.lbPublicNetworkID) for _, subnet := range foundSubnets { floatIPOpts.SubnetID = subnet.ID - floatIP, err = lbaas.createFloatingIP(fmt.Sprintf("Trying subnet %s for creating", subnet.Name), floatIPOpts) + floatIP, err = lbaas.createFloatingIP(fmt.Sprintf("Trying subnet %s for creating", subnet.Name), floatIPOpts, &svcConf.lbName) if err == nil { foundSubnet = subnet break @@ -721,7 +732,7 @@ func (lbaas *LbaasV2) ensureFloatingIP(clusterName string, service *corev1.Servi floatIPOpts.SubnetID = svcConf.lbPublicSubnetSpec.subnetID } floatIPOpts.FloatingIP = loadBalancerIP - floatIP, err = lbaas.createFloatingIP("Creating", floatIPOpts) + floatIP, err = lbaas.createFloatingIP("Creating", floatIPOpts, &svcConf.lbName) if err != nil { return "", err } From 9180e6d8a55de28620c50b6058c41cf4f116e9bc Mon Sep 17 00:00:00 2001 From: lidongsheng Date: Fri, 26 Apr 2024 19:04:46 +0800 Subject: [PATCH 2/8] [occm] ensure tag exist on floating ip create or lookup remove add tag on floating create --- pkg/openstack/loadbalancer.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/pkg/openstack/loadbalancer.go b/pkg/openstack/loadbalancer.go index e9ffb6209f..5ad495b8e6 100644 --- a/pkg/openstack/loadbalancer.go +++ b/pkg/openstack/loadbalancer.go @@ -564,7 +564,7 @@ func (lbaas *LbaasV2) deleteOctaviaListeners(lbID string, listenerList []listene return nil } -func (lbaas *LbaasV2) createFloatingIP(msg string, floatIPOpts floatingips.CreateOpts, tag *string) (*floatingips.FloatingIP, error) { +func (lbaas *LbaasV2) createFloatingIP(msg string, floatIPOpts floatingips.CreateOpts) (*floatingips.FloatingIP, error) { klog.V(4).Infof("%s floating ip with opts %+v", msg, floatIPOpts) mc := metrics.NewMetricContext("floating_ip", "create") floatIP, err := floatingips.Create(lbaas.network, floatIPOpts).Extract() @@ -573,15 +573,6 @@ func (lbaas *LbaasV2) createFloatingIP(msg string, floatIPOpts floatingips.Creat return floatIP, fmt.Errorf("error creating LB floatingip: %v", err) } - if tag != nil { - err = attributestags.Add(lbaas.network, "floatingips", floatIP.ID, *tag).ExtractErr() - err = PreserveGopherError(err) - if mc.ObserveRequest(err) != nil { - floatingips.Delete(lbaas.network, floatIP.ID) - return floatIP, fmt.Errorf("error update LB floatingip tag: %v", err) - } - } - return floatIP, err } @@ -715,7 +706,7 @@ func (lbaas *LbaasV2) ensureFloatingIP(clusterName string, service *corev1.Servi svcConf.lbPublicSubnetSpec, svcConf.lbPublicNetworkID) for _, subnet := range foundSubnets { floatIPOpts.SubnetID = subnet.ID - floatIP, err = lbaas.createFloatingIP(fmt.Sprintf("Trying subnet %s for creating", subnet.Name), floatIPOpts, &svcConf.lbName) + floatIP, err = lbaas.createFloatingIP(fmt.Sprintf("Trying subnet %s for creating", subnet.Name), floatIPOpts) if err == nil { foundSubnet = subnet break @@ -732,7 +723,7 @@ func (lbaas *LbaasV2) ensureFloatingIP(clusterName string, service *corev1.Servi floatIPOpts.SubnetID = svcConf.lbPublicSubnetSpec.subnetID } floatIPOpts.FloatingIP = loadBalancerIP - floatIP, err = lbaas.createFloatingIP("Creating", floatIPOpts, &svcConf.lbName) + floatIP, err = lbaas.createFloatingIP("Creating", floatIPOpts) if err != nil { return "", err } @@ -746,6 +737,20 @@ func (lbaas *LbaasV2) ensureFloatingIP(clusterName string, service *corev1.Servi } if floatIP != nil { + tags, err := attributestags.List(lbaas.network, "floatingips", floatIP.ID).Extract() + found := false + for _, tag := range tags { + if tag == svcConf.lbName { + found = true + } + } + if !found { + err = attributestags.Add(lbaas.network, "floatingips", floatIP.ID, svcConf.lbName).ExtractErr() + if err != nil { + klog.V(3).Infof("Cannot update floatIP tag %s for floating %s", svcConf.lbName, floatIP.ID) + } + } + return floatIP.FloatingIP, nil } From a7a5daf72e372942dbb7734585273afda4daff91 Mon Sep 17 00:00:00 2001 From: lidongsheng Date: Fri, 26 Apr 2024 19:08:28 +0800 Subject: [PATCH 3/8] [occm] handling tag list error --- pkg/openstack/loadbalancer.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/openstack/loadbalancer.go b/pkg/openstack/loadbalancer.go index 5ad495b8e6..84baf3fab2 100644 --- a/pkg/openstack/loadbalancer.go +++ b/pkg/openstack/loadbalancer.go @@ -738,6 +738,10 @@ func (lbaas *LbaasV2) ensureFloatingIP(clusterName string, service *corev1.Servi if floatIP != nil { tags, err := attributestags.List(lbaas.network, "floatingips", floatIP.ID).Extract() + if err != nil { + klog.V(3).Infof("Cannot get floatIP tags for floating %s", floatIP.ID) + return floatIP.FloatingIP, nil + } found := false for _, tag := range tags { if tag == svcConf.lbName { From 4319a8865044175ea772f0d2fb143e18537d0fae Mon Sep 17 00:00:00 2001 From: lidongsheng Date: Fri, 26 Apr 2024 20:07:36 +0800 Subject: [PATCH 4/8] [occm] untag FIP when force CCM to not delete the FIP --- pkg/openstack/loadbalancer.go | 56 ++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/pkg/openstack/loadbalancer.go b/pkg/openstack/loadbalancer.go index 84baf3fab2..e6eb942769 100644 --- a/pkg/openstack/loadbalancer.go +++ b/pkg/openstack/loadbalancer.go @@ -593,6 +593,42 @@ func (lbaas *LbaasV2) updateFloatingIP(floatingip *floatingips.FloatingIP, portI return floatingip, nil } +func (lbaas *LbaasV2) updateFloatingIPTag(floatingip *floatingips.FloatingIP, Tag string, delete bool) error { + if Tag == "" { + return fmt.Errorf("Error input tag argument ") + } + tags, err := attributestags.List(lbaas.network, "floatingips", floatingip.ID).Extract() + if err != nil { + klog.V(3).Infof("Cannot get floatIP tags for floating %s", floatingip.ID) + return err + } + found := false + for _, tag := range tags { + if tag == Tag { + found = true + } + } + if delete { + if !found { + return nil + } + err = attributestags.Delete(lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() + if err != nil { + klog.V(3).Infof("Cannot update floatIP tag %s for floating %s", Tag, floatingip.ID) + } + return err + } + + if !found { + err = attributestags.Add(lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() + if err != nil { + klog.V(3).Infof("Cannot update floatIP tag %s for floating %s", Tag, floatingip.ID) + } + return err + } + return nil +} + // ensureFloatingIP manages a FIP for a Service and returns the address that should be advertised in the // .Status.LoadBalancer. In particular it will: // 1. Lookup if any FIP is already attached to the VIP port of the LB. @@ -638,6 +674,7 @@ func (lbaas *LbaasV2) ensureFloatingIP(clusterName string, service *corev1.Servi if err != nil { return "", err } + _ = lbaas.updateFloatingIPTag(floatIP, svcConf.lbName, true) } } return lb.VipAddress, nil @@ -737,24 +774,7 @@ func (lbaas *LbaasV2) ensureFloatingIP(clusterName string, service *corev1.Servi } if floatIP != nil { - tags, err := attributestags.List(lbaas.network, "floatingips", floatIP.ID).Extract() - if err != nil { - klog.V(3).Infof("Cannot get floatIP tags for floating %s", floatIP.ID) - return floatIP.FloatingIP, nil - } - found := false - for _, tag := range tags { - if tag == svcConf.lbName { - found = true - } - } - if !found { - err = attributestags.Add(lbaas.network, "floatingips", floatIP.ID, svcConf.lbName).ExtractErr() - if err != nil { - klog.V(3).Infof("Cannot update floatIP tag %s for floating %s", svcConf.lbName, floatIP.ID) - } - } - + _ = lbaas.updateFloatingIPTag(floatIP, svcConf.lbName, false) return floatIP.FloatingIP, nil } From 51c738b65bb4c13531182ed2ffcef96f78a6716c Mon Sep 17 00:00:00 2001 From: lidongshengxdayu Date: Wed, 3 Jul 2024 10:27:31 +0800 Subject: [PATCH 5/8] [occm] fix build issue with Upgrade Gophercloud to v2 --- pkg/openstack/loadbalancer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/openstack/loadbalancer.go b/pkg/openstack/loadbalancer.go index 49f86edecc..942bd7864c 100644 --- a/pkg/openstack/loadbalancer.go +++ b/pkg/openstack/loadbalancer.go @@ -32,6 +32,7 @@ import ( "github.com/gophercloud/gophercloud/v2/openstack/loadbalancer/v2/loadbalancers" v2monitors "github.com/gophercloud/gophercloud/v2/openstack/loadbalancer/v2/monitors" v2pools "github.com/gophercloud/gophercloud/v2/openstack/loadbalancer/v2/pools" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/attributestags" "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/floatingips" "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/subnets" corev1 "k8s.io/api/core/v1" @@ -607,7 +608,7 @@ func (lbaas *LbaasV2) updateFloatingIPTag(floatingip *floatingips.FloatingIP, Ta if Tag == "" { return fmt.Errorf("Error input tag argument ") } - tags, err := attributestags.List(lbaas.network, "floatingips", floatingip.ID).Extract() + tags, err := attributestags.List(context.TODO(), lbaas.network, "floatingips", floatingip.ID).Extract() if err != nil { klog.V(3).Infof("Cannot get floatIP tags for floating %s", floatingip.ID) return err @@ -622,7 +623,7 @@ func (lbaas *LbaasV2) updateFloatingIPTag(floatingip *floatingips.FloatingIP, Ta if !found { return nil } - err = attributestags.Delete(lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() + err = attributestags.Delete(context.TODO(), lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() if err != nil { klog.V(3).Infof("Cannot update floatIP tag %s for floating %s", Tag, floatingip.ID) } @@ -630,7 +631,7 @@ func (lbaas *LbaasV2) updateFloatingIPTag(floatingip *floatingips.FloatingIP, Ta } if !found { - err = attributestags.Add(lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() + err = attributestags.Add(context.TODO(), lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() if err != nil { klog.V(3).Infof("Cannot update floatIP tag %s for floating %s", Tag, floatingip.ID) } From 7196a75751761e714eb20a1e9675c725891c9ccc Mon Sep 17 00:00:00 2001 From: lidongshengxdayu Date: Thu, 7 Nov 2024 10:46:18 +0800 Subject: [PATCH 6/8] [occm] fix context argument pass to `updateFloatingIPTag` optimize tag traversal --- pkg/openstack/loadbalancer.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/pkg/openstack/loadbalancer.go b/pkg/openstack/loadbalancer.go index c51f358fd1..94c4971e3a 100644 --- a/pkg/openstack/loadbalancer.go +++ b/pkg/openstack/loadbalancer.go @@ -619,26 +619,23 @@ func (lbaas *LbaasV2) updateFloatingIP(ctx context.Context, floatingip *floating return floatingip, nil } -func (lbaas *LbaasV2) updateFloatingIPTag(floatingip *floatingips.FloatingIP, Tag string, delete bool) error { +func (lbaas *LbaasV2) updateFloatingIPTag(ctx context.Context, floatingip *floatingips.FloatingIP, Tag string, delete bool) error { if Tag == "" { return fmt.Errorf("Error input tag argument ") } - tags, err := attributestags.List(context.TODO(), lbaas.network, "floatingips", floatingip.ID).Extract() + tags, err := attributestags.List(ctx, lbaas.network, "floatingips", floatingip.ID).Extract() if err != nil { klog.V(3).Infof("Cannot get floatIP tags for floating %s", floatingip.ID) return err } - found := false - for _, tag := range tags { - if tag == Tag { - found = true - } - } + + found := slices.Contains(tags, Tag) + if delete { if !found { return nil } - err = attributestags.Delete(context.TODO(), lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() + err = attributestags.Delete(ctx, lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() if err != nil { klog.V(3).Infof("Cannot update floatIP tag %s for floating %s", Tag, floatingip.ID) } @@ -646,7 +643,7 @@ func (lbaas *LbaasV2) updateFloatingIPTag(floatingip *floatingips.FloatingIP, Ta } if !found { - err = attributestags.Add(context.TODO(), lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() + err = attributestags.Add(ctx, lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() if err != nil { klog.V(3).Infof("Cannot update floatIP tag %s for floating %s", Tag, floatingip.ID) } @@ -700,7 +697,7 @@ func (lbaas *LbaasV2) ensureFloatingIP(ctx context.Context, clusterName string, if err != nil { return "", err } - _ = lbaas.updateFloatingIPTag(floatIP, svcConf.lbName, true) + _ = lbaas.updateFloatingIPTag(ctx, floatIP, svcConf.lbName, true) } } return lb.VipAddress, nil @@ -800,7 +797,7 @@ func (lbaas *LbaasV2) ensureFloatingIP(ctx context.Context, clusterName string, } if floatIP != nil { - _ = lbaas.updateFloatingIPTag(floatIP, svcConf.lbName, false) + _ = lbaas.updateFloatingIPTag(ctx, floatIP, svcConf.lbName, false) return floatIP.FloatingIP, nil } From c16cc039b650aaa43d6f8d1366426af71db188b0 Mon Sep 17 00:00:00 2001 From: lidongshengxdayu Date: Thu, 7 Nov 2024 19:19:24 +0800 Subject: [PATCH 7/8] [occm] fix `updateFloatingIPTag` typo and error print --- pkg/openstack/loadbalancer.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/pkg/openstack/loadbalancer.go b/pkg/openstack/loadbalancer.go index 94c4971e3a..d4c0e39a96 100644 --- a/pkg/openstack/loadbalancer.go +++ b/pkg/openstack/loadbalancer.go @@ -619,33 +619,34 @@ func (lbaas *LbaasV2) updateFloatingIP(ctx context.Context, floatingip *floating return floatingip, nil } -func (lbaas *LbaasV2) updateFloatingIPTag(ctx context.Context, floatingip *floatingips.FloatingIP, Tag string, delete bool) error { +func (lbaas *LbaasV2) updateFloatingIPTag(ctx context.Context, fip *floatingips.FloatingIP, Tag string, del bool) error { if Tag == "" { return fmt.Errorf("Error input tag argument ") } - tags, err := attributestags.List(ctx, lbaas.network, "floatingips", floatingip.ID).Extract() + tags, err := attributestags.List(ctx, lbaas.network, "floatingips", fip.ID).Extract() if err != nil { - klog.V(3).Infof("Cannot get floatIP tags for floating %s", floatingip.ID) + klog.V(3).Infof("Cannot get floatIP tags for floating %s", fip.ID) return err } found := slices.Contains(tags, Tag) - if delete { - if !found { - return nil - } - err = attributestags.Delete(ctx, lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() + if (found && !del) || (!found && del) { + return nil + } + + if found && del { + err = attributestags.Delete(ctx, lbaas.network, "floatingips", fip.ID, Tag).ExtractErr() if err != nil { - klog.V(3).Infof("Cannot update floatIP tag %s for floating %s", Tag, floatingip.ID) + klog.V(2).ErrorS(err, "Cannot update floating IP tag %s for floating %s", Tag, fip.ID) } return err } - if !found { - err = attributestags.Add(ctx, lbaas.network, "floatingips", floatingip.ID, Tag).ExtractErr() + if !found && !del { + err = attributestags.Add(ctx, lbaas.network, "floatingips", fip.ID, Tag).ExtractErr() if err != nil { - klog.V(3).Infof("Cannot update floatIP tag %s for floating %s", Tag, floatingip.ID) + klog.V(2).ErrorS(err, "Cannot update floating IP tag %s for floating %s", Tag, fip.ID) } return err } From f6189ee85792ab8eb1e56dffe918f7fee09519cb Mon Sep 17 00:00:00 2001 From: lidongshengxdayu Date: Fri, 8 Nov 2024 10:50:11 +0800 Subject: [PATCH 8/8] [occm] add check `standard-attr-tag` when update floating ip tags --- pkg/openstack/loadbalancer.go | 10 ++++++++++ pkg/openstack/openstack.go | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/openstack/loadbalancer.go b/pkg/openstack/loadbalancer.go index d4c0e39a96..feb72c0157 100644 --- a/pkg/openstack/loadbalancer.go +++ b/pkg/openstack/loadbalancer.go @@ -620,6 +620,9 @@ func (lbaas *LbaasV2) updateFloatingIP(ctx context.Context, floatingip *floating } func (lbaas *LbaasV2) updateFloatingIPTag(ctx context.Context, fip *floatingips.FloatingIP, Tag string, del bool) error { + if enabled, ok := lbaas.netExtensions["standard-attr-tag"]; !ok || !enabled { + return nil + } if Tag == "" { return fmt.Errorf("Error input tag argument ") } @@ -1797,6 +1800,13 @@ func (lbaas *LbaasV2) ensureOctaviaLoadBalancer(ctx context.Context, clusterName if err != nil { return nil, err } + extensions, err := openstackutil.GetNetworkExtensions(ctx, lbaas.network) + if err == nil { + klog.V(2).ErrorS(err, "cannot get network extensions") + lbaas.netExtensions = extensions + } else { + lbaas.netExtensions = map[string]bool{} + } klog.V(4).InfoS("Load balancer ensured", "lbID", loadbalancer.ID, "isLBOwner", isLBOwner, "createNewLB", createNewLB) diff --git a/pkg/openstack/openstack.go b/pkg/openstack/openstack.go index ef1a7bc57b..e0a14533c0 100644 --- a/pkg/openstack/openstack.go +++ b/pkg/openstack/openstack.go @@ -91,6 +91,7 @@ type LoadBalancer struct { opts LoadBalancerOpts kclient kubernetes.Interface eventRecorder record.EventRecorder + netExtensions map[string]bool } // LoadBalancerOpts have the options to talk to Neutron LBaaSV2 or Octavia @@ -375,7 +376,7 @@ func (os *OpenStack) LoadBalancer() (cloudprovider.LoadBalancer, bool) { klog.V(1).Info("Claiming to support LoadBalancer") - return &LbaasV2{LoadBalancer{secret, network, lb, os.lbOpts, os.kclient, os.eventRecorder}}, true + return &LbaasV2{LoadBalancer{secret, network, lb, os.lbOpts, os.kclient, os.eventRecorder, map[string]bool{}}}, true } // Zones indicates that we support zones