From f0c7a18c6bea3dff93f248787cbdbc48227e1f89 Mon Sep 17 00:00:00 2001 From: Cybwan Date: Tue, 26 Sep 2023 16:36:46 +0800 Subject: [PATCH] fix issues (#194) * update version to 1.3.9 * add HTTP1PerRequestLoadBalancing and HTTP2PerRequestLoadBalancing to mesh config. * Changed implementation of outbound per-request load balancing. * [fix] Explicitly release _session in onEnd event * upgrade pipy to 0.90.3-30. * upgrade pipy to 0.90.3-38. --------- Co-authored-by: wanpf --- charts/osm/Chart.yaml | 4 ++-- charts/osm/README.md | 14 +++++++------ charts/osm/templates/preset-mesh-config.yaml | 2 ++ charts/osm/values.schema.json | 20 +++++++++++++++++++ charts/osm/values.yaml | 12 ++++++++--- cmd/osm-bootstrap/crds/config_meshconfig.yaml | 12 +++++++++++ pkg/apis/config/v1alpha1/mesh_config.go | 6 ++++++ pkg/apis/config/v1alpha2/mesh_config.go | 6 ++++++ pkg/messaging/broker.go | 2 ++ .../modules/outbound-http-load-balancing.js | 19 +++++++++++++++++- .../codebase/modules/outbound-http-routing.js | 8 ++++---- pkg/sidecar/providers/pipy/repo/jobs.go | 2 ++ pkg/sidecar/providers/pipy/repo/policy.go | 14 +++++++++++++ pkg/sidecar/providers/pipy/repo/types.go | 2 ++ 14 files changed, 107 insertions(+), 16 deletions(-) diff --git a/charts/osm/Chart.yaml b/charts/osm/Chart.yaml index 4ef56fffa..e252bf266 100644 --- a/charts/osm/Chart.yaml +++ b/charts/osm/Chart.yaml @@ -14,11 +14,11 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 1.3.8 +version: 1.3.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v1.3.8 +appVersion: v1.3.9 # This specifies the minimum Kubernetes version OSM is compatible with. kubeVersion: ">= 1.19.0-0" diff --git a/charts/osm/README.md b/charts/osm/README.md index 9ddeb4a6f..0087f7709 100644 --- a/charts/osm/README.md +++ b/charts/osm/README.md @@ -1,6 +1,6 @@ # Open Service Mesh Edge Helm Chart -![Version: 1.3.8](https://img.shields.io/badge/Version-1.3.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.3.8](https://img.shields.io/badge/AppVersion-v1.3.8-informational?style=flat-square) +![Version: 1.3.9](https://img.shields.io/badge/Version-1.3.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.3.9](https://img.shields.io/badge/AppVersion-v1.3.9-informational?style=flat-square) A Helm chart to install the [osm-edge](https://github.com/flomesh-io/osm-edge) control plane on Kubernetes. @@ -130,6 +130,8 @@ The following table lists the configurable parameters of the osm chart and their | osm.grafana.port | int | `3000` | Grafana service's port | | osm.grafana.rendererImage | string | `"grafana/grafana-image-renderer:3.2.1"` | Image used for Grafana Renderer | | osm.grafana.tolerations | list | `[]` | Node tolerations applied to control plane pods. The specified tolerations allow pods to schedule onto nodes with matching taints. | +| osm.http1PerRequestLoadBalancing | bool | `false` | Specifies a boolean indicating if load balancing based on request is enabled for http1. | +| osm.http2PerRequestLoadBalancing | bool | `true` | Specifies a boolean indicating if load balancing based on request is enabled for http2. | | osm.image.digest | object | `{"osmBootstrap":"","osmCRDs":"","osmController":"","osmHealthcheck":"","osmInjector":"","osmInterceptor":"","osmPreinstall":"","osmSidecarInit":""}` | Image digest (defaults to latest compatible tag) | | osm.image.digest.osmBootstrap | string | `""` | osm-boostrap's image digest | | osm.image.digest.osmCRDs | string | `""` | osm-crds' image digest | @@ -150,7 +152,7 @@ The following table lists the configurable parameters of the osm chart and their | osm.image.name.osmSidecarInit | string | `"osm-edge-sidecar-init"` | Sidecar init container's image name | | osm.image.pullPolicy | string | `"IfNotPresent"` | Container image pull policy for control plane containers | | osm.image.registry | string | `"flomesh"` | Container image registry for control plane images | -| osm.image.tag | string | `"1.3.8"` | Container image tag for control plane images | +| osm.image.tag | string | `"1.3.9"` | Container image tag for control plane images | | osm.imagePullSecrets | list | `[]` | `osm-controller` image pull secret | | osm.inboundPortExclusionList | list | `[]` | Specifies a global list of ports to exclude from inbound traffic interception by the sidecar proxy. If specified, must be a list of positive integers. | | osm.injector.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"kubernetes.io/os"` | | @@ -330,15 +332,15 @@ The following table lists the configurable parameters of the osm chart and their | osm.remoteLogging.level | int | `2` | Level of the remote logging service | | osm.remoteLogging.port | int | `30514` | Port of the remote logging service | | osm.remoteLogging.sampledFraction | string | `"1.0"` | Sampled Fraction | -| osm.repoServer | object | `{"codebase":"","image":"flomesh/pipy-repo:0.90.2-14","ipaddr":"127.0.0.1","standalone":false}` | Pipy RepoServer | +| osm.repoServer | object | `{"codebase":"","image":"flomesh/pipy-repo:0.90.3-38","ipaddr":"127.0.0.1","standalone":false}` | Pipy RepoServer | | osm.repoServer.codebase | string | `""` | codebase is the folder used by osmController. | -| osm.repoServer.image | string | `"flomesh/pipy-repo:0.90.2-14"` | Image used for Pipy RepoServer | +| osm.repoServer.image | string | `"flomesh/pipy-repo:0.90.3-38"` | Image used for Pipy RepoServer | | osm.repoServer.ipaddr | string | `"127.0.0.1"` | ipaddr of host/service where Pipy RepoServer is installed | | osm.repoServer.standalone | bool | `false` | if false , Pipy RepoServer is installed within osmController pod. | | osm.sidecarClass | string | `"pipy"` | The class of the OSM Sidecar Driver | -| osm.sidecarDrivers | list | `[{"proxyServerPort":6060,"sidecarImage":"flomesh/pipy:0.90.2-14","sidecarName":"pipy"},{"proxyServerPort":15128,"sidecarImage":"envoyproxy/envoy:v1.19.3","sidecarName":"envoy","sidecarWindowsImage":"envoyproxy/envoy-windows:latest"}]` | Sidecar drivers supported by osm-edge | +| osm.sidecarDrivers | list | `[{"proxyServerPort":6060,"sidecarImage":"flomesh/pipy:0.90.3-38","sidecarName":"pipy"},{"proxyServerPort":15128,"sidecarImage":"envoyproxy/envoy:v1.19.3","sidecarName":"envoy","sidecarWindowsImage":"envoyproxy/envoy-windows:latest"}]` | Sidecar drivers supported by osm-edge | | osm.sidecarDrivers[0].proxyServerPort | int | `6060` | Remote destination port on which the Discovery Service listens for new connections from Sidecars. | -| osm.sidecarDrivers[0].sidecarImage | string | `"flomesh/pipy:0.90.2-14"` | Sidecar image for Linux workloads | +| osm.sidecarDrivers[0].sidecarImage | string | `"flomesh/pipy:0.90.3-38"` | Sidecar image for Linux workloads | | osm.sidecarDrivers[1].proxyServerPort | int | `15128` | Remote destination port on which the Discovery Service listens for new connections from Sidecars. | | osm.sidecarDrivers[1].sidecarImage | string | `"envoyproxy/envoy:v1.19.3"` | Sidecar image for Linux workloads | | osm.sidecarDrivers[1].sidecarWindowsImage | string | `"envoyproxy/envoy-windows:latest"` | Sidecar image for Windows workloads | diff --git a/charts/osm/templates/preset-mesh-config.yaml b/charts/osm/templates/preset-mesh-config.yaml index 5d68f25a8..b72c62e85 100644 --- a/charts/osm/templates/preset-mesh-config.yaml +++ b/charts/osm/templates/preset-mesh-config.yaml @@ -25,6 +25,8 @@ data: "interceptionMode": {{.Values.osm.trafficInterceptionMode | mustToJson}}, "enableEgress": {{.Values.osm.enableEgress | mustToJson}}, "enablePermissiveTrafficPolicyMode": {{.Values.osm.enablePermissiveTrafficPolicy | mustToJson}}, + "http1PerRequestLoadBalancing": {{.Values.osm.http1PerRequestLoadBalancing | mustToJson}}, + "http2PerRequestLoadBalancing": {{.Values.osm.http2PerRequestLoadBalancing | mustToJson}}, "outboundPortExclusionList": {{.Values.osm.outboundPortExclusionList | mustToJson}}, "inboundPortExclusionList": {{.Values.osm.inboundPortExclusionList | mustToJson}}, "outboundIPRangeExclusionList": {{.Values.osm.outboundIPRangeExclusionList | mustToJson}}, diff --git a/charts/osm/values.schema.json b/charts/osm/values.schema.json index bc8e483de..4a09e6c1c 100644 --- a/charts/osm/values.schema.json +++ b/charts/osm/values.schema.json @@ -175,6 +175,8 @@ "caBundleSecretName", "enableDebugServer", "enablePermissiveTrafficPolicy", + "http1PerRequestLoadBalancing", + "http2PerRequestLoadBalancing", "trafficInterceptionMode", "enableEgress", "enableReconciler", @@ -786,6 +788,24 @@ false ] }, + "http1PerRequestLoadBalancing": { + "$id": "#/properties/osm/properties/http1PerRequestLoadBalancing", + "type": "boolean", + "title": "The http1PerRequestLoadBalancing schema", + "description": "Indicating if load balancing based on request is enabled for http1.", + "examples": [ + false + ] + }, + "http2PerRequestLoadBalancing": { + "$id": "#/properties/osm/properties/http2PerRequestLoadBalancing", + "type": "boolean", + "title": "The http2PerRequestLoadBalancing schema", + "description": "Indicating if load balancing based on request is enabled for http2.", + "examples": [ + true + ] + }, "trafficInterceptionMode": { "$id": "#/properties/osm/properties/trafficInterceptionMode", "type": "string", diff --git a/charts/osm/values.yaml b/charts/osm/values.yaml index 8c28cb6d3..3f1649e9c 100644 --- a/charts/osm/values.yaml +++ b/charts/osm/values.yaml @@ -12,7 +12,7 @@ osm: # -- Container image pull policy for control plane containers pullPolicy: IfNotPresent # -- Container image tag for control plane images - tag: "1.3.8" + tag: "1.3.9" # -- Image name defaults name: # -- osm-controller's image name @@ -60,7 +60,7 @@ osm: sidecarDrivers: - sidecarName: pipy # -- Sidecar image for Linux workloads - sidecarImage: flomesh/pipy:0.90.2-14 + sidecarImage: flomesh/pipy:0.90.3-38 # -- Remote destination port on which the Discovery Service listens for new connections from Sidecars. proxyServerPort: 6060 - sidecarName: envoy @@ -76,7 +76,7 @@ osm: # -- Pipy RepoServer repoServer: # -- Image used for Pipy RepoServer - image: flomesh/pipy-repo:0.90.2-14 + image: flomesh/pipy-repo:0.90.3-38 # -- if false , Pipy RepoServer is installed within osmController pod. standalone: false # -- ipaddr of host/service where Pipy RepoServer is installed @@ -385,6 +385,12 @@ osm: # -- Enable permissive traffic policy mode enablePermissiveTrafficPolicy: true + # -- Specifies a boolean indicating if load balancing based on request is enabled for http1. + http1PerRequestLoadBalancing: false + + # -- Specifies a boolean indicating if load balancing based on request is enabled for http2. + http2PerRequestLoadBalancing: true + # -- Traffic interception mode in the mesh trafficInterceptionMode: iptables diff --git a/cmd/osm-bootstrap/crds/config_meshconfig.yaml b/cmd/osm-bootstrap/crds/config_meshconfig.yaml index 1f5529e6b..7743bd804 100644 --- a/cmd/osm-bootstrap/crds/config_meshconfig.yaml +++ b/cmd/osm-bootstrap/crds/config_meshconfig.yaml @@ -249,6 +249,12 @@ spec: type: array items: type: string + http1PerRequestLoadBalancing: + description: True for load balancing based on request is enabled for http1. + type: boolean + http2PerRequestLoadBalancing: + description: True for load balancing based on request is enabled for http1. + type: boolean enablePermissiveTrafficPolicyMode: description: True for allowing traffic to flow between client and service pods within the mesh without SMI traffic policies, i.e. no traffic policy enforcement in the mesh. If set to false, enables deny-all traffic policy in mesh i.e. an SMI Traffic Target is necessary for services to communicate. type: boolean @@ -622,6 +628,12 @@ spec: type: integer minimum: 1 maximum: 65535 + http1PerRequestLoadBalancing: + description: True for load balancing based on request is enabled for http1. + type: boolean + http2PerRequestLoadBalancing: + description: True for load balancing based on request is enabled for http1. + type: boolean enablePermissiveTrafficPolicyMode: description: True for allowing traffic to flow between client and service pods within the mesh without SMI traffic policies, i.e. no traffic policy enforcement in the mesh. If set to false, enables deny-all traffic policy in mesh i.e. an SMI Traffic Target is necessary for services to communicate. type: boolean diff --git a/pkg/apis/config/v1alpha1/mesh_config.go b/pkg/apis/config/v1alpha1/mesh_config.go index eb1a48cfd..b988b6e2d 100644 --- a/pkg/apis/config/v1alpha1/mesh_config.go +++ b/pkg/apis/config/v1alpha1/mesh_config.go @@ -108,6 +108,12 @@ type TrafficSpec struct { // InboundExternalAuthorization defines a ruleset that, if enabled, will configure a remote external authorization endpoint // for all inbound and ingress traffic in the mesh. InboundExternalAuthorization ExternalAuthzSpec `json:"inboundExternalAuthorization,omitempty"` + + // HTTP1PerRequestLoadBalancing defines a boolean indicating if load balancing based on request is enabled for http1. + HTTP1PerRequestLoadBalancing bool `json:"http1PerRequestLoadBalancing"` + + // HTTP1PerRequestLoadBalancing defines a boolean indicating if load balancing based on request is enabled for http2. + HTTP2PerRequestLoadBalancing bool `json:"http2PerRequestLoadBalancing"` } // ObservabilitySpec is the type to represent OSM's observability configurations. diff --git a/pkg/apis/config/v1alpha2/mesh_config.go b/pkg/apis/config/v1alpha2/mesh_config.go index 41e5f3fba..c2e89400a 100644 --- a/pkg/apis/config/v1alpha2/mesh_config.go +++ b/pkg/apis/config/v1alpha2/mesh_config.go @@ -157,6 +157,12 @@ type TrafficSpec struct { // names to exclude from inbound and outbound traffic interception by the // sidecar proxy. NetworkInterfaceExclusionList []string `json:"networkInterfaceExclusionList"` + + // HTTP1PerRequestLoadBalancing defines a boolean indicating if load balancing based on request is enabled for http1. + HTTP1PerRequestLoadBalancing bool `json:"http1PerRequestLoadBalancing"` + + // HTTP1PerRequestLoadBalancing defines a boolean indicating if load balancing based on request is enabled for http2. + HTTP2PerRequestLoadBalancing bool `json:"http2PerRequestLoadBalancing"` } // ObservabilitySpec is the type to represent OSM's observability configurations. diff --git a/pkg/messaging/broker.go b/pkg/messaging/broker.go index c46fb2abb..1a6ee2aa0 100644 --- a/pkg/messaging/broker.go +++ b/pkg/messaging/broker.go @@ -341,6 +341,8 @@ func getProxyUpdateEvent(msg events.PubSubMessage) *proxyUpdateEvent { // changes. if prevSpec.Traffic.EnableEgress != newSpec.Traffic.EnableEgress || prevSpec.Traffic.EnablePermissiveTrafficPolicyMode != newSpec.Traffic.EnablePermissiveTrafficPolicyMode || + prevSpec.Traffic.HTTP1PerRequestLoadBalancing != newSpec.Traffic.HTTP1PerRequestLoadBalancing || + prevSpec.Traffic.HTTP2PerRequestLoadBalancing != newSpec.Traffic.HTTP2PerRequestLoadBalancing || prevSpec.Observability.Tracing != newSpec.Observability.Tracing || prevSpec.Observability.RemoteLogging != newSpec.Observability.RemoteLogging || prevSpec.Sidecar.LogLevel != newSpec.Sidecar.LogLevel || diff --git a/pkg/sidecar/providers/pipy/repo/codebase/modules/outbound-http-load-balancing.js b/pkg/sidecar/providers/pipy/repo/codebase/modules/outbound-http-load-balancing.js index 7283c87cc..b305c9b40 100644 --- a/pkg/sidecar/providers/pipy/repo/codebase/modules/outbound-http-load-balancing.js +++ b/pkg/sidecar/providers/pipy/repo/codebase/modules/outbound-http-load-balancing.js @@ -1,5 +1,7 @@ (( config = pipy.solve('config.js'), + http1PerRequestLoadBalancing = Boolean(config?.Spec?.Traffic?.HTTP1PerRequestLoadBalancing), + http2PerRequestLoadBalancing = Boolean(config?.Spec?.Traffic?.HTTP2PerRequestLoadBalancing), certChain = config?.Certificate?.CertChain, privateKey = config?.Certificate?.PrivateKey, isDebugEnabled = config?.Spec?.SidecarLogLevel === 'debug', @@ -86,6 +88,7 @@ _failoverObject: null, _targetObject: null, _muxHttpOptions: null, + _session: null, }) .import({ @@ -103,6 +106,7 @@ .pipeline() .onStart( () => void ( + _session = {}, (_clusterConfig = clusterConfigs.get(__cluster)) && ( _muxHttpOptions = _clusterConfig.muxHttpOptions, _clusterConfig.failoverBalancer && ( @@ -111,10 +115,23 @@ ) ) ) +.onEnd(() => void ( _session = null)) .handleMessageStart( msg => ( _clusterConfig && ( - _targetObject = _clusterConfig.targetBalancer?.next?.({}), + __isHTTP2 ? ( + http2PerRequestLoadBalancing ? ( + _targetObject = _clusterConfig.targetBalancer?.next?.({}) + ) : ( + _targetObject = _clusterConfig.targetBalancer?.next?.() + ) + ) : ( + http1PerRequestLoadBalancing ? ( + _targetObject = _clusterConfig.targetBalancer?.next?.(_session) + ) : ( + _targetObject = _clusterConfig.targetBalancer?.next?.() + ) + ), __target = _targetObject?.id ) && ( ( diff --git a/pkg/sidecar/providers/pipy/repo/codebase/modules/outbound-http-routing.js b/pkg/sidecar/providers/pipy/repo/codebase/modules/outbound-http-routing.js index 193c30c64..1677442b0 100644 --- a/pkg/sidecar/providers/pipy/repo/codebase/modules/outbound-http-routing.js +++ b/pkg/sidecar/providers/pipy/repo/codebase/modules/outbound-http-routing.js @@ -43,9 +43,9 @@ (path, headers) => matchPath(path) && headerRules.every(([k, v]) => v.test(headers[k] || '')) && ( __route = config, __service = service, - __cluster = clusterCache.get(balancer.next({})?.id), + __cluster = clusterCache.get(balancer.next()?.id), failoverBalancer && ( - _failoverCluster = clusterCache.get(failoverBalancer.next({})?.id) + _failoverCluster = clusterCache.get(failoverBalancer.next()?.id) ), true ) @@ -53,9 +53,9 @@ (path) => matchPath(path) && ( __route = config, __service = service, - __cluster = clusterCache.get(balancer.next({})?.id), + __cluster = clusterCache.get(balancer.next()?.id), failoverBalancer && ( - _failoverCluster = clusterCache.get(failoverBalancer.next({})?.id) + _failoverCluster = clusterCache.get(failoverBalancer.next()?.id) ), true ) diff --git a/pkg/sidecar/providers/pipy/repo/jobs.go b/pkg/sidecar/providers/pipy/repo/jobs.go index 993dbfbc5..8d2c7e839 100644 --- a/pkg/sidecar/providers/pipy/repo/jobs.go +++ b/pkg/sidecar/providers/pipy/repo/jobs.go @@ -287,6 +287,8 @@ func features(s *Server, proxy *pipy.Proxy, pipyConf *PipyConf) { pipyConf.setRemoteLoggingLevel((*meshConf).GetMeshConfig().Spec.Observability.RemoteLogging.Level) pipyConf.setEnableSidecarActiveHealthChecks((*meshConf).GetFeatureFlags().EnableSidecarActiveHealthChecks) pipyConf.setEnableEgress((*meshConf).IsEgressEnabled()) + pipyConf.setHTTP1PerRequestLoadBalancing((*meshConf).GetMeshConfig().Spec.Traffic.HTTP1PerRequestLoadBalancing) + pipyConf.setHTTP2PerRequestLoadBalancing((*meshConf).GetMeshConfig().Spec.Traffic.HTTP2PerRequestLoadBalancing) pipyConf.setEnablePermissiveTrafficPolicyMode((*meshConf).IsPermissiveTrafficPolicyMode()) pipyConf.setLocalDNSProxy((*meshConf).IsLocalDNSProxyEnabled(), (*meshConf).GetLocalDNSProxyPrimaryUpstream(), (*meshConf).GetLocalDNSProxySecondaryUpstream()) clusterProps := (*meshConf).GetMeshConfig().Spec.ClusterSet.Properties diff --git a/pkg/sidecar/providers/pipy/repo/policy.go b/pkg/sidecar/providers/pipy/repo/policy.go index 9678c5eb2..56a74dbb8 100644 --- a/pkg/sidecar/providers/pipy/repo/policy.go +++ b/pkg/sidecar/providers/pipy/repo/policy.go @@ -78,6 +78,20 @@ func (p *PipyConf) setEnableEgress(enableEgress bool) (update bool) { return } +func (p *PipyConf) setHTTP1PerRequestLoadBalancing(http1PerRequestLoadBalancing bool) (update bool) { + if update = p.Spec.Traffic.HTTP1PerRequestLoadBalancing != http1PerRequestLoadBalancing; update { + p.Spec.Traffic.HTTP1PerRequestLoadBalancing = http1PerRequestLoadBalancing + } + return +} + +func (p *PipyConf) setHTTP2PerRequestLoadBalancing(http2PerRequestLoadBalancing bool) (update bool) { + if update = p.Spec.Traffic.HTTP2PerRequestLoadBalancing != http2PerRequestLoadBalancing; update { + p.Spec.Traffic.HTTP2PerRequestLoadBalancing = http2PerRequestLoadBalancing + } + return +} + func (p *PipyConf) setEnablePermissiveTrafficPolicyMode(enablePermissiveTrafficPolicyMode bool) (update bool) { if update = p.Spec.Traffic.enablePermissiveTrafficPolicyMode != enablePermissiveTrafficPolicyMode; update { p.Spec.Traffic.enablePermissiveTrafficPolicyMode = enablePermissiveTrafficPolicyMode diff --git a/pkg/sidecar/providers/pipy/repo/types.go b/pkg/sidecar/providers/pipy/repo/types.go index fd623c2b0..e9fb9965e 100644 --- a/pkg/sidecar/providers/pipy/repo/types.go +++ b/pkg/sidecar/providers/pipy/repo/types.go @@ -191,6 +191,8 @@ type FeatureFlags struct { type TrafficSpec struct { EnableEgress bool enablePermissiveTrafficPolicyMode bool + HTTP1PerRequestLoadBalancing bool + HTTP2PerRequestLoadBalancing bool } // UpstreamDNSServers defines upstream DNS servers for local DNS Proxy.