diff --git a/go.mod b/go.mod index 3d607a621..151b7feca 100644 --- a/go.mod +++ b/go.mod @@ -161,6 +161,7 @@ require ( ) replace ( + github.com/kubewharf/katalyst-api => github.com/yadzhang/katalyst-api v0.0.0-20240902073003-a72eb9ae209c k8s.io/api => k8s.io/api v0.24.6 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.24.6 k8s.io/apimachinery => k8s.io/apimachinery v0.24.6 diff --git a/go.sum b/go.sum index 637b0ff7a..3cb429382 100644 --- a/go.sum +++ b/go.sum @@ -568,8 +568,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubewharf/katalyst-api v0.5.1-0.20240702044746-be552fd7ea7d h1:6CuK3axf2B63zIkEu5XyxbaC+JArE/3Jo3QHvb+Hn0M= -github.com/kubewharf/katalyst-api v0.5.1-0.20240702044746-be552fd7ea7d/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k= github.com/kubewharf/kubelet v1.24.6-kubewharf.9 h1:jOTYZt7h/J7I8xQMKMUcJjKf5UFBv37jHWvNp5VRFGc= github.com/kubewharf/kubelet v1.24.6-kubewharf.9/go.mod h1:MxbSZUx3wXztFneeelwWWlX7NAAStJ6expqq7gY2J3c= github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8= @@ -913,6 +911,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yadzhang/katalyst-api v0.0.0-20240902073003-a72eb9ae209c h1:AaS5COARc7Hl1Flu209pTushx87DWTHLM4rY15gCCEc= +github.com/yadzhang/katalyst-api v0.0.0-20240902073003-a72eb9ae209c/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/advisor_test.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/advisor_test.go index 72db43452..e8a63407a 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/advisor_test.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/advisor_test.go @@ -816,6 +816,11 @@ func TestUpdate(t *testing.T) { 0: machine.MustParse("1"), 1: machine.MustParse("25"), }, 200<<30), + makeContainerInfo("uid4", "default", "pod4", "c4", consts.PodAnnotationQoSLevelReclaimedCores, nil, + map[int]machine.CPUSet{ + 0: machine.MustParse("1"), + 1: machine.MustParse("25"), + }, 200<<30), }, pods: []*v1.Pod{ { @@ -874,6 +879,62 @@ func TestUpdate(t *testing.T) { }, }, }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod3", + Namespace: "default", + UID: "uid3", + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "c3", + }, + }, + }, + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "c3", + ContainerID: "containerd://c3", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod4", + Namespace: "default", + UID: "uid4", + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "c4", + }, + }, + }, + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "c4", + ContainerID: "containerd://c4", + }, + }, + }, + }, }, wantHeadroom: *resource.NewQuantity(996<<30, resource.DecimalSI), nodeMetrics: defaultNodeMetrics, @@ -891,6 +952,18 @@ func TestUpdate(t *testing.T) { podUID: "uid1", containerName: "c1", }, + { + metricName: coreconsts.MetricMemCacheContainer, + metricValue: metricutil.MetricData{Value: 5 << 30}, + podUID: "uid1", + containerName: "c1", + }, + { + metricName: coreconsts.MetricMemMappedContainer, + metricValue: metricutil.MetricData{Value: 2 << 30}, + podUID: "uid1", + containerName: "c1", + }, { metricName: coreconsts.MetricMemInactiveAnonContainer, metricValue: metricutil.MetricData{Value: 1 << 30}, @@ -939,9 +1012,21 @@ func TestUpdate(t *testing.T) { podUID: "uid2", containerName: "c2", }, + { + metricName: coreconsts.MetricMemCacheContainer, + metricValue: metricutil.MetricData{Value: 3 << 30}, + podUID: "uid2", + containerName: "c2", + }, + { + metricName: coreconsts.MetricMemMappedContainer, + metricValue: metricutil.MetricData{Value: 2 << 30}, + podUID: "uid2", + containerName: "c2", + }, { metricName: coreconsts.MetricMemInactiveAnonContainer, - metricValue: metricutil.MetricData{Value: 1 << 30}, + metricValue: metricutil.MetricData{Value: 2 << 30}, podUID: "uid2", containerName: "c2", }, @@ -975,6 +1060,126 @@ func TestUpdate(t *testing.T) { podUID: "uid2", containerName: "c2", }, + { + metricName: coreconsts.MetricMemPsiAvg60Container, + metricValue: metricutil.MetricData{Value: 0.01}, + podUID: "uid3", + containerName: "c3", + }, + { + metricName: coreconsts.MetricMemUsageContainer, + metricValue: metricutil.MetricData{Value: 10 << 30}, + podUID: "uid3", + containerName: "c3", + }, + { + metricName: coreconsts.MetricMemCacheContainer, + metricValue: metricutil.MetricData{Value: 3 << 30}, + podUID: "uid3", + containerName: "c3", + }, + { + metricName: coreconsts.MetricMemMappedContainer, + metricValue: metricutil.MetricData{Value: 2 << 30}, + podUID: "uid3", + containerName: "c3", + }, + { + metricName: coreconsts.MetricMemInactiveAnonContainer, + metricValue: metricutil.MetricData{Value: 2 << 30}, + podUID: "uid3", + containerName: "c3", + }, + { + metricName: coreconsts.MetricMemInactiveFileContainer, + metricValue: metricutil.MetricData{Value: 1 << 30}, + podUID: "uid3", + containerName: "c3", + }, + { + metricName: coreconsts.MetricMemPgscanContainer, + metricValue: metricutil.MetricData{Value: 15000}, + podUID: "uid3", + containerName: "c3", + }, + { + metricName: coreconsts.MetricMemPgstealContainer, + metricValue: metricutil.MetricData{Value: 10000}, + podUID: "uid3", + containerName: "c3", + }, + { + metricName: coreconsts.MetricMemWorkingsetRefaultContainer, + metricValue: metricutil.MetricData{Value: 1000}, + podUID: "uid3", + containerName: "c3", + }, + { + metricName: coreconsts.MetricMemWorkingsetActivateContainer, + metricValue: metricutil.MetricData{Value: 1000}, + podUID: "uid3", + containerName: "c3", + }, + { + metricName: coreconsts.MetricMemPsiAvg60Container, + metricValue: metricutil.MetricData{Value: 0.01}, + podUID: "uid4", + containerName: "c4", + }, + { + metricName: coreconsts.MetricMemUsageContainer, + metricValue: metricutil.MetricData{Value: 10 << 30}, + podUID: "uid4", + containerName: "c4", + }, + { + metricName: coreconsts.MetricMemCacheContainer, + metricValue: metricutil.MetricData{Value: 2 << 30}, + podUID: "uid4", + containerName: "c4", + }, + { + metricName: coreconsts.MetricMemMappedContainer, + metricValue: metricutil.MetricData{Value: 2 << 30}, + podUID: "uid4", + containerName: "c4", + }, + { + metricName: coreconsts.MetricMemInactiveAnonContainer, + metricValue: metricutil.MetricData{Value: 1 << 30}, + podUID: "uid4", + containerName: "c4", + }, + { + metricName: coreconsts.MetricMemInactiveFileContainer, + metricValue: metricutil.MetricData{Value: 1 << 30}, + podUID: "uid4", + containerName: "c4", + }, + { + metricName: coreconsts.MetricMemPgscanContainer, + metricValue: metricutil.MetricData{Value: 15000}, + podUID: "uid4", + containerName: "c4", + }, + { + metricName: coreconsts.MetricMemPgstealContainer, + metricValue: metricutil.MetricData{Value: 10000}, + podUID: "uid4", + containerName: "c4", + }, + { + metricName: coreconsts.MetricMemWorkingsetRefaultContainer, + metricValue: metricutil.MetricData{Value: 1000}, + podUID: "uid4", + containerName: "c4", + }, + { + metricName: coreconsts.MetricMemWorkingsetActivateContainer, + metricValue: metricutil.MetricData{Value: 1000}, + podUID: "uid4", + containerName: "c4", + }, }, cgroupMetrics: []cgroupMetric{ { @@ -1007,6 +1212,16 @@ func TestUpdate(t *testing.T) { metricValue: metricutil.MetricData{Value: 4 << 30}, cgroupPath: "/hdfs", }, + { + metricName: coreconsts.MetricMemCacheCgroup, + metricValue: metricutil.MetricData{Value: 3 << 30}, + cgroupPath: "/hdfs", + }, + { + metricName: coreconsts.MetricMemMappedCgroup, + metricValue: metricutil.MetricData{Value: 1 << 30}, + cgroupPath: "/hdfs", + }, { metricName: coreconsts.MetricMemInactiveAnonCgroup, metricValue: metricutil.MetricData{Value: 1 << 30}, @@ -1024,19 +1239,37 @@ func TestUpdate(t *testing.T) { { CgroupPath: "/hdfs", Values: map[string]string{ - string(memoryadvisor.ControlKnobKeySwapMax): coreconsts.ControlKnobON, + string(memoryadvisor.ControlKnobKeySwapMax): coreconsts.ControlKnobOFF, string(memoryadvisor.ControlKnowKeyMemoryOffloading): "38654705", }, }, }, - ContainerEntries: []types.ContainerMemoryAdvices{{ - PodUID: "uid1", - ContainerName: "c1", - Values: map[string]string{ - string(memoryadvisor.ControlKnobKeySwapMax): coreconsts.ControlKnobON, - string(memoryadvisor.ControlKnowKeyMemoryOffloading): "96636764", + ContainerEntries: []types.ContainerMemoryAdvices{ + { + PodUID: "uid1", + ContainerName: "c1", + Values: map[string]string{ + string(memoryadvisor.ControlKnobKeySwapMax): coreconsts.ControlKnobOFF, + string(memoryadvisor.ControlKnowKeyMemoryOffloading): "96636764", + }, }, - }}, + { + PodUID: "uid3", + ContainerName: "c3", + Values: map[string]string{ + string(memoryadvisor.ControlKnobKeySwapMax): coreconsts.ControlKnobON, + string(memoryadvisor.ControlKnowKeyMemoryOffloading): "96636764", + }, + }, + //{ + // PodUID: "uid4", + // ContainerName: "c4", + // Values: map[string]string{ + // string(memoryadvisor.ControlKnobKeySwapMax): coreconsts.ControlKnobOFF, + // string(memoryadvisor.ControlKnowKeyMemoryOffloading): "96636764", + // }, + //}, + }, }, }, { @@ -2226,12 +2459,16 @@ func TestUpdate(t *testing.T) { transparentMemoryOffloadingConfiguration := tmo.NewTransparentMemoryOffloadingConfiguration() transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelReclaimedCores] = tmo.NewTMOConfigDetail(transparentMemoryOffloadingConfiguration.DefaultConfigurations) transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelReclaimedCores].EnableTMO = true - transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelReclaimedCores].EnableSwap = true + transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelReclaimedCores].EnableSwap = false + + transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelSharedCores] = tmo.NewTMOConfigDetail(transparentMemoryOffloadingConfiguration.DefaultConfigurations) + transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelSharedCores].EnableTMO = true + transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelSharedCores].EnableSwap = true // cgroup level transparentMemoryOffloadingConfiguration.CgroupConfigs["/sys/fs/cgroup/hdfs"] = tmo.NewTMOConfigDetail(transparentMemoryOffloadingConfiguration.DefaultConfigurations) transparentMemoryOffloadingConfiguration.CgroupConfigs["/sys/fs/cgroup/hdfs"].EnableTMO = true - transparentMemoryOffloadingConfiguration.CgroupConfigs["/sys/fs/cgroup/hdfs"].EnableSwap = true + transparentMemoryOffloadingConfiguration.CgroupConfigs["/sys/fs/cgroup/hdfs"].EnableSwap = false advisor.conf.GetDynamicConfiguration().TransparentMemoryOffloadingConfiguration = transparentMemoryOffloadingConfiguration _, advisorRecvChInterface := advisor.GetChannels() @@ -2256,7 +2493,7 @@ func TestUpdate(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) advisor.Run(ctx) - time.Sleep(10 * time.Millisecond) // Wait some time because no signal will be sent to channel + time.Sleep(100 * time.Millisecond) // Wait some time because no signal will be sent to channel if tt.needRecvAdvices { result := <-recvCh diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/plugin/memory_offloading.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/plugin/memory_offloading.go index e046d83d3..920247c1d 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/plugin/memory_offloading.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/plugin/memory_offloading.go @@ -47,15 +47,19 @@ import ( const ( TransparentMemoryOffloading = "transparent-memory-offloading" + + MetricMemoryOffloading = "memory_offloading" ) const ( InactiveProbe = 0.1 OffloadingSizeScaleCoeff = 1.05 + CacheMappedCoeff = 2 ) const ( - DummyTMOBlockFnName string = "dummy-tmo-block-fn" + DummyTMOBlockFnName string = "dummy-tmo-block-fn" + TMOBlockFuncFromDynamicConfig = "tmo-block-func-from-dynamic-config" ) // TMO policy funcs to calculate memory offloading size @@ -73,22 +77,34 @@ type TmoStats struct { pgsteal float64 refault float64 refaultActivate float64 + cache float64 + mapped float64 offloadingTargetSize float64 } type TmoPolicyFn func( lastStats TmoStats, currStats TmoStats, - conf *tmoconf.TMOConfigDetail) (error, float64) + conf *tmoconf.TMOConfigDetail, + emitter metrics.MetricEmitter) (error, float64) + +func psiPolicyFunc(lastStats TmoStats, currStats TmoStats, conf *tmoconf.TMOConfigDetail, emitter metrics.MetricEmitter) (error, float64) { -func psiPolicyFunc(lastStats TmoStats, currStats TmoStats, conf *tmoconf.TMOConfigDetail) (error, float64) { if conf.PSIPolicyConf == nil { return errors.New("psi policy requires psi policy configuration"), 0 } - return nil, math.Max(0, 1-(currStats.memPsiAvg60)/(conf.PSIPolicyConf.PsiAvg60Threshold)) * conf.PSIPolicyConf.MaxProbe * currStats.memUsage + result := math.Max(0, 1-(currStats.memPsiAvg60)/(conf.PSIPolicyConf.PsiAvg60Threshold)) * conf.PSIPolicyConf.MaxProbe * currStats.memUsage + + general.InfoS("psi info", "obj", currStats.obj, "memPsiAvg60", currStats.memPsiAvg60, "psiAvg60Threshold", + conf.PSIPolicyConf.PsiAvg60Threshold, "maxProbe", conf.PSIPolicyConf.MaxProbe, "memUsage", currStats.memUsage, + "result", result) + _ = emitter.StoreFloat64(MetricMemoryOffloading, result, metrics.MetricTypeNameRaw, + metrics.MetricTag{Key: "policy", Val: string(v1alpha1.TMOPolicyNamePSI)}, + metrics.MetricTag{Key: "obj", Val: currStats.obj}) + return nil, result } -func refaultPolicyFunc(lastStats TmoStats, currStats TmoStats, conf *tmoconf.TMOConfigDetail) (error, float64) { +func refaultPolicyFunc(lastStats TmoStats, currStats TmoStats, conf *tmoconf.TMOConfigDetail, emitter metrics.MetricEmitter) (error, float64) { if conf.RefaultPolicyConf == nil { return errors.New("refault policy requires refault policy configurations"), 0 } @@ -115,16 +131,39 @@ func refaultPolicyFunc(lastStats TmoStats, currStats TmoStats, conf *tmoconf.TMO "reclaimScanEfficiencyRatio", reclaimScanEfficiencyRatio, "ReclaimScanEfficiencyTarget", conf.RefaultPolicyConf.ReclaimScanEfficiencyTarget, "refaultDelta", refaultDelta, "pgstealDelta", pgstealDelta, "pgscanDelta", pgscanDelta, "lastOffloadingTargetSize", general.FormatMemoryQuantity(lastStats.offloadingTargetSize), "result", general.FormatMemoryQuantity(result)) + _ = emitter.StoreFloat64(MetricMemoryOffloading, result, metrics.MetricTypeNameRaw, + metrics.MetricTag{Key: "policy", Val: string(v1alpha1.TMOPolicyNameRefault)}, + metrics.MetricTag{Key: "obj", Val: currStats.obj}) return nil, result } -type TMOBlockFn func(ci *types.ContainerInfo, conf interface{}) bool +func integratedPolicyFunc(lastStats TmoStats, currStats TmoStats, conf *tmoconf.TMOConfigDetail, emitter metrics.MetricEmitter) (error, float64) { + err, targetSizeFromPSI := psiPolicyFunc(lastStats, currStats, conf, emitter) + if err != nil { + return err, 0 + } + err, targetSizeFromRefault := refaultPolicyFunc(lastStats, currStats, conf, emitter) + if err != nil { + return err, 0 + } + result := math.Min(targetSizeFromPSI, targetSizeFromRefault) + + _ = emitter.StoreFloat64(MetricMemoryOffloading, result, metrics.MetricTypeNameRaw, + metrics.MetricTag{Key: "policy", Val: string(v1alpha1.TMOPolicyNameIntegrated)}, + metrics.MetricTag{Key: "obj", Val: currStats.obj}) + return err, result +} + +type TMOBlockFn func(ci *types.ContainerInfo, conf interface{}, dynamicConf interface{}) bool -func DummyTMOBlockFn(ci *types.ContainerInfo, conf interface{}) bool { return false } +func DummyTMOBlockFn(ci *types.ContainerInfo, conf interface{}, dynamicConf interface{}) bool { + return false +} func init() { RegisterTMOPolicyFunc(v1alpha1.TMOPolicyNamePSI, psiPolicyFunc) RegisterTMOPolicyFunc(v1alpha1.TMOPolicyNameRefault, refaultPolicyFunc) + RegisterTMOPolicyFunc(v1alpha1.TMOPolicyNameIntegrated, integratedPolicyFunc) RegisterTMOBlockFunc(DummyTMOBlockFnName, DummyTMOBlockFn) } @@ -136,6 +175,28 @@ func RegisterTMOBlockFunc(blockFnName string, blockFn TMOBlockFn) { tmoBlockFuncs.Store(blockFnName, blockFn) } +func TMOBlockFnFromDynamicConfig(ci *types.ContainerInfo, extraConf interface{}, dynamicConf interface{}) bool { + if conf, ok := dynamicConf.(*tmoconf.TMOBlockConfig); ok { + if checkBlocked(conf.BlockLabels, ci.Labels) || checkBlocked(conf.BlockAnnotations, ci.Annotations) { + return true + } + } + return false +} + +func checkBlocked(blockMap map[string][]string, originalMap map[string]string) bool { + for key, originalVal := range originalMap { + if blockValues, ok := blockMap[key]; ok { + for _, blockVal := range blockValues { + if originalVal == blockVal { + return true + } + } + } + } + return false +} + type transparentMemoryOffloading struct { conf *config.Configuration extraConf interface{} @@ -231,6 +292,14 @@ func (tmoEngine *tmoEngineInstance) getStats() (TmoStats, error) { if err != nil { return err } + memCache, err := metaserver.GetCgroupMetric(relativePath, consts.MetricMemCacheCgroup) + if err != nil { + return err + } + memMappedFile, err := metaserver.GetCgroupMetric(relativePath, consts.MetricMemMappedCgroup) + if err != nil { + return err + } tmoStats.memUsage = memUsage.Value tmoStats.memInactive = memInactiveFile.Value + memInactiveAnon.Value tmoStats.memPsiAvg60 = psiAvg60.Value @@ -238,8 +307,10 @@ func (tmoEngine *tmoEngineInstance) getStats() (TmoStats, error) { tmoStats.pgscan = pgscan.Value tmoStats.refault = refault.Value tmoStats.refaultActivate = refaultActivate.Value + tmoStats.cache = memCache.Value + tmoStats.mapped = memMappedFile.Value tmoStats.offloadingTargetSize = tmoEngine.offloadingTargetSize - general.Infof("Memory Usage of Cgroup %s, memUsage: %v", tmoEngine.cgpath, memUsage.Value) + general.Infof("Memory Usage of Cgroup %s, memUsage: %v, cache: %v, mapped: %v", tmoEngine.cgpath, memUsage.Value, memCache.Value, memMappedFile.Value) return nil } getContainerMetrics := func(metaserver *metaserver.MetaServer, podUID string, containerName string) error { @@ -275,6 +346,14 @@ func (tmoEngine *tmoEngineInstance) getStats() (TmoStats, error) { if err != nil { return err } + memCache, err := metaserver.GetContainerMetric(podUID, containerName, consts.MetricMemCacheContainer) + if err != nil { + return err + } + memMappedFile, err := metaserver.GetContainerMetric(podUID, containerName, consts.MetricMemMappedContainer) + if err != nil { + return err + } tmoStats.memUsage = memUsage.Value tmoStats.memInactive = memInactiveFile.Value + memInactiveAnon.Value tmoStats.memPsiAvg60 = psiAvg60.Value @@ -282,8 +361,10 @@ func (tmoEngine *tmoEngineInstance) getStats() (TmoStats, error) { tmoStats.pgscan = pgscan.Value tmoStats.refault = refault.Value tmoStats.refaultActivate = refaultActivate.Value + tmoStats.cache = memCache.Value + tmoStats.mapped = memMappedFile.Value tmoStats.offloadingTargetSize = tmoEngine.offloadingTargetSize - general.Infof("Memory Usage of Pod %v, Container %v, memUsage: %v", podUID, containerName, memUsage.Value) + general.Infof("Memory Usage of Pod %v, Container %v, memUsage: %v, cache: %v, mapped: %v", podUID, containerName, memUsage.Value, memCache.Value, memMappedFile.Value) return nil } @@ -330,7 +411,6 @@ func (tmoEngine *tmoEngineInstance) CalculateOffloadingTargetSize() { if !tmoEngine.conf.EnableTMO { return } - currTime := time.Now() if currTime.Sub(tmoEngine.lastTime) < tmoEngine.conf.Interval { tmoEngine.offloadingTargetSize = 0 @@ -342,14 +422,24 @@ func (tmoEngine *tmoEngineInstance) CalculateOffloadingTargetSize() { general.Infof("Failed to get metrics %v", err) return } + // TODO: get result from qrm to make sure last offloading action finished if fn, ok := tmoPolicyFuncs.Load(tmoEngine.conf.PolicyName); ok { if policyFunc, ok := fn.(TmoPolicyFn); ok { - err, targetSize := policyFunc(tmoEngine.lastStats, currStats, tmoEngine.conf) + if !tmoEngine.conf.EnableSwap && currStats.cache < CacheMappedCoeff*currStats.mapped { + general.Infof("Tmo obj: %s cache is close to mapped, skip reclaim", currStats.obj) + tmoEngine.offloadingTargetSize = 0 + return + } + err, targetSize := policyFunc(tmoEngine.lastStats, currStats, tmoEngine.conf, tmoEngine.emitter) if err != nil { general.ErrorS(err, "Failed to calculate offloading memory size") return } + + cacheExceptMapped := currStats.cache - currStats.mapped + general.InfoS("Handle targetSize from policy", "Tmo obj:", currStats.obj, "targetSize:", targetSize, "cacheExceptMapped", cacheExceptMapped) + targetSize = math.Max(0, math.Min(cacheExceptMapped, targetSize)) tmoEngine.offloadingTargetSize = targetSize currStats.offloadingTargetSize = targetSize tmoEngine.lastStats = currStats @@ -371,10 +461,14 @@ func NewTransparentMemoryOffloading(conf *config.Configuration, extraConfig inte } func (tmo *transparentMemoryOffloading) Reconcile(status *types.MemoryPressureStatus) error { + if tmo.conf.GetDynamicConfiguration().BlockConfig != nil { + RegisterTMOBlockFunc(TMOBlockFuncFromDynamicConfig, TMOBlockFnFromDynamicConfig) + } podContainerNamesMap := make(map[katalystcoreconsts.PodContainerName]bool) podList, err := tmo.metaServer.GetPodList(context.Background(), native.PodIsActive) if err != nil { general.Infof("Failed to get pod list: %v", err) + return err } for _, pod := range podList { @@ -449,7 +543,7 @@ func (tmo *transparentMemoryOffloading) Reconcile(status *types.MemoryPressureSt return true }) for tmoBlockFnName, tmoBlockFn := range funcs { - if tmoBlockFn(containerInfo, tmo.extraConf) { + if tmoBlockFn(containerInfo, tmo.extraConf, tmo.conf.GetDynamicConfiguration().BlockConfig) { tmo.containerTmoEngines[podContainerName].GetConf().EnableTMO = false general.Infof("container with podContainerName: %s is required to disable TMO by TMOBlockFn: %s", podContainerName, tmoBlockFnName) } diff --git a/pkg/config/agent/dynamic/tmo/tmo_base.go b/pkg/config/agent/dynamic/tmo/tmo_base.go index 2246bddc2..98ad1f224 100644 --- a/pkg/config/agent/dynamic/tmo/tmo_base.go +++ b/pkg/config/agent/dynamic/tmo/tmo_base.go @@ -39,6 +39,7 @@ type TransparentMemoryOffloadingConfiguration struct { DefaultConfigurations *TMODefaultConfigurations QoSLevelConfigs map[consts.QoSLevel]*TMOConfigDetail CgroupConfigs map[string]*TMOConfigDetail + BlockConfig *TMOBlockConfig } func NewTransparentMemoryOffloadingConfiguration() *TransparentMemoryOffloadingConfiguration { @@ -46,6 +47,7 @@ func NewTransparentMemoryOffloadingConfiguration() *TransparentMemoryOffloadingC DefaultConfigurations: NewTMODefaultConfigurations(), QoSLevelConfigs: map[consts.QoSLevel]*TMOConfigDetail{}, CgroupConfigs: map[string]*TMOConfigDetail{}, + BlockConfig: &TMOBlockConfig{}, } } @@ -100,6 +102,11 @@ func NewTMOConfigDetail(defaultConfigs *TMODefaultConfigurations) *TMOConfigDeta } } +type TMOBlockConfig struct { + BlockLabels map[string][]string + BlockAnnotations map[string][]string +} + type PSIPolicyConf struct { MaxProbe float64 PsiAvg60Threshold float64 @@ -162,5 +169,9 @@ func (c *TransparentMemoryOffloadingConfiguration) ApplyConfiguration(conf *crd. c.CgroupConfigs[cgroupConfig.CgroupPath] = tmoConfigDetail } } + if tmoConf.Spec.Config.BlockConfig != nil { + c.BlockConfig.BlockLabels = tmoConf.Spec.Config.BlockConfig.Labels + c.BlockConfig.BlockAnnotations = tmoConf.Spec.Config.BlockConfig.Annotations + } } } diff --git a/pkg/consts/metric.go b/pkg/consts/metric.go index ab3fc2f02..a9a569585 100644 --- a/pkg/consts/metric.go +++ b/pkg/consts/metric.go @@ -224,6 +224,7 @@ const ( MetricMemPsiAvg60Container = "mem.psiavg60.container" MetricMemInactiveAnonContainer = "mem.inactiveanon.container" MetricMemInactiveFileContainer = "mem.inactivefile.container" + MetricMemMappedContainer = "mem.mapped.container" ) // container blkio metrics @@ -344,6 +345,7 @@ const ( MetricMemPsiAvg60Cgroup = "mem.psiavg60.cgroup" MetricMemInactiveAnonCgroup = "mem.inactiveanon.cgroup" MetricMemInactiveFileCgroup = "mem.inactivefile.cgroup" + MetricMemMappedCgroup = "mem.mapped.cgroup" ) // Cgroup blkio metrics diff --git a/pkg/metaserver/agent/metric/provisioner/malachite/provisioner.go b/pkg/metaserver/agent/metric/provisioner/malachite/provisioner.go index 00d847fdb..e9be37dda 100644 --- a/pkg/metaserver/agent/metric/provisioner/malachite/provisioner.go +++ b/pkg/metaserver/agent/metric/provisioner/malachite/provisioner.go @@ -587,6 +587,7 @@ func (m *MalachiteMetricsProvisioner) processCgroupMemoryData(cgroupPath string, m.metricStore.SetCgroupMetric(cgroupPath, consts.MetricMemPsiAvg60Cgroup, utilmetric.MetricData{Value: float64(mem.BpfMemStat.MemReclaimSettingSum), Time: &updateTime}) m.metricStore.SetCgroupMetric(cgroupPath, consts.MetricMemInactiveAnonCgroup, utilmetric.MetricData{Value: float64(mem.MemStats.InactiveAnon), Time: &updateTime}) m.metricStore.SetCgroupMetric(cgroupPath, consts.MetricMemInactiveFileCgroup, utilmetric.MetricData{Value: float64(mem.MemStats.InactiveFile), Time: &updateTime}) + m.metricStore.SetCgroupMetric(cgroupPath, consts.MetricMemMappedCgroup, utilmetric.MetricData{Value: float64(mem.MemStats.FileMapped), Time: &updateTime}) } } @@ -896,6 +897,8 @@ func (m *MalachiteMetricsProvisioner) processContainerMemoryData(podUID, contain m.metricStore.SetContainerMetric(podUID, containerName, consts.MetricMemPgmajfaultContainer, utilmetric.MetricData{Value: float64(mem.MemStats.Pgmajfault), Time: &updateTime}) + m.metricStore.SetContainerMetric(podUID, containerName, consts.MetricMemMappedCgroup, + utilmetric.MetricData{Value: float64(mem.MemStats.FileDirty), Time: &updateTime}) m.metricStore.SetContainerMetric(podUID, containerName, consts.MetricMemDirtyContainer, utilmetric.MetricData{Value: float64(mem.MemStats.FileDirty), Time: &updateTime}) m.metricStore.SetContainerMetric(podUID, containerName, consts.MetricMemOomContainer, @@ -920,6 +923,8 @@ func (m *MalachiteMetricsProvisioner) processContainerMemoryData(podUID, contain utilmetric.MetricData{Value: float64(mem.MemStats.InactiveAnon), Time: &updateTime}) m.metricStore.SetContainerMetric(podUID, containerName, consts.MetricMemInactiveFileContainer, utilmetric.MetricData{Value: float64(mem.MemStats.InactiveFile), Time: &updateTime}) + m.metricStore.SetContainerMetric(podUID, containerName, consts.MetricMemMappedContainer, + utilmetric.MetricData{Value: float64(mem.MemStats.FileMapped), Time: &updateTime}) } }