Skip to content

Commit

Permalink
YARN-11692. Support mixed cgroup v1/v2 controller structure (#6821)
Browse files Browse the repository at this point in the history
  • Loading branch information
@p-szucs
p-szucs authored May 15, 2024
1 parent cfdf1f5 commit 129d91f
Show file tree
Hide file tree
Showing 7 changed files with 237 additions and 117 deletions.
4 changes: 4 additions & 0 deletions ...oop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2733,6 +2733,10 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH =
NM_PREFIX + "linux-container-executor.cgroups.mount-path";

/** Where the linux container executor should mount cgroups v2 if not found. */
public static final String NM_LINUX_CONTAINER_CGROUPS_V2_MOUNT_PATH =
NM_PREFIX + "linux-container-executor.cgroups.v2.mount-path";

/**
* Whether the apps should run in strict resource usage mode(not allowed to
* use spare CPU)
Expand Down
14 changes: 14 additions & 0 deletions hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2087,6 +2087,20 @@
<name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
</property>

<property>
<description>This property sets the mount path for CGroups v2.
This parameter is optional, and needed to be set only in mixed mode,
when CGroups v2 is mounted alongside with Cgroups v1.
For example, in hybrid mode, CGroups v1 controllers can be mounted under /sys/fs/cgroup/
(for example /sys/fs/cgroup/cpu,cpuacct), while v2 can be mounted in /sys/fs/cgroup/unified folder.

If this value is not set, the value of
yarn.nodemanager.linux-container-executor.cgroups.mount-path
will be used for CGroups v2 as well.
</description>
<name>yarn.nodemanager.linux-container-executor.cgroups.v2.mount-path</name>
</property>

<property>
<description>Delay in ms between attempts to remove linux cgroup</description>
<name>yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms</name>
Expand Down
13 changes: 12 additions & 1 deletion ...e/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMountConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,17 @@ public class CGroupsMountConfig {
private final boolean enableMount;
private final String mountPath;

// CGroups v2 mount path is only relevant in mixed CGroups v1/v2 mode,
// where v2 is mounted alongside with v1.
private final String v2MountPath;

public CGroupsMountConfig(Configuration conf) {
this.enableMount = conf.getBoolean(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_MOUNT, false);
this.mountPath = conf.get(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, null);
this.v2MountPath = conf.get(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_V2_MOUNT_PATH, mountPath);
}

public boolean ensureMountPathIsDefined() throws ResourceHandlerException {
Expand Down Expand Up @@ -62,11 +68,16 @@ public String getMountPath() {
return mountPath;
}

public String getV2MountPath() {
return v2MountPath;
}

@Override
public String toString() {
return "CGroupsMountConfig{" +
"enableMount=" + enableMount +
", mountPath='" + mountPath + '\'' +
", mountPath='" + mountPath +
", v2MountPath='" + v2MountPath + '\'' +
'}';
}
}
4 changes: 2 additions & 2 deletions ...hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsV2HandlerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@ protected List<CGroupController> getCGroupControllers() {
@Override
protected Map<String, Set<String>> parsePreConfiguredMountPath() throws IOException {
Map<String, Set<String>> controllerMappings = new HashMap<>();
controllerMappings.put(this.cGroupsMountConfig.getMountPath(),
readControllersFile(this.cGroupsMountConfig.getMountPath()));
controllerMappings.put(this.cGroupsMountConfig.getV2MountPath(),
readControllersFile(this.cGroupsMountConfig.getV2MountPath()));
return controllerMappings;
}

Expand Down
102 changes: 60 additions & 42 deletions ...adoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,35 +63,55 @@ public class ResourceHandlerModule {
* as resource metrics functionality. We need to ensure that the same
* instance is used for both.
*/
private static volatile CGroupsHandler cGroupV1Handler;
private static volatile CGroupsHandler cGroupV2Handler;
private static volatile TrafficControlBandwidthHandlerImpl
trafficControlBandwidthHandler;
private static volatile NetworkPacketTaggingHandlerImpl
networkPacketTaggingHandlerImpl;
private static volatile CGroupsHandler cGroupsHandler;
private static volatile CGroupsBlkioResourceHandlerImpl
cGroupsBlkioResourceHandler;
private static volatile MemoryResourceHandler
cGroupsMemoryResourceHandler;
private static volatile CpuResourceHandler
cGroupsCpuResourceHandler;

/**
* Returns an initialized, thread-safe CGroupsHandler instance.
*/
private static CGroupsHandler getInitializedCGroupsHandler(Configuration conf)
private static void initializeCGroupHandlers(Configuration conf)
throws ResourceHandlerException {
initializeCGroupV1Handler(conf);
if (cgroupsV2Enabled) {
initializeCGroupV2Handler(conf);
}
}

private static void initializeCGroupV1Handler(Configuration conf)
throws ResourceHandlerException {
if (cGroupsHandler == null) {
if (cGroupV1Handler == null) {
synchronized (CGroupsHandler.class) {
if (cGroupsHandler == null) {
cGroupsHandler = cgroupsV2Enabled
? new CGroupsV2HandlerImpl(conf, PrivilegedOperationExecutor.getInstance(conf))
: new CGroupsHandlerImpl(conf, PrivilegedOperationExecutor.getInstance(conf));
LOG.debug("Value of CGroupsHandler is: {}", cGroupsHandler);
if (cGroupV1Handler == null) {
cGroupV1Handler = new CGroupsHandlerImpl(
conf, PrivilegedOperationExecutor.getInstance(conf));
LOG.debug("Value of CGroupsV1Handler is: {}", cGroupV1Handler);
}
}
}
}

return cGroupsHandler;
private static void initializeCGroupV2Handler(Configuration conf)
throws ResourceHandlerException {
if (cGroupV2Handler == null) {
synchronized (CGroupsHandler.class) {
if (cGroupV2Handler == null) {
cGroupV2Handler = new CGroupsV2HandlerImpl(
conf, PrivilegedOperationExecutor.getInstance(conf));
LOG.debug("Value of CGroupsV2Handler is: {}", cGroupV2Handler);
}
}
}
}

private static boolean isMountedInCGroupsV2(CGroupsHandler.CGroupController controller) {
return (cGroupV2Handler != null && cGroupV2Handler.getControllerPath(controller) != null);
}

/**
Expand All @@ -101,18 +121,18 @@ private static CGroupsHandler getInitializedCGroupsHandler(Configuration conf)
*/

public static CGroupsHandler getCGroupsHandler() {
return cGroupsHandler;
return cGroupV1Handler;
}

/**
* Returns relative root for cgroups. Returns null if cGroupsHandler is
* not initialized, or if the path is empty.
*/
public static String getCgroupsRelativeRoot() {
if (cGroupsHandler == null) {
if (getCGroupsHandler() == null) {
return null;
}
String cGroupPath = cGroupsHandler.getRelativePathForCGroup("");
String cGroupPath = getCGroupsHandler().getRelativePathForCGroup("");
if (cGroupPath == null || cGroupPath.isEmpty()) {
return null;
}
Expand Down Expand Up @@ -153,9 +173,13 @@ private static CpuResourceHandler initCGroupsCpuResourceHandler(
synchronized (CpuResourceHandler.class) {
if (cGroupsCpuResourceHandler == null) {
LOG.debug("Creating new cgroups cpu handler");
cGroupsCpuResourceHandler = cgroupsV2Enabled
? new CGroupsV2CpuResourceHandlerImpl(getInitializedCGroupsHandler(conf))
: new CGroupsCpuResourceHandlerImpl(getInitializedCGroupsHandler(conf));

initializeCGroupHandlers(conf);
if (isMountedInCGroupsV2(CGroupsHandler.CGroupController.CPU)) {
cGroupsCpuResourceHandler = new CGroupsV2CpuResourceHandlerImpl(cGroupV2Handler);
} else {
cGroupsCpuResourceHandler = new CGroupsCpuResourceHandlerImpl(cGroupV1Handler);
}
return cGroupsCpuResourceHandler;
}
}
Expand All @@ -173,9 +197,11 @@ private static CpuResourceHandler initCGroupsCpuResourceHandler(
synchronized (OutboundBandwidthResourceHandler.class) {
if (trafficControlBandwidthHandler == null) {
LOG.info("Creating new traffic control bandwidth handler.");

initializeCGroupHandlers(conf);
trafficControlBandwidthHandler = new
TrafficControlBandwidthHandlerImpl(PrivilegedOperationExecutor
.getInstance(conf), getInitializedCGroupsHandler(conf),
.getInstance(conf), cGroupV1Handler,
new TrafficController(conf, PrivilegedOperationExecutor
.getInstance(conf)));
}
Expand Down Expand Up @@ -208,10 +234,11 @@ public static ResourceHandler getNetworkTaggingHandler(Configuration conf)
synchronized (OutboundBandwidthResourceHandler.class) {
if (networkPacketTaggingHandlerImpl == null) {
LOG.info("Creating new network-tagging-handler.");

initializeCGroupHandlers(conf);
networkPacketTaggingHandlerImpl =
new NetworkPacketTaggingHandlerImpl(
PrivilegedOperationExecutor.getInstance(conf),
getInitializedCGroupsHandler(conf));
PrivilegedOperationExecutor.getInstance(conf), cGroupV1Handler);
}
}
}
Expand Down Expand Up @@ -239,9 +266,10 @@ private static CGroupsBlkioResourceHandlerImpl getCgroupsBlkioResourceHandler(
synchronized (DiskResourceHandler.class) {
if (cGroupsBlkioResourceHandler == null) {
LOG.debug("Creating new cgroups blkio handler");

initializeCGroupHandlers(conf);
cGroupsBlkioResourceHandler =
new CGroupsBlkioResourceHandlerImpl(
getInitializedCGroupsHandler(conf));
new CGroupsBlkioResourceHandlerImpl(cGroupV1Handler);
}
}
}
Expand All @@ -263,9 +291,13 @@ public static MemoryResourceHandler initMemoryResourceHandler(
if (cGroupsMemoryResourceHandler == null) {
synchronized (MemoryResourceHandler.class) {
if (cGroupsMemoryResourceHandler == null) {
cGroupsMemoryResourceHandler = cgroupsV2Enabled
? new CGroupsV2MemoryResourceHandlerImpl(getInitializedCGroupsHandler(conf))
: new CGroupsMemoryResourceHandlerImpl(getInitializedCGroupsHandler(conf));

initializeCGroupHandlers(conf);
if (isMountedInCGroupsV2(CGroupsHandler.CGroupController.MEMORY)) {
cGroupsMemoryResourceHandler = new CGroupsV2MemoryResourceHandlerImpl(cGroupV2Handler);
} else {
cGroupsMemoryResourceHandler = new CGroupsMemoryResourceHandlerImpl(cGroupV1Handler);
}
}
}
}
Expand Down Expand Up @@ -327,9 +359,10 @@ private static void addHandlersFromConfiguredResourcePlugins(
}

for (ResourcePlugin plugin : pluginMap.values()) {
initializeCGroupHandlers(conf);
addHandlerIfNotNull(handlerList,
plugin.createResourceHandler(nmContext,
getInitializedCGroupsHandler(conf),
cGroupV1Handler,
PrivilegedOperationExecutor.getInstance(conf)));
}
}
Expand Down Expand Up @@ -360,21 +393,6 @@ static void nullifyResourceHandlerChain() throws ResourceHandlerException {
resourceHandlerChain = null;
}

@VisibleForTesting
static void resetCgroupsHandler() {
cGroupsHandler = null;
}

@VisibleForTesting
static void resetCpuResourceHandler() {
cGroupsCpuResourceHandler = null;
}

@VisibleForTesting
static void resetMemoryResourceHandler() {
cGroupsMemoryResourceHandler = null;
}

/**
* If a cgroup mount directory is specified, it returns cgroup directories
* with valid names.
Expand Down
Loading

0 comments on commit 129d91f

Please sign in to comment.